Deprecate object metadata maps in favor of flat entities (#16080)

## Context
Deprecating the old objectMetadataMap type in favour of split flat
entities to match with our new caching.
In the long run, trying to achieve:
- Better performance through caching
- Consistent data access patterns across the codebase
- Reduced database queries

Now that everything is based on flat entities, which are cached, we can
finish the refactoring of workspace context cache which should already
improve performances.
Then the last step will be to consume that new cache in the new global
datasource to get rid of the many workspace datasources stored in the
server
This commit is contained in:
Weiko
2025-11-27 13:43:34 +01:00
committed by GitHub
parent 978c0acb90
commit 1607aebcc6
281 changed files with 6510 additions and 6627 deletions
@@ -11,6 +11,7 @@ import { AgentService } from 'src/engine/metadata-modules/ai/ai-agent/agent.serv
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
@@ -145,6 +146,12 @@ describe('WorkspaceManagerService', () => {
}),
},
},
{
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
useValue: {
invalidateEntireCache: jest.fn(),
},
},
],
}).compile();
@@ -8,8 +8,9 @@ import { DataSource } from 'typeorm';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import {
@@ -268,6 +269,7 @@ export class DevSeederDataService {
private readonly objectMetadataService: ObjectMetadataServiceV2,
private readonly timelineActivitySeederService: TimelineActivitySeederService,
private readonly fileStorageService: FileStorageService,
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
) {}
public async seed({
@@ -282,6 +284,14 @@ export class DevSeederDataService {
const objectMetadataItems =
await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
},
);
await this.coreDataSource.transaction(
async (entityManager: WorkspaceEntityManager) => {
await this.seedRecordsInBatches({
@@ -300,7 +310,12 @@ export class DevSeederDataService {
await this.seedAttachmentFiles(workspaceId);
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
await prefillWorkflows(
entityManager,
schemaName,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
);
},
);
}
@@ -316,7 +331,7 @@ export class DevSeederDataService {
schemaName: string;
workspaceId: string;
featureFlags?: Record<FeatureFlagKey, boolean>;
objectMetadataItems: ObjectMetadataEntity[];
objectMetadataItems: FlatObjectMetadata[];
}) {
const batches = getRecordSeedsBatches(workspaceId, featureFlags);
@@ -193,13 +193,19 @@ export class DevSeederMetadataService {
featureFlags?: Record<string, boolean>;
twentyStandardFlatApplication: FlatApplication;
}): Promise<void> {
const createdObjectMetadata =
await this.objectMetadataServiceV2.findManyWithinWorkspace(workspaceId);
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
},
);
await prefillCoreViews({
coreDataSource: this.coreDataSource,
workspaceId,
objectMetadataItems: createdObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
workspaceSchemaName: dataSourceMetadata.schema,
featureFlags,
twentyStandardFlatApplication,
@@ -1,7 +1,11 @@
import { isString } from '@sniptt/guards';
import { type DataSource, type QueryRunner } from 'typeorm';
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ViewFieldEntity } from 'src/engine/metadata-modules/view-field/entities/view-field.entity';
import { ViewFilterEntity } from 'src/engine/metadata-modules/view-filter/entities/view-filter.entity';
@@ -37,19 +41,46 @@ import { workspaceMembersAllView } from 'src/engine/workspace-manager/standard-o
type PrefillCoreViewsArgs = {
coreDataSource: DataSource;
workspaceId: string;
objectMetadataItems: ObjectMetadataEntity[];
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
featureFlags?: Record<string, boolean>;
workspaceSchemaName: string;
twentyStandardFlatApplication: FlatApplication;
};
// This is a temporary function to build the object metadata items from the flat maps.
// We should use the maps in the seeders instead.
const buildObjectMetadataItemsFromFlatMaps = (
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
): ObjectMetadataEntity[] => {
return Object.values(flatObjectMetadataMaps.byId)
.filter((flatObjectMetadata) => flatObjectMetadata !== undefined)
.map((flatObjectMetadata) => {
const fields = getFlatFieldsFromFlatObjectMetadata(
flatObjectMetadata,
flatFieldMetadataMaps,
);
return {
...flatObjectMetadata,
fields,
} as unknown as ObjectMetadataEntity;
});
};
export const prefillCoreViews = async ({
coreDataSource,
workspaceId,
objectMetadataItems,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
workspaceSchemaName,
twentyStandardFlatApplication,
}: PrefillCoreViewsArgs): Promise<ViewEntity[]> => {
const objectMetadataItems = buildObjectMetadataItemsFromFlatMaps(
flatObjectMetadataMaps,
flatFieldMetadataMaps,
);
const views = [
companiesAllView,
peopleAllView,
@@ -1,9 +1,11 @@
import { FieldActorSource } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { type EntityManager } from 'typeorm';
import { FieldActorSource } from 'twenty-shared/types';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { generateObjectMetadataMaps } from 'src/engine/metadata-modules/utils/generate-object-metadata-maps.util';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
import { generateObjectRecordFields } from 'src/modules/workflow/workflow-builder/workflow-schema/utils/generate-object-record-fields';
const QUICK_LEAD_WORKFLOW_ID = '8b213cac-a68b-4ffe-817a-3ec994e9932d';
@@ -12,13 +14,15 @@ const QUICK_LEAD_WORKFLOW_VERSION_ID = 'ac67974f-c524-4288-9d88-af8515400b68';
export const prefillWorkflows = async (
entityManager: EntityManager,
schemaName: string,
objectMetadataItems: ObjectMetadataEntity[],
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
) => {
const objectMetadataMaps = generateObjectMetadataMaps(objectMetadataItems);
const companyObjectMetadataId =
objectMetadataMaps.idByNameSingular['company'];
const { idByNameSingular: objectIdByNameSingular } = buildObjectIdByNameMaps(
flatObjectMetadataMaps,
);
const personObjectMetadataId = objectMetadataMaps.idByNameSingular['person'];
const companyObjectMetadataId = objectIdByNameSingular['company'];
const personObjectMetadataId = objectIdByNameSingular['person'];
if (
!isDefined(companyObjectMetadataId) ||
@@ -28,9 +32,10 @@ export const prefillWorkflows = async (
}
const companyObjectMetadata =
objectMetadataMaps.byId[companyObjectMetadataId];
flatObjectMetadataMaps.byId[companyObjectMetadataId];
const personObjectMetadata = objectMetadataMaps.byId[personObjectMetadataId];
const personObjectMetadata =
flatObjectMetadataMaps.byId[personObjectMetadataId];
if (!isDefined(companyObjectMetadata) || !isDefined(personObjectMetadata)) {
throw new Error('Company or person object metadata not found');
@@ -220,8 +225,9 @@ export const prefillWorkflows = async (
_outputSchemaType: 'RECORD',
fields: generateObjectRecordFields({
objectMetadataInfo: {
objectMetadataItemWithFieldsMaps: companyObjectMetadata,
objectMetadataMaps,
flatObjectMetadata: companyObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
},
depth: 0,
}),
@@ -262,8 +268,9 @@ export const prefillWorkflows = async (
outputSchema: {
fields: generateObjectRecordFields({
objectMetadataInfo: {
objectMetadataItemWithFieldsMaps: personObjectMetadata,
objectMetadataMaps,
flatObjectMetadata: personObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
},
depth: 0,
}),
@@ -1,6 +1,8 @@
import { type DataSource, type EntityManager } from 'typeorm';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { prefillCompanies } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-companies';
import { prefillPeople } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-people';
import { prefillWorkflows } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-workflows';
@@ -8,13 +10,19 @@ import { prefillWorkflows } from 'src/engine/workspace-manager/standard-objects-
export const standardObjectsPrefillData = async (
dataSource: DataSource,
schemaName: string,
objectMetadataItems: ObjectMetadataEntity[],
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
) => {
dataSource.transaction(async (entityManager: EntityManager) => {
await prefillCompanies(entityManager, schemaName);
await prefillPeople(entityManager, schemaName);
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
await prefillWorkflows(
entityManager,
schemaName,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
);
});
};
@@ -1,10 +0,0 @@
import { Module } from '@nestjs/common';
import { WorkspaceHealthCommand } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health.command';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
@Module({
imports: [WorkspaceHealthModule],
providers: [WorkspaceHealthCommand],
})
export class WorkspaceHealthCommandModule {}
@@ -1,145 +0,0 @@
import { Logger } from '@nestjs/common';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import { WorkspaceHealthFixKind } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-fix-kind.interface';
import { WorkspaceHealthMode } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-options.interface';
import { CommandLogger } from 'src/command/command-logger';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
interface WorkspaceHealthCommandOptions {
workspaceId: string;
mode?: WorkspaceHealthMode;
fix?: WorkspaceHealthFixKind;
dryRun?: boolean;
}
@Command({
name: 'workspace:health',
description: 'Check health of the given workspace.',
})
export class WorkspaceHealthCommand extends CommandRunner {
private readonly logger = new Logger(WorkspaceHealthCommand.name);
private readonly commandLogger = new CommandLogger(
WorkspaceHealthCommand.name,
);
constructor(private readonly workspaceHealthService: WorkspaceHealthService) {
super();
}
async run(
_passedParam: string[],
options: WorkspaceHealthCommandOptions,
): Promise<void> {
const issues = await this.workspaceHealthService.healthCheck(
options.workspaceId,
{
mode: options.mode ?? WorkspaceHealthMode.All,
},
);
if (issues.length === 0) {
this.logger.log(chalk.green('Workspace is healthy'));
} else {
this.logger.log(
chalk.red(`Workspace is not healthy, found ${issues.length} issues`),
);
for (let issueIndex = 0; issueIndex < issues.length; issueIndex++) {
this.logger.log(
chalk.red(`Issue #${issueIndex + 1} : ${issues[issueIndex].message}`),
);
}
const logFilePath = await this.commandLogger.writeLog(
`workspace-health-issues-${options.workspaceId}`,
issues,
);
this.logger.log(
chalk.yellow(`Issues written to log at : ${logFilePath}`),
);
}
if (options.fix) {
this.logger.log(chalk.yellow('Fixing issues'));
const { workspaceMigrations, metadataEntities } =
await this.workspaceHealthService.fixIssues(
options.workspaceId,
issues,
{
type: options.fix,
applyChanges: !options.dryRun,
},
);
const totalCount = workspaceMigrations.length + metadataEntities.length;
if (options.dryRun) {
await this.commandLogger.writeLog(
`workspace-health-${options.fix}-migrations`,
workspaceMigrations,
);
await this.commandLogger.writeLog(
`workspace-health-${options.fix}-metadata-entities`,
metadataEntities,
);
} else {
this.logger.log(
chalk.green(`Fixed ${totalCount}/${issues.length} issues`),
);
}
}
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
@Option({
flags: '-f, --fix [kind]',
description: 'fix issues',
required: false,
})
fix(value: string): WorkspaceHealthFixKind {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (!Object.values(WorkspaceHealthFixKind).includes(value as any)) {
throw new Error(`Invalid fix kind ${value}`);
}
return value as WorkspaceHealthFixKind;
}
@Option({
flags: '-m, --mode [mode]',
description: 'Mode of the health check [structure, metadata, all]',
required: false,
defaultValue: WorkspaceHealthMode.All,
})
parseMode(value: string): WorkspaceHealthMode {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (!Object.values(WorkspaceHealthMode).includes(value as any)) {
throw new Error(`Invalid mode ${value}`);
}
return value as WorkspaceHealthMode;
}
@Option({
flags: '-d, --dry-run',
description: 'Dry run without applying changes',
required: false,
})
dryRun(): boolean {
return true;
}
}
@@ -1,68 +0,0 @@
import { type EntityManager } from 'typeorm';
import {
type WorkspaceHealthIssue,
type WorkspaceHealthIssueType,
type WorkspaceIssueTypeToInterface,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
export class CompareEntity<T> {
current: T | null;
altered: T | null;
}
export abstract class AbstractWorkspaceFixer<
IssueTypes extends WorkspaceHealthIssueType,
UpdateRecordEntities = unknown,
> {
private issueTypes: IssueTypes[];
protected constructor(...issueTypes: IssueTypes[]) {
this.issueTypes = issueTypes;
}
filterIssues(
issues: WorkspaceHealthIssue[],
): WorkspaceIssueTypeToInterface<IssueTypes>[] {
return issues.filter(
(issue): issue is WorkspaceIssueTypeToInterface<IssueTypes> =>
this.issueTypes.includes(issue.type as IssueTypes),
);
}
protected splitIssuesByType(
issues: WorkspaceIssueTypeToInterface<IssueTypes>[],
): Record<IssueTypes, WorkspaceIssueTypeToInterface<IssueTypes>[]> {
return issues.reduce(
(
acc: Record<IssueTypes, WorkspaceIssueTypeToInterface<IssueTypes>[]>,
issue,
) => {
const type = issue.type as IssueTypes;
if (!acc[type]) {
acc[type] = [];
}
acc[type].push(issue);
return acc;
},
{} as Record<IssueTypes, WorkspaceIssueTypeToInterface<IssueTypes>[]>,
);
}
async createWorkspaceMigrations?(
manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceIssueTypeToInterface<IssueTypes>[],
): Promise<Partial<WorkspaceMigrationEntity>[]>;
async createMetadataUpdates?(
manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceIssueTypeToInterface<IssueTypes>[],
): Promise<CompareEntity<UpdateRecordEntities>[]>;
}
@@ -1,12 +0,0 @@
import { WorkspaceMissingColumnFixer } from 'src/engine/workspace-manager/workspace-health/fixer/workspace-missing-column.fixer';
import { WorkspaceNullableFixer } from './workspace-nullable.fixer';
import { WorkspaceDefaultValueFixer } from './workspace-default-value.fixer';
import { WorkspaceTypeFixer } from './workspace-type.fixer';
export const workspaceFixers = [
WorkspaceNullableFixer,
WorkspaceDefaultValueFixer,
WorkspaceTypeFixer,
WorkspaceMissingColumnFixer,
];
@@ -1,229 +0,0 @@
import { Injectable } from '@nestjs/common';
import { type EntityManager } from 'typeorm';
import { type FieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-default-value.interface';
import {
type WorkspaceHealthColumnIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import {
type FieldMetadataDefaultValueFunctionNames,
fieldMetadataDefaultValueFunctionName,
} from 'src/engine/metadata-modules/field-metadata/dtos/default-value.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationFieldFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import {
AbstractWorkspaceFixer,
type CompareEntity,
} from './abstract-workspace.fixer';
type WorkspaceDefaultValueFixerType =
| WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT
| WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID;
@Injectable()
export class WorkspaceDefaultValueFixer extends AbstractWorkspaceFixer<WorkspaceDefaultValueFixerType> {
constructor(
private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory,
) {
super(
WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT,
WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
);
}
async createWorkspaceMigrations(
_manager: EntityManager,
_objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceDefaultValueFixerType>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
if (issues.length <= 0) {
return [];
}
const splittedIssues = this.splitIssuesByType(issues);
const issueNeedingMigration =
splittedIssues[WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT] ??
[];
return this.fixColumnDefaultValueConflictIssues(
_objectMetadataCollection,
issueNeedingMigration as WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT>[],
);
}
async createMetadataUpdates(
_manager: EntityManager,
_objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceDefaultValueFixerType>[],
): Promise<CompareEntity<FieldMetadataEntity>[]> {
if (issues.length <= 0) {
return [];
}
const splittedIssues = this.splitIssuesByType(issues);
const issueNeedingMetadataUpdate =
splittedIssues[WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID] ??
[];
return this.fixColumnDefaultValueNotValidIssues(
_manager,
issueNeedingMetadataUpdate as WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID>[],
);
}
private async fixColumnDefaultValueConflictIssues(
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const fieldMetadataUpdateCollection = issues.map((issue) => {
const oldDefaultValue =
this.computeFieldMetadataDefaultValueFromColumnDefault(
issue.columnStructure?.columnDefault,
);
return {
current: {
...issue.fieldMetadata,
defaultValue: oldDefaultValue,
},
altered: issue.fieldMetadata,
};
});
return this.workspaceMigrationFieldFactory.create(
objectMetadataCollection,
fieldMetadataUpdateCollection,
WorkspaceMigrationBuilderAction.UPDATE,
);
}
private async fixColumnDefaultValueNotValidIssues(
manager: EntityManager,
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID>[],
): Promise<CompareEntity<FieldMetadataEntity>[]> {
const fieldMetadataRepository = manager.getRepository(FieldMetadataEntity);
const updatedEntities: CompareEntity<FieldMetadataEntity>[] = [];
for (const issue of issues) {
const currentDefaultValue:
| FieldMetadataDefaultValue
// Old format for default values
// TODO: Remove this after all workspaces are migrated
| { type: FieldMetadataDefaultValueFunctionNames }
| null = issue.fieldMetadata.defaultValue;
let alteredDefaultValue: FieldMetadataDefaultValue | null = null;
// Check if it's an old function default value
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
if (currentDefaultValue && 'type' in currentDefaultValue) {
alteredDefaultValue =
currentDefaultValue.type as FieldMetadataDefaultValueFunctionNames;
}
// Check if it's an old string default value
if (currentDefaultValue) {
for (const key of Object.keys(currentDefaultValue)) {
if (key === 'type') {
continue;
}
// @ts-expect-error legacy noImplicitAny
const value = currentDefaultValue[key];
const newValue =
typeof value === 'string' &&
!value.startsWith("'") &&
!Object.values(fieldMetadataDefaultValueFunctionName).includes(
value as FieldMetadataDefaultValueFunctionNames,
)
? `'${value}'`
: value;
alteredDefaultValue = {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...(currentDefaultValue as any),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...(alteredDefaultValue as any),
[key]: newValue,
};
}
}
// Old formart default values
if (
alteredDefaultValue &&
typeof alteredDefaultValue === 'object' &&
'value' in alteredDefaultValue
) {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
alteredDefaultValue = alteredDefaultValue.value;
}
if (alteredDefaultValue === null) {
continue;
}
await fieldMetadataRepository.update(issue.fieldMetadata.id, {
defaultValue: alteredDefaultValue,
});
const alteredEntity = await fieldMetadataRepository.findOne({
where: {
id: issue.fieldMetadata.id,
},
});
updatedEntities.push({
current: issue.fieldMetadata,
altered: alteredEntity as FieldMetadataEntity | null,
});
}
return updatedEntities;
}
private computeFieldMetadataDefaultValueFromColumnDefault(
columnDefault: string | undefined,
): FieldMetadataDefaultValue {
if (
columnDefault === undefined ||
columnDefault === null ||
columnDefault === 'NULL'
) {
return null;
}
if (!isNaN(Number(columnDefault))) {
return +columnDefault;
}
if (columnDefault === 'true') {
return true;
}
if (columnDefault === 'false') {
return false;
}
if (columnDefault === '') {
return "''";
}
if (columnDefault === 'now()') {
return 'now';
}
if (columnDefault.startsWith('public.uuid_generate_v4')) {
return 'uuid';
}
return columnDefault;
}
}
@@ -1,82 +0,0 @@
import { Injectable } from '@nestjs/common';
import { type EntityManager } from 'typeorm';
import {
type WorkspaceHealthColumnIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import {
type FieldMetadataUpdate,
WorkspaceMigrationFieldFactory,
} from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import { AbstractWorkspaceFixer } from './abstract-workspace.fixer';
@Injectable()
export class WorkspaceMissingColumnFixer extends AbstractWorkspaceFixer<WorkspaceHealthIssueType.MISSING_COLUMN> {
constructor(
private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory,
) {
super(WorkspaceHealthIssueType.MISSING_COLUMN);
}
async createWorkspaceMigrations(
_manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.MISSING_COLUMN>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
if (issues.length <= 0) {
return [];
}
return this.fixMissingColumnIssues(objectMetadataCollection, issues);
}
private async fixMissingColumnIssues(
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.MISSING_COLUMN>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const fieldMetadataUpdateCollection: FieldMetadataUpdate[] = [];
for (const issue of issues) {
if (!issue.columnStructures) {
continue;
}
/**
* Check if the column is prefixed with an underscore as it was the old convention
*/
const oldColumnName = `_${issue.fieldMetadata.name}`;
const oldColumnStructure = issue.columnStructures.find(
(columnStructure) => columnStructure.columnName === oldColumnName,
);
if (!oldColumnStructure) {
continue;
}
fieldMetadataUpdateCollection.push({
current: {
...issue.fieldMetadata,
name: oldColumnName,
},
altered: issue.fieldMetadata,
});
}
if (fieldMetadataUpdateCollection.length <= 0) {
return [];
}
return this.workspaceMigrationFieldFactory.create(
objectMetadataCollection,
fieldMetadataUpdateCollection,
WorkspaceMigrationBuilderAction.UPDATE,
);
}
}
@@ -1,57 +0,0 @@
import { Injectable } from '@nestjs/common';
import { type EntityManager } from 'typeorm';
import {
type WorkspaceHealthColumnIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationFieldFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import { AbstractWorkspaceFixer } from './abstract-workspace.fixer';
@Injectable()
export class WorkspaceNullableFixer extends AbstractWorkspaceFixer<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT> {
constructor(
private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory,
) {
super(WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT);
}
async createWorkspaceMigrations(
_manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
if (issues.length <= 0) {
return [];
}
return this.fixColumnNullabilityIssues(objectMetadataCollection, issues);
}
private async fixColumnNullabilityIssues(
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const fieldMetadataUpdateCollection = issues.map((issue) => {
return {
current: {
...issue.fieldMetadata,
isNullable: issue.columnStructure?.isNullable ?? false,
},
altered: issue.fieldMetadata,
};
});
return this.workspaceMigrationFieldFactory.create(
objectMetadataCollection,
fieldMetadataUpdateCollection,
WorkspaceMigrationBuilderAction.UPDATE,
);
}
}
@@ -1,90 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { type EntityManager } from 'typeorm';
import {
type WorkspaceHealthColumnIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { DatabaseStructureService } from 'src/engine/workspace-manager/workspace-health/services/database-structure.service';
import {
type FieldMetadataUpdate,
WorkspaceMigrationFieldFactory,
} from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import { AbstractWorkspaceFixer } from './abstract-workspace.fixer';
const oldDataTypes = ['integer'];
@Injectable()
export class WorkspaceTypeFixer extends AbstractWorkspaceFixer<WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT> {
private readonly logger = new Logger(WorkspaceTypeFixer.name);
constructor(
private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory,
private readonly databaseStructureService: DatabaseStructureService,
) {
super(WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT);
}
async createWorkspaceMigrations(
_manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
if (issues.length <= 0) {
return [];
}
return this.fixColumnTypeIssues(objectMetadataCollection, issues);
}
private async fixColumnTypeIssues(
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const fieldMetadataUpdateCollection: FieldMetadataUpdate[] = [];
for (const issue of issues) {
const dataType = issue.columnStructure?.dataType;
if (!dataType) {
throw new Error('Column structure data type is missing');
}
const type =
this.databaseStructureService.getFieldMetadataTypeFromPostgresDataType(
dataType,
);
if (oldDataTypes.includes(dataType)) {
this.logger.warn(
`Old data type detected for column ${issue.columnStructure?.columnName} with data type ${dataType}. Please update the column data type manually.`,
);
continue;
}
if (!type) {
throw new Error("Can't find field metadata type from column structure");
}
fieldMetadataUpdateCollection.push({
current: {
...issue.fieldMetadata,
type,
},
altered: issue.fieldMetadata,
});
}
return this.workspaceMigrationFieldFactory.create(
objectMetadataCollection,
fieldMetadataUpdateCollection,
WorkspaceMigrationBuilderAction.UPDATE,
);
}
}
@@ -1,6 +0,0 @@
export enum WorkspaceHealthFixKind {
Nullable = 'nullable',
Type = 'type',
DefaultValue = 'default-value',
MissingColumn = 'missing-column',
}
@@ -1,104 +0,0 @@
import { type WorkspaceTableStructure } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-table-definition.interface';
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
export enum WorkspaceHealthIssueType {
MISSING_TABLE = 'MISSING_TABLE',
TABLE_NAME_SHOULD_BE_CUSTOM = 'TABLE_NAME_SHOULD_BE_CUSTOM',
TABLE_TARGET_TABLE_NAME_NOT_VALID = 'TABLE_TARGET_TABLE_NAME_NOT_VALID',
TABLE_DATA_SOURCE_ID_NOT_VALID = 'TABLE_DATA_SOURCE_ID_NOT_VALID',
TABLE_NAME_NOT_VALID = 'TABLE_NAME_NOT_VALID',
MISSING_COLUMN = 'MISSING_COLUMN',
MISSING_INDEX = 'MISSING_INDEX',
MISSING_FOREIGN_KEY = 'MISSING_FOREIGN_KEY',
MISSING_COMPOSITE_TYPE = 'MISSING_COMPOSITE_TYPE',
COLUMN_NAME_SHOULD_NOT_BE_PREFIXED = 'COLUMN_NAME_SHOULD_NOT_BE_PREFIXED',
COLUMN_NAME_SHOULD_NOT_BE_CUSTOM = 'COLUMN_NAME_SHOULD_NOT_BE_CUSTOM',
COLUMN_OBJECT_REFERENCE_INVALID = 'COLUMN_OBJECT_REFERENCE_INVALID',
COLUMN_NAME_NOT_VALID = 'COLUMN_NAME_NOT_VALID',
COLUMN_TYPE_NOT_VALID = 'COLUMN_TYPE_NOT_VALID',
COLUMN_DATA_TYPE_CONFLICT = 'COLUMN_DATA_TYPE_CONFLICT',
COLUMN_NULLABILITY_CONFLICT = 'COLUMN_NULLABILITY_CONFLICT',
COLUMN_DEFAULT_VALUE_CONFLICT = 'COLUMN_DEFAULT_VALUE_CONFLICT',
COLUMN_DEFAULT_VALUE_NOT_VALID = 'COLUMN_DEFAULT_VALUE_NOT_VALID',
COLUMN_OPTIONS_NOT_VALID = 'COLUMN_OPTIONS_NOT_VALID',
RELATION_METADATA_NOT_VALID = 'RELATION_METADATA_NOT_VALID',
RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID = 'RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID',
RELATION_FOREIGN_KEY_NOT_VALID = 'RELATION_FOREIGN_KEY_NOT_VALID',
RELATION_FOREIGN_KEY_CONFLICT = 'RELATION_FOREIGN_KEY_CONFLICT',
RELATION_FOREIGN_KEY_ON_DELETE_ACTION_CONFLICT = 'RELATION_FOREIGN_KEY_ON_DELETE_ACTION_CONFLICT',
RELATION_TYPE_NOT_VALID = 'RELATION_TYPE_NOT_VALID',
}
/**
* Table issues
*/
export type WorkspaceTableIssueTypes =
| WorkspaceHealthIssueType.MISSING_TABLE
| WorkspaceHealthIssueType.TABLE_NAME_SHOULD_BE_CUSTOM
| WorkspaceHealthIssueType.TABLE_TARGET_TABLE_NAME_NOT_VALID
| WorkspaceHealthIssueType.TABLE_DATA_SOURCE_ID_NOT_VALID
| WorkspaceHealthIssueType.TABLE_NAME_NOT_VALID;
export interface WorkspaceHealthTableIssue<T extends WorkspaceTableIssueTypes> {
type: T;
objectMetadata: ObjectMetadataEntity;
message: string;
}
/**
* Column issues
*/
export type WorkspaceColumnIssueTypes =
| WorkspaceHealthIssueType.MISSING_COLUMN
| WorkspaceHealthIssueType.MISSING_INDEX
| WorkspaceHealthIssueType.MISSING_FOREIGN_KEY
| WorkspaceHealthIssueType.MISSING_COMPOSITE_TYPE
| WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_PREFIXED
| WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_CUSTOM
| WorkspaceHealthIssueType.COLUMN_OBJECT_REFERENCE_INVALID
| WorkspaceHealthIssueType.COLUMN_NAME_NOT_VALID
| WorkspaceHealthIssueType.COLUMN_TYPE_NOT_VALID
| WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT
| WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT
| WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT
| WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID
| WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID;
export interface WorkspaceHealthColumnIssue<
T extends WorkspaceColumnIssueTypes,
> {
type: T;
fieldMetadata: FieldMetadataEntity;
columnStructure?: WorkspaceTableStructure;
columnStructures?: WorkspaceTableStructure[];
message: string;
}
/**
* Relation issues
*/
export type WorkspaceRelationIssueTypes =
| WorkspaceHealthIssueType.RELATION_METADATA_NOT_VALID
| WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_ON_DELETE_ACTION_CONFLICT
| WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID;
/**
* Get the interface for the issue type
*/
export type WorkspaceIssueTypeToInterface<T extends WorkspaceHealthIssueType> =
T extends WorkspaceTableIssueTypes
? WorkspaceHealthTableIssue<T>
: T extends WorkspaceColumnIssueTypes
? WorkspaceHealthColumnIssue<T>
: never;
/**
* Union of all issues
*/
export type WorkspaceHealthIssue =
WorkspaceIssueTypeToInterface<WorkspaceHealthIssueType>;
@@ -1,9 +0,0 @@
export enum WorkspaceHealthMode {
Structure = 'structure',
Metadata = 'metadata',
All = 'all',
}
export interface WorkspaceHealthOptions {
mode: WorkspaceHealthMode;
}
@@ -1,18 +0,0 @@
export interface WorkspaceTableStructure {
tableSchema: string;
tableName: string;
columnName: string;
dataType: string;
columnDefault: string;
isNullable: boolean;
isUnique: boolean;
isPrimaryKey: boolean;
isForeignKey: boolean;
isArray: boolean;
onUpdateAction: string;
onDeleteAction: string;
}
export type WorkspaceTableStructureResult = {
[P in keyof WorkspaceTableStructure]: string;
};
@@ -1,328 +0,0 @@
import { Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import {
FieldMetadataType,
compositeTypeDefinitions,
} from 'twenty-shared/types';
import { DataSource, type ColumnType } from 'typeorm';
import { type ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
import {
type FieldMetadataDefaultValue,
type FieldMetadataFunctionDefaultValue,
} from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-default-value.interface';
import {
type WorkspaceTableStructure,
type WorkspaceTableStructureResult,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-table-definition.interface';
import { type FieldMetadataDefaultValueFunctionNames } from 'src/engine/metadata-modules/field-metadata/dtos/default-value.input';
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { isFunctionDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/is-function-default-value.util';
import { serializeFunctionDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-function-default-value.util';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { isMorphOrRelationFieldMetadataType } from 'src/engine/utils/is-morph-or-relation-field-metadata-type.util';
@Injectable()
export class DatabaseStructureService {
constructor(
@InjectDataSource()
private readonly coreDataSource: DataSource,
) {}
async getWorkspaceTableColumns(
schemaName: string,
tableName: string,
): Promise<WorkspaceTableStructure[]> {
const results = await this.coreDataSource.query<
WorkspaceTableStructureResult[]
>(`
WITH foreign_keys AS (
SELECT
kcu.table_schema AS schema_name,
kcu.table_name AS table_name,
kcu.column_name AS column_name,
tc.constraint_name AS constraint_name
FROM
information_schema.key_column_usage AS kcu
JOIN
information_schema.table_constraints AS tc
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = '${schemaName}'
AND tc.table_name = '${tableName}'
),
unique_constraints AS (
SELECT
tc.table_schema AS schema_name,
tc.table_name AS table_name,
kcu.column_name AS column_name
FROM
information_schema.key_column_usage AS kcu
JOIN
information_schema.table_constraints AS tc
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'UNIQUE'
AND tc.table_schema = '${schemaName}'
AND tc.table_name = '${tableName}'
)
SELECT
c.table_schema AS "tableSchema",
c.table_name AS "tableName",
c.column_name AS "columnName",
CASE
WHEN c.data_type = 'ARRAY' THEN
(SELECT typname FROM pg_type WHERE oid = t.typelem)
WHEN c.data_type = 'USER-DEFINED' THEN
c.udt_name
ELSE
c.data_type
END AS "dataType",
c.is_nullable AS "isNullable",
c.column_default AS "columnDefault",
CASE
WHEN pk.constraint_type = 'PRIMARY KEY' THEN 'TRUE'
ELSE 'FALSE'
END AS "isPrimaryKey",
CASE
WHEN fk.constraint_name IS NOT NULL THEN 'TRUE'
ELSE 'FALSE'
END AS "isForeignKey",
CASE
WHEN uc.column_name IS NOT NULL THEN 'TRUE'
ELSE 'FALSE'
END AS "isUnique",
CASE
WHEN c.data_type = 'ARRAY' THEN 'TRUE'
ELSE 'FALSE'
END AS "isArray",
rc.update_rule AS "onUpdateAction",
rc.delete_rule AS "onDeleteAction"
FROM
information_schema.columns AS c
LEFT JOIN pg_type t ON t.typname = c.udt_name
LEFT JOIN
information_schema.constraint_column_usage AS ccu
ON c.column_name = ccu.column_name
AND c.table_name = ccu.table_name
AND c.table_schema = ccu.table_schema
LEFT JOIN
information_schema.table_constraints AS pk
ON pk.constraint_name = ccu.constraint_name
AND pk.constraint_type = 'PRIMARY KEY'
AND pk.table_name = c.table_name
AND pk.table_schema = c.table_schema
LEFT JOIN
foreign_keys AS fk
ON c.table_schema = fk.schema_name
AND c.table_name = fk.table_name
AND c.column_name = fk.column_name
LEFT JOIN
unique_constraints AS uc
ON c.table_schema = uc.schema_name
AND c.table_name = uc.table_name
AND c.column_name = uc.column_name
LEFT JOIN
information_schema.referential_constraints AS rc
ON rc.constraint_name = fk.constraint_name
AND rc.constraint_schema = '${schemaName}'
WHERE
c.table_schema = '${schemaName}'
AND c.table_name = '${tableName}';
`);
if (!results || results.length === 0) {
return [];
}
return results.map((item) => ({
...item,
dataType: item.isArray === 'TRUE' ? `${item.dataType}[]` : item.dataType,
isNullable: item.isNullable === 'YES',
isPrimaryKey: item.isPrimaryKey === 'TRUE',
isForeignKey: item.isForeignKey === 'TRUE',
isUnique: item.isUnique === 'TRUE',
isArray: item.isArray === 'TRUE',
}));
}
getPostgresDataTypes(fieldMetadata: FieldMetadataEntity): string[] {
const normalizer = (
type: FieldMetadataType,
isArray: boolean | undefined,
columnName: string,
) => {
const typeORMType = fieldMetadataTypeToColumnType(type);
// Compute enum name to compare data type properly
if (typeORMType === 'enum') {
const objectName = computeObjectTargetTable(fieldMetadata.object);
return `${objectName}_${columnName}_enum${isArray ? '[]' : ''}`;
}
return this.coreDataSource.driver.normalizeType({
type: typeORMType,
});
};
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);
if (!compositeType) {
throw new Error(
`Composite type definition not found for ${fieldMetadata.type}`,
);
}
return compositeType.properties.map((compositeProperty) =>
normalizer(
compositeProperty.type,
compositeProperty.isArray,
compositeProperty.name,
),
);
}
return [
normalizer(
fieldMetadata.type,
fieldMetadata.type === FieldMetadataType.MULTI_SELECT,
fieldMetadata.name,
),
];
}
getFieldMetadataTypeFromPostgresDataType(
postgresDataType: string,
): FieldMetadataType | null {
const types = Object.values(FieldMetadataType).filter((type) => {
// We're skipping composite and relation types, as they're not directly mapped to a column type
if (isCompositeFieldMetadataType(type)) {
return false;
}
if (isMorphOrRelationFieldMetadataType(type)) {
return false;
}
return true;
});
for (const type of types) {
const typeORMType = fieldMetadataTypeToColumnType(
FieldMetadataType[type],
) as ColumnType;
const dataType = this.coreDataSource.driver.normalizeType({
type: typeORMType,
});
if (postgresDataType === dataType) {
return FieldMetadataType[type];
}
}
return null;
}
getPostgresDefaults(
fieldMetadataType: FieldMetadataType,
initialDefaultValue:
| FieldMetadataDefaultValue
// Old format for default values
// TODO: Should be removed once all default values are migrated
| { type: FieldMetadataDefaultValueFunctionNames }
| null,
): (string | null | undefined)[] {
const normalizer = (
type: FieldMetadataType,
defaultValue:
| FieldMetadataDefaultValue
| { type: FieldMetadataDefaultValueFunctionNames }
| null,
) => {
const typeORMType = fieldMetadataTypeToColumnType(type) as ColumnType;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let value: any =
// Old formart default values
defaultValue &&
typeof defaultValue === 'object' &&
'value' in defaultValue
? defaultValue.value
: defaultValue;
// Old format for default values
// TODO: Should be removed once all default values are migrated
if (
defaultValue &&
typeof defaultValue === 'object' &&
'type' in defaultValue
) {
return this.computeFunctionDefaultValue(defaultValue.type);
}
if (isFunctionDefaultValue(value)) {
return this.computeFunctionDefaultValue(value);
}
if (typeof value === 'number') {
return value.toString();
}
// Remove leading and trailing single quotes for string default values as it's already handled by TypeORM
if (typeof value === 'string' && value.match(/^'.*'$/)) {
value = value.replace(/^'/, '').replace(/'$/, '');
}
return this.coreDataSource.driver.normalizeDefault({
type: typeORMType,
default: value,
isArray: false,
// Workaround to use normalizeDefault without a complete ColumnMetadata object
} as ColumnMetadata);
};
if (isCompositeFieldMetadataType(fieldMetadataType)) {
const compositeType = compositeTypeDefinitions.get(fieldMetadataType);
if (!compositeType) {
throw new Error(
`Composite type definition not found for ${fieldMetadataType}`,
);
}
return compositeType.properties.map((compositeProperty) =>
normalizer(
compositeProperty.type,
typeof initialDefaultValue === 'object'
? // @ts-expect-error legacy noImplicitAny
initialDefaultValue?.[compositeProperty.name]
: null,
),
);
}
return [normalizer(fieldMetadataType, initialDefaultValue)];
}
private computeFunctionDefaultValue(
value: FieldMetadataFunctionDefaultValue,
) {
const serializedDefaultValue = serializeFunctionDefaultValue(value);
// Special case for uuid_generate_v4() default value
if (serializedDefaultValue === 'public.uuid_generate_v4()') {
return 'uuid_generate_v4()';
}
return serializedDefaultValue;
}
}
@@ -1,347 +0,0 @@
import { Injectable } from '@nestjs/common';
import {
FieldMetadataType,
compositeTypeDefinitions,
} from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import {
type WorkspaceHealthIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { type WorkspaceHealthOptions } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-options.interface';
import { type WorkspaceTableStructure } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-table-definition.interface';
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import {
computeColumnName,
computeCompositeColumnName,
} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
import { validateDefaultValueForType } from 'src/engine/metadata-modules/field-metadata/utils/validate-default-value-for-type.util';
import { validateOptionsForType } from 'src/engine/metadata-modules/field-metadata/utils/validate-options-for-type.util';
import { customNamePrefix } from 'src/engine/utils/compute-table-name.util';
import { isMorphOrRelationFieldMetadataType } from 'src/engine/utils/is-morph-or-relation-field-metadata-type.util';
import { DatabaseStructureService } from 'src/engine/workspace-manager/workspace-health/services/database-structure.service';
import { validName } from 'src/engine/workspace-manager/workspace-health/utils/valid-name.util';
@Injectable()
export class FieldMetadataHealthService {
constructor(
private readonly databaseStructureService: DatabaseStructureService,
) {}
async healthCheck(
tableName: string,
workspaceTableColumns: WorkspaceTableStructure[],
fieldMetadataCollection: FieldMetadataEntity[],
options: WorkspaceHealthOptions,
): Promise<WorkspaceHealthIssue[]> {
const issues: WorkspaceHealthIssue[] = [];
for (const fieldMetadata of fieldMetadataCollection) {
// Relation metadata are checked in another service
if (isMorphOrRelationFieldMetadataType(fieldMetadata.type)) {
continue;
}
const fieldIssues = await this.healthCheckField(
tableName,
workspaceTableColumns,
fieldMetadata,
options,
);
issues.push(...fieldIssues);
}
return issues;
}
private async healthCheckField(
tableName: string,
workspaceTableColumns: WorkspaceTableStructure[],
fieldMetadata: FieldMetadataEntity,
options: WorkspaceHealthOptions,
): Promise<WorkspaceHealthIssue[]> {
const issues: WorkspaceHealthIssue[] = [];
if (options.mode === 'structure' || options.mode === 'all') {
const structureIssues = this.structureFieldCheck(
tableName,
workspaceTableColumns,
fieldMetadata,
);
issues.push(...structureIssues);
}
if (options.mode === 'metadata' || options.mode === 'all') {
const metadataIssues = this.metadataFieldCheck(tableName, fieldMetadata);
issues.push(...metadataIssues);
}
return issues;
}
private structureFieldCheck(
tableName: string,
workspaceTableColumns: WorkspaceTableStructure[],
fieldMetadata: FieldMetadataEntity,
): WorkspaceHealthIssue[] {
const dataTypes =
this.databaseStructureService.getPostgresDataTypes(fieldMetadata);
const issues: WorkspaceHealthIssue[] = [];
let columnNames: string[] = [];
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);
if (!compositeType) {
throw new Error(`Composite type ${fieldMetadata.type} is not defined`);
}
columnNames = compositeType.properties.map((compositeProperty) =>
computeCompositeColumnName(fieldMetadata, compositeProperty),
);
} else {
columnNames = [computeColumnName(fieldMetadata)];
}
const defaultValues = this.databaseStructureService.getPostgresDefaults(
fieldMetadata.type,
fieldMetadata.defaultValue,
);
// Check if column exist in database
const columnStructureMap = workspaceTableColumns.reduce(
(acc, workspaceTableColumn) => {
const columnName = workspaceTableColumn.columnName;
if (columnNames.includes(columnName)) {
acc[columnName] = workspaceTableColumn;
}
return acc;
},
{} as { [key: string]: WorkspaceTableStructure },
);
for (const [index, columnName] of columnNames.entries()) {
const columnStructure = columnStructureMap[columnName];
if (!columnStructure) {
issues.push({
type: WorkspaceHealthIssueType.MISSING_COLUMN,
fieldMetadata,
columnStructures: workspaceTableColumns,
message: `Column ${columnName} not found in table ${tableName}`,
});
continue;
}
const columnDefaultValue =
columnStructure.columnDefault?.split('::')?.[0];
// Check if column data type is the same
if (!dataTypes[index] || columnStructure.dataType !== dataTypes[index]) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT,
fieldMetadata,
columnStructure,
message: `Column ${columnName} type is not the same as the field metadata type "${columnStructure.dataType}" !== "${dataTypes[index]}"`,
});
}
if (columnStructure.isNullable !== fieldMetadata.isNullable) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT,
fieldMetadata,
columnStructure,
message: `Column ${columnName} is expected to be ${
fieldMetadata.isNullable ? 'nullable' : 'not nullable'
} but is ${columnStructure.isNullable ? 'nullable' : 'not nullable'}`,
});
}
if (columnDefaultValue && isEnumFieldMetadataType(fieldMetadata.type)) {
const enumValues = fieldMetadata.options?.map((option) =>
serializeDefaultValue(`'${option.value}'`),
);
if (!isDefined(enumValues)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID,
fieldMetadata,
columnStructure,
message: `Column options of ${fieldMetadata.name} are not defined`,
});
}
if (isDefined(enumValues) && !enumValues.includes(columnDefaultValue)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
fieldMetadata,
columnStructure,
message: `Column ${columnName} default value is not in the enum values "${columnDefaultValue}" NOT IN "${enumValues}"`,
});
}
}
if (columnDefaultValue !== defaultValues[index]) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT,
fieldMetadata,
columnStructure,
message: `Column ${columnName} default value is not the same as the field metadata default value "${columnStructure.columnDefault}" !== "${defaultValues[index]}"`,
});
}
}
return issues;
}
private metadataFieldCheck(
tableName: string,
fieldMetadata: FieldMetadataEntity,
): WorkspaceHealthIssue[] {
const issues: WorkspaceHealthIssue[] = [];
const columnName = fieldMetadata.name;
const defaultValueIssues = this.defaultValueHealthCheck(fieldMetadata);
if (fieldMetadata.name.startsWith(customNamePrefix)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_PREFIXED,
fieldMetadata,
message: `Column ${columnName} should not be prefixed with "${customNamePrefix}"`,
});
}
if (fieldMetadata.isCustom && columnName?.startsWith(customNamePrefix)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_CUSTOM,
fieldMetadata,
message: `Column ${columnName} is marked as custom in table ${tableName} but and start with "_", this behavior has been removed. Please remove the prefix.`,
});
}
if (!fieldMetadata.objectMetadataId) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_OBJECT_REFERENCE_INVALID,
fieldMetadata,
message: `Column ${columnName} doesn't have a valid object metadata id`,
});
}
if (!Object.values(FieldMetadataType).includes(fieldMetadata.type)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_TYPE_NOT_VALID,
fieldMetadata,
message: `Column ${columnName} doesn't have a valid field metadata type`,
});
}
if (
!fieldMetadata.name ||
!validName(fieldMetadata.name) ||
!fieldMetadata.label
) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NAME_NOT_VALID,
fieldMetadata,
message: `Column ${columnName} doesn't have a valid name or label`,
});
}
if (!isDefined(fieldMetadata.options)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID,
fieldMetadata,
message: `Column options of ${fieldMetadata.name} are not defined`,
});
}
if (
isEnumFieldMetadataType(fieldMetadata.type) &&
isDefined(fieldMetadata.options) &&
!validateOptionsForType(fieldMetadata.type, fieldMetadata.options)
) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID,
fieldMetadata,
message: `Column options of ${fieldMetadata.name} is not valid`,
});
}
issues.push(...defaultValueIssues);
return issues;
}
private defaultValueHealthCheck(
fieldMetadata: FieldMetadataEntity,
): WorkspaceHealthIssue[] {
const issues: WorkspaceHealthIssue[] = [];
if (
!validateDefaultValueForType(
fieldMetadata.type,
fieldMetadata.defaultValue,
).isValid
) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
fieldMetadata,
message: `Column default value for composite type ${fieldMetadata.type} is not well structured`,
});
}
if (
isEnumFieldMetadataType(fieldMetadata.type) &&
fieldMetadata.defaultValue
) {
const enumValues = fieldMetadata.options?.map((option) =>
serializeDefaultValue(`'${option.value}'`),
);
const metadataDefaultValue = fieldMetadata.defaultValue;
if (isDefined(metadataDefaultValue)) {
if (fieldMetadata.type === FieldMetadataType.MULTI_SELECT) {
if (!Array.isArray(metadataDefaultValue)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
fieldMetadata,
message: `Column default value for multi-select must be an array, got "${metadataDefaultValue}"`,
});
} else {
metadataDefaultValue.forEach((value) => {
if (isDefined(enumValues) && !enumValues.includes(value)) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
fieldMetadata,
message: `Column default value "${value}" is not in the enum values "${enumValues}"`,
});
}
});
}
} else if (
isDefined(enumValues) &&
!enumValues.includes(metadataDefaultValue as string)
) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID,
fieldMetadata,
message: `Column default value is not in the enum values "${metadataDefaultValue}" NOT IN "${enumValues}"`,
});
}
}
}
return issues;
}
}
@@ -1,120 +0,0 @@
import { Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
import {
type WorkspaceHealthIssue,
WorkspaceHealthIssueType,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { type WorkspaceHealthOptions } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-options.interface';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { validName } from 'src/engine/workspace-manager/workspace-health/utils/valid-name.util';
@Injectable()
export class ObjectMetadataHealthService {
constructor(
@InjectDataSource()
private readonly coreDataSource: DataSource,
) {}
async healthCheck(
schemaName: string,
objectMetadata: ObjectMetadataEntity,
options: WorkspaceHealthOptions,
): Promise<WorkspaceHealthIssue[]> {
const issues: WorkspaceHealthIssue[] = [];
if (options.mode === 'structure' || options.mode === 'all') {
const structureIssues = await this.structureObjectCheck(
schemaName,
objectMetadata,
);
issues.push(...structureIssues);
}
if (options.mode === 'metadata' || options.mode === 'all') {
const metadataIssues = this.metadataObjectCheck(objectMetadata);
issues.push(...metadataIssues);
}
return issues;
}
/**
* Check the structure health of the table based on metadata
* @param schemaName
* @param objectMetadata
* @returns WorkspaceHealthIssue[]
*/
private async structureObjectCheck(
schemaName: string,
objectMetadata: ObjectMetadataEntity,
): Promise<WorkspaceHealthIssue[]> {
const issues: WorkspaceHealthIssue[] = [];
// Check if the table exist in database
const tableExist = await this.coreDataSource.query(
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = '${schemaName}'
AND table_name = '${computeObjectTargetTable(objectMetadata)}')`,
);
if (!tableExist) {
issues.push({
type: WorkspaceHealthIssueType.MISSING_TABLE,
objectMetadata,
message: `Table ${computeObjectTargetTable(
objectMetadata,
)} not found in schema ${schemaName}`,
});
return issues;
}
return issues;
}
/**
* Check ObjectMetadata health
* @param objectMetadata
* @returns WorkspaceHealthIssue[]
*/
private metadataObjectCheck(
objectMetadata: ObjectMetadataEntity,
): WorkspaceHealthIssue[] {
const issues: WorkspaceHealthIssue[] = [];
if (!objectMetadata.dataSourceId) {
issues.push({
type: WorkspaceHealthIssueType.TABLE_DATA_SOURCE_ID_NOT_VALID,
objectMetadata,
message: `Table ${computeObjectTargetTable(
objectMetadata,
)} doesn't have a data source`,
});
}
if (
!objectMetadata.nameSingular ||
!objectMetadata.namePlural ||
!validName(objectMetadata.nameSingular) ||
!validName(objectMetadata.namePlural) ||
!objectMetadata.labelSingular ||
!objectMetadata.labelPlural
) {
issues.push({
type: WorkspaceHealthIssueType.TABLE_NAME_NOT_VALID,
objectMetadata,
message: `Table ${computeObjectTargetTable(
objectMetadata,
)} doesn't have a valid name or label`,
});
}
return issues;
}
}
@@ -1,99 +0,0 @@
import { Injectable } from '@nestjs/common';
import { type EntityManager } from 'typeorm';
import { WorkspaceHealthFixKind } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-fix-kind.interface';
import { type WorkspaceHealthIssue } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import { type WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceNullableFixer } from 'src/engine/workspace-manager/workspace-health/fixer/workspace-nullable.fixer';
import { WorkspaceDefaultValueFixer } from 'src/engine/workspace-manager/workspace-health/fixer/workspace-default-value.fixer';
import { WorkspaceTypeFixer } from 'src/engine/workspace-manager/workspace-health/fixer/workspace-type.fixer';
import { type CompareEntity } from 'src/engine/workspace-manager/workspace-health/fixer/abstract-workspace.fixer';
import { WorkspaceMissingColumnFixer } from 'src/engine/workspace-manager/workspace-health/fixer/workspace-missing-column.fixer';
@Injectable()
export class WorkspaceFixService {
constructor(
private readonly workspaceNullableFixer: WorkspaceNullableFixer,
private readonly workspaceDefaultValueFixer: WorkspaceDefaultValueFixer,
private readonly workspaceTypeFixer: WorkspaceTypeFixer,
private readonly workspaceMissingColumnFixer: WorkspaceMissingColumnFixer,
) {}
async createWorkspaceMigrations(
manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
type: WorkspaceHealthFixKind,
issues: WorkspaceHealthIssue[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
switch (type) {
case WorkspaceHealthFixKind.Nullable: {
const filteredIssues = this.workspaceNullableFixer.filterIssues(issues);
return this.workspaceNullableFixer.createWorkspaceMigrations(
manager,
objectMetadataCollection,
filteredIssues,
);
}
case WorkspaceHealthFixKind.DefaultValue: {
const filteredIssues =
this.workspaceDefaultValueFixer.filterIssues(issues);
return this.workspaceDefaultValueFixer.createWorkspaceMigrations(
manager,
objectMetadataCollection,
filteredIssues,
);
}
case WorkspaceHealthFixKind.Type: {
const filteredIssues = this.workspaceTypeFixer.filterIssues(issues);
return this.workspaceTypeFixer.createWorkspaceMigrations(
manager,
objectMetadataCollection,
filteredIssues,
);
}
case WorkspaceHealthFixKind.MissingColumn: {
const filteredIssues =
this.workspaceMissingColumnFixer.filterIssues(issues);
return this.workspaceMissingColumnFixer.createWorkspaceMigrations(
manager,
objectMetadataCollection,
filteredIssues,
);
}
default: {
return [];
}
}
}
async createMetadataUpdates(
manager: EntityManager,
objectMetadataCollection: ObjectMetadataEntity[],
type: WorkspaceHealthFixKind,
issues: WorkspaceHealthIssue[],
): Promise<CompareEntity<unknown>[]> {
switch (type) {
case WorkspaceHealthFixKind.DefaultValue: {
const filteredIssues =
this.workspaceDefaultValueFixer.filterIssues(issues);
return this.workspaceDefaultValueFixer.createMetadataUpdates(
manager,
objectMetadataCollection,
filteredIssues,
);
}
default: {
return [];
}
}
}
}
@@ -1,3 +0,0 @@
export const validName = (name: string): boolean => {
return /^[a-zA-Z0-9_]+$/.test(name);
};
@@ -1,36 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { DatabaseStructureService } from 'src/engine/workspace-manager/workspace-health/services/database-structure.service';
import { FieldMetadataHealthService } from 'src/engine/workspace-manager/workspace-health/services/field-metadata-health.service';
import { ObjectMetadataHealthService } from 'src/engine/workspace-manager/workspace-health/services/object-metadata-health.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceFixService } from 'src/engine/workspace-manager/workspace-health/services/workspace-fix.service';
import { workspaceFixers } from './fixer';
@Module({
imports: [
DataSourceModule,
TypeORMModule,
ObjectMetadataModule,
WorkspaceDataSourceModule,
WorkspaceMigrationRunnerModule,
WorkspaceMigrationBuilderModule,
],
providers: [
...workspaceFixers,
WorkspaceHealthService,
DatabaseStructureService,
ObjectMetadataHealthService,
FieldMetadataHealthService,
WorkspaceFixService,
],
exports: [WorkspaceHealthService, DatabaseStructureService],
})
export class WorkspaceHealthModule {}
@@ -1,184 +0,0 @@
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
import { type WorkspaceHealthFixKind } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-fix-kind.interface';
import { type WorkspaceHealthIssue } from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-issue.interface';
import {
WorkspaceHealthMode,
type WorkspaceHealthOptions,
} from 'src/engine/workspace-manager/workspace-health/interfaces/workspace-health-options.interface';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
import { DatabaseStructureService } from 'src/engine/workspace-manager/workspace-health/services/database-structure.service';
import { FieldMetadataHealthService } from 'src/engine/workspace-manager/workspace-health/services/field-metadata-health.service';
import { ObjectMetadataHealthService } from 'src/engine/workspace-manager/workspace-health/services/object-metadata-health.service';
import { WorkspaceFixService } from 'src/engine/workspace-manager/workspace-health/services/workspace-fix.service';
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
@Injectable()
export class WorkspaceHealthService {
private readonly logger = new Logger(WorkspaceHealthService.name);
constructor(
@InjectDataSource()
private readonly coreDataSource: DataSource,
private readonly dataSourceService: DataSourceService,
private readonly objectMetadataService: ObjectMetadataServiceV2,
private readonly databaseStructureService: DatabaseStructureService,
private readonly objectMetadataHealthService: ObjectMetadataHealthService,
private readonly fieldMetadataHealthService: FieldMetadataHealthService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceFixService: WorkspaceFixService,
) {}
async healthCheck(
workspaceId: string,
options: WorkspaceHealthOptions = { mode: WorkspaceHealthMode.All },
): Promise<WorkspaceHealthIssue[]> {
const schemaName = getWorkspaceSchemaName(workspaceId);
const issues: WorkspaceHealthIssue[] = [];
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
);
// Check if a data source exists for this workspace
if (!dataSourceMetadata) {
throw new NotFoundException(
`DataSource for workspace id ${workspaceId} not found`,
);
}
const objectMetadataCollection =
await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
// Check if object metadata exists for this workspace
if (!objectMetadataCollection || objectMetadataCollection.length === 0) {
throw new NotFoundException(`Workspace with id ${workspaceId} not found`);
}
for (const objectMetadata of objectMetadataCollection) {
const tableName = computeObjectTargetTable(objectMetadata);
const workspaceTableColumns =
await this.databaseStructureService.getWorkspaceTableColumns(
schemaName,
tableName,
);
if (!workspaceTableColumns || workspaceTableColumns.length === 0) {
throw new NotFoundException(
`Table ${tableName} not found in schema ${schemaName}`,
);
}
// Check object metadata health
const objectIssues = await this.objectMetadataHealthService.healthCheck(
schemaName,
objectMetadata,
options,
);
issues.push(...objectIssues);
// Check fields metadata health
const fieldIssues = await this.fieldMetadataHealthService.healthCheck(
computeObjectTargetTable(objectMetadata),
workspaceTableColumns,
objectMetadata.fields,
options,
);
issues.push(...fieldIssues);
}
return issues;
}
async fixIssues(
workspaceId: string,
issues: WorkspaceHealthIssue[],
options: {
type: WorkspaceHealthFixKind;
applyChanges?: boolean;
},
): Promise<{
workspaceMigrations: Partial<WorkspaceMigrationEntity>[];
metadataEntities: unknown[];
}> {
let workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
let metadataEntities: unknown[] = [];
// Set default options
options.applyChanges ??= true;
const queryRunner = this.coreDataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
const manager = queryRunner.manager;
try {
const workspaceMigrationRepository = manager.getRepository(
WorkspaceMigrationEntity,
);
const objectMetadataCollection =
await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
workspaceMigrations =
await this.workspaceFixService.createWorkspaceMigrations(
manager,
objectMetadataCollection,
options.type,
issues,
);
metadataEntities = await this.workspaceFixService.createMetadataUpdates(
manager,
objectMetadataCollection,
options.type,
issues,
);
// Save workspace migrations into the database
await workspaceMigrationRepository.save(workspaceMigrations);
if (!options.applyChanges) {
// Rollback transactions
await queryRunner.rollbackTransaction();
await queryRunner.release();
return {
workspaceMigrations,
metadataEntities,
};
}
// Commit the transaction
await queryRunner.commitTransaction();
// Apply pending migrations
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
);
} catch (error) {
await queryRunner.rollbackTransaction();
this.logger.error('Fix of issues failed with:', error);
} finally {
await queryRunner.release();
}
return {
workspaceMigrations,
metadataEntities,
};
}
}
@@ -8,6 +8,7 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent
import { AiAgentModule } from 'src/engine/metadata-modules/ai/ai-agent/ai-agent.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
@@ -17,7 +18,6 @@ import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
@@ -32,10 +32,10 @@ import { WorkspaceManagerService } from './workspace-manager.service';
DevSeederModule,
DataSourceModule,
WorkspaceSyncMetadataModule,
WorkspaceHealthModule,
FeatureFlagModule,
PermissionsModule,
AiAgentModule,
WorkspaceManyOrAllFlatEntityMapsCacheModule,
TypeOrmModule.forFeature([UserWorkspaceEntity, WorkspaceEntity]),
RoleModule,
UserRoleModule,
@@ -10,6 +10,7 @@ import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { type DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
@@ -48,6 +49,7 @@ export class WorkspaceManagerService {
private readonly roleTargetsRepository: Repository<RoleTargetsEntity>,
protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly applicationService: ApplicationService,
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
) {}
public async init({
@@ -133,20 +135,27 @@ export class WorkspaceManagerService {
featureFlags: Record<string, boolean>,
twentyStandardFlatApplication: FlatApplication,
) {
const createdObjectMetadata =
await this.objectMetadataServiceV2.findManyWithinWorkspace(workspaceId);
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
},
);
await standardObjectsPrefillData(
this.coreDataSource,
dataSourceMetadata.schema,
createdObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
);
await prefillCoreViews({
twentyStandardFlatApplication,
coreDataSource: this.coreDataSource,
workspaceId,
objectMetadataItems: createdObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
workspaceSchemaName: dataSourceMetadata.schema,
featureFlags,
});
@@ -1,22 +1,20 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module';
import { SyncWorkspaceLoggerModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command';
@Module({
imports: [
WorkspaceSyncMetadataModule,
WorkspaceHealthModule,
WorkspaceModule,
DataSourceModule,
WorkspaceDataSourceModule,
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
import {
FieldMetadataType,
compositeTypeDefinitions,
@@ -15,7 +16,6 @@ import {
} from 'typeorm';
import { type DeepPartial } from 'typeorm/common/DeepPartial';
import { v4 as uuidV4 } from 'uuid';
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
import { type PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { type PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';