Update unique fields on standard field - include soft deleted records (#14562)

Done : 
- migrate standard unique fields (add : -old)
- deactivate defaultWhereClause
- restore soft deleted if upserted (same for contact creation in mailing
sync)

fixes https://github.com/twentyhq/twenty/issues/14443

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Etienne
2025-10-11 15:50:29 +02:00
committed by GitHub
parent c681fb7fb6
commit 560e24fa4b
26 changed files with 1119 additions and 316 deletions
@@ -0,0 +1,360 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Command } from 'nest-commander';
import { IsNull, Not, Repository } from 'typeorm';
import {
ActiveOrSuspendedWorkspacesMigrationCommandRunner,
type RunOnWorkspaceArgs,
} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service';
import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationIndexAction,
WorkspaceMigrationIndexActionType,
WorkspaceMigrationTableAction,
WorkspaceMigrationTableActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
@Command({
name: 'upgrade:1-10:deduplicate-unique-fields',
description:
'Deduplicate unique fields for workspaceMembers, companies and people because we changed the unique constraint',
})
export class DeduplicateUniqueFieldsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
protected readonly logger = new Logger(DeduplicateUniqueFieldsCommand.name);
constructor(
@InjectRepository(Workspace)
protected readonly workspaceRepository: Repository<Workspace>,
protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
protected readonly indexMetadataService: IndexMetadataService,
protected readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
protected readonly workspaceMigrationService: WorkspaceMigrationService,
@InjectRepository(ObjectMetadataEntity)
protected readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
@InjectRepository(IndexMetadataEntity)
protected readonly indexMetadataRepository: Repository<IndexMetadataEntity>,
) {
super(workspaceRepository, twentyORMGlobalManager);
}
override async runOnWorkspace({
workspaceId,
dataSource,
options,
}: RunOnWorkspaceArgs): Promise<void> {
this.logger.log(
`Deduplicating indexed fields for workspace ${workspaceId}`,
);
await this.deduplicateUniqueUserEmailFieldForWorkspaceMembers({
dataSource,
dryRun: options.dryRun ?? false,
});
await this.deduplicateUniqueDomainNameFieldForCompanies({
dataSource,
dryRun: options.dryRun ?? false,
});
await this.deduplicateUniqueEmailFieldForPeople({
dataSource,
dryRun: options.dryRun ?? false,
});
if (!options.dryRun) {
await this.updateExistingIndexedFields({
workspaceId,
objectMetadataNameSingular: 'workspaceMember',
columnName: 'userEmail',
});
await this.updateExistingIndexedFields({
workspaceId,
objectMetadataNameSingular: 'company',
columnName: 'domainNamePrimaryLinkUrl',
});
await this.updateExistingIndexedFields({
workspaceId,
objectMetadataNameSingular: 'person',
columnName: 'emailsPrimaryEmail',
});
}
}
private computeExistingUniqueIndexName({
objectMetadata,
fieldMetadataToIndex,
}: {
objectMetadata: ObjectMetadataEntity;
fieldMetadataToIndex: Partial<FieldMetadataEntity>[];
}) {
const tableName = computeObjectTargetTable(objectMetadata);
const columnNames: string[] = fieldMetadataToIndex.map(
(fieldMetadata) => fieldMetadata.name as string,
);
return `IDX_UNIQUE_${generateDeterministicIndexName([tableName, ...columnNames])}`;
}
private async computeExistingIndexDeletionMigration({
objectMetadata,
fieldMetadataToIndex,
}: {
objectMetadata: ObjectMetadataEntity;
fieldMetadataToIndex: Partial<FieldMetadataEntity>[];
}) {
const tableName = computeObjectTargetTable(objectMetadata);
const indexName = this.computeExistingUniqueIndexName({
objectMetadata,
fieldMetadataToIndex,
});
return {
name: tableName,
action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
indexes: [
{
action: WorkspaceMigrationIndexActionType.DROP,
name: indexName,
columns: [],
isUnique: true,
} satisfies WorkspaceMigrationIndexAction,
],
} satisfies WorkspaceMigrationTableAction;
}
private async updateExistingIndexedFields({
workspaceId,
objectMetadataNameSingular,
columnName,
}: {
workspaceId: string;
objectMetadataNameSingular: string;
columnName: string;
}) {
this.logger.log(
`Updating existing indexed fields for workspace members for workspace ${workspaceId}`,
);
const workspaceMemberObjectMetadata =
await this.objectMetadataRepository.findOneByOrFail({
nameSingular: objectMetadataNameSingular,
});
await this.indexMetadataRepository.delete({
workspaceId,
name: this.computeExistingUniqueIndexName({
objectMetadata: workspaceMemberObjectMetadata,
fieldMetadataToIndex: [{ name: columnName }],
}),
});
const indexDeletionMigration =
await this.computeExistingIndexDeletionMigration({
objectMetadata: workspaceMemberObjectMetadata,
fieldMetadataToIndex: [{ name: columnName }],
});
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`delete-${objectMetadataNameSingular}-index`),
workspaceId,
[indexDeletionMigration],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
);
}
private async deduplicateUniqueUserEmailFieldForWorkspaceMembers({
dataSource,
dryRun,
}: {
dataSource: WorkspaceDataSource;
dryRun: boolean;
}) {
const workspaceMemberRepository = dataSource.getRepository(
'workspaceMember',
true,
);
const duplicates = await workspaceMemberRepository
.createQueryBuilder('workspaceMember')
.select('workspaceMember.userEmail', 'userEmail')
.addSelect('COUNT(*)', 'count')
.andWhere('workspaceMember.userEmail IS NOT NULL')
.andWhere("workspaceMember.userEmail != ''")
.withDeleted()
.groupBy('workspaceMember.userEmail')
.having('COUNT(*) > 1')
.getRawMany();
for (const duplicate of duplicates) {
const { userEmail } = duplicate;
const softDeletedWorkspaceMembers = await workspaceMemberRepository.find({
where: {
userEmail,
deletedAt: Not(IsNull()),
},
withDeleted: true,
});
for (const [
i,
softDeletedWorkspaceMember,
] of softDeletedWorkspaceMembers.entries()) {
const newUserEmail = this.computeNewFieldValues(
softDeletedWorkspaceMember.userEmail,
i,
);
if (!dryRun) {
await workspaceMemberRepository
.createQueryBuilder('workspaceMember')
.update()
.set({
userEmail: newUserEmail,
})
.where('id = :id', { id: softDeletedWorkspaceMember.id })
.execute();
}
this.logger.log(
`Updated workspaceMember ${softDeletedWorkspaceMembers[i].id} userEmail from ${userEmail} to ${newUserEmail}`,
);
}
}
}
private async deduplicateUniqueDomainNameFieldForCompanies({
dataSource,
dryRun,
}: {
dataSource: WorkspaceDataSource;
dryRun: boolean;
}) {
const companyRepository = dataSource.getRepository('company', true);
const duplicates = await companyRepository
.createQueryBuilder('company')
.select('company.domainNamePrimaryLinkUrl', 'domainNamePrimaryLinkUrl')
.addSelect('COUNT(*)', 'count')
.andWhere('company.domainNamePrimaryLinkUrl IS NOT NULL')
.andWhere("company.domainNamePrimaryLinkUrl != ''")
.withDeleted()
.groupBy('company.domainNamePrimaryLinkUrl')
.having('COUNT(*) > 1')
.getRawMany();
for (const duplicate of duplicates) {
const { domainNamePrimaryLinkUrl } = duplicate;
const softDeletedCompanies = await companyRepository.find({
where: {
domainName: {
primaryLinkUrl: domainNamePrimaryLinkUrl,
},
deletedAt: Not(IsNull()),
},
withDeleted: true,
});
for (const [i, softDeletedCompany] of softDeletedCompanies.entries()) {
const newDomainNamePrimaryLinkUrl = this.computeNewFieldValues(
softDeletedCompany.domainName.primaryLinkUrl,
i,
);
if (!dryRun) {
await companyRepository
.createQueryBuilder('company')
.update()
.set({
domainName: {
primaryLinkUrl: newDomainNamePrimaryLinkUrl,
},
})
.where('id = :id', { id: softDeletedCompany.id })
.execute();
}
this.logger.log(
`Updated company ${softDeletedCompany.id} domainNamePrimaryLinkUrl from ${domainNamePrimaryLinkUrl} to ${newDomainNamePrimaryLinkUrl}`,
);
}
}
}
private async deduplicateUniqueEmailFieldForPeople({
dataSource,
dryRun,
}: {
dataSource: WorkspaceDataSource;
dryRun: boolean;
}) {
const personRepository = dataSource.getRepository('person', true);
const duplicates = await personRepository
.createQueryBuilder('person')
.select('person.emailsPrimaryEmail', 'emailsPrimaryEmail')
.addSelect('COUNT(*)', 'count')
.andWhere('person.emailsPrimaryEmail IS NOT NULL')
.andWhere("person.emailsPrimaryEmail != ''")
.withDeleted()
.groupBy('person.emailsPrimaryEmail')
.having('COUNT(*) > 1')
.getRawMany();
for (const duplicate of duplicates) {
const { emailsPrimaryEmail } = duplicate;
const softDeletedPersons = await personRepository.find({
where: {
emails: {
primaryEmail: emailsPrimaryEmail,
},
deletedAt: Not(IsNull()),
},
withDeleted: true,
});
for (const [i, softDeletedPerson] of softDeletedPersons.entries()) {
const newEmailsPrimaryEmail = this.computeNewFieldValues(
softDeletedPerson.emails.primaryEmail,
i,
);
if (!dryRun) {
await personRepository
.createQueryBuilder('person')
.update()
.set({
emails: {
primaryEmail: newEmailsPrimaryEmail,
},
})
.where('id = :id', { id: softDeletedPerson.id })
.execute();
}
this.logger.log(
`Updated person ${softDeletedPerson.id} emailsPrimaryEmail from ${emailsPrimaryEmail} to ${newEmailsPrimaryEmail}`,
);
}
}
}
private computeNewFieldValues(fieldValue: string, i: number) {
return `${fieldValue}-old-${i}`;
}
}
@@ -14,7 +14,7 @@ import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-mana
import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity';
@Command({
name: 'upgrade:1-7:regenerate-person-search-vector-with-phones',
name: 'upgrade:1-10:regenerate-person-search-vector-with-phones',
description:
'Regenerate person search vector to include phone number indexing for existing workspaces',
})
@@ -1,13 +1,38 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { DeduplicateUniqueFieldsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-deduplicate-unique-fields.command';
import { MigrateWorkflowStepFilterOperandValueCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-workflow-step-filter-operand-value';
import { RegeneratePersonSearchVectorWithPhonesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-person-search-vector-with-phones.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@Module({
imports: [TypeOrmModule.forFeature([Workspace]), WorkspaceDataSourceModule],
providers: [MigrateWorkflowStepFilterOperandValueCommand],
exports: [MigrateWorkflowStepFilterOperandValueCommand],
imports: [
TypeOrmModule.forFeature([
Workspace,
ObjectMetadataEntity,
IndexMetadataEntity,
]),
WorkspaceDataSourceModule,
IndexMetadataModule,
WorkspaceMigrationRunnerModule,
WorkspaceMigrationModule,
],
providers: [
MigrateWorkflowStepFilterOperandValueCommand,
DeduplicateUniqueFieldsCommand,
RegeneratePersonSearchVectorWithPhonesCommand,
],
exports: [
MigrateWorkflowStepFilterOperandValueCommand,
DeduplicateUniqueFieldsCommand,
RegeneratePersonSearchVectorWithPhonesCommand,
],
})
export class V1_10_UpgradeVersionCommandModule {}
@@ -2,19 +2,12 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BackfillWorkflowManualTriggerAvailabilityCommand } from 'src/database/commands/upgrade-version-command/1-7/1-7-backfill-workflow-manual-trigger-availability.command';
import { RegeneratePersonSearchVectorWithPhonesCommand } from 'src/database/commands/upgrade-version-command/1-7/1-7-regenerate-person-search-vector-with-phones.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [TypeOrmModule.forFeature([Workspace]), WorkspaceDataSourceModule],
providers: [
RegeneratePersonSearchVectorWithPhonesCommand,
BackfillWorkflowManualTriggerAvailabilityCommand,
],
exports: [
RegeneratePersonSearchVectorWithPhonesCommand,
BackfillWorkflowManualTriggerAvailabilityCommand,
],
providers: [BackfillWorkflowManualTriggerAvailabilityCommand],
exports: [BackfillWorkflowManualTriggerAvailabilityCommand],
})
export class V1_7_UpgradeVersionCommandModule {}
@@ -19,7 +19,9 @@ import { AddEnqueuedStatusToWorkflowRunCommand } from 'src/database/commands/upg
import { FixSchemaArrayTypeCommand } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-schema-array-type.command';
import { FixUpdateStandardFieldsIsLabelSyncedWithName } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-update-standard-field-is-label-synced-with-name.command';
import { MigrateWorkflowRunStatesCommand } from 'src/database/commands/upgrade-version-command/1-1/1-1-migrate-workflow-run-state.command';
import { DeduplicateUniqueFieldsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-deduplicate-unique-fields.command';
import { MigrateWorkflowStepFilterOperandValueCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-workflow-step-filter-operand-value';
import { RegeneratePersonSearchVectorWithPhonesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-person-search-vector-with-phones.command';
import { AddEnqueuedStatusToWorkflowRunV2Command } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run-v2.command';
import { AddNextStepIdsToWorkflowVersionTriggers } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-next-step-ids-to-workflow-version-triggers.command';
import { RemoveWorkflowRunsWithoutState } from 'src/database/commands/upgrade-version-command/1-2/1-2-remove-workflow-runs-without-state.command';
@@ -87,6 +89,8 @@ export class UpgradeCommand extends UpgradeCommandRunner {
// 1.10 Commands
protected readonly migrateWorkflowStepFilterOperandValueCommand: MigrateWorkflowStepFilterOperandValueCommand,
protected readonly deduplicateUniqueFieldsCommand: DeduplicateUniqueFieldsCommand,
protected readonly regeneratePersonSearchVectorWithPhonesCommand: RegeneratePersonSearchVectorWithPhonesCommand,
) {
super(
workspaceRepository,
@@ -181,7 +185,11 @@ export class UpgradeCommand extends UpgradeCommandRunner {
};
const commands_1100: VersionCommands = {
beforeSyncMetadata: [this.migrateWorkflowStepFilterOperandValueCommand],
beforeSyncMetadata: [
this.migrateWorkflowStepFilterOperandValueCommand,
this.deduplicateUniqueFieldsCommand,
this.regeneratePersonSearchVectorWithPhonesCommand,
],
afterSyncMetadata: [],
};
@@ -388,7 +388,7 @@ export class GraphqlQueryCreateManyResolverService extends GraphqlQueryBaseResol
const savedRecords = await repository.updateMany(
partialRecordsToUpdateWithoutCreatedByUpdate.map((record) => ({
criteria: record.id,
partialEntity: record,
partialEntity: { ...record, deletedAt: null },
})),
undefined,
columnsToReturn,
@@ -269,7 +269,10 @@ export class IndexMetadataService {
queryRunner,
}: {
workspaceId: string;
objectMetadata: ObjectMetadataEntity;
objectMetadata: Pick<
ObjectMetadataEntity,
'nameSingular' | 'isCustom' | 'id'
>;
fieldMetadataToIndex: Partial<FieldMetadataEntity>[];
queryRunner?: QueryRunner;
}) {
@@ -19,7 +19,7 @@ export function WorkspaceIsUnique(): PropertyDecorator {
const columns = [propertyKey.toString()];
metadataArgsStorage.addIndexes({
name: `IDX_UNIQUE_${generateDeterministicIndexName([
name: `IDX_${generateDeterministicIndexName([
convertClassNameToObjectMetadataName(target.constructor.name),
...columns,
])}`,
@@ -46,9 +46,7 @@ describe('WorkspaceMigrationIndexFactory', () => {
expect(firstMigration.indexes[0].columns).toEqual(['simpleField']);
expect(firstMigration.indexes[0].type).toBe('BTREE');
expect(firstMigration.indexes[0].isUnique).toBe(true);
expect(firstMigration.indexes[0].where).toBe(
'"simpleField" != \'\' AND "deletedAt" IS NULL',
);
expect(firstMigration.indexes[0].where).toBe('"simpleField" != \'\'');
});
it('should create index migrations for relation fields', async () => {
@@ -88,7 +88,7 @@ export const createIndexMigration = async (
.filter(isDefined);
const defaultWhereClause = indexMetadata.isUnique
? `${columns.map((column) => `"${column}"`).join(" != '' AND ")} != '' AND "deletedAt" IS NULL`
? `${columns.map((column) => `"${column}"`).join(" != '' AND ")} != ''`
: null;
return {
@@ -8,7 +8,7 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarCreateCompanyAndContactAfterSyncJob } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job';
import { CalendarCreateCompanyAndPersonAfterSyncJob } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job';
import { CalendarEventParticipantMatchParticipantJob } from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-event-participant-match-participant.job';
import { CalendarEventParticipantPersonListener } from 'src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-person.listener';
import { CalendarEventParticipantWorkspaceMemberListener } from 'src/modules/calendar/calendar-event-participant-manager/listeners/calendar-event-participant-workspace-member.listener';
@@ -28,7 +28,7 @@ import { MatchParticipantModule } from 'src/modules/match-participant/match-part
],
providers: [
CalendarEventParticipantService,
CalendarCreateCompanyAndContactAfterSyncJob,
CalendarCreateCompanyAndPersonAfterSyncJob,
CalendarEventParticipantMatchParticipantJob,
CalendarEventParticipantListener,
CalendarEventParticipantPersonListener,
@@ -9,9 +9,9 @@ import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/com
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
export type CalendarCreateCompanyAndContactAfterSyncJobData = {
export type CalendarCreateCompanyAndPersonAfterSyncJobData = {
workspaceId: string;
calendarChannelId: string;
};
@@ -20,15 +20,15 @@ export type CalendarCreateCompanyAndContactAfterSyncJobData = {
queueName: MessageQueue.calendarQueue,
scope: Scope.REQUEST,
})
export class CalendarCreateCompanyAndContactAfterSyncJob {
export class CalendarCreateCompanyAndPersonAfterSyncJob {
constructor(
private readonly twentyORMManager: TwentyORMManager,
private readonly createCompanyAndContactService: CreateCompanyAndContactService,
private readonly createCompanyAndPersonService: CreateCompanyAndPersonService,
) {}
@Process(CalendarCreateCompanyAndContactAfterSyncJob.name)
@Process(CalendarCreateCompanyAndPersonAfterSyncJob.name)
async handle(
data: CalendarCreateCompanyAndContactAfterSyncJobData,
data: CalendarCreateCompanyAndPersonAfterSyncJobData,
): Promise<void> {
const { workspaceId, calendarChannelId } = data;
@@ -87,7 +87,7 @@ export class CalendarCreateCompanyAndContactAfterSyncJob {
],
});
await this.createCompanyAndContactService.createCompaniesAndContactsAndUpdateParticipants(
await this.createCompanyAndPersonService.createCompaniesAndPeopleAndUpdateParticipants(
connectedAccount,
calendarEventParticipantsWithoutPersonIdAndWorkspaceMemberId,
workspaceId,
@@ -7,9 +7,9 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { AutoCompaniesAndContactsCreationCalendarChannelListener } from 'src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener';
import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
import { CreateContactService } from 'src/modules/contact-creation-manager/services/create-contact.service';
import { CreatePersonService } from 'src/modules/contact-creation-manager/services/create-person.service';
@Module({
imports: [
@@ -19,11 +19,11 @@ import { CreateContactService } from 'src/modules/contact-creation-manager/servi
],
providers: [
CreateCompanyService,
CreateContactService,
CreateCompanyAndContactService,
CreatePersonService,
CreateCompanyAndPersonService,
AutoCompaniesAndContactsCreationMessageChannelListener,
AutoCompaniesAndContactsCreationCalendarChannelListener,
],
exports: [CreateCompanyAndContactService],
exports: [CreateCompanyAndPersonService],
})
export class ContactCreationManagerModule {}
@@ -3,7 +3,7 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/proc
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { type FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
export type CreateCompanyAndContactJobData = {
workspaceId: string;
@@ -18,14 +18,14 @@ export type CreateCompanyAndContactJobData = {
@Processor(MessageQueue.contactCreationQueue)
export class CreateCompanyAndContactJob {
constructor(
private readonly createCompanyAndContactService: CreateCompanyAndContactService,
private readonly createCompanyAndPersonService: CreateCompanyAndPersonService,
) {}
@Process(CreateCompanyAndContactJob.name)
async handle(data: CreateCompanyAndContactJobData): Promise<void> {
const { workspaceId, connectedAccount, contactsToCreate, source } = data;
await this.createCompanyAndContactService.createCompaniesAndContactsAndUpdateParticipants(
await this.createCompanyAndPersonService.createCompaniesAndPeopleAndUpdateParticipants(
connectedAccount,
contactsToCreate.map((contact) => ({
handle: contact.handle,
@@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { objectRecordChangedProperties } from 'src/engine/core-modules/event-emitter/utils/object-record-changed-properties.util';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
@@ -7,12 +9,10 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import {
CalendarCreateCompanyAndContactAfterSyncJob,
type CalendarCreateCompanyAndContactAfterSyncJobData,
CalendarCreateCompanyAndPersonAfterSyncJob,
CalendarCreateCompanyAndPersonAfterSyncJobData,
} from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
@Injectable()
export class AutoCompaniesAndContactsCreationCalendarChannelListener {
@@ -36,8 +36,8 @@ export class AutoCompaniesAndContactsCreationCalendarChannelListener {
).includes('isContactAutoCreationEnabled') &&
eventPayload.properties.after.isContactAutoCreationEnabled
) {
return this.messageQueueService.add<CalendarCreateCompanyAndContactAfterSyncJobData>(
CalendarCreateCompanyAndContactAfterSyncJob.name,
return this.messageQueueService.add<CalendarCreateCompanyAndPersonAfterSyncJobData>(
CalendarCreateCompanyAndPersonAfterSyncJob.name,
{
workspaceId: payload.workspaceId,
calendarChannelId: eventPayload.recordId,
@@ -0,0 +1,134 @@
import { Test, type TestingModule } from '@nestjs/testing';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
import { CreatePersonService } from 'src/modules/contact-creation-manager/services/create-person.service';
import { type Contact } from 'src/modules/contact-creation-manager/types/contact.type';
import { type PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
describe('CreateCompanyAndPersonService', () => {
let service: CreateCompanyAndPersonService;
const mockConnectedAccount = {
id: 'connected-account-1',
accountOwner: {
id: 'workspace-member-1',
},
} as unknown as ConnectedAccountWorkspaceEntity;
beforeEach(async () => {
const mockCreateCompaniesService = {
createOrRestoreCompanies: jest.fn(),
};
const mockCreatePersonService = {
restorePeople: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
CreateCompanyAndPersonService,
{
provide: CreateCompanyService,
useValue: mockCreateCompaniesService,
},
{
provide: CreatePersonService,
useValue: mockCreatePersonService,
},
{
provide: TwentyORMGlobalManager,
useValue: {},
},
{
provide: ExceptionHandlerService,
useValue: {},
},
],
}).compile();
service = module.get<CreateCompanyAndPersonService>(
CreateCompanyAndPersonService,
);
});
describe('computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate', () => {
const mockContacts: Contact[] = [
{
handle: 'john.doe@company.com',
displayName: 'John Doe',
},
{
handle: 'jane.smith@company.com',
displayName: 'Jane Smith',
},
{
handle: 'personal@email.com',
displayName: 'Personal Contact',
},
];
const mockExistingPeople: PersonWorkspaceEntity[] = [
{
id: 'soft-deleted-person-1',
emails: {
primaryEmail: 'john.doe@company.com',
additionalEmails: null,
},
deletedAt: new Date(),
} as unknown as PersonWorkspaceEntity,
{
id: 'soft-deleted-person-2',
emails: {
primaryEmail: 'different@company.com',
additionalEmails: ['jane.smith@company.com'],
},
deletedAt: new Date(),
} as unknown as PersonWorkspaceEntity,
{
id: 'active-person-3',
emails: {
primaryEmail: 'active@company.com',
additionalEmails: null,
},
deletedAt: null,
} as unknown as PersonWorkspaceEntity,
];
it('should identify contacts that need person creation for new contacts', () => {
const result =
service.computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
mockContacts,
mockExistingPeople,
FieldActorSource.CALENDAR,
mockConnectedAccount,
);
expect(result.contactsThatNeedPersonCreate).toHaveLength(1);
expect(result.contactsThatNeedPersonCreate[0].handle).toBe(
'personal@email.com',
);
});
it('should identify contacts that need person restoration for soft-deleted contacts', () => {
const result =
service.computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
mockContacts,
mockExistingPeople,
FieldActorSource.CALENDAR,
mockConnectedAccount,
);
expect(result.contactsThatNeedPersonRestore).toHaveLength(2);
expect(
result.contactsThatNeedPersonRestore.map((c) => c.handle),
).toContain('john.doe@company.com');
expect(
result.contactsThatNeedPersonRestore.map((c) => c.handle),
).toContain('jane.smith@company.com');
});
});
});
@@ -46,6 +46,13 @@ describe('CreateCompanyService', () => {
provider: ConnectedAccountProvider.GOOGLE,
},
};
const companyToRestore: CompanyToCreate = {
domainName: 'soft-deleted-company.com',
createdBySource: FieldActorSource.MANUAL,
createdByContext: {
provider: ConnectedAccountProvider.GOOGLE,
},
};
const inputForCompanyToCreate1 = {
address: {
addressCity: undefined,
@@ -89,6 +96,7 @@ describe('CreateCompanyService', () => {
find: jest.fn(),
save: jest.fn(),
maximum: jest.fn().mockResolvedValue(0),
updateMany: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
@@ -135,10 +143,14 @@ describe('CreateCompanyService', () => {
mockCompanyRepository.find.mockResolvedValue([]);
// it is useless to check results here, we can only check the input it was called with
mockCompanyRepository.save.mockResolvedValue([]);
mockCompanyRepository.updateMany.mockResolvedValue({
raw: [],
});
});
it('should successfully create a company', async () => {
await service.createCompanies([companyToCreate1], workspaceId);
await service.createOrRestoreCompanies([companyToCreate1], workspaceId);
expect(mockCompanyRepository.find).toHaveBeenCalled();
expect(mockCompanyRepository.save).toHaveBeenCalledWith([
@@ -147,7 +159,7 @@ describe('CreateCompanyService', () => {
});
it('should successfully two companies', async () => {
await service.createCompanies(
await service.createOrRestoreCompanies(
[companyToCreate1, companyToCreate2],
workspaceId,
);
@@ -160,7 +172,7 @@ describe('CreateCompanyService', () => {
});
it('should create only one of example.com & example.com/ ', async () => {
await service.createCompanies(
await service.createOrRestoreCompanies(
[companyToCreate1, companyToCreate1withSlash],
workspaceId,
);
@@ -187,13 +199,59 @@ describe('CreateCompanyService', () => {
},
]);
mockCompanyRepository.save.mockResolvedValue([]);
mockCompanyRepository.updateMany.mockResolvedValue({
raw: [],
});
});
it('should not create a company if it already exists', async () => {
await service.createCompanies([companyToCreateExisting], workspaceId);
await service.createOrRestoreCompanies(
[companyToCreateExisting],
workspaceId,
);
expect(mockCompanyRepository.find).toHaveBeenCalled();
expect(mockCompanyRepository.save).not.toHaveBeenCalled();
expect(mockCompanyRepository.save).toHaveBeenCalledWith([]);
});
});
describe('With existing companies and some deleted', () => {
beforeEach(() => {
mockCompanyRepository.find.mockResolvedValue([
{
id: 'soft-deleted-company-1',
domainName: { primaryLinkUrl: 'https://soft-deleted-company.com' },
deletedAt: new Date(),
},
]);
mockCompanyRepository.save.mockResolvedValue([]);
mockCompanyRepository.updateMany.mockResolvedValue({
raw: [
{
id: 'soft-deleted-company-1',
domainNamePrimaryLinkUrl: 'https://soft-deleted-company.com',
},
],
});
});
it('should restore the soft deleted company', async () => {
await service.createOrRestoreCompanies([companyToRestore], workspaceId);
expect(mockCompanyRepository.find).toHaveBeenCalled();
expect(mockCompanyRepository.save).toHaveBeenCalledWith([]);
expect(mockCompanyRepository.updateMany).toHaveBeenCalledWith(
[
{
criteria: 'soft-deleted-company-1',
partialEntity: {
deletedAt: null,
},
},
],
undefined,
['domainNamePrimaryLinkUrl', 'id'],
);
});
});
});
@@ -1,9 +1,12 @@
import { Injectable } from '@nestjs/common';
import { isNonEmptyString } from '@sniptt/guards';
import { isNonEmptyString, isNull } from '@sniptt/guards';
import chunk from 'lodash.chunk';
import compact from 'lodash.compact';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { type DeepPartial } from 'typeorm';
import { v4 } from 'uuid';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { type FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
@@ -11,26 +14,27 @@ import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/contact-creation-manager/constants/contacts-creation-batch-size.constant';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
import { CreateContactService } from 'src/modules/contact-creation-manager/services/create-contact.service';
import { CreatePersonService } from 'src/modules/contact-creation-manager/services/create-person.service';
import { type Contact } from 'src/modules/contact-creation-manager/types/contact.type';
import { filterOutSelfAndContactsFromCompanyOrWorkspace } from 'src/modules/contact-creation-manager/utils/filter-out-contacts-from-company-or-workspace.util';
import { filterOutContactsThatBelongToSelfOrWorkspaceMembers } from 'src/modules/contact-creation-manager/utils/filter-out-contacts-that-belong-to-self-or-workspace-members.util';
import { getDomainNameFromHandle } from 'src/modules/contact-creation-manager/utils/get-domain-name-from-handle.util';
import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/modules/contact-creation-manager/utils/get-first-name-and-last-name-from-handle-and-display-name.util';
import { getUniqueContactsAndHandles } from 'src/modules/contact-creation-manager/utils/get-unique-contacts-and-handles.util';
import { addPersonEmailFiltersToQueryBuilder } from 'src/modules/match-participant/utils/add-person-email-filters-to-query-builder';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { computeDisplayName } from 'src/utils/compute-display-name';
import { isWorkDomain, isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class CreateCompanyAndContactService {
export class CreateCompanyAndPersonService {
constructor(
private readonly createContactService: CreateContactService,
private readonly createPersonService: CreatePersonService,
private readonly createCompaniesService: CreateCompanyService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
private async createCompaniesAndPeople(
async createCompaniesAndPeople(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contact[],
workspaceId: string,
@@ -57,15 +61,15 @@ export class CreateCompanyAndContactService {
const workspaceMembers = await workspaceMemberRepository.find();
const contactsToCreateFromOtherCompanies =
filterOutSelfAndContactsFromCompanyOrWorkspace(
const peopleToCreateFromOtherCompanies =
filterOutContactsThatBelongToSelfOrWorkspaceMembers(
contactsToCreate,
connectedAccount,
workspaceMembers,
);
const { uniqueContacts, uniqueHandles } = getUniqueContactsAndHandles(
contactsToCreateFromOtherCompanies,
peopleToCreateFromOtherCompanies,
);
if (uniqueHandles.length === 0) {
@@ -77,96 +81,62 @@ export class CreateCompanyAndContactService {
emails: uniqueHandles,
});
const alreadyCreatedContacts = await queryBuilder
const alreadyCreatedPeople = await queryBuilder
.orderBy('person.createdAt', 'ASC')
.withDeleted()
.getMany();
const alreadyCreatedContactEmails: string[] =
alreadyCreatedContacts?.reduce<string[]>((acc, { emails }) => {
const currentContactEmails: string[] = [];
const {
contactsThatNeedPersonCreate,
contactsThatNeedPersonRestore,
workDomainNamesToCreate,
shouldCreateOrRestorePeopleByHandleMap,
} =
this.computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
uniqueContacts,
alreadyCreatedPeople,
source,
connectedAccount,
);
if (isNonEmptyString(emails?.primaryEmail)) {
currentContactEmails.push(emails.primaryEmail.toLowerCase());
}
if (Array.isArray(emails?.additionalEmails)) {
const additionalEmails = emails.additionalEmails
.filter(isNonEmptyString)
.map((email) => email.toLowerCase());
const companiesMap =
await this.createCompaniesService.createOrRestoreCompanies(
workDomainNamesToCreate,
workspaceId,
);
currentContactEmails.push(...additionalEmails);
}
return [...acc, ...currentContactEmails];
}, []);
const filteredContactsToCreate = uniqueContacts.filter(
(participant) =>
!alreadyCreatedContactEmails.includes(
participant.handle.toLowerCase(),
) && participant.handle.includes('@'),
);
const filteredContactsToCreateWithCompanyDomainNames =
filteredContactsToCreate?.map((participant) => ({
handle: participant.handle,
displayName: participant.displayName,
companyDomainName: isWorkEmail(participant.handle)
? getDomainNameFromHandle(participant.handle)
: undefined,
}));
const domainNamesToCreate = compact(
filteredContactsToCreateWithCompanyDomainNames
.filter((participant) => participant.companyDomainName)
.map((participant) => ({
domainName: participant.companyDomainName,
createdBySource: source,
createdByWorkspaceMember: connectedAccount.accountOwner,
})),
);
const workDomainNamesToCreate = domainNamesToCreate.filter(
(domainName) =>
domainName?.domainName && isWorkDomain(domainName.domainName),
);
const workDomainNamesToCreateFormatted = workDomainNamesToCreate.map(
(domainName) => ({
...domainName,
createdBySource: source,
createdByWorkspaceMember: connectedAccount.accountOwner,
createdByContext: {
const peopleToCreate = this.formatPeopleToCreateFromContacts({
contactsToCreate: contactsThatNeedPersonCreate,
createdBy: {
source: source,
workspaceMember: connectedAccount.accountOwner,
context: {
provider: connectedAccount.provider,
},
}),
);
},
companiesMap,
});
const companiesObject = await this.createCompaniesService.createCompanies(
workDomainNamesToCreateFormatted,
const createdPeople = await this.createPersonService.createPeople(
peopleToCreate,
workspaceId,
);
const formattedContactsToCreate =
filteredContactsToCreateWithCompanyDomainNames.map((contact) => ({
handle: contact.handle,
displayName: contact.displayName,
companyId: isNonEmptyString(contact.companyDomainName)
? companiesObject[contact.companyDomainName]
: undefined,
createdBySource: source,
createdByWorkspaceMember: connectedAccount.accountOwner,
createdByContext: {
provider: connectedAccount.provider,
},
}));
const peopleToRestore = this.formatPeopleToRestoreFromContacts({
contactsToRestore: contactsThatNeedPersonRestore,
companiesMap,
shouldCreateOrRestorePeopleByHandleMap,
});
return this.createContactService.createPeople(
formattedContactsToCreate,
const restoredPeople = await this.createPersonService.restorePeople(
peopleToRestore,
workspaceId,
);
return { ...createdPeople, ...restoredPeople };
}
async createCompaniesAndContactsAndUpdateParticipants(
async createCompaniesAndPeopleAndUpdateParticipants(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contact[],
workspaceId: string,
@@ -177,7 +147,6 @@ export class CreateCompanyAndContactService {
CONTACTS_CREATION_BATCH_SIZE,
);
// In some jobs the accountOwner is not populated
if (!connectedAccount.accountOwner) {
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
@@ -217,4 +186,195 @@ export class CreateCompanyAndContactService {
}
}
}
computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
uniqueContacts: Contact[],
alreadyCreatedPeople: PersonWorkspaceEntity[],
source: FieldActorSource,
connectedAccount: ConnectedAccountWorkspaceEntity,
) {
const shouldCreateOrRestorePeopleByHandleMap = new Map<
string,
{ existingPerson: PersonWorkspaceEntity }
>();
for (const contact of uniqueContacts) {
if (!contact.handle.includes('@')) {
continue;
}
const existingPersonOnPrimaryEmail = alreadyCreatedPeople.find(
(person) => {
return (
isNonEmptyString(person.emails?.primaryEmail) &&
person.emails.primaryEmail.toLowerCase() ===
contact.handle.toLowerCase()
);
},
);
if (isDefined(existingPersonOnPrimaryEmail)) {
shouldCreateOrRestorePeopleByHandleMap.set(
contact.handle.toLowerCase(),
{
existingPerson: existingPersonOnPrimaryEmail,
},
);
continue;
}
const existingPersonOnAdditionalEmails = alreadyCreatedPeople.find(
(person) => {
return (
Array.isArray(person.emails?.additionalEmails) &&
person.emails.additionalEmails.some(
(email) => email.toLowerCase() === contact.handle.toLowerCase(),
)
);
},
);
if (!isDefined(existingPersonOnAdditionalEmails)) continue;
shouldCreateOrRestorePeopleByHandleMap.set(contact.handle.toLowerCase(), {
existingPerson: existingPersonOnAdditionalEmails,
});
}
const contactsThatNeedPersonCreate = uniqueContacts.filter(
(contact) =>
!shouldCreateOrRestorePeopleByHandleMap.has(
contact.handle.toLowerCase(),
),
);
const contactsThatNeedPersonRestore = uniqueContacts.filter((contact) => {
const existingPerson = shouldCreateOrRestorePeopleByHandleMap.get(
contact.handle.toLowerCase(),
)?.existingPerson;
if (!isDefined(existingPerson)) {
return false;
}
return !isNull(existingPerson.deletedAt);
});
const workDomainNamesToCreate = compact(
[...contactsThatNeedPersonCreate, ...contactsThatNeedPersonRestore]
.map((contact) => {
const companyDomainName = isWorkEmail(contact.handle)
? getDomainNameFromHandle(contact.handle)
: undefined;
if (!isDefined(companyDomainName) || !isWorkDomain(companyDomainName))
return undefined;
return {
domainName: companyDomainName,
createdBySource: source,
createdByWorkspaceMember: connectedAccount.accountOwner,
createdByContext: {
provider: connectedAccount.provider,
},
};
})
.filter(isDefined),
);
return {
contactsThatNeedPersonCreate,
contactsThatNeedPersonRestore,
workDomainNamesToCreate,
shouldCreateOrRestorePeopleByHandleMap,
};
}
formatPeopleToCreateFromContacts({
contactsToCreate,
createdBy,
companiesMap,
}: {
contactsToCreate: {
handle: string;
displayName: string;
}[];
createdBy: {
source: FieldActorSource;
workspaceMember?: WorkspaceMemberWorkspaceEntity | null;
context: {
provider: ConnectedAccountProvider;
};
};
companiesMap: Record<string, string>;
}): Partial<PersonWorkspaceEntity>[] {
return contactsToCreate.map((contact) => {
const id = v4();
const { handle, displayName } = contact;
const { firstName, lastName } =
getFirstNameAndLastNameFromHandleAndDisplayName(handle, displayName);
const createdByName = computeDisplayName(createdBy.workspaceMember?.name);
const companyId = companiesMap[getDomainNameFromHandle(handle)];
return {
id,
emails: {
primaryEmail: handle.toLowerCase(),
additionalEmails: null,
},
name: {
firstName,
lastName,
},
companyId,
createdBy: {
source: createdBy.source,
workspaceMemberId: createdBy.workspaceMember?.id ?? null,
name: createdByName,
context: createdBy.context,
},
};
});
}
formatPeopleToRestoreFromContacts({
contactsToRestore,
companiesMap,
shouldCreateOrRestorePeopleByHandleMap,
}: {
contactsToRestore: {
handle: string;
displayName: string;
}[];
companiesMap: Record<string, string>;
shouldCreateOrRestorePeopleByHandleMap: Map<
string,
{ existingPerson: PersonWorkspaceEntity | undefined }
>;
}): { personId: string; companyId: string | undefined }[] {
const peopleToRestore = [];
for (const contact of contactsToRestore) {
const { handle } = contact;
const existingPerson = shouldCreateOrRestorePeopleByHandleMap.get(
handle.toLowerCase(),
)?.existingPerson;
if (!isDefined(existingPerson) || isNull(existingPerson.deletedAt))
continue;
const companyId = companiesMap[getDomainNameFromHandle(handle)];
peopleToRestore.push({
personId: existingPerson.id,
companyId,
});
}
return peopleToRestore;
}
}
@@ -4,7 +4,10 @@ import axios, { type AxiosInstance } from 'axios';
import uniqBy from 'lodash.uniqby';
import { TWENTY_COMPANIES_BASE_URL } from 'twenty-shared/constants';
import { type ConnectedAccountProvider } from 'twenty-shared/types';
import { lowercaseUrlOriginAndRemoveTrailingSlash } from 'twenty-shared/utils';
import {
isDefined,
lowercaseUrlOriginAndRemoveTrailingSlash,
} from 'twenty-shared/utils';
import { type DeepPartial, ILike } from 'typeorm';
import { type FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
@@ -35,7 +38,7 @@ export class CreateCompanyService {
});
}
async createCompanies(
async createOrRestoreCompanies(
companies: CompanyToCreate[],
workspaceId: string,
): Promise<{
@@ -54,7 +57,6 @@ export class CreateCompanyService {
},
);
// Remove trailing slash from domain names
const companiesWithoutTrailingSlash = companies.map((company) => ({
...company,
domainName: company.domainName
@@ -62,7 +64,6 @@ export class CreateCompanyService {
: undefined,
}));
// Avoid creating duplicate companies, e.g. example.com and example.com/
const uniqueCompanies = uniqBy(companiesWithoutTrailingSlash, 'domainName');
const conditions = uniqueCompanies.map((companyToCreate) => ({
domainName: {
@@ -70,13 +71,12 @@ export class CreateCompanyService {
},
}));
// Find existing companies
const existingCompanies = await companyRepository.find({
where: conditions,
withDeleted: true,
});
const existingCompanyIdsMap = this.createCompanyMap(existingCompanies);
// Filter out companies that already exist
const newCompaniesToCreate = uniqueCompanies.filter(
(company) =>
!existingCompanies.some(
@@ -87,11 +87,15 @@ export class CreateCompanyService {
),
);
if (newCompaniesToCreate.length === 0) {
const companiesToRestore = this.filterCompaniesToRestore(
uniqueCompanies,
existingCompanies,
);
if (newCompaniesToCreate.length === 0 && companiesToRestore.length === 0) {
return existingCompanyIdsMap;
}
// Retrieve the last company position
let lastCompanyPosition =
await this.getLastCompanyPosition(companyRepository);
const newCompaniesData = await Promise.all(
@@ -100,17 +104,67 @@ export class CreateCompanyService {
),
);
// Create new companies
const createdCompanies = await companyRepository.save(newCompaniesData);
const createdCompanyIdsMap = this.createCompanyMap(createdCompanies);
const restoredCompanies = await companyRepository.updateMany(
companiesToRestore.map((company) => {
return {
criteria: company.id,
partialEntity: {
deletedAt: null,
},
};
}),
undefined,
['domainNamePrimaryLinkUrl', 'id'],
);
const formattedRestoredCompanies = restoredCompanies.raw.map(
(row: { id: string; domainNamePrimaryLinkUrl: string }) => {
return {
id: row.id,
domainName: {
primaryLinkUrl: row.domainNamePrimaryLinkUrl,
},
};
},
);
return {
...existingCompanyIdsMap,
...createdCompanyIdsMap,
...(createdCompanies.length > 0
? this.createCompanyMap(createdCompanies)
: {}),
...(formattedRestoredCompanies.length > 0
? this.createCompanyMap(formattedRestoredCompanies)
: {}),
};
}
private filterCompaniesToRestore(
uniqueCompanies: CompanyToCreate[],
existingCompanies: CompanyWorkspaceEntity[],
) {
return uniqueCompanies
.map((company) => {
const existingCompany = existingCompanies.find(
(existingCompany) =>
existingCompany.domainName &&
extractDomainFromLink(existingCompany.domainName.primaryLinkUrl) ===
company.domainName,
);
return isDefined(existingCompany)
? {
domainName: company.domainName,
id: existingCompany.id,
deletedAt: null,
}
: undefined;
})
.filter(isDefined);
}
private async prepareCompanyData(
company: CompanyToCreate,
position: number,
@@ -142,7 +196,9 @@ export class CreateCompanyService {
};
}
private createCompanyMap(companies: DeepPartial<CompanyWorkspaceEntity>[]) {
private createCompanyMap(
companies: Pick<CompanyWorkspaceEntity, 'id' | 'domainName'>[],
) {
return companies.reduce(
(acc, company) => {
if (!company.domainName?.primaryLinkUrl || !company.id) {
@@ -1,110 +0,0 @@
import { Injectable } from '@nestjs/common';
import { type ConnectedAccountProvider } from 'twenty-shared/types';
import { type DeepPartial } from 'typeorm';
import { v4 } from 'uuid';
import { type FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/modules/contact-creation-manager/utils/get-first-name-and-last-name-from-handle-and-display-name.util';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { computeDisplayName } from 'src/utils/compute-display-name';
type ContactToCreate = {
handle: string;
displayName: string;
companyId?: string;
createdBySource: FieldActorSource;
createdByWorkspaceMember?: WorkspaceMemberWorkspaceEntity | null;
createdByContext?: {
provider?: ConnectedAccountProvider;
};
};
@Injectable()
export class CreateContactService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
private formatContacts(
contactsToCreate: ContactToCreate[],
lastPersonPosition: number,
): DeepPartial<PersonWorkspaceEntity>[] {
return contactsToCreate.map((contact) => {
const id = v4();
const {
handle,
displayName,
companyId,
createdBySource,
createdByWorkspaceMember,
createdByContext,
} = contact;
const { firstName, lastName } =
getFirstNameAndLastNameFromHandleAndDisplayName(handle, displayName);
const createdByName = computeDisplayName(createdByWorkspaceMember?.name);
return {
id,
emails: {
primaryEmail: handle.toLowerCase(),
additionalEmails: null,
},
name: {
firstName,
lastName,
},
companyId,
createdBy: {
source: createdBySource,
workspaceMemberId: contact.createdByWorkspaceMember?.id,
name: createdByName,
context: createdByContext,
},
position: ++lastPersonPosition,
};
});
}
public async createPeople(
contactsToCreate: ContactToCreate[],
workspaceId: string,
): Promise<DeepPartial<PersonWorkspaceEntity>[]> {
if (contactsToCreate.length === 0) return [];
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const lastPersonPosition =
await this.getLastPersonPosition(personRepository);
const formattedContacts = this.formatContacts(
contactsToCreate,
lastPersonPosition,
);
return personRepository.save(formattedContacts, undefined);
}
private async getLastPersonPosition(
personRepository: WorkspaceRepository<PersonWorkspaceEntity>,
): Promise<number> {
const lastPersonPosition = await personRepository.maximum(
'position',
undefined,
);
return lastPersonPosition ?? 0;
}
}
@@ -0,0 +1,87 @@
import { Injectable } from '@nestjs/common';
import { DeepPartial } from 'typeorm';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
@Injectable()
export class CreatePersonService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
public async createPeople(
peopleToCreate: Partial<PersonWorkspaceEntity>[],
workspaceId: string,
): Promise<DeepPartial<PersonWorkspaceEntity>[]> {
if (peopleToCreate.length === 0) return [];
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const lastPersonPosition =
await this.getLastPersonPosition(personRepository);
const createdPeople = await personRepository.insert(
peopleToCreate.map((person, index) => ({
...person,
position: lastPersonPosition + index,
})),
undefined,
['companyId', 'id'],
);
return createdPeople.raw;
}
public async restorePeople(
people: { personId: string; companyId: string | undefined }[],
workspaceId: string,
): Promise<DeepPartial<PersonWorkspaceEntity>[]> {
if (people.length === 0) {
return [];
}
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const restoredPeople = await personRepository.updateMany(
people.map(({ personId, companyId }) => ({
criteria: personId,
partialEntity: {
deletedAt: null,
companyId,
},
})),
undefined,
['companyId', 'id'],
);
return restoredPeople.raw;
}
private async getLastPersonPosition(
personRepository: WorkspaceRepository<PersonWorkspaceEntity>,
): Promise<number> {
const lastPersonPosition = await personRepository.maximum(
'position',
undefined,
);
return lastPersonPosition ?? 0;
}
}
@@ -4,7 +4,7 @@ import { getDomainNameFromHandle } from 'src/modules/contact-creation-manager/ut
import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { isWorkDomain } from 'src/utils/is-work-email';
export function filterOutSelfAndContactsFromCompanyOrWorkspace(
export function filterOutContactsThatBelongToSelfOrWorkspaceMembers(
contacts: Contact[],
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceMembers: WorkspaceMemberWorkspaceEntity[],
@@ -8,6 +8,7 @@ exports[`addPersonEmailFiltersToQueryBuilder case-insensitive email normalizatio
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -24,6 +25,10 @@ exports[`addPersonEmailFiltersToQueryBuilder case-insensitive email normalizatio
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.emailsAdditionalEmails @> :email0::jsonb",
@@ -57,6 +62,7 @@ exports[`addPersonEmailFiltersToQueryBuilder emails with empty exclusion array:
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -72,6 +78,10 @@ exports[`addPersonEmailFiltersToQueryBuilder emails with empty exclusion array:
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.emailsAdditionalEmails @> :email0::jsonb",
@@ -96,6 +106,7 @@ exports[`addPersonEmailFiltersToQueryBuilder empty emails array: should handle e
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -113,6 +124,10 @@ exports[`addPersonEmailFiltersToQueryBuilder empty emails array: should handle e
"args": [],
"method": "withDeleted",
},
{
"args": [],
"method": "withDeleted",
},
]
`;
@@ -124,6 +139,7 @@ exports[`addPersonEmailFiltersToQueryBuilder multiple emails with exclusions: sh
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -140,6 +156,10 @@ exports[`addPersonEmailFiltersToQueryBuilder multiple emails with exclusions: sh
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.id NOT IN (:...excludePersonIds)",
@@ -190,6 +210,7 @@ exports[`addPersonEmailFiltersToQueryBuilder multiple emails without exclusions:
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -206,6 +227,10 @@ exports[`addPersonEmailFiltersToQueryBuilder multiple emails without exclusions:
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.emailsAdditionalEmails @> :email0::jsonb",
@@ -239,6 +264,7 @@ exports[`addPersonEmailFiltersToQueryBuilder single email with person ID exclusi
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -254,6 +280,10 @@ exports[`addPersonEmailFiltersToQueryBuilder single email with person ID exclusi
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.id NOT IN (:...excludePersonIds)",
@@ -294,6 +324,7 @@ exports[`addPersonEmailFiltersToQueryBuilder single email without exclusions: sh
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -309,6 +340,10 @@ exports[`addPersonEmailFiltersToQueryBuilder single email without exclusions: sh
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.emailsAdditionalEmails @> :email0::jsonb",
@@ -333,6 +368,7 @@ exports[`addPersonEmailFiltersToQueryBuilder three emails with unique parameter
"person.id",
"person.emailsPrimaryEmail",
"person.emailsAdditionalEmails",
"person.deletedAt",
],
],
"method": "select",
@@ -350,6 +386,10 @@ exports[`addPersonEmailFiltersToQueryBuilder three emails with unique parameter
],
"method": "where",
},
{
"args": [],
"method": "withDeleted",
},
{
"args": [
"person.emailsAdditionalEmails @> :email0::jsonb",
@@ -30,10 +30,12 @@ export function addPersonEmailFiltersToQueryBuilder({
'person.id',
'person.emailsPrimaryEmail',
'person.emailsAdditionalEmails',
'person.deletedAt',
])
.where('LOWER(person.emailsPrimaryEmail) IN (:...emails)', {
emails: normalizedEmails,
});
})
.withDeleted();
if (excludePersonIds.length > 0) {
queryBuilder = queryBuilder.andWhere(
@@ -8,7 +8,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { FieldActorSource } from 'src/engine/metadata-modules/field-metadata/composite-types/actor.composite-type';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
import {
MessageChannelContactAutoCreationPolicy,
@@ -27,7 +27,7 @@ export class MessagingCreateCompanyAndContactAfterSyncJob {
MessagingCreateCompanyAndContactAfterSyncJob.name,
);
constructor(
private readonly createCompanyAndContactService: CreateCompanyAndContactService,
private readonly createCompanyAndPersonService: CreateCompanyAndPersonService,
private readonly twentyORMManager: TwentyORMManager,
) {}
@@ -36,7 +36,7 @@ export class MessagingCreateCompanyAndContactAfterSyncJob {
data: MessagingCreateCompanyAndContactAfterSyncJobData,
): Promise<void> {
this.logger.log(
`create contacts and companies after sync for workspace ${data.workspaceId} and messageChannel ${data.messageChannelId}`,
`create people and companies after sync for workspace ${data.workspaceId} and messageChannel ${data.messageChannelId}`,
);
const { workspaceId, messageChannelId } = data;
@@ -100,7 +100,7 @@ export class MessagingCreateCompanyAndContactAfterSyncJob {
},
});
await this.createCompanyAndContactService.createCompaniesAndContactsAndUpdateParticipants(
await this.createCompanyAndPersonService.createCompaniesAndPeopleAndUpdateParticipants(
connectedAccount,
contactsToCreate,
workspaceId,
@@ -171,60 +171,6 @@ describe('upsert (createMany with upsert:true)', () => {
});
});
it('should update soft-deleted records', async () => {
// Create a record
const createResponse = await makeGraphqlAPIRequest({
query: createRecordsQuery,
variables: {
data: [
{
firstUniqueTestField: 'softDeletedRecord',
secondUniqueTestField: 'softDeletedSecondField',
name: 'originalRecord',
},
],
upsert: false,
},
});
const createdRecord = createResponse.body.data.createTestRecordObjects[0];
// Soft delete the record
await makeGraphqlAPIRequest({
query: deleteRecordsQuery,
variables: {
filter: {
id: {
eq: createdRecord.id,
},
},
},
});
const upsertResponse = await makeGraphqlAPIRequest({
query: createRecordsQuery,
variables: {
data: [
{
firstUniqueTestField: 'softDeletedRecord',
name: 'restoredRecord',
},
],
upsert: true,
},
});
const upsertedRecord = upsertResponse.body.data.createTestRecordObjects[0];
expect(upsertedRecord).toEqual({
id: createdRecord.id,
firstUniqueTestField: 'softDeletedRecord',
secondUniqueTestField: 'softDeletedSecondField',
name: 'restoredRecord',
deletedAt: expect.any(String),
});
});
it('should throw an error when multiple records with the same unique field values are found', async () => {
await makeGraphqlAPIRequest({
query: createRecordsQuery,
@@ -267,4 +213,47 @@ describe('upsert (createMany with upsert:true)', () => {
'BAD_USER_INPUT',
);
});
it('should update and restore updated soft-deleted record', async () => {
const createResponse = await makeGraphqlAPIRequest({
query: createRecordsQuery,
variables: {
data: [
{
firstUniqueTestField: 'softDeletedRecord',
secondUniqueTestField: 'softDeletedSecondField',
name: 'originalRecord',
},
],
upsert: false,
},
});
const createdRecord = createResponse.body.data.createTestRecordObjects[0];
const deleteResponse = await makeGraphqlAPIRequest({
query: deleteRecordsQuery,
variables: {
filter: { id: { eq: createdRecord.id } },
},
});
const updateResponse = await makeGraphqlAPIRequest({
query: createRecordsQuery,
variables: {
data: [{ id: createdRecord.id, name: 'updatedRecord' }],
upsert: true,
},
});
expect(
deleteResponse.body.data.deleteTestRecordObjects[0].deletedAt,
).toEqual(expect.any(String));
expect(
updateResponse.body.data.createTestRecordObjects[0].deletedAt,
).toBeNull();
expect(updateResponse.body.data.createTestRecordObjects[0].id).toEqual(
createdRecord.id,
);
});
});