Refactor global datasource part 3 (#16447)

## Context
Following https://github.com/twentyhq/twenty/pull/16399
Now using the new global orm manager everywhere and returning a
GlobalDatasource/WorkspaceDatasource based on a feature flag.
This means we now need to wrap all our ORM calls within
executeInWorkspaceContext callback (at least for now) so the global
datasource can dynamically hydrate its context via the new store (the
global datasource does not store anything related to workspaces as it is
now a unique singleton). If feature flag is off it still uses local data
stored in the workspace datasource.
This commit is contained in:
Weiko
2025-12-10 17:17:33 +01:00
committed by GitHub
parent 4f13022774
commit 9bd8f94b3a
203 changed files with 8887 additions and 7237 deletions
@@ -8,7 +8,8 @@ import {
} from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { isDomain } from 'src/engine/utils/is-domain';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
@@ -28,7 +29,7 @@ export class BlocklistValidationService {
constructor(
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async validateBlocklistForCreateMany(
@@ -83,15 +84,23 @@ export class BlocklistValidationService {
userId: string,
workspaceId: string,
) {
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
const authContext = buildSystemAuthContext(workspaceId);
const currentWorkspaceMember =
await workspaceMemberRepository.findOneByOrFail({
userId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
return workspaceMemberRepository.findOneByOrFail({
userId,
});
},
);
const currentBlocklist =
await this.blocklistRepository.getByWorkspaceMemberId(
@@ -132,16 +141,23 @@ export class BlocklistValidationService {
return;
}
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
const authContext = buildSystemAuthContext(workspaceId);
const currentWorkspaceMember =
await workspaceMemberRepository.findOneByOrFail({
userId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
return workspaceMemberRepository.findOneByOrFail({
userId,
});
},
);
const currentBlocklist =
await this.blocklistRepository.getByWorkspaceMemberId(
@@ -1,46 +1,61 @@
import { Injectable } from '@nestjs/common';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
@Injectable()
export class BlocklistRepository {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async getById(
id: string,
workspaceId: string,
): Promise<BlocklistWorkspaceEntity | null> {
const blockListRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
BlocklistWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceId);
return blockListRepository.findOneBy({
id,
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const blockListRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
BlocklistWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
return blockListRepository.findOneBy({
id,
});
},
);
}
public async getByWorkspaceMemberId(
workspaceMemberId: string,
workspaceId: string,
): Promise<BlocklistWorkspaceEntity[]> {
const blockListRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
BlocklistWorkspaceEntity,
);
const authContext = buildSystemAuthContext(workspaceId);
return blockListRepository.find({
where: {
workspaceMemberId,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const blockListRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
BlocklistWorkspaceEntity,
);
return blockListRepository.find({
where: {
workspaceMemberId,
},
});
},
});
);
}
}
@@ -7,7 +7,8 @@ import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emit
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
@@ -24,7 +25,7 @@ export type BlocklistItemDeleteCalendarEventsJobData = WorkspaceEventBatch<
})
export class BlocklistItemDeleteCalendarEventsJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {}
@@ -32,121 +33,128 @@ export class BlocklistItemDeleteCalendarEventsJob {
async handle(data: BlocklistItemDeleteCalendarEventsJobData): Promise<void> {
const workspaceId = data.workspaceId;
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const authContext = buildSystemAuthContext(workspaceId);
const blocklistRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannelEventAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const calendarChannels = await calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
if (calendarChannel.connectedAccount.handleAliases) {
calendarChannelHandles.push(
...calendarChannel.connectedAccount.handleAliases.split(','),
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const calendarChannels = await calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
if (calendarEventsAssociationsToDelete.length === 0) {
continue;
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
if (calendarChannel.connectedAccount.handleAliases) {
calendarChannelHandles.push(
...calendarChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
});
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
},
},
});
if (calendarEventsAssociationsToDelete.length === 0) {
continue;
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
}
}
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
},
);
}
}
@@ -6,7 +6,8 @@ import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emit
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
@@ -25,7 +26,7 @@ export type BlocklistReimportCalendarEventsJobData = WorkspaceEventBatch<
})
export class BlocklistReimportCalendarEventsJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
) {}
@@ -33,32 +34,39 @@ export class BlocklistReimportCalendarEventsJob {
async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
const workspaceId = data.workspaceId;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannels = await calendarChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
),
},
});
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
const calendarChannels = await calendarChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
),
},
});
await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
},
);
}
}
@@ -2,40 +2,49 @@ import { Injectable } from '@nestjs/common';
import { Any, IsNull } from 'typeorm';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/utils/delete-using-pagination.util';
@Injectable()
export class CalendarEventCleanerService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async cleanWorkspaceCalendarEvents(workspaceId: string) {
const calendarEventRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'calendarEvent',
);
const authContext = buildSystemAuthContext(workspaceId);
await deleteUsingPagination(
workspaceId,
500,
async (limit, offset) => {
const nonAssociatedCalendarEvents = await calendarEventRepository.find({
where: {
calendarChannelEventAssociations: {
id: IsNull(),
},
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'calendarEvent',
);
await deleteUsingPagination(
workspaceId,
500,
async (limit, offset) => {
const nonAssociatedCalendarEvents =
await calendarEventRepository.find({
where: {
calendarChannelEventAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
});
return nonAssociatedCalendarEvents.map(({ id }) => id);
},
take: limit,
skip: offset,
});
return nonAssociatedCalendarEvents.map(({ id }) => id);
},
async (ids) => {
await calendarEventRepository.delete({ id: Any(ids) });
async (ids) => {
await calendarEventRepository.delete({ id: Any(ids) });
},
);
},
);
}
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
@@ -23,7 +24,7 @@ export type CalendarEventListFetchJobData = {
})
export class CalendarEventListFetchJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly calendarFetchEventsService: CalendarFetchEventsService,
) {}
@@ -32,50 +33,57 @@ export class CalendarEventListFetchJob {
async handle(data: CalendarEventListFetchJobData): Promise<void> {
const { workspaceId, calendarChannelId } = data;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
},
relations: ['connectedAccount'],
});
if (!calendarChannel) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED
) {
return;
}
if (
isThrottled(
calendarChannel.syncStageStartedAt,
calendarChannel.throttleFailureCount,
)
) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
true,
);
return;
}
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
);
},
relations: ['connectedAccount'],
});
if (!calendarChannel) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED
) {
return;
}
if (
isThrottled(
calendarChannel.syncStageStartedAt,
calendarChannel.throttleFailureCount,
)
) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
true,
);
return;
}
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
);
}
}
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
@@ -25,56 +26,63 @@ export class CalendarEventsImportJob {
constructor(
private readonly calendarEventsImportService: CalendarEventsImportService,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(CalendarEventsImportJob.name)
async handle(data: CalendarEventsImportJobData): Promise<void> {
const { calendarChannelId, workspaceId } = data;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
},
relations: ['connectedAccount'],
});
if (!calendarChannel?.isSyncEnabled) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED
) {
return;
}
if (
isThrottled(
calendarChannel.syncStageStartedAt,
calendarChannel.throttleFailureCount,
)
) {
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
true,
);
return;
}
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
);
},
relations: ['connectedAccount'],
});
if (!calendarChannel?.isSyncEnabled) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED
) {
return;
}
if (
isThrottled(
calendarChannel.syncStageStartedAt,
calendarChannel.throttleFailureCount,
)
) {
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
true,
);
return;
}
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
calendarChannel.connectedAccount,
workspaceId,
);
}
}
@@ -5,7 +5,8 @@ import { In } from 'typeorm';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { isSyncStale } from 'src/modules/calendar/calendar-event-import-manager/utils/is-sync-stale.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
@@ -24,7 +25,7 @@ export type CalendarOngoingStaleJobData = {
export class CalendarOngoingStaleJob {
private readonly logger = new Logger(CalendarOngoingStaleJob.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
) {}
@@ -32,58 +33,65 @@ export class CalendarOngoingStaleJob {
async handle(data: CalendarOngoingStaleJobData): Promise<void> {
const { workspaceId } = data;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannels = await calendarChannelRepository.find({
where: {
syncStage: In([
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
]),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
for (const calendarChannel of calendarChannels) {
if (
calendarChannel.syncStageStartedAt &&
isSyncStale(calendarChannel.syncStageStartedAt)
) {
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
[calendarChannel.id],
workspaceId,
);
const calendarChannels = await calendarChannelRepository.find({
where: {
syncStage: In([
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
]),
},
});
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
for (const calendarChannel of calendarChannels) {
if (
calendarChannel.syncStageStartedAt &&
isSyncStale(calendarChannel.syncStageStartedAt)
) {
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
[calendarChannel.id],
workspaceId,
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
break;
default:
break;
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
break;
default:
break;
}
}
}
}
}
},
);
}
}
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
CalendarChannelSyncStage,
CalendarChannelSyncStatus,
@@ -21,42 +22,50 @@ export type CalendarRelaunchFailedCalendarChannelJobData = {
})
export class CalendarRelaunchFailedCalendarChannelJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(CalendarRelaunchFailedCalendarChannelJob.name)
async handle(data: CalendarRelaunchFailedCalendarChannelJobData) {
const { workspaceId, calendarChannelId } = data;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
{ shouldBypassPermissionChecks: true },
);
const calendarChannel = await calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
if (
!calendarChannel ||
calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED ||
calendarChannel.syncStatus !==
CalendarChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await calendarChannelRepository.update(calendarChannelId, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
});
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
if (
!calendarChannel ||
calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED ||
calendarChannel.syncStatus !== CalendarChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await calendarChannelRepository.update(calendarChannelId, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
});
);
}
}
@@ -5,7 +5,8 @@ import {
type TwentyORMException,
TwentyORMExceptionCode,
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { CALENDAR_THROTTLE_MAX_ATTEMPTS } from 'src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts';
import {
type CalendarEventImportDriverException,
@@ -28,7 +29,7 @@ export class CalendarEventImportErrorHandlerService {
CalendarEventImportErrorHandlerService.name,
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@@ -130,20 +131,27 @@ export class CalendarEventImportErrorHandlerService {
throw calendarEventImportException;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.increment(
{
id: calendarChannel.id,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.increment(
{
id: calendarChannel.id,
},
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
);
},
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
);
switch (syncStep) {
@@ -7,7 +7,8 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
@@ -34,7 +35,7 @@ export class CalendarEventsImportService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly calendarEventCleanerService: CalendarEventCleanerService,
@@ -55,115 +56,123 @@ export class CalendarEventsImportService {
workspaceId,
);
let calendarEvents: FetchedCalendarEvent[] = [];
const authContext = buildSystemAuthContext(workspaceId);
try {
if (fetchedCalendarEvents) {
calendarEvents = fetchedCalendarEvents;
} else {
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
let calendarEvents: FetchedCalendarEvent[] = [];
try {
if (fetchedCalendarEvents) {
calendarEvents = fetchedCalendarEvents;
} else {
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
);
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
return;
}
switch (connectedAccount.provider) {
case 'microsoft':
calendarEvents =
await this.microsoftCalendarImportEventService.getCalendarEvents(
connectedAccount,
eventIdsToFetch,
);
break;
default:
break;
}
}
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
}
const blocklist =
await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccount.accountOwnerId,
workspaceId,
);
if (
!isDefined(connectedAccount.handleAliases) ||
!isDefined(calendarChannel.handle)
) {
throw new CalendarEventImportDriverException(
'Calendar channel handle or Handle aliases are required',
CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
[
calendarChannel.handle,
...connectedAccount.handleAliases.split(','),
],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle ?? ''),
);
const cancelledEventExternalIds = cancelledEvents.map(
(event) => event.id,
);
const BATCH_SIZE = 1000;
for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) {
const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE);
await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob(
eventsBatch,
calendarChannel,
connectedAccount,
workspaceId,
);
}
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
await calendarChannelEventAssociationRepository.delete({
eventExternalId: Any(cancelledEventExternalIds),
calendarChannel: {
id: calendarChannel.id,
},
});
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
return;
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT,
calendarChannel,
workspaceId,
);
}
switch (connectedAccount.provider) {
case 'microsoft':
calendarEvents =
await this.microsoftCalendarImportEventService.getCalendarEvents(
connectedAccount,
eventIdsToFetch,
);
break;
default:
break;
}
}
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
}
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccount.accountOwnerId,
workspaceId,
);
if (
!isDefined(connectedAccount.handleAliases) ||
!isDefined(calendarChannel.handle)
) {
throw new CalendarEventImportDriverException(
'Calendar channel handle or Handle aliases are required',
CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
[
calendarChannel.handle,
...connectedAccount.handleAliases.split(','),
],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle ?? ''),
);
const cancelledEventExternalIds = cancelledEvents.map(
(event) => event.id,
);
const BATCH_SIZE = 1000;
for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) {
const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE);
await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob(
eventsBatch,
calendarChannel,
connectedAccount,
workspaceId,
);
}
const calendarChannelEventAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
await calendarChannelEventAssociationRepository.delete({
eventExternalId: Any(cancelledEventExternalIds),
calendarChannel: {
id: calendarChannel.id,
},
});
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT,
calendarChannel,
workspaceId,
);
}
},
);
}
}
@@ -5,7 +5,8 @@ import { isDefined } from 'twenty-shared/utils';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
CalendarEventImportDriverException,
CalendarEventImportDriverExceptionCode,
@@ -27,7 +28,7 @@ export class CalendarFetchEventsService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
@@ -45,111 +46,115 @@ export class CalendarFetchEventsService {
workspaceId,
);
try {
const { accessToken, refreshToken } =
await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const { accessToken, refreshToken } =
await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
if (!isDefined(calendarChannel.syncCursor)) {
throw new CalendarEventImportDriverException(
'Sync cursor is required',
CalendarEventImportDriverExceptionCode.SYNC_CURSOR_ERROR,
);
}
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccountWithFreshTokens,
calendarChannel.syncCursor,
);
const hasFullEvents = getCalendarEventsResponse.fullEvents;
const calendarEvents = hasFullEvents
? getCalendarEventsResponse.calendarEvents
: null;
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
if (!calendarEvents || calendarEvents?.length === 0) {
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
}
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
if (hasFullEvents && calendarEvents) {
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
connectedAccount,
workspaceId,
calendarEvents,
);
} else if (!hasFullEvents && calendarEventIds) {
await this.cacheStorage.setAdd(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
calendarEventIds,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
} else {
throw new CalendarEventImportDriverException(
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
} catch (error) {
this.logger.log(
`Calendar event fetch error for workspace ${workspaceId} and calendar channel ${calendarChannel.id}`,
);
this.logger.error(error);
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH,
calendarChannel,
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
if (!isDefined(calendarChannel.syncCursor)) {
throw new CalendarEventImportDriverException(
'Sync cursor is required',
CalendarEventImportDriverExceptionCode.SYNC_CURSOR_ERROR,
);
}
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccountWithFreshTokens,
calendarChannel.syncCursor,
);
const hasFullEvents = getCalendarEventsResponse.fullEvents;
const calendarEvents = hasFullEvents
? getCalendarEventsResponse.calendarEvents
: null;
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
if (!calendarEvents || calendarEvents?.length === 0) {
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
}
await calendarChannelRepository.update(
{
id: calendarChannel.id,
},
{
syncCursor: nextSyncCursor,
},
);
if (hasFullEvents && calendarEvents) {
// Event Import already done
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
connectedAccount,
workspaceId,
calendarEvents,
);
} else if (!hasFullEvents && calendarEventIds) {
// Event Import still needed
await this.cacheStorage.setAdd(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
calendarEventIds,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
} else {
throw new CalendarEventImportDriverException(
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
} catch (error) {
this.logger.log(
`Calendar event fetch error for workspace ${workspaceId} and calendar channel ${calendarChannel.id}`,
);
this.logger.error(error);
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH,
calendarChannel,
workspaceId,
);
}
);
}
},
);
}
}
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { CalendarEventParticipantService } from 'src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service';
import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
@@ -20,7 +21,7 @@ type FetchedCalendarEventWithDBEvent = {
@Injectable()
export class CalendarSaveEventsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly calendarEventParticipantService: CalendarEventParticipantService,
) {}
@@ -30,227 +31,246 @@ export class CalendarSaveEventsService {
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
): Promise<void> {
const calendarEventRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarEventWorkspaceEntity>(
workspaceId,
'calendarEvent',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannelEventAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventWorkspaceEntity>(
workspaceId,
'calendarEvent',
);
const existingCalendarEvents = await calendarEventRepository.find({
where: {
iCalUid: Any(
fetchedCalendarEvents.map((event) => event.iCalUid as string),
),
},
});
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEvents.map((event): FetchedCalendarEventWithDBEvent => {
const existingEventWithSameiCalUid = existingCalendarEvents.find(
(existingEvent) => existingEvent.iCalUid === event.iCalUid,
);
return {
fetchedCalendarEvent: event,
existingCalendarEvent: existingEventWithSameiCalUid ?? null,
newlyCreatedCalendarEvent: null,
};
});
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const savedCalendarEvents = await calendarEventRepository.save(
fetchedCalendarEventsWithDBEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent === null,
)
.map(
({ fetchedCalendarEvent }) =>
({
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
}) satisfies Omit<
CalendarEventWorkspaceEntity,
| 'id'
| 'calendarChannelEventAssociations'
| 'calendarEventParticipants'
| 'createdAt'
| 'updatedAt'
| 'deletedAt'
>,
const existingCalendarEvents = await calendarEventRepository.find({
where: {
iCalUid: Any(
fetchedCalendarEvents.map((event) => event.iCalUid as string),
),
{},
transactionManager,
);
},
});
const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEventsWithDBEvents.map(
({ fetchedCalendarEvent, existingCalendarEvent }) => {
const savedCalendarEvent = savedCalendarEvents.find(
(savedCalendarEvent) =>
savedCalendarEvent.iCalUid === fetchedCalendarEvent.iCalUid,
const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEvents.map(
(event): FetchedCalendarEventWithDBEvent => {
const existingEventWithSameiCalUid = existingCalendarEvents.find(
(existingEvent) => existingEvent.iCalUid === event.iCalUid,
);
return {
fetchedCalendarEvent,
existingCalendarEvent: existingCalendarEvent,
newlyCreatedCalendarEvent: savedCalendarEvent ?? null,
fetchedCalendarEvent: event,
existingCalendarEvent: existingEventWithSameiCalUid ?? null,
newlyCreatedCalendarEvent: null,
};
},
);
await calendarEventRepository.save(
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return {
id: existingCalendarEvent.id,
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
} satisfies Omit<
CalendarEventWorkspaceEntity,
| 'calendarChannelEventAssociations'
| 'calendarEventParticipants'
| 'createdAt'
| 'updatedAt'
| 'deletedAt'
>;
}),
{},
transactionManager,
);
const calendarChannelEventAssociationsToSave: Pick<
CalendarChannelEventAssociationWorkspaceEntity,
| 'calendarEventId'
| 'eventExternalId'
| 'calendarChannelId'
| 'recurringEventExternalId'
>[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents.map(
({
fetchedCalendarEvent,
existingCalendarEvent,
newlyCreatedCalendarEvent,
}) => {
const calendarEventId =
existingCalendarEvent?.id ?? newlyCreatedCalendarEvent?.id;
if (!calendarEventId) {
throw new Error(
`Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`,
);
}
return {
calendarEventId,
eventExternalId: fetchedCalendarEvent.id,
calendarChannelId: calendarChannel.id,
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
};
},
);
await calendarChannelEventAssociationRepository.save(
calendarChannelEventAssociationsToSave,
{},
transactionManager,
);
const participantsToCreate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ newlyCreatedCalendarEvent }) =>
newlyCreatedCalendarEvent !== null,
)
.flatMap(({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
throw new Error(
`Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map((participant) => ({
...participant,
calendarEventId: newlyCreatedCalendarEvent.id,
}));
});
// todo: we should prevent duplicate rows on calendarEventAssociation by creating
// an index on calendarChannelId and calendarEventId
const participantsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent?.id) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map((participant) => ({
...participant,
calendarEventId: existingCalendarEvent.id,
}));
});
await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants(
{
participantsToCreate,
participantsToUpdate,
transactionManager,
calendarChannel,
connectedAccount,
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const savedCalendarEvents = await calendarEventRepository.save(
fetchedCalendarEventsWithDBEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent === null,
)
.map(
({ fetchedCalendarEvent }) =>
({
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution:
fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel:
fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
}) satisfies Omit<
CalendarEventWorkspaceEntity,
| 'id'
| 'calendarChannelEventAssociations'
| 'calendarEventParticipants'
| 'createdAt'
| 'updatedAt'
| 'deletedAt'
>,
),
{},
transactionManager,
);
const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEventsWithDBEvents.map(
({ fetchedCalendarEvent, existingCalendarEvent }) => {
const savedCalendarEvent = savedCalendarEvents.find(
(savedCalendarEvent) =>
savedCalendarEvent.iCalUid ===
fetchedCalendarEvent.iCalUid,
);
return {
fetchedCalendarEvent,
existingCalendarEvent: existingCalendarEvent,
newlyCreatedCalendarEvent: savedCalendarEvent ?? null,
};
},
);
await calendarEventRepository.save(
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return {
id: existingCalendarEvent.id,
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel:
fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
} satisfies Omit<
CalendarEventWorkspaceEntity,
| 'calendarChannelEventAssociations'
| 'calendarEventParticipants'
| 'createdAt'
| 'updatedAt'
| 'deletedAt'
>;
}),
{},
transactionManager,
);
const calendarChannelEventAssociationsToSave: Pick<
CalendarChannelEventAssociationWorkspaceEntity,
| 'calendarEventId'
| 'eventExternalId'
| 'calendarChannelId'
| 'recurringEventExternalId'
>[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents.map(
({
fetchedCalendarEvent,
existingCalendarEvent,
newlyCreatedCalendarEvent,
}) => {
const calendarEventId =
existingCalendarEvent?.id ?? newlyCreatedCalendarEvent?.id;
if (!calendarEventId) {
throw new Error(
`Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`,
);
}
return {
calendarEventId,
eventExternalId: fetchedCalendarEvent.id,
calendarChannelId: calendarChannel.id,
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
};
},
);
await calendarChannelEventAssociationRepository.save(
calendarChannelEventAssociationsToSave,
{},
transactionManager,
);
const participantsToCreate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ newlyCreatedCalendarEvent }) =>
newlyCreatedCalendarEvent !== null,
)
.flatMap(
({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
throw new Error(
`Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map(
(participant) => ({
...participant,
calendarEventId: newlyCreatedCalendarEvent.id,
}),
);
},
);
// todo: we should prevent duplicate rows on calendarEventAssociation by creating
// an index on calendarChannelId and calendarEventId
const participantsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent?.id) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map(
(participant) => ({
...participant,
calendarEventId: existingCalendarEvent.id,
}),
);
});
await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants(
{
participantsToCreate,
participantsToUpdate,
transactionManager,
calendarChannel,
connectedAccount,
workspaceId,
},
);
},
);
},
@@ -10,7 +10,8 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { type FetchedCalendarEventParticipant } from 'src/modules/calendar/common/types/fetched-calendar-event';
@@ -34,7 +35,7 @@ type FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId =
@Injectable()
export class CalendarEventParticipantService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly matchParticipantService: MatchParticipantService<CalendarEventParticipantWorkspaceEntity>,
@InjectMessageQueue(MessageQueue.contactCreationQueue)
private readonly messageQueueService: MessageQueueService,
@@ -55,125 +56,133 @@ export class CalendarEventParticipantService {
connectedAccount: ConnectedAccountWorkspaceEntity;
workspaceId: string;
}): Promise<void> {
const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200);
const authContext = buildSystemAuthContext(workspaceId);
const calendarEventParticipantRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarEventParticipantWorkspaceEntity>(
workspaceId,
'calendarEventParticipant',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200);
for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) {
const existingCalendarEventParticipants =
await calendarEventParticipantRepository.find({
where: {
calendarEventId: Any(
participantsToUpdateChunk
.map((participant) => participant.calendarEventId)
.filter(isDefined),
),
},
});
const calendarEventParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventParticipantWorkspaceEntity>(
workspaceId,
'calendarEventParticipant',
);
const {
calendarEventParticipantsToUpdate,
newCalendarEventParticipants,
} = participantsToUpdateChunk.reduce<{
calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[];
newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[];
}>(
(acc, calendarEventParticipant) => {
const existingCalendarEventParticipant =
existingCalendarEventParticipants.find(
(existingCalendarEventParticipant) =>
existingCalendarEventParticipant.handle ===
calendarEventParticipant.handle &&
existingCalendarEventParticipant.calendarEventId ===
calendarEventParticipant.calendarEventId,
for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) {
const existingCalendarEventParticipants =
await calendarEventParticipantRepository.find({
where: {
calendarEventId: Any(
participantsToUpdateChunk
.map((participant) => participant.calendarEventId)
.filter(isDefined),
),
},
});
const {
calendarEventParticipantsToUpdate,
newCalendarEventParticipants,
} = participantsToUpdateChunk.reduce<{
calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[];
newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[];
}>(
(acc, calendarEventParticipant) => {
const existingCalendarEventParticipant =
existingCalendarEventParticipants.find(
(existingCalendarEventParticipant) =>
existingCalendarEventParticipant.handle ===
calendarEventParticipant.handle &&
existingCalendarEventParticipant.calendarEventId ===
calendarEventParticipant.calendarEventId,
);
if (existingCalendarEventParticipant) {
acc.calendarEventParticipantsToUpdate.push({
...calendarEventParticipant,
id: existingCalendarEventParticipant.id,
});
} else {
acc.newCalendarEventParticipants.push(calendarEventParticipant);
}
return acc;
},
{
calendarEventParticipantsToUpdate: [],
newCalendarEventParticipants: [],
},
);
const calendarEventParticipantsToDelete = differenceWith(
existingCalendarEventParticipants,
participantsToUpdateChunk,
(existingCalendarEventParticipant, participantToUpdate) =>
existingCalendarEventParticipant.handle ===
participantToUpdate.handle &&
existingCalendarEventParticipant.calendarEventId ===
participantToUpdate.calendarEventId,
);
await calendarEventParticipantRepository.delete(
{
id: Any(
calendarEventParticipantsToDelete.map(
(calendarEventParticipant) => calendarEventParticipant.id,
),
),
},
transactionManager,
);
await calendarEventParticipantRepository.updateMany(
calendarEventParticipantsToUpdate.map((participant) => ({
criteria: participant.id,
partialEntity: participant,
})),
transactionManager,
);
participantsToCreate.push(...newCalendarEventParticipants);
}
const chunkedParticipantsToCreate = chunk(participantsToCreate, 200);
const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = [];
for (const participantsToCreateChunk of chunkedParticipantsToCreate) {
const savedParticipantsChunk =
await calendarEventParticipantRepository.insert(
participantsToCreateChunk,
transactionManager,
);
if (existingCalendarEventParticipant) {
acc.calendarEventParticipantsToUpdate.push({
...calendarEventParticipant,
id: existingCalendarEventParticipant.id,
});
} else {
acc.newCalendarEventParticipants.push(calendarEventParticipant);
}
savedParticipants.push(...savedParticipantsChunk.raw);
}
return acc;
},
{
calendarEventParticipantsToUpdate: [],
newCalendarEventParticipants: [],
},
);
if (calendarChannel.isContactAutoCreationEnabled) {
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
CreateCompanyAndContactJob.name,
{
workspaceId,
connectedAccount,
contactsToCreate: savedParticipants.map((participant) => ({
handle: participant.handle ?? '',
displayName:
participant.displayName ?? participant.handle ?? '',
})),
source: FieldActorSource.CALENDAR,
},
);
}
const calendarEventParticipantsToDelete = differenceWith(
existingCalendarEventParticipants,
participantsToUpdateChunk,
(existingCalendarEventParticipant, participantToUpdate) =>
existingCalendarEventParticipant.handle ===
participantToUpdate.handle &&
existingCalendarEventParticipant.calendarEventId ===
participantToUpdate.calendarEventId,
);
await calendarEventParticipantRepository.delete(
{
id: Any(
calendarEventParticipantsToDelete.map(
(calendarEventParticipant) => calendarEventParticipant.id,
),
),
},
transactionManager,
);
await calendarEventParticipantRepository.updateMany(
calendarEventParticipantsToUpdate.map((participant) => ({
criteria: participant.id,
partialEntity: participant,
})),
transactionManager,
);
participantsToCreate.push(...newCalendarEventParticipants);
}
const chunkedParticipantsToCreate = chunk(participantsToCreate, 200);
const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = [];
for (const participantsToCreateChunk of chunkedParticipantsToCreate) {
const savedParticipantsChunk =
await calendarEventParticipantRepository.insert(
participantsToCreateChunk,
await this.matchParticipantService.matchParticipants({
participants: savedParticipants,
objectMetadataName: 'calendarEventParticipant',
transactionManager,
);
savedParticipants.push(...savedParticipantsChunk.raw);
}
if (calendarChannel.isContactAutoCreationEnabled) {
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
CreateCompanyAndContactJob.name,
{
matchWith: 'workspaceMemberAndPerson',
workspaceId,
connectedAccount,
contactsToCreate: savedParticipants.map((participant) => ({
handle: participant.handle ?? '',
displayName: participant.displayName ?? participant.handle ?? '',
})),
source: FieldActorSource.CALENDAR,
},
);
}
await this.matchParticipantService.matchParticipants({
participants: savedParticipants,
objectMetadataName: 'calendarEventParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
});
},
);
}
}
@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED } from 'twenty-shared/constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { CalendarChannelVisibility } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity';
@@ -52,20 +52,21 @@ describe('ApplyCalendarEventsVisibilityRestrictionsService', () => {
findOneByOrFail: jest.fn(),
};
const mockTwentyORMGlobalManager = {
getRepositoryForWorkspace: jest
const mockGlobalWorkspaceOrmManager = {
getRepository: jest.fn().mockImplementation((workspaceId, name) => {
if (name === 'calendarChannelEventAssociation') {
return mockCalendarEventAssociationRepository;
}
if (name === 'connectedAccount') {
return mockConnectedAccountRepository;
}
if (name === 'workspaceMember') {
return mockWorkspaceMemberRepository;
}
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((workspaceId, name) => {
if (name === 'calendarChannelEventAssociation') {
return mockCalendarEventAssociationRepository;
}
if (name === 'connectedAccount') {
return mockConnectedAccountRepository;
}
if (name === 'workspaceMember') {
return mockWorkspaceMemberRepository;
}
}),
.mockImplementation((_authContext: any, fn: () => any) => fn()),
};
beforeEach(async () => {
@@ -73,8 +74,8 @@ describe('ApplyCalendarEventsVisibilityRestrictionsService', () => {
providers: [
ApplyCalendarEventsVisibilityRestrictionsService,
{
provide: TwentyORMGlobalManager,
useValue: mockTwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: mockGlobalWorkspaceOrmManager,
},
],
}).compile();
@@ -5,7 +5,8 @@ import { FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED } from 'twenty-shared/
import { isDefined } from 'twenty-shared/utils';
import { In } from 'typeorm';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelVisibility } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity';
@@ -15,7 +16,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
@Injectable()
export class ApplyCalendarEventsVisibilityRestrictionsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async applyCalendarEventsVisibilityRestrictions(
@@ -23,90 +24,99 @@ export class ApplyCalendarEventsVisibilityRestrictionsService {
workspaceId: string,
userId?: string, // undefined when request is made with api key
) {
const calendarChannelEventAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannelCalendarEventsAssociations =
await calendarChannelEventAssociationRepository.find({
where: {
calendarEventId: In(calendarEvents.map((event) => event.id)),
},
relations: ['calendarChannel'],
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
for (let i = calendarEvents.length - 1; i >= 0; i--) {
const calendarChannelCalendarEventAssociations =
calendarChannelCalendarEventsAssociations.filter(
(association) => association.calendarEventId === calendarEvents[i].id,
);
const calendarChannels = calendarChannelCalendarEventAssociations.map(
(association) => association.calendarChannel,
);
const calendarChannelsGroupByVisibility = groupBy(
calendarChannels,
(channel) => channel.visibility,
);
if (
calendarChannelsGroupByVisibility[
CalendarChannelVisibility.SHARE_EVERYTHING
]
) {
continue;
}
if (isDefined(userId)) {
const workspaceMember = await workspaceMemberRepository.findOneByOrFail(
{
userId: userId,
},
);
const connectedAccounts = await connectedAccountRepository.find({
select: ['id'],
where: {
calendarChannels: {
id: In(calendarChannels.map((channel) => channel.id)),
const calendarChannelCalendarEventsAssociations =
await calendarChannelEventAssociationRepository.find({
where: {
calendarEventId: In(calendarEvents.map((event) => event.id)),
},
accountOwnerId: workspaceMember.id,
},
});
relations: ['calendarChannel'],
});
if (connectedAccounts.length > 0) {
continue;
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
for (let i = calendarEvents.length - 1; i >= 0; i--) {
const calendarChannelCalendarEventAssociations =
calendarChannelCalendarEventsAssociations.filter(
(association) =>
association.calendarEventId === calendarEvents[i].id,
);
const calendarChannels = calendarChannelCalendarEventAssociations.map(
(association) => association.calendarChannel,
);
const calendarChannelsGroupByVisibility = groupBy(
calendarChannels,
(channel) => channel.visibility,
);
if (
calendarChannelsGroupByVisibility[
CalendarChannelVisibility.SHARE_EVERYTHING
]
) {
continue;
}
if (isDefined(userId)) {
const workspaceMember =
await workspaceMemberRepository.findOneByOrFail({
userId: userId,
});
const connectedAccounts = await connectedAccountRepository.find({
select: ['id'],
where: {
calendarChannels: {
id: In(calendarChannels.map((channel) => channel.id)),
},
accountOwnerId: workspaceMember.id,
},
});
if (connectedAccounts.length > 0) {
continue;
}
}
if (
calendarChannelsGroupByVisibility[
CalendarChannelVisibility.METADATA
]
) {
calendarEvents[i].title =
FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
calendarEvents[i].description =
FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
calendarEvents.splice(i, 1);
}
}
if (
calendarChannelsGroupByVisibility[CalendarChannelVisibility.METADATA]
) {
calendarEvents[i].title =
FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
calendarEvents[i].description =
FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
calendarEvents.splice(i, 1);
}
return calendarEvents;
return calendarEvents;
},
);
}
}
@@ -7,7 +7,8 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
CalendarChannelSyncStage,
CalendarChannelSyncStatus,
@@ -20,7 +21,7 @@ import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/acc
@Injectable()
export class CalendarChannelSyncStatusService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly accountsToReconnectService: AccountsToReconnectService,
@@ -36,16 +37,23 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
}
public async markAsCalendarEventListFetchOngoing(
@@ -56,17 +64,24 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
},
);
}
public async resetAndMarkAsCalendarEventListFetchPending(
@@ -83,17 +98,24 @@ export class CalendarChannelSyncStatusService {
);
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
});
},
);
await this.markAsCalendarEventListFetchPending(
calendarChannelIds,
@@ -109,15 +131,22 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStageStartedAt: null,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStageStartedAt: null,
});
},
);
}
public async markAsCalendarEventsImportPending(
@@ -129,16 +158,23 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
}
public async markAsCalendarEventsImportOngoing(
@@ -149,16 +185,23 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
},
);
}
public async markAsCompletedAndMarkAsCalendarEventListFetchPending(
@@ -169,19 +212,26 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
},
);
await this.markAsCalendarEventListFetchPending(
calendarChannelIds,
@@ -202,22 +252,29 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:${calendarChannelId}`,
);
}
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
});
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
});
},
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedUnknown,
@@ -233,49 +290,57 @@ export class CalendarChannelSyncStatusService {
return;
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:${calendarChannelId}`,
);
}
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
});
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannels = await calendarChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds) },
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const connectedAccountIds = calendarChannels.map(
(calendarChannel) => calendarChannel.connectedAccountId,
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
});
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const calendarChannels = await calendarChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds) },
});
const connectedAccountIds = calendarChannels.map(
(calendarChannel) => calendarChannel.connectedAccountId,
);
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
},
);
await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions,
eventIds: calendarChannelIds,
@@ -291,7 +356,7 @@ export class CalendarChannelSyncStatusService {
}
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
CalendarEventListFetchJob,
type CalendarEventListFetchJobData,
@@ -31,7 +32,7 @@ export type StartChannelSyncInput = {
@Injectable()
export class ChannelSyncService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.calendarQueue)
@@ -50,65 +51,80 @@ export class ChannelSyncService {
connectedAccountId: string,
workspaceId: string,
): Promise<void> {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannels = await messageChannelRepository.find({
where: {
connectedAccountId,
syncStage: MessageChannelSyncStage.PENDING_CONFIGURATION,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
where: {
connectedAccountId,
syncStage: MessageChannelSyncStage.PENDING_CONFIGURATION,
},
});
for (const messageChannel of messageChannels) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchScheduled(
[messageChannel.id],
workspaceId,
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
workspaceId,
messageChannelId: messageChannel.id,
},
);
}
},
});
for (const messageChannel of messageChannels) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchScheduled(
[messageChannel.id],
workspaceId,
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
workspaceId,
messageChannelId: messageChannel.id,
},
);
}
);
}
private async startCalendarChannelSync(
connectedAccountId: string,
workspaceId: string,
): Promise<void> {
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const calendarChannels = await calendarChannelRepository.find({
where: {
connectedAccountId,
syncStage: CalendarChannelSyncStage.PENDING_CONFIGURATION,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const calendarChannels = await calendarChannelRepository.find({
where: {
connectedAccountId,
syncStage: CalendarChannelSyncStage.PENDING_CONFIGURATION,
},
});
for (const calendarChannel of calendarChannels) {
await calendarChannelRepository.update(calendarChannel.id, {
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
}
},
});
for (const calendarChannel of calendarChannels) {
await calendarChannelRepository.update(calendarChannel.id, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
await this.calendarQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
}
);
}
}
@@ -3,7 +3,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { type Repository } from 'typeorm';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { GoogleEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service';
import { microsoftGraphMeResponseWithProxyAddresses } from 'src/modules/connected-account/email-alias-manager/drivers/microsoft/mocks/microsoft-api-examples';
import { MicrosoftEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/microsoft/services/microsoft-email-alias-manager.service';
@@ -28,11 +28,14 @@ describe('Email Alias Manager Service', () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getRepositoryForWorkspace: jest
getRepository: jest
.fn()
.mockResolvedValue(connectedAccountRepository),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
EmailAliasManagerService,
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { assertUnreachable } from 'twenty-shared/utils';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { GoogleEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service';
import { MicrosoftEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/microsoft/services/microsoft-email-alias-manager.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -13,7 +14,7 @@ export class EmailAliasManagerService {
constructor(
private readonly googleEmailAliasManagerService: GoogleEmailAliasManagerService,
private readonly microsoftEmailAliasManagerService: MicrosoftEmailAliasManagerService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async refreshHandleAliases(
@@ -46,16 +47,23 @@ export class EmailAliasManagerService {
);
}
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const authContext = buildSystemAuthContext(workspaceId);
await connectedAccountRepository.update(
{ id: connectedAccount.id },
{
handleAliases: handleAliases.join(','), // TODO: modify handleAliases to be of fieldmetadatatype array
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
await connectedAccountRepository.update(
{ id: connectedAccount.id },
{
handleAliases: handleAliases.join(','), // TODO: modify handleAliases to be of fieldmetadatatype array
},
);
},
);
}
@@ -1,7 +1,8 @@
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
export type DeleteWorkspaceMemberConnectedAccountsCleanupJobData = {
@@ -12,7 +13,7 @@ export type DeleteWorkspaceMemberConnectedAccountsCleanupJobData = {
@Processor(MessageQueue.deleteCascadeQueue)
export class DeleteWorkspaceMemberConnectedAccountsCleanupJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(DeleteWorkspaceMemberConnectedAccountsCleanupJob.name)
@@ -21,14 +22,21 @@ export class DeleteWorkspaceMemberConnectedAccountsCleanupJob {
): Promise<void> {
const { workspaceId, workspaceMemberId } = data;
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const authContext = buildSystemAuthContext(workspaceId);
await connectedAccountRepository.delete({
accountOwnerId: workspaceMemberId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
await connectedAccountRepository.delete({
accountOwnerId: workspaceMemberId,
});
},
);
}
}
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -12,7 +13,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
@Injectable()
export class ConnectedAccountListener {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}
@@ -22,28 +23,39 @@ export class ConnectedAccountListener {
ObjectRecordDeleteEvent<ConnectedAccountWorkspaceEntity>
>,
) {
for (const eventPayload of payload.events) {
const workspaceMemberId = eventPayload.properties.before.accountOwnerId;
const workspaceId = payload.workspaceId;
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = await workspaceMemberRepository.findOneOrFail({
where: { id: workspaceMemberId },
});
const workspaceId = payload.workspaceId;
const authContext = buildSystemAuthContext(workspaceId);
const userId = workspaceMember.userId;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
for (const eventPayload of payload.events) {
const workspaceMemberId =
eventPayload.properties.before.accountOwnerId;
const connectedAccountId = eventPayload.properties.before.id;
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = await workspaceMemberRepository.findOneOrFail(
{
where: { id: workspaceMemberId },
},
);
await this.accountsToReconnectService.removeAccountToReconnect(
userId,
workspaceId,
connectedAccountId,
);
}
const userId = workspaceMember.userId;
const connectedAccountId = eventPayload.properties.before.id;
await this.accountsToReconnectService.removeAccountToReconnect(
userId,
workspaceId,
connectedAccountId,
);
}
},
);
}
}
@@ -5,6 +5,7 @@ import { Repository } from 'typeorm';
import { type WorkspacePreQueryHookInstance } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/interfaces/workspace-query-hook.interface';
import { type DeleteOneResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator';
@@ -12,7 +13,7 @@ import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-contex
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { fromObjectMetadataEntityToFlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/utils/from-object-metadata-entity-to-flat-object-metadata.util';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@@ -21,7 +22,7 @@ export class ConnectedAccountDeleteOnePreQueryHook
implements WorkspacePreQueryHookInstance
{
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workspaceEventEmitter: WorkspaceEventEmitter,
@InjectRepository(ObjectMetadataEntity)
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
@@ -38,15 +39,21 @@ export class ConnectedAccountDeleteOnePreQueryHook
assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
const messageChannels =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext as WorkspaceAuthContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
const messageChannels = await messageChannelRepository.findBy({
connectedAccountId,
});
return messageChannelRepository.findBy({
connectedAccountId,
});
},
);
const objectMetadataEntity =
await this.objectMetadataRepository.findOneOrFail({
@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/refresh-tokens-manager/drivers/google/services/google-api-refresh-tokens.service';
import { MicrosoftAPIRefreshAccessTokenService } from 'src/modules/connected-account/refresh-tokens-manager/drivers/microsoft/services/microsoft-api-refresh-tokens.service';
import {
@@ -17,7 +17,7 @@ describe('ConnectedAccountRefreshTokensService', () => {
let service: ConnectedAccountRefreshTokensService;
let googleAPIRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService;
let microsoftAPIRefreshAccessTokenService: MicrosoftAPIRefreshAccessTokenService;
let twentyORMGlobalManager: TwentyORMGlobalManager;
let globalWorkspaceOrmManager: GlobalWorkspaceOrmManager;
const mockWorkspaceId = 'workspace-123';
const mockConnectedAccountId = 'account-456';
@@ -42,9 +42,13 @@ describe('ConnectedAccountRefreshTokensService', () => {
},
},
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getRepositoryForWorkspace: jest.fn(),
getRepository: jest.fn(),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
],
@@ -61,8 +65,8 @@ describe('ConnectedAccountRefreshTokensService', () => {
module.get<MicrosoftAPIRefreshAccessTokenService>(
MicrosoftAPIRefreshAccessTokenService,
);
twentyORMGlobalManager = module.get<TwentyORMGlobalManager>(
TwentyORMGlobalManager,
globalWorkspaceOrmManager = module.get<GlobalWorkspaceOrmManager>(
GlobalWorkspaceOrmManager,
);
});
@@ -92,9 +96,7 @@ describe('ConnectedAccountRefreshTokensService', () => {
expect(
microsoftAPIRefreshAccessTokenService.refreshTokens,
).not.toHaveBeenCalled();
expect(
twentyORMGlobalManager.getRepositoryForWorkspace,
).not.toHaveBeenCalled();
expect(globalWorkspaceOrmManager.getRepository).not.toHaveBeenCalled();
});
it('should refresh and save new Microsoft token when expired (lastCredentialsRefreshedAt is old)', async () => {
@@ -116,7 +118,7 @@ describe('ConnectedAccountRefreshTokensService', () => {
.spyOn(microsoftAPIRefreshAccessTokenService, 'refreshTokens')
.mockResolvedValue(newTokens);
jest
.spyOn(twentyORMGlobalManager, 'getRepositoryForWorkspace')
.spyOn(globalWorkspaceOrmManager, 'getRepository')
.mockResolvedValue(mockRepository as any);
const result = await service.refreshAndSaveTokens(
@@ -156,7 +158,7 @@ describe('ConnectedAccountRefreshTokensService', () => {
.spyOn(googleAPIRefreshAccessTokenService, 'refreshTokens')
.mockResolvedValue(newTokens);
jest
.spyOn(twentyORMGlobalManager, 'getRepositoryForWorkspace')
.spyOn(globalWorkspaceOrmManager, 'getRepository')
.mockResolvedValue(mockRepository as any);
const result = await service.refreshAndSaveTokens(
@@ -196,7 +198,7 @@ describe('ConnectedAccountRefreshTokensService', () => {
.spyOn(microsoftAPIRefreshAccessTokenService, 'refreshTokens')
.mockResolvedValue(newTokens);
jest
.spyOn(twentyORMGlobalManager, 'getRepositoryForWorkspace')
.spyOn(globalWorkspaceOrmManager, 'getRepository')
.mockResolvedValue(mockRepository as any);
const result = await service.refreshAndSaveTokens(
@@ -3,7 +3,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { assertUnreachable, isDefined } from 'twenty-shared/utils';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/refresh-tokens-manager/drivers/google/services/google-api-refresh-tokens.service';
import { MicrosoftAPIRefreshAccessTokenService } from 'src/modules/connected-account/refresh-tokens-manager/drivers/microsoft/services/microsoft-api-refresh-tokens.service';
import {
@@ -29,7 +30,7 @@ export class ConnectedAccountRefreshTokensService {
constructor(
private readonly googleAPIRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
private readonly microsoftAPIRefreshAccessTokenService: MicrosoftAPIRefreshAccessTokenService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async refreshAndSaveTokens(
@@ -75,17 +76,24 @@ export class ConnectedAccountRefreshTokensService {
workspaceId,
);
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const authContext = buildSystemAuthContext(workspaceId);
await connectedAccountRepository.update(
{ id: connectedAccount.id },
{
...connectedAccountTokens,
lastCredentialsRefreshedAt: new Date(),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
await connectedAccountRepository.update(
{ id: connectedAccount.id },
{
...connectedAccountTokens,
lastCredentialsRefreshedAt: new Date(),
},
);
},
);
@@ -5,7 +5,7 @@ import { ConnectedAccountProvider } from 'twenty-shared/types';
import { CreateCalendarChannelService } from 'src/engine/core-modules/auth/services/create-calendar-channel.service';
import { CreateMessageChannelService } from 'src/engine/core-modules/auth/services/create-message-channel.service';
import { type EmailAccountConnectionParameters } from 'src/engine/core-modules/imap-smtp-caldav-connection/dtos/imap-smtp-caldav-connection.dto';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { ImapSmtpCalDavAPIService } from 'src/modules/connected-account/services/imap-smtp-caldav-apis.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -50,9 +50,9 @@ describe('ImapSmtpCalDavAPIService', () => {
providers: [
ImapSmtpCalDavAPIService,
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getRepositoryForWorkspace: jest
getRepository: jest
.fn()
.mockImplementation((_workspaceId, entity) => {
if (entity === 'connectedAccount')
@@ -67,6 +67,10 @@ describe('ImapSmtpCalDavAPIService', () => {
getDataSourceForWorkspace: jest
.fn()
.mockImplementation(() => mockWorkspaceDataSource),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
{
@@ -8,7 +8,8 @@ import { CreateCalendarChannelService } from 'src/engine/core-modules/auth/servi
import { CreateMessageChannelService } from 'src/engine/core-modules/auth/services/create-message-channel.service';
import { type EmailAccountConnectionParameters } from 'src/engine/core-modules/imap-smtp-caldav-connection/dtos/imap-smtp-caldav-connection.dto';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@@ -16,11 +17,33 @@ import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common
@Injectable()
export class ImapSmtpCalDavAPIService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly createMessageChannelService: CreateMessageChannelService,
private readonly createCalendarChannelService: CreateCalendarChannelService,
) {}
async getImapSmtpCaldavConnectedAccount(
workspaceId: string,
id: string,
): Promise<ConnectedAccountWorkspaceEntity | null> {
const authContext = buildSystemAuthContext(workspaceId);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
return connectedAccountRepository.findOne({
where: { id, provider: ConnectedAccountProvider.IMAP_SMTP_CALDAV },
});
},
);
}
async processAccount(input: {
handle: string;
workspaceMemberId: string;
@@ -31,94 +54,101 @@ export class ImapSmtpCalDavAPIService {
const { handle, workspaceId, workspaceMemberId, connectedAccountId } =
input;
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const calendarChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const existingAccount = connectedAccountId
? await connectedAccountRepository.findOne({
where: { id: connectedAccountId },
})
: await connectedAccountRepository.findOne({
where: { handle, accountOwnerId: workspaceMemberId },
});
const calendarChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const newOrExistingAccountId =
existingAccount?.id ?? connectedAccountId ?? v4();
const existingAccount = connectedAccountId
? await connectedAccountRepository.findOne({
where: { id: connectedAccountId },
})
: await connectedAccountRepository.findOne({
where: { handle, accountOwnerId: workspaceMemberId },
});
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const newOrExistingAccountId =
existingAccount?.id ?? connectedAccountId ?? v4();
const existingMessageChannel = existingAccount
? await messageChannelRepository.findOne({
where: { connectedAccountId: existingAccount.id },
})
: null;
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
const existingCalendarChannel = existingAccount
? await calendarChannelRepository.findOne({
where: { connectedAccountId: existingAccount.id },
})
: null;
const existingMessageChannel = existingAccount
? await messageChannelRepository.findOne({
where: { connectedAccountId: existingAccount.id },
})
: null;
const shouldCreateMessageChannel =
!isDefined(existingMessageChannel) &&
Boolean(input.connectionParameters.IMAP);
const existingCalendarChannel = existingAccount
? await calendarChannelRepository.findOne({
where: { connectedAccountId: existingAccount.id },
})
: null;
const shouldCreateCalendarChannel =
!isDefined(existingCalendarChannel) &&
Boolean(input.connectionParameters.CALDAV);
const shouldCreateMessageChannel =
!isDefined(existingMessageChannel) &&
Boolean(input.connectionParameters.IMAP);
await workspaceDataSource.transaction(
async (manager: WorkspaceEntityManager) => {
await connectedAccountRepository.save(
{
id: newOrExistingAccountId,
handle,
provider: ConnectedAccountProvider.IMAP_SMTP_CALDAV,
connectionParameters: input.connectionParameters,
accountOwnerId: workspaceMemberId,
const shouldCreateCalendarChannel =
!isDefined(existingCalendarChannel) &&
Boolean(input.connectionParameters.CALDAV);
await workspaceDataSource.transaction(
async (manager: WorkspaceEntityManager) => {
await connectedAccountRepository.save(
{
id: newOrExistingAccountId,
handle,
provider: ConnectedAccountProvider.IMAP_SMTP_CALDAV,
connectionParameters: input.connectionParameters,
accountOwnerId: workspaceMemberId,
},
{},
manager,
);
if (shouldCreateMessageChannel) {
await this.createMessageChannelService.createMessageChannel({
workspaceId,
connectedAccountId: newOrExistingAccountId,
handle,
manager,
});
}
if (shouldCreateCalendarChannel) {
await this.createCalendarChannelService.createCalendarChannel({
workspaceId,
connectedAccountId: newOrExistingAccountId,
handle,
manager,
});
}
},
{},
manager,
);
if (shouldCreateMessageChannel) {
await this.createMessageChannelService.createMessageChannel({
workspaceId,
connectedAccountId: newOrExistingAccountId,
handle,
manager,
});
}
if (shouldCreateCalendarChannel) {
await this.createCalendarChannelService.createCalendarChannel({
workspaceId,
connectedAccountId: newOrExistingAccountId,
handle,
manager,
});
}
return newOrExistingAccountId;
},
);
return newOrExistingAccountId;
}
}
@@ -3,7 +3,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { FieldActorSource } from 'twenty-shared/types';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndPersonService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
@@ -41,7 +41,7 @@ describe('CreateCompanyAndPersonService', () => {
useValue: mockCreatePersonService,
},
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {},
},
{
@@ -2,14 +2,14 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import axios from 'axios';
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
import {
ConnectedAccountProvider,
FieldActorSource,
} from 'twenty-shared/types';
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import {
type CompanyToCreate,
CreateCompanyService,
@@ -115,11 +115,12 @@ describe('CreateCompanyService', () => {
providers: [
CreateCompanyService,
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getRepositoryForWorkspace: jest
getRepository: jest.fn().mockResolvedValue(mockCompanyRepository),
executeInWorkspaceContext: jest
.fn()
.mockResolvedValue(mockCompanyRepository),
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
{
@@ -12,7 +12,8 @@ import { type DeepPartial } from 'typeorm';
import { v4 } from 'uuid';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/contact-creation-manager/constants/contacts-creation-batch-size.constant';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
@@ -27,12 +28,13 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { computeDisplayName } from 'src/utils/compute-display-name';
import { isWorkDomain, isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class CreateCompanyAndPersonService {
constructor(
private readonly createPersonService: CreatePersonService,
private readonly createCompaniesService: CreateCompanyService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@@ -46,96 +48,103 @@ export class CreateCompanyAndPersonService {
return [];
}
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceId);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const personRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const workspaceMembers = await workspaceMemberRepository.find();
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
const peopleToCreateFromOtherCompanies =
filterOutContactsThatBelongToSelfOrWorkspaceMembers(
contactsToCreate,
connectedAccount,
workspaceMembers,
);
const workspaceMembers = await workspaceMemberRepository.find();
const { uniqueContacts, uniqueHandles } = getUniqueContactsAndHandles(
peopleToCreateFromOtherCompanies,
);
const peopleToCreateFromOtherCompanies =
filterOutContactsThatBelongToSelfOrWorkspaceMembers(
contactsToCreate,
connectedAccount,
workspaceMembers,
);
if (uniqueHandles.length === 0) {
return [];
}
const { uniqueContacts, uniqueHandles } = getUniqueContactsAndHandles(
peopleToCreateFromOtherCompanies,
);
const queryBuilder = addPersonEmailFiltersToQueryBuilder({
queryBuilder: personRepository.createQueryBuilder('person'),
emails: uniqueHandles,
});
if (uniqueHandles.length === 0) {
return [];
}
const alreadyCreatedPeople = await queryBuilder
.orderBy('person.createdAt', 'ASC')
.withDeleted()
.getMany();
const queryBuilder = addPersonEmailFiltersToQueryBuilder({
queryBuilder: personRepository.createQueryBuilder('person'),
emails: uniqueHandles,
});
const {
contactsThatNeedPersonCreate,
contactsThatNeedPersonRestore,
workDomainNamesToCreate,
shouldCreateOrRestorePeopleByHandleMap,
} =
this.computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
uniqueContacts,
alreadyCreatedPeople,
source,
connectedAccount,
);
const alreadyCreatedPeople = await queryBuilder
.orderBy('person.createdAt', 'ASC')
.withDeleted()
.getMany();
const companiesMap =
await this.createCompaniesService.createOrRestoreCompanies(
workDomainNamesToCreate,
workspaceId,
);
const {
contactsThatNeedPersonCreate,
contactsThatNeedPersonRestore,
workDomainNamesToCreate,
shouldCreateOrRestorePeopleByHandleMap,
} =
this.computeContactsThatNeedPersonCreateAndRestoreAndWorkDomainNamesToCreate(
uniqueContacts,
alreadyCreatedPeople,
source,
connectedAccount,
);
const peopleToCreate = this.formatPeopleToCreateFromContacts({
contactsToCreate: contactsThatNeedPersonCreate,
createdBy: {
source: source,
workspaceMember: connectedAccount.accountOwner,
context: {
provider: connectedAccount.provider,
},
const companiesMap =
await this.createCompaniesService.createOrRestoreCompanies(
workDomainNamesToCreate,
workspaceId,
);
const peopleToCreate = this.formatPeopleToCreateFromContacts({
contactsToCreate: contactsThatNeedPersonCreate,
createdBy: {
source: source,
workspaceMember: connectedAccount.accountOwner,
context: {
provider: connectedAccount.provider,
},
},
companiesMap,
});
const createdPeople = await this.createPersonService.createPeople(
peopleToCreate,
workspaceId,
);
const peopleToRestore = this.formatPeopleToRestoreFromContacts({
contactsToRestore: contactsThatNeedPersonRestore,
companiesMap,
shouldCreateOrRestorePeopleByHandleMap,
});
const restoredPeople = await this.createPersonService.restorePeople(
peopleToRestore,
workspaceId,
);
return { ...createdPeople, ...restoredPeople };
},
companiesMap,
});
const createdPeople = await this.createPersonService.createPeople(
peopleToCreate,
workspaceId,
);
const peopleToRestore = this.formatPeopleToRestoreFromContacts({
contactsToRestore: contactsThatNeedPersonRestore,
companiesMap,
shouldCreateOrRestorePeopleByHandleMap,
});
const restoredPeople = await this.createPersonService.restorePeople(
peopleToRestore,
workspaceId,
);
return { ...createdPeople, ...restoredPeople };
}
async createCompaniesAndPeopleAndUpdateParticipants(
@@ -149,27 +158,34 @@ export class CreateCompanyAndPersonService {
CONTACTS_CREATION_BATCH_SIZE,
);
if (!connectedAccount.accountOwner) {
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
const authContext = buildSystemAuthContext(workspaceId);
const workspaceMember = await workspaceMemberRepository.findOne({
where: {
id: connectedAccount.accountOwnerId,
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
if (!connectedAccount.accountOwner) {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkspaceMemberWorkspaceEntity,
);
if (!workspaceMember) {
throw new Error(
`Workspace member with id ${connectedAccount.accountOwnerId} not found in workspace ${workspaceId}`,
);
}
const workspaceMember = await workspaceMemberRepository.findOne({
where: {
id: connectedAccount.accountOwnerId,
},
});
connectedAccount.accountOwner = workspaceMember;
}
if (!workspaceMember) {
throw new Error(
`Workspace member with id ${connectedAccount.accountOwnerId} not found in workspace ${workspaceId}`,
);
}
connectedAccount.accountOwner = workspaceMember;
}
},
);
for (const contactsBatch of contactsBatches) {
try {
@@ -13,8 +13,9 @@ import {
} from 'twenty-shared/utils';
import { type DeepPartial, ILike } from 'typeorm';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity';
import { extractDomainFromLink } from 'src/modules/contact-creation-manager/utils/extract-domain-from-link.util';
import { getCompanyNameFromDomainName } from 'src/modules/contact-creation-manager/utils/get-company-name-from-domain-name.util';
@@ -34,7 +35,9 @@ export type CompanyToCreate = {
export class CreateCompanyService {
private readonly httpService: AxiosInstance;
constructor(private readonly twentyORMGlobalManager: TwentyORMGlobalManager) {
constructor(
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {
this.httpService = axios.create({
baseURL: TWENTY_COMPANIES_BASE_URL,
});
@@ -50,97 +53,111 @@ export class CreateCompanyService {
return {};
}
const companyRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
CompanyWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceId);
const companiesWithoutTrailingSlash = companies.map((company) => ({
...company,
domainName: company.domainName
? lowercaseUrlOriginAndRemoveTrailingSlash(company.domainName)
: undefined,
}));
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const companyRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
CompanyWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const uniqueCompanies = uniqBy(companiesWithoutTrailingSlash, 'domainName');
const conditions = uniqueCompanies.map((companyToCreate) => ({
domainName: {
primaryLinkUrl: ILike(`%${companyToCreate.domainName}%`),
},
}));
const companiesWithoutTrailingSlash = companies.map((company) => ({
...company,
domainName: company.domainName
? lowercaseUrlOriginAndRemoveTrailingSlash(company.domainName)
: undefined,
}));
const existingCompanies = await companyRepository.find({
where: conditions,
withDeleted: true,
});
const existingCompanyIdsMap = this.createCompanyMap(existingCompanies);
const newCompaniesToCreate = uniqueCompanies.filter(
(company) =>
!existingCompanies.some(
(existingCompany) =>
existingCompany.domainName &&
extractDomainFromLink(existingCompany.domainName.primaryLinkUrl) ===
company.domainName,
),
);
const companiesToRestore = this.filterCompaniesToRestore(
uniqueCompanies,
existingCompanies,
);
if (newCompaniesToCreate.length === 0 && companiesToRestore.length === 0) {
return existingCompanyIdsMap;
}
let lastCompanyPosition =
await this.getLastCompanyPosition(companyRepository);
const newCompaniesData = await Promise.all(
newCompaniesToCreate.map((company) =>
this.prepareCompanyData(company, ++lastCompanyPosition),
),
);
const createdCompanies = await companyRepository.save(newCompaniesData);
const restoredCompanies = await companyRepository.updateMany(
companiesToRestore.map((company) => {
return {
criteria: company.id,
partialEntity: {
deletedAt: null,
},
};
}),
undefined,
['domainNamePrimaryLinkUrl', 'id'],
);
const formattedRestoredCompanies = restoredCompanies.raw.map(
(row: { id: string; domainNamePrimaryLinkUrl: string }) => {
return {
id: row.id,
const uniqueCompanies = uniqBy(
companiesWithoutTrailingSlash,
'domainName',
);
const conditions = uniqueCompanies.map((companyToCreate) => ({
domainName: {
primaryLinkUrl: row.domainNamePrimaryLinkUrl,
primaryLinkUrl: ILike(`%${companyToCreate.domainName}%`),
},
}));
const existingCompanies = await companyRepository.find({
where: conditions,
withDeleted: true,
});
const existingCompanyIdsMap = this.createCompanyMap(existingCompanies);
const newCompaniesToCreate = uniqueCompanies.filter(
(company) =>
!existingCompanies.some(
(existingCompany) =>
existingCompany.domainName &&
extractDomainFromLink(
existingCompany.domainName.primaryLinkUrl,
) === company.domainName,
),
);
const companiesToRestore = this.filterCompaniesToRestore(
uniqueCompanies,
existingCompanies,
);
if (
newCompaniesToCreate.length === 0 &&
companiesToRestore.length === 0
) {
return existingCompanyIdsMap;
}
let lastCompanyPosition =
await this.getLastCompanyPosition(companyRepository);
const newCompaniesData = await Promise.all(
newCompaniesToCreate.map((company) =>
this.prepareCompanyData(company, ++lastCompanyPosition),
),
);
const createdCompanies = await companyRepository.save(newCompaniesData);
const restoredCompanies = await companyRepository.updateMany(
companiesToRestore.map((company) => {
return {
criteria: company.id,
partialEntity: {
deletedAt: null,
},
};
}),
undefined,
['domainNamePrimaryLinkUrl', 'id'],
);
const formattedRestoredCompanies = restoredCompanies.raw.map(
(row: { id: string; domainNamePrimaryLinkUrl: string }) => {
return {
id: row.id,
domainName: {
primaryLinkUrl: row.domainNamePrimaryLinkUrl,
},
};
},
);
return {
...existingCompanyIdsMap,
...(createdCompanies.length > 0
? this.createCompanyMap(createdCompanies)
: {}),
...(formattedRestoredCompanies.length > 0
? this.createCompanyMap(formattedRestoredCompanies)
: {}),
};
},
);
return {
...existingCompanyIdsMap,
...(createdCompanies.length > 0
? this.createCompanyMap(createdCompanies)
: {}),
...(formattedRestoredCompanies.length > 0
? this.createCompanyMap(formattedRestoredCompanies)
: {}),
};
}
private filterCompaniesToRestore(
@@ -2,14 +2,15 @@ import { Injectable } from '@nestjs/common';
import { DeepPartial } from 'typeorm';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
@Injectable()
export class CreatePersonService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async createPeople(
@@ -18,28 +19,35 @@ export class CreatePersonService {
): Promise<DeepPartial<PersonWorkspaceEntity>[]> {
if (peopleToCreate.length === 0) return [];
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceId);
const lastPersonPosition =
await this.getLastPersonPosition(personRepository);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const personRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const createdPeople = await personRepository.insert(
peopleToCreate.map((person, index) => ({
...person,
position: lastPersonPosition + index,
})),
undefined,
['companyId', 'id'],
const lastPersonPosition =
await this.getLastPersonPosition(personRepository);
const createdPeople = await personRepository.insert(
peopleToCreate.map((person, index) => ({
...person,
position: lastPersonPosition + index,
})),
undefined,
['companyId', 'id'],
);
return createdPeople.raw;
},
);
return createdPeople.raw;
}
public async restorePeople(
@@ -50,28 +58,35 @@ export class CreatePersonService {
return [];
}
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceId);
const restoredPeople = await personRepository.updateMany(
people.map(({ personId, companyId }) => ({
criteria: personId,
partialEntity: {
deletedAt: null,
companyId,
},
})),
undefined,
['companyId', 'id'],
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const personRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
PersonWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const restoredPeople = await personRepository.updateMany(
people.map(({ personId, companyId }) => ({
criteria: personId,
partialEntity: {
deletedAt: null,
companyId,
},
})),
undefined,
['companyId', 'id'],
);
return restoredPeople.raw;
},
);
return restoredPeople.raw;
}
private async getLastPersonPosition(
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type FavoriteFolderWorkspaceEntity } from 'src/modules/favorite-folder/standard-objects/favorite-folder.workspace-entity';
import { type FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
@@ -11,7 +12,7 @@ import { type FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-obje
@Injectable()
export class FavoriteFolderDeletionListener {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@OnDatabaseBatchEvent('favoriteFolder', DatabaseEventAction.DELETED)
@@ -20,17 +21,25 @@ export class FavoriteFolderDeletionListener {
ObjectRecordDeleteEvent<FavoriteFolderWorkspaceEntity>
>,
) {
for (const eventPayload of payload.events) {
const favoriteRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<FavoriteWorkspaceEntity>(
payload.workspaceId,
'favorite',
);
const workspaceId = payload.workspaceId;
const authContext = buildSystemAuthContext(workspaceId);
await favoriteRepository.update(
{ favoriteFolderId: eventPayload.recordId },
{ deletedAt: new Date().toISOString() },
);
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
for (const eventPayload of payload.events) {
const favoriteRepository =
await this.globalWorkspaceOrmManager.getRepository<FavoriteWorkspaceEntity>(
workspaceId,
'favorite',
);
await favoriteRepository.update(
{ favoriteFolderId: eventPayload.recordId },
{ deletedAt: new Date().toISOString() },
);
}
},
);
}
}
@@ -6,7 +6,8 @@ import { In, Repository } from 'typeorm';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { FAVORITE_DELETION_BATCH_SIZE } from 'src/modules/favorite/constants/favorite-deletion-batch-size';
import { type FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
@@ -18,69 +19,77 @@ export class FavoriteDeletionService {
@InjectRepository(FieldMetadataEntity)
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async deleteFavoritesForDeletedRecords(
deletedRecordIds: string[],
workspaceId: string,
): Promise<void> {
const favoriteRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<FavoriteWorkspaceEntity>(
workspaceId,
'favorite',
);
const authContext = buildSystemAuthContext(workspaceId);
const favoriteObjectMetadata = await this.objectMetadataRepository.findOne({
where: {
nameSingular: 'favorite',
workspaceId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const favoriteRepository =
await this.globalWorkspaceOrmManager.getRepository<FavoriteWorkspaceEntity>(
workspaceId,
'favorite',
);
const favoriteObjectMetadata =
await this.objectMetadataRepository.findOne({
where: {
nameSingular: 'favorite',
workspaceId,
},
});
if (!favoriteObjectMetadata) {
throw new Error('Favorite object metadata not found');
}
const favoriteFields = await this.fieldMetadataRepository.find({
where: {
objectMetadataId: favoriteObjectMetadata.id,
type: FieldMetadataType.RELATION,
},
});
const favoritesToDelete = await favoriteRepository.find({
select: {
id: true,
},
where: favoriteFields.map((field) => ({
[`${field.name}Id`]: In(deletedRecordIds),
})),
withDeleted: true,
});
if (favoritesToDelete.length === 0) {
return;
}
const favoriteIdsToDelete = favoritesToDelete.map(
(favorite) => favorite.id,
);
const batches: string[][] = [];
for (
let i = 0;
i < favoriteIdsToDelete.length;
i += FAVORITE_DELETION_BATCH_SIZE
) {
batches.push(
favoriteIdsToDelete.slice(i, i + FAVORITE_DELETION_BATCH_SIZE),
);
}
for (const batch of batches) {
await favoriteRepository.delete(batch);
}
},
});
if (!favoriteObjectMetadata) {
throw new Error('Favorite object metadata not found');
}
const favoriteFields = await this.fieldMetadataRepository.find({
where: {
objectMetadataId: favoriteObjectMetadata.id,
type: FieldMetadataType.RELATION,
},
});
const favoritesToDelete = await favoriteRepository.find({
select: {
id: true,
},
where: favoriteFields.map((field) => ({
[`${field.name}Id`]: In(deletedRecordIds),
})),
withDeleted: true,
});
if (favoritesToDelete.length === 0) {
return;
}
const favoriteIdsToDelete = favoritesToDelete.map(
(favorite) => favorite.id,
);
const batches: string[][] = [];
for (
let i = 0;
i < favoriteIdsToDelete.length;
i += FAVORITE_DELETION_BATCH_SIZE
) {
batches.push(
favoriteIdsToDelete.slice(i, i + FAVORITE_DELETION_BATCH_SIZE),
);
}
for (const batch of batches) {
await favoriteRepository.delete(batch);
}
}
}
@@ -5,7 +5,8 @@ import { isDefined } from 'twenty-shared/utils';
import { Any, In } from 'typeorm';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { type CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { addPersonEmailFiltersToQueryBuilder } from 'src/modules/match-participant/utils/add-person-email-filters-to-query-builder';
@@ -59,7 +60,7 @@ export class MatchParticipantService<
> {
constructor(
private readonly workspaceEventEmitter: WorkspaceEventEmitter,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
private async getParticipantRepository(
@@ -67,13 +68,13 @@ export class MatchParticipantService<
objectMetadataName: 'messageParticipant' | 'calendarEventParticipant',
) {
if (objectMetadataName === 'messageParticipant') {
return await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageParticipantWorkspaceEntity>(
return await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
objectMetadataName,
);
}
return await this.twentyORMGlobalManager.getRepositoryForWorkspace<CalendarEventParticipantWorkspaceEntity>(
return await this.globalWorkspaceOrmManager.getRepository<CalendarEventParticipantWorkspaceEntity>(
workspaceId,
objectMetadataName,
);
@@ -91,7 +92,7 @@ export class MatchParticipantService<
}
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<PersonWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<PersonWorkspaceEntity>(
workspaceId,
'person',
{ shouldBypassPermissionChecks: true },
@@ -103,7 +104,7 @@ export class MatchParticipantService<
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
@@ -207,30 +208,38 @@ export class MatchParticipantService<
objectMetadataName,
workspaceId,
}: MatchParticipantsForWorkspaceMembersArgs) {
const participantRepository = await this.getParticipantRepository(
workspaceId,
objectMetadataName,
);
const authContext = buildSystemAuthContext(workspaceId);
const participants = await participantRepository.find({
where: {
workspaceMemberId: In(participantMatching.workspaceMemberIds),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const participantRepository = await this.getParticipantRepository(
workspaceId,
objectMetadataName,
);
const participants = await participantRepository.find({
where: {
workspaceMemberId: In(participantMatching.workspaceMemberIds),
},
});
const tobeRematchedParticipants = participants.map((participant) => {
return {
...participant,
workspaceMemberId: null,
};
});
await this.matchParticipants({
matchWith: 'workspaceMemberOnly',
participants:
tobeRematchedParticipants as ParticipantWorkspaceEntity[],
objectMetadataName,
workspaceId,
});
},
});
const tobeRematchedParticipants = participants.map((participant) => {
return {
...participant,
workspaceMemberId: null,
};
});
await this.matchParticipants({
matchWith: 'workspaceMemberOnly',
participants: tobeRematchedParticipants as ParticipantWorkspaceEntity[],
objectMetadataName,
workspaceId,
});
);
}
public async matchParticipantsForPeople({
@@ -238,49 +247,58 @@ export class MatchParticipantService<
objectMetadataName,
workspaceId,
}: MatchParticipantsForPeopleArgs) {
const participantRepository = await this.getParticipantRepository(
workspaceId,
objectMetadataName,
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const participantRepository = await this.getParticipantRepository(
workspaceId,
objectMetadataName,
);
let participantsMatchingPersonEmails: ParticipantWorkspaceEntity[] = [];
let participantsMatchingPersonId: ParticipantWorkspaceEntity[] = [];
if (participantMatching.personIds.length > 0) {
participantsMatchingPersonId = (await participantRepository.find({
where: {
personId: In(participantMatching.personIds),
},
})) as ParticipantWorkspaceEntity[];
}
if (participantMatching.personEmails.length > 0) {
participantsMatchingPersonEmails = (await participantRepository.find({
where: {
handle: In(participantMatching.personEmails),
},
})) as ParticipantWorkspaceEntity[];
}
const uniqueParticipants = [
...new Set([
...participantsMatchingPersonId,
...participantsMatchingPersonEmails,
]),
];
const tobeRematchedParticipants = uniqueParticipants.map(
(participant) => {
return {
...participant,
personId: null,
};
},
);
await this.matchParticipants({
matchWith: 'personOnly',
participants: tobeRematchedParticipants,
objectMetadataName,
workspaceId,
});
},
);
let participantsMatchingPersonEmails: ParticipantWorkspaceEntity[] = [];
let participantsMatchingPersonId: ParticipantWorkspaceEntity[] = [];
if (participantMatching.personIds.length > 0) {
participantsMatchingPersonId = (await participantRepository.find({
where: {
personId: In(participantMatching.personIds),
},
})) as ParticipantWorkspaceEntity[];
}
if (participantMatching.personEmails.length > 0) {
participantsMatchingPersonEmails = (await participantRepository.find({
where: {
handle: In(participantMatching.personEmails),
},
})) as ParticipantWorkspaceEntity[];
}
const uniqueParticipants = [
...new Set([
...participantsMatchingPersonId,
...participantsMatchingPersonEmails,
]),
];
const tobeRematchedParticipants = uniqueParticipants.map((participant) => {
return {
...participant,
personId: null,
};
});
await this.matchParticipants({
matchWith: 'personOnly',
participants: tobeRematchedParticipants,
objectMetadataName,
workspaceId,
});
}
}
@@ -1,14 +1,15 @@
import { Scope } from '@nestjs/common';
import { MessageParticipantRole } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { And, Any, ILike, In, Not, Or } from 'typeorm';
import { MessageParticipantRole } from 'twenty-shared/types';
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
@@ -26,132 +27,141 @@ export type BlocklistItemDeleteMessagesJobData = WorkspaceEventBatch<
export class BlocklistItemDeleteMessagesJob {
constructor(
private readonly threadCleanerService: MessagingMessageCleanerService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(BlocklistItemDeleteMessagesJob.name)
async handle(data: BlocklistItemDeleteMessagesJobData): Promise<void> {
const workspaceId = data.workspaceId;
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const authContext = buildSystemAuthContext(workspaceId);
const blocklistRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const messageChannels = await messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
if (messageChannel.connectedAccount.handleAliases) {
messageChannelHandles.push(
...messageChannel.connectedAccount.handleAliases.split(','),
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const messageChannels = await messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
if (messageChannel.connectedAccount.handleAliases) {
messageChannelHandles.push(
...messageChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
}
await this.threadCleanerService.cleanOrphanMessagesAndThreads(workspaceId);
},
);
}
}
@@ -6,7 +6,8 @@ import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emit
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
@@ -25,7 +26,7 @@ export type BlocklistReimportMessagesJobData = WorkspaceEventBatch<
})
export class BlocklistReimportMessagesJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
) {}
@@ -33,30 +34,39 @@ export class BlocklistReimportMessagesJob {
async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
const workspaceId = data.workspaceId;
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING),
},
});
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
const messageChannels = await messageChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
),
},
});
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
);
}
}
@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED } from 'twenty-shared/constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
@@ -42,20 +42,21 @@ describe('ApplyMessagesVisibilityRestrictionsService', () => {
findOneByOrFail: jest.fn(),
};
const mockTwentyORMGlobalManager = {
getRepositoryForWorkspace: jest
const mockGlobalWorkspaceOrmManager = {
getRepository: jest.fn().mockImplementation((workspaceId, name) => {
if (name === 'messageChannelMessageAssociation') {
return mockMessageChannelMessageAssociationRepository;
}
if (name === 'connectedAccount') {
return mockConnectedAccountRepository;
}
if (name === 'workspaceMember') {
return mockWorkspaceMemberRepository;
}
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((workspaceId, name) => {
if (name === 'messageChannelMessageAssociation') {
return mockMessageChannelMessageAssociationRepository;
}
if (name === 'connectedAccount') {
return mockConnectedAccountRepository;
}
if (name === 'workspaceMember') {
return mockWorkspaceMemberRepository;
}
}),
.mockImplementation((_authContext: any, fn: () => any) => fn()),
};
beforeEach(async () => {
@@ -63,8 +64,8 @@ describe('ApplyMessagesVisibilityRestrictionsService', () => {
providers: [
ApplyMessagesVisibilityRestrictionsService,
{
provide: TwentyORMGlobalManager,
useValue: mockTwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: mockGlobalWorkspaceOrmManager,
},
],
}).compile();
@@ -6,7 +6,8 @@ import { isDefined } from 'twenty-shared/utils';
import { In } from 'typeorm';
import { NotFoundError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelVisibility } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@@ -16,105 +17,117 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
@Injectable()
export class ApplyMessagesVisibilityRestrictionsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async applyMessagesVisibilityRestrictions(
messages: MessageWorkspaceEntity[],
workspaceId: string,
userId?: string, // undefined when request is made with api key
userId?: string,
) {
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannelMessagesAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageId: In(messages.map((message) => message.id)),
},
relations: ['messageChannel'],
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
for (let i = messages.length - 1; i >= 0; i--) {
const messageChannelMessageAssociations =
messageChannelMessagesAssociations.filter(
(association) => association.messageId === messages[i].id,
);
const messageChannels = messageChannelMessageAssociations
.map((association) => association.messageChannel)
.filter(
(channel): channel is NonNullable<typeof channel> => channel !== null,
);
if (messageChannels.length === 0) {
throw new NotFoundError('Associated message channels not found');
}
const messageChannelsGroupByVisibility = groupBy(
messageChannels,
(channel) => channel.visibility,
);
if (
messageChannelsGroupByVisibility[
MessageChannelVisibility.SHARE_EVERYTHING
]
) {
continue;
}
if (isDefined(userId)) {
const workspaceMember = await workspaceMemberRepository.findOneByOrFail(
{
userId,
},
);
const connectedAccounts = await connectedAccountRepository.find({
select: ['id'],
where: {
messageChannels: {
id: In(messageChannels.map((channel) => channel.id)),
const messageChannelMessagesAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageId: In(messages.map((message) => message.id)),
},
accountOwnerId: workspaceMember.id,
},
});
relations: ['messageChannel'],
});
if (connectedAccounts.length > 0) {
continue;
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
for (let i = messages.length - 1; i >= 0; i--) {
const messageChannelMessageAssociations =
messageChannelMessagesAssociations.filter(
(association) => association.messageId === messages[i].id,
);
const messageChannels = messageChannelMessageAssociations
.map((association) => association.messageChannel)
.filter(
(channel): channel is NonNullable<typeof channel> =>
channel !== null,
);
if (messageChannels.length === 0) {
throw new NotFoundError('Associated message channels not found');
}
const messageChannelsGroupByVisibility = groupBy(
messageChannels,
(channel) => channel.visibility,
);
if (
messageChannelsGroupByVisibility[
MessageChannelVisibility.SHARE_EVERYTHING
]
) {
continue;
}
if (isDefined(userId)) {
const workspaceMember =
await workspaceMemberRepository.findOneByOrFail({
userId,
});
const connectedAccounts = await connectedAccountRepository.find({
select: ['id'],
where: {
messageChannels: {
id: In(messageChannels.map((channel) => channel.id)),
},
accountOwnerId: workspaceMember.id,
},
});
if (connectedAccounts.length > 0) {
continue;
}
}
if (
messageChannelsGroupByVisibility[MessageChannelVisibility.SUBJECT]
) {
messages[i].text = FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
if (
messageChannelsGroupByVisibility[MessageChannelVisibility.METADATA]
) {
messages[i].subject =
FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
messages[i].text = FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
messages.splice(i, 1);
}
}
if (messageChannelsGroupByVisibility[MessageChannelVisibility.SUBJECT]) {
messages[i].text = FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
if (messageChannelsGroupByVisibility[MessageChannelVisibility.METADATA]) {
messages[i].subject = FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
messages[i].text = FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED;
continue;
}
messages.splice(i, 1);
}
return messages;
return messages;
},
);
}
}
@@ -14,7 +14,8 @@ import {
} from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.exception';
import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
MessageChannelPendingGroupEmailsAction,
MessageChannelSyncStage,
@@ -39,7 +40,7 @@ export class MessageChannelUpdateOnePreQueryHook
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingProcessGroupEmailActionsService: MessagingProcessGroupEmailActionsService,
) {}
@@ -52,89 +53,97 @@ export class MessageChannelUpdateOnePreQueryHook
assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
const systemAuthContext = buildSystemAuthContext(workspace.id);
const messageChannel = await messageChannelRepository.findOne({
where: { id: payload.id },
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
systemAuthContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
if (!isDefined(messageChannel)) {
throw new WorkspaceQueryRunnerException(
'Message channel not found',
WorkspaceQueryRunnerExceptionCode.DATA_NOT_FOUND,
{
userFriendlyMessage: msg`Message channel not found`,
},
);
}
const messageChannel = await messageChannelRepository.findOne({
where: { id: payload.id },
});
const isSyncOngoing = ONGOING_SYNC_STAGES.includes(
messageChannel.syncStage,
if (!isDefined(messageChannel)) {
throw new WorkspaceQueryRunnerException(
'Message channel not found',
WorkspaceQueryRunnerExceptionCode.DATA_NOT_FOUND,
{
userFriendlyMessage: msg`Message channel not found`,
},
);
}
const isSyncOngoing = ONGOING_SYNC_STAGES.includes(
messageChannel.syncStage,
);
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspace.id,
'messageFolder',
);
const messageFoldersWithPendingActionCount =
await messageFolderRepository.count({
where: {
messageChannelId: messageChannel.id,
pendingSyncAction: Not(MessageFolderPendingSyncAction.NONE),
},
});
const hasPendingFolderActions =
messageFoldersWithPendingActionCount > 0;
const hasPendingGroupEmailsAction =
messageChannel.pendingGroupEmailsAction !==
MessageChannelPendingGroupEmailsAction.NONE;
if (
isSyncOngoing &&
(hasPendingFolderActions || hasPendingGroupEmailsAction)
) {
throw new WorkspaceQueryRunnerException(
'Cannot update message channel while sync is ongoing with pending actions',
WorkspaceQueryRunnerExceptionCode.INVALID_QUERY_INPUT,
{
userFriendlyMessage: msg`Cannot update message channel while sync is ongoing. Please wait for the sync to complete.`,
},
);
}
const hasCompletedConfiguration =
messageChannel.syncStage !==
MessageChannelSyncStage.PENDING_CONFIGURATION;
if (!hasCompletedConfiguration) {
this.logger.log(
`MessageChannelId: ${messageChannel.id} - Skipping pending action for message channel in PENDING_CONFIGURATION state`,
);
return payload;
}
const excludeGroupEmailsChanged =
isDefined(payload.data.excludeGroupEmails) &&
payload.data.excludeGroupEmails !== messageChannel.excludeGroupEmails;
if (excludeGroupEmailsChanged) {
await this.messagingProcessGroupEmailActionsService.markMessageChannelAsPendingGroupEmailsAction(
messageChannel,
workspace.id,
payload.data.excludeGroupEmails
? MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION
: MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT,
);
}
return payload;
},
);
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
workspace.id,
'messageFolder',
);
const messageFoldersWithPendingActionCount =
await messageFolderRepository.count({
where: {
messageChannelId: messageChannel.id,
pendingSyncAction: Not(MessageFolderPendingSyncAction.NONE),
},
});
const hasPendingFolderActions = messageFoldersWithPendingActionCount > 0;
const hasPendingGroupEmailsAction =
messageChannel.pendingGroupEmailsAction !==
MessageChannelPendingGroupEmailsAction.NONE;
if (
isSyncOngoing &&
(hasPendingFolderActions || hasPendingGroupEmailsAction)
) {
throw new WorkspaceQueryRunnerException(
'Cannot update message channel while sync is ongoing with pending actions',
WorkspaceQueryRunnerExceptionCode.INVALID_QUERY_INPUT,
{
userFriendlyMessage: msg`Cannot update message channel while sync is ongoing. Please wait for the sync to complete.`,
},
);
}
const hasCompletedConfiguration =
messageChannel.syncStage !==
MessageChannelSyncStage.PENDING_CONFIGURATION;
if (!hasCompletedConfiguration) {
this.logger.log(
`MessageChannelId: ${messageChannel.id} - Skipping pending action for message channel in PENDING_CONFIGURATION state`,
);
return payload;
}
const excludeGroupEmailsChanged =
isDefined(payload.data.excludeGroupEmails) &&
payload.data.excludeGroupEmails !== messageChannel.excludeGroupEmails;
if (excludeGroupEmailsChanged) {
await this.messagingProcessGroupEmailActionsService.markMessageChannelAsPendingGroupEmailsAction(
messageChannel,
workspace.id,
payload.data.excludeGroupEmails
? MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION
: MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT,
);
}
return payload;
}
}
@@ -7,7 +7,8 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/accounts-to-reconnect-key-value.type';
@@ -27,7 +28,7 @@ export class MessageChannelSyncStatusService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly accountsToReconnectService: AccountsToReconnectService,
private readonly metricsService: MetricsService,
) {}
@@ -41,16 +42,23 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
}
public async markAsMessagesImportPending(
@@ -62,16 +70,23 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
}
public async resetAndMarkAsMessagesListFetchPending(
@@ -88,30 +103,37 @@ export class MessageChannelSyncStatusService {
);
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
});
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
await messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds) },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
await messageChannelRepository.update(messageChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
});
await messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds) },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
},
);
@@ -126,15 +148,22 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStageStartedAt: null,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStageStartedAt: null,
});
},
);
}
public async markAsMessagesListFetchScheduled(
@@ -145,17 +174,24 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
},
);
}
public async markAsMessagesListFetchOngoing(
@@ -166,16 +202,23 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
});
},
);
}
public async markAsCompletedAndMarkAsMessagesListFetchPending(
@@ -186,19 +229,26 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
},
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.MessageChannelSyncJobActive,
@@ -214,15 +264,22 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
});
},
);
}
public async markAsMessagesImportOngoing(
@@ -233,16 +290,23 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
},
);
}
public async markAsFailed(
@@ -256,57 +320,66 @@ export class MessageChannelSyncStatusService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const metricsKey =
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
});
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
const metricsKey =
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
if (
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const connectedAccountRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
const messageChannels = await messageChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(messageChannelIds) },
});
if (
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
const messageChannels = await messageChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(messageChannelIds) },
});
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
);
}
private async addToAccountsToReconnect(
@@ -318,7 +391,7 @@ export class MessageChannelSyncStatusService {
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
@@ -7,7 +7,7 @@ import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
@Command({
@@ -18,11 +18,11 @@ export class MessagingMessageCleanerRemoveOrphansCommand extends ActiveOrSuspend
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
protected readonly dataSourceService: DataSourceService,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService);
}
override async runOnWorkspace({
@@ -3,7 +3,8 @@ import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { isDefined } from 'twenty-shared/utils';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
@@ -22,7 +23,7 @@ export class MessagingResetChannelCommand extends CommandRunner {
private readonly logger = new Logger(MessagingResetChannelCommand.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
) {
@@ -35,44 +36,53 @@ export class MessagingResetChannelCommand extends CommandRunner {
): Promise<void> {
const { workspaceId, messageChannelId } = options;
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
const messageChannels = await messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
},
});
if (messageChannels.length === 0) {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
},
});
if (messageChannels.length === 0) {
this.logger.log(`No message channels found in workspace ${workspaceId}`);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
}
@@ -4,7 +4,8 @@ import chunk from 'lodash.chunk';
import { In, IsNull } from 'typeorm';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { type MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
@@ -14,7 +15,7 @@ import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/uti
export class MessagingMessageCleanerService {
private readonly logger = new Logger(MessagingMessageCleanerService.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async deleteMessagesChannelMessageAssociationsAndRelatedOrphans({
@@ -26,183 +27,199 @@ export class MessagingMessageCleanerService {
messageExternalIds: string[];
messageChannelId: string;
}) {
const messageRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const messageThreadRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
const orphanMessages = await messageRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
},
});
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
if (orphanMessages.length <= 0) {
continue;
}
const orphanMessages = await messageRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
},
});
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
if (orphanMessages.length <= 0) {
continue;
}
await messageRepository.delete(orphanMessages.map(({ id }) => id));
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(orphanMessages.map(({ messageThreadId }) => messageThreadId)),
messages: {
id: IsNull(),
},
},
});
await messageRepository.delete(orphanMessages.map(({ id }) => id));
if (orphanMessageThreads.length <= 0) {
continue;
}
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
messages: {
id: IsNull(),
},
},
});
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
if (orphanMessageThreads.length <= 0) {
continue;
}
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
},
);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const messageThreadRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageWorkspaceEntity>(
workspaceId,
'message',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
},
take: limit,
skip: offset,
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
@@ -12,7 +12,8 @@ import {
} from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.exception';
import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
MessageFolderImportPolicy,
type MessageChannelWorkspaceEntity,
@@ -24,7 +25,7 @@ export class MessageFolderUpdateOnePreQueryHook
implements WorkspacePreQueryHookInstance
{
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async execute(
@@ -36,68 +37,75 @@ export class MessageFolderUpdateOnePreQueryHook
assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError);
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
workspace.id,
'messageFolder',
);
const systemAuthContext = buildSystemAuthContext(workspace.id);
const messageFolder = await messageFolderRepository.findOne({
where: { id: payload.id },
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
systemAuthContext,
async () => {
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspace.id,
'messageFolder',
);
if (!messageFolder) {
throw new WorkspaceQueryRunnerException(
'Message folder not found',
WorkspaceQueryRunnerExceptionCode.DATA_NOT_FOUND,
{
userFriendlyMessage: msg`Message folder not found`,
},
);
}
const messageFolder = await messageFolderRepository.findOne({
where: { id: payload.id },
});
if (payload.data.isSynced !== false) {
return payload;
}
if (!messageFolder) {
throw new WorkspaceQueryRunnerException(
'Message folder not found',
WorkspaceQueryRunnerExceptionCode.DATA_NOT_FOUND,
{
userFriendlyMessage: msg`Message folder not found`,
},
);
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
if (payload.data.isSynced !== false) {
return payload;
}
const messageChannel = await messageChannelRepository.findOne({
where: { id: messageFolder.messageChannelId },
});
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspace.id,
'messageChannel',
);
if (
messageChannel?.messageFolderImportPolicy !==
MessageFolderImportPolicy.SELECTED_FOLDERS
) {
return payload;
}
const messageChannel = await messageChannelRepository.findOne({
where: { id: messageFolder.messageChannelId },
});
const syncedFoldersCount = await messageFolderRepository.count({
where: {
messageChannelId: messageFolder.messageChannelId,
isSynced: true,
if (
messageChannel?.messageFolderImportPolicy !==
MessageFolderImportPolicy.SELECTED_FOLDERS
) {
return payload;
}
const syncedFoldersCount = await messageFolderRepository.count({
where: {
messageChannelId: messageFolder.messageChannelId,
isSynced: true,
},
});
if (
isDefined(syncedFoldersCount) &&
isNumber(syncedFoldersCount) &&
syncedFoldersCount <= 1
) {
throw new WorkspaceQueryRunnerException(
'Cannot unsync the last folder when folder import policy is set to selected folders',
WorkspaceQueryRunnerExceptionCode.INVALID_QUERY_INPUT,
{
userFriendlyMessage: msg`At least one folder must be synced.`,
},
);
}
return payload;
},
});
if (
isDefined(syncedFoldersCount) &&
isNumber(syncedFoldersCount) &&
syncedFoldersCount <= 1
) {
throw new WorkspaceQueryRunnerException(
'Cannot unsync the last folder when folder import policy is set to selected folders',
WorkspaceQueryRunnerExceptionCode.INVALID_QUERY_INPUT,
{
userFriendlyMessage: msg`At least one folder must be synced.`,
},
);
}
return payload;
);
}
}
@@ -9,8 +9,9 @@ import { v4 } from 'uuid';
import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessageFolderPendingSyncAction,
@@ -52,7 +53,7 @@ type MessageFolderToUpdate = Partial<
@Injectable()
export class SyncMessageFoldersService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly gmailGetAllFoldersService: GmailGetAllFoldersService,
private readonly microsoftGetAllFoldersService: MicrosoftGetAllFoldersService,
private readonly imapGetAllFoldersService: ImapGetAllFoldersService,
@@ -61,17 +62,24 @@ export class SyncMessageFoldersService {
async syncMessageFolders(input: SyncMessageFoldersInput): Promise<void> {
const { workspaceId, messageChannel, manager } = input;
const folders = await this.discoverAllFolders(
messageChannel.connectedAccount,
messageChannel,
);
const authContext = buildSystemAuthContext(workspaceId);
await this.upsertDiscoveredFolders({
workspaceId,
messageChannelId: messageChannel.id,
folders,
manager,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const folders = await this.discoverAllFolders(
messageChannel.connectedAccount,
messageChannel,
);
await this.upsertDiscoveredFolders({
workspaceId,
messageChannelId: messageChannel.id,
folders,
manager,
});
},
);
}
private async upsertDiscoveredFolders({
@@ -86,7 +94,7 @@ export class SyncMessageFoldersService {
manager: WorkspaceEntityManager;
}): Promise<void> {
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
@@ -30,7 +31,7 @@ export class MessagingMessageListFetchJob {
constructor(
private readonly messagingMessageListFetchService: MessagingMessageListFetchService,
private readonly messagingMonitoringService: MessagingMonitoringService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
) {}
@@ -45,77 +46,84 @@ export class MessagingMessageListFetchJob {
workspaceId,
});
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount', 'messageFolders'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
try {
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
},
relations: ['connectedAccount', 'messageFolders'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
try {
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
);
}
}
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
@@ -12,6 +13,7 @@ import {
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMessagesImportService } from 'src/modules/messaging/message-import-manager/services/messaging-messages-import.service';
import { MessagingMonitoringService } from 'src/modules/messaging/monitoring/services/messaging-monitoring.service';
export type MessagingMessagesImportJobData = {
messageChannelId: string;
workspaceId: string;
@@ -26,7 +28,7 @@ export class MessagingMessagesImportJob {
private readonly messagingMessagesImportService: MessagingMessagesImportService,
private readonly messagingMonitoringService: MessagingMonitoringService,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(MessagingMessagesImportJob.name)
@@ -39,59 +41,66 @@ export class MessagingMessagesImportJob {
messageChannelId,
});
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
},
relations: ['connectedAccount'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
}
}
@@ -5,7 +5,8 @@ import { In } from 'typeorm';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageChannelSyncStage,
@@ -24,7 +25,7 @@ export type MessagingOngoingStaleJobData = {
export class MessagingOngoingStaleJob {
private readonly logger = new Logger(MessagingOngoingStaleJob.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
) {}
@@ -32,58 +33,65 @@ export class MessagingOngoingStaleJob {
async handle(data: MessagingOngoingStaleJobData): Promise<void> {
const { workspaceId } = data;
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannels = await messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
for (const messageChannel of messageChannels) {
if (
messageChannel.syncStageStartedAt &&
isSyncStale(messageChannel.syncStageStartedAt)
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
);
const messageChannels = await messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
},
});
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
for (const messageChannel of messageChannels) {
if (
messageChannel.syncStageStartedAt &&
isSyncStale(messageChannel.syncStageStartedAt)
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
}
}
}
}
}
},
);
}
}
@@ -3,7 +3,8 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
MessageChannelSyncStage,
MessageChannelSyncStatus,
@@ -21,42 +22,49 @@ export type MessagingRelaunchFailedMessageChannelJobData = {
})
export class MessagingRelaunchFailedMessageChannelJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(MessagingRelaunchFailedMessageChannelJob.name)
async handle(data: MessagingRelaunchFailedMessageChannelJobData) {
const { workspaceId, messageChannelId } = data;
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
{ shouldBypassPermissionChecks: true },
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await messageChannelRepository.update(messageChannelId, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
});
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await messageChannelRepository.update(messageChannelId, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
});
);
}
}
@@ -4,7 +4,7 @@ import { ConnectedAccountProvider } from 'twenty-shared/types';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
@@ -24,7 +24,7 @@ describe('MessagingMessageListFetchService', () => {
let messagingGetMessageListService: MessagingGetMessageListService;
let messagingAccountAuthenticationService: MessagingAccountAuthenticationService;
let messageChannelSyncStatusService: MessageChannelSyncStatusService;
let twentyORMGlobalManager: TwentyORMGlobalManager;
let globalWorkspaceOrmManager: GlobalWorkspaceOrmManager;
let messagingCursorService: MessagingCursorService;
let mockMicrosoftMessageChannel: MessageChannelWorkspaceEntity;
@@ -196,21 +196,23 @@ describe('MessagingMessageListFetchService', () => {
},
},
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getDataSourceForWorkspace: jest.fn().mockResolvedValue({
manager: {},
}),
getRepositoryForWorkspace: jest
executeInWorkspaceContext: jest
.fn()
.mockImplementation((workspaceId, name) => {
if (name === 'messageChannelMessageAssociation') {
return mockMessageChannelMessageAssociationRepository;
}
if (name === 'messageFolder') {
return mockMessageFolderRepository;
}
}),
.mockImplementation((_authContext: any, fn: () => any) => fn()),
getRepository: jest.fn().mockImplementation((workspaceId, name) => {
if (name === 'messageChannelMessageAssociation') {
return mockMessageChannelMessageAssociationRepository;
}
if (name === 'messageFolder') {
return mockMessageFolderRepository;
}
}),
},
},
{
@@ -275,8 +277,8 @@ describe('MessagingMessageListFetchService', () => {
module.get<MessageChannelSyncStatusService>(
MessageChannelSyncStatusService,
);
twentyORMGlobalManager = module.get<TwentyORMGlobalManager>(
TwentyORMGlobalManager,
globalWorkspaceOrmManager = module.get<GlobalWorkspaceOrmManager>(
GlobalWorkspaceOrmManager,
);
messagingCursorService = module.get<MessagingCursorService>(
MessagingCursorService,
@@ -320,9 +322,10 @@ describe('MessagingMessageListFetchService', () => {
],
);
expect(
twentyORMGlobalManager.getRepositoryForWorkspace,
).toHaveBeenCalledWith(workspaceId, 'messageChannelMessageAssociation');
expect(globalWorkspaceOrmManager.getRepository).toHaveBeenCalledWith(
workspaceId,
'messageChannelMessageAssociation',
);
expect(messagingCursorService.updateCursor).toHaveBeenCalledWith(
{
@@ -380,9 +383,10 @@ describe('MessagingMessageListFetchService', () => {
],
);
expect(
twentyORMGlobalManager.getRepositoryForWorkspace,
).toHaveBeenCalledWith(workspaceId, 'messageChannelMessageAssociation');
expect(globalWorkspaceOrmManager.getRepository).toHaveBeenCalledWith(
workspaceId,
'messageChannelMessageAssociation',
);
expect(messagingCursorService.updateCursor).toHaveBeenCalledWith(
{
@@ -5,7 +5,7 @@ import { ConnectedAccountProvider } from 'twenty-shared/types';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { ConnectedAccountRefreshTokensService } from 'src/modules/connected-account/refresh-tokens-manager/services/connected-account-refresh-tokens.service';
@@ -106,11 +106,14 @@ describe('MessagingMessagesImportService', () => {
},
},
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getRepositoryForWorkspace: jest.fn().mockResolvedValue({
getRepository: jest.fn().mockResolvedValue({
update: jest.fn().mockResolvedValue(undefined),
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
{
@@ -1,13 +1,14 @@
import { Injectable } from '@nestjs/common';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
@Injectable()
export class MessagingCursorService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async updateCursor(
@@ -16,50 +17,57 @@ export class MessagingCursorService {
workspaceId: string,
folderId?: string,
) {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const folderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
const authContext = buildSystemAuthContext(workspaceId);
if (!folderId) {
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await folderRepository.update(
{
id: folderId,
},
{
syncCursor: nextSyncCursor,
},
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const folderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
if (!folderId) {
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await folderRepository.update(
{
id: folderId,
},
{
syncCursor: nextSyncCursor,
},
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
}
},
);
}
}
@@ -1,10 +1,11 @@
import { Injectable, Logger } from '@nestjs/common';
import chunk from 'lodash.chunk';
import { isDefined } from 'twenty-shared/utils';
import { MessageParticipantRole } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email';
@@ -24,7 +25,7 @@ export class MessagingDeleteGroupEmailMessagesService {
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
) {}
@@ -36,92 +37,99 @@ export class MessagingDeleteGroupEmailMessagesService {
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Deleting messages from group email addresses`,
);
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const authContext = buildSystemAuthContext(workspaceId);
let offset = 0;
let totalDeletedCount = 0;
while (true) {
const batch = await messageChannelMessageAssociationRepository
.createQueryBuilder('mcma')
.select('mcma.messageId', 'messageId')
.addSelect('mcma.messageExternalId', 'messageExternalId')
.addSelect('participant.handle', 'participantHandle')
.innerJoin('mcma.message', 'message')
.innerJoin(
'message.messageParticipants',
'participant',
'participant.role = :role',
{ role: MessageParticipantRole.FROM },
)
.where('mcma.messageChannelId = :messageChannelId', {
messageChannelId,
})
.skip(offset)
.take(MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE)
.getRawMany<MessageBatchRawResult>();
if (batch.length === 0) {
break;
}
const groupEmailRecords = batch.filter(
(record) =>
isDefined(record.participantHandle) &&
isGroupEmail(record.participantHandle),
);
if (groupEmailRecords.length > 0) {
const uniqueMessageIds = new Set(
groupEmailRecords.map((r) => r.messageId),
);
const messageExternalIdsToDelete = batch
.filter((record) => uniqueMessageIds.has(record.messageId))
.map((record) => record.messageExternalId)
.filter(isDefined);
if (messageExternalIdsToDelete.length > 0) {
const messageExternalIdsChunks = chunk(
messageExternalIdsToDelete,
200,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: messageExternalIdsChunk,
messageChannelId,
},
let offset = 0;
let totalDeletedCount = 0;
while (true) {
const batch = await messageChannelMessageAssociationRepository
.createQueryBuilder('mcma')
.select('mcma.messageId', 'messageId')
.addSelect('mcma.messageExternalId', 'messageExternalId')
.addSelect('participant.handle', 'participantHandle')
.innerJoin('mcma.message', 'message')
.innerJoin(
'message.messageParticipants',
'participant',
'participant.role = :role',
{ role: MessageParticipantRole.FROM },
)
.where('mcma.messageChannelId = :messageChannelId', {
messageChannelId,
})
.skip(offset)
.take(MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE)
.getRawMany<MessageBatchRawResult>();
if (batch.length === 0) {
break;
}
const groupEmailRecords = batch.filter(
(record) =>
isDefined(record.participantHandle) &&
isGroupEmail(record.participantHandle),
);
if (groupEmailRecords.length > 0) {
const uniqueMessageIds = new Set(
groupEmailRecords.map((r) => r.messageId),
);
totalDeletedCount += messageExternalIdsChunk.length;
const messageExternalIdsToDelete = batch
.filter((record) => uniqueMessageIds.has(record.messageId))
.map((record) => record.messageExternalId)
.filter(isDefined);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Deleted ${messageExternalIdsChunk.length} group email messages`,
);
if (messageExternalIdsToDelete.length > 0) {
const messageExternalIdsChunks = chunk(
messageExternalIdsToDelete,
200,
);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: messageExternalIdsChunk,
messageChannelId,
},
);
totalDeletedCount += messageExternalIdsChunk.length;
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Deleted ${messageExternalIdsChunk.length} group email messages`,
);
}
}
}
if (batch.length < MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE) {
break;
}
if (groupEmailRecords.length === 0) {
offset += MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE;
}
}
}
if (batch.length < MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE) {
break;
}
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Completed deleting ${totalDeletedCount} group email messages`,
);
if (groupEmailRecords.length === 0) {
offset += MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE;
}
}
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Completed deleting ${totalDeletedCount} group email messages`,
return totalDeletedCount;
},
);
return totalDeletedCount;
}
}
@@ -5,7 +5,8 @@ import {
type TwentyORMException,
TwentyORMExceptionCode,
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageChannelSyncStatus,
@@ -27,7 +28,7 @@ export enum MessageImportSyncStep {
@Injectable()
export class MessageImportExceptionHandlerService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@@ -148,18 +149,25 @@ export class MessageImportExceptionHandlerService {
return;
}
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.increment(
{ id: messageChannel.id },
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.increment(
{ id: messageChannel.id },
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
);
},
);
switch (syncStep) {
@@ -8,7 +8,9 @@ import { In, MoreThanOrEqual } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import {
@@ -43,7 +45,7 @@ export class MessagingMessageListFetchService {
@InjectCacheStorage(CacheStorageNamespace.ModuleMessaging)
private readonly cacheStorage: CacheStorageService,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingGetMessageListService: MessagingGetMessageListService,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
@@ -59,264 +61,275 @@ export class MessagingMessageListFetchService {
messageChannel: MessageChannelWorkspaceEntity,
workspaceId: string,
) {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(messageChannel, workspaceId);
const authContext = buildSystemAuthContext(workspaceId);
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
);
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
this.logger.log(
`messageChannelId: ${messageChannel.id} Processing message list fetch`,
);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await messageChannelRepository.findOne({
where: {
id: messageChannel.id,
},
relations: ['connectedAccount', 'messageFolders'],
})
: messageChannel;
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`error processing message list fetch: messageChannelId: ${messageChannel.id} Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
messageChannelId: freshMessageChannel.id,
},
);
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
this.logger.log(
`messageChannelId: ${messageChannel.id} Processing message list fetch`,
);
const datasource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.syncMessageFoldersService.syncMessageFolders({
workspaceId,
messageChannel: messageChannelWithFreshTokens,
manager: datasource.manager,
});
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await messageChannelRepository.findOne({
where: {
id: messageChannel.id,
},
relations: ['connectedAccount', 'messageFolders'],
})
: messageChannel;
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`error processing message list fetch: messageChannelId: ${messageChannel.id} Message channel not found`,
);
const messageFolders = await messageFolderRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
});
return;
}
const messageFoldersToSync =
messageChannelWithFreshTokens.messageFolderImportPolicy ===
MessageFolderImportPolicy.ALL_FOLDERS
? messageFolders
: messageFolders.filter((folder) => folder.isSynced);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
workspaceId,
messageChannelId: freshMessageChannel.id,
},
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const datasource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
await this.syncMessageFoldersService.syncMessageFolders({
workspaceId,
messageChannel: messageChannelWithFreshTokens,
manager: datasource.manager as WorkspaceEntityManager,
});
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
const isFullSync =
messageLists.every(
(messageList) => !isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
(messageList) => {
messageList.nextSyncCursor;
},
)}`,
);
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
const messageFolders = await messageFolderRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
const messageFoldersToSync =
messageChannelWithFreshTokens.messageFolderImportPolicy ===
MessageFolderImportPolicy.ALL_FOLDERS
? messageFolders
: messageFolders.filter((folder) => folder.isSynced);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) =>
!isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
if (messageExternalIdsToImport.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
`messageChannelId: ${freshMessageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
(messageList) => {
messageList.nextSyncCursor;
},
)}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
messageExternalIds: toDeleteChunk.filter((messageExternalId) =>
isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageExternalIdsToImport.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter(
(messageExternalId) => isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
}
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
);
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
workspaceId,
);
return;
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
);
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
workspaceId,
);
return;
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
},
);
}
private async processPendingGroupEmailActions(
@@ -350,7 +363,7 @@ export class MessagingMessageListFetchService {
workspaceId: string,
): Promise<boolean> {
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
@@ -385,7 +398,7 @@ export class MessagingMessageListFetchService {
workspaceId: string,
) {
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
@@ -5,7 +5,8 @@ import { In } from 'typeorm';
import { v4 } from 'uuid';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { type MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
@@ -39,7 +40,7 @@ export class MessagingMessageService {
private readonly logger = new Logger(MessagingMessageService.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
public async saveMessagesWithinTransaction(
@@ -51,183 +52,200 @@ export class MessagingMessageService {
createdMessages: Partial<MessageWorkspaceEntity>[];
messageExternalIdsAndIdsMap: Map<string, string>;
}> {
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const authContext = buildSystemAuthContext(workspaceId);
const messageRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageWorkspaceEntity>(
workspaceId,
'message',
);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageThreadRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const messageAccumulatorMap = new Map<string, MessageAccumulator>();
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const existingMessagesInDB = await messageRepository.find({
where: {
headerMessageId: In(messages.map((message) => message.headerMessageId)),
},
});
const messageAccumulatorMap = new Map<string, MessageAccumulator>();
const messageChannelMessageAssociationsReferencingMessageThread =
await messageChannelMessageAssociationRepository.find(
{
const existingMessagesInDB = await messageRepository.find({
where: {
messageThreadExternalId: In(
messages.map((message) => message.messageThreadExternalId),
headerMessageId: In(
messages.map((message) => message.headerMessageId),
),
messageChannelId,
},
relations: ['message'],
},
transactionManager,
);
});
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageId: In(existingMessagesInDB.map((message) => message.id)),
messageChannelId,
},
});
const messageChannelMessageAssociationsReferencingMessageThread =
await messageChannelMessageAssociationRepository.find(
{
where: {
messageThreadExternalId: In(
messages.map((message) => message.messageThreadExternalId),
),
messageChannelId,
},
relations: ['message'],
},
transactionManager,
);
await this.enrichMessageAccumulatorWithExistingMessages(
messages,
messageAccumulatorMap,
existingMessagesInDB,
);
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageId: In(existingMessagesInDB.map((message) => message.id)),
messageChannelId,
},
});
await this.enrichMessageAccumulatorWithExistingMessageThreadIds(
messages,
messageAccumulatorMap,
messageChannelMessageAssociationsReferencingMessageThread,
workspaceId,
);
await this.enrichMessageAccumulatorWithExistingMessageChannelMessageAssociations(
messages,
messageAccumulatorMap,
existingMessageChannelMessageAssociations,
);
await this.enrichMessageAccumulatorWithMessageThreadToCreate(
messages,
messageAccumulatorMap,
);
for (const message of messages) {
const messageAccumulator = messageAccumulatorMap.get(message.externalId);
if (!isDefined(messageAccumulator)) {
throw new Error(
`Message accumulator should reference the message, this should never happen`,
await this.enrichMessageAccumulatorWithExistingMessages(
messages,
messageAccumulatorMap,
existingMessagesInDB,
);
}
const messageThreadId =
messageAccumulator.threadToCreate?.id ??
messageAccumulator.existingThreadInDB?.id;
if (!isDefined(messageThreadId)) {
throw new Error(
`Message thread id should be defined, either in the threadToCreate or existingThreadInDB`,
await this.enrichMessageAccumulatorWithExistingMessageThreadIds(
messages,
messageAccumulatorMap,
messageChannelMessageAssociationsReferencingMessageThread,
workspaceId,
);
}
let newOrExistingMessageId: string;
await this.enrichMessageAccumulatorWithExistingMessageChannelMessageAssociations(
messages,
messageAccumulatorMap,
existingMessageChannelMessageAssociations,
);
if (!isDefined(messageAccumulator.existingMessageInDB)) {
newOrExistingMessageId = v4();
await this.enrichMessageAccumulatorWithMessageThreadToCreate(
messages,
messageAccumulatorMap,
);
const messageToCreate = {
id: newOrExistingMessageId,
headerMessageId: message.headerMessageId,
subject: message.subject,
receivedAt: message.receivedAt,
text: message.text,
messageThreadId,
};
for (const message of messages) {
const messageAccumulator = messageAccumulatorMap.get(
message.externalId,
);
messageAccumulator.messageToCreate = messageToCreate;
} else {
newOrExistingMessageId = messageAccumulator.existingMessageInDB.id;
}
if (!isDefined(messageAccumulator)) {
throw new Error(
`Message accumulator should reference the message, this should never happen`,
);
}
if (
!isDefined(
messageAccumulator.existingMessageChannelMessageAssociationInDB,
const messageThreadId =
messageAccumulator.threadToCreate?.id ??
messageAccumulator.existingThreadInDB?.id;
if (!isDefined(messageThreadId)) {
throw new Error(
`Message thread id should be defined, either in the threadToCreate or existingThreadInDB`,
);
}
let newOrExistingMessageId: string;
if (!isDefined(messageAccumulator.existingMessageInDB)) {
newOrExistingMessageId = v4();
const messageToCreate = {
id: newOrExistingMessageId,
headerMessageId: message.headerMessageId,
subject: message.subject,
receivedAt: message.receivedAt,
text: message.text,
messageThreadId,
};
messageAccumulator.messageToCreate = messageToCreate;
} else {
newOrExistingMessageId = messageAccumulator.existingMessageInDB.id;
}
if (
!isDefined(
messageAccumulator.existingMessageChannelMessageAssociationInDB,
)
) {
messageAccumulator.messageChannelMessageAssociationToCreate = {
messageChannelId,
messageId: newOrExistingMessageId,
messageExternalId: message.externalId,
messageThreadExternalId: message.messageThreadExternalId,
direction: message.direction,
};
messageAccumulatorMap.set(message.externalId, messageAccumulator);
}
}
const messageThreadsToCreate = Array.from(
messageAccumulatorMap.values(),
)
) {
messageAccumulator.messageChannelMessageAssociationToCreate = {
messageChannelId,
messageId: newOrExistingMessageId,
messageExternalId: message.externalId,
messageThreadExternalId: message.messageThreadExternalId,
direction: message.direction,
.map((accumulator) => accumulator.threadToCreate)
.filter(isDefined);
await messageThreadRepository.insert(
messageThreadsToCreate,
transactionManager,
);
const messagesToCreate = Array.from(messageAccumulatorMap.values())
.map((accumulator) => accumulator.messageToCreate)
.filter(isDefined);
await messageRepository.insert(messagesToCreate, transactionManager);
const messageChannelMessageAssociationsToCreate = Array.from(
messageAccumulatorMap.values(),
)
.map(
(accumulator) =>
accumulator.messageChannelMessageAssociationToCreate,
)
.filter(isDefined);
await messageChannelMessageAssociationRepository.insert(
messageChannelMessageAssociationsToCreate,
transactionManager,
);
const messageExternalIdsAndIdsMap = new Map<string, string>();
for (const [
externalId,
accumulator,
] of messageAccumulatorMap.entries()) {
if (isDefined(accumulator.messageToCreate)) {
messageExternalIdsAndIdsMap.set(
externalId,
accumulator.messageToCreate.id,
);
}
if (isDefined(accumulator.existingMessageInDB)) {
messageExternalIdsAndIdsMap.set(
externalId,
accumulator.existingMessageInDB.id,
);
}
}
return {
createdMessages: messagesToCreate,
messageExternalIdsAndIdsMap,
};
messageAccumulatorMap.set(message.externalId, messageAccumulator);
}
}
const messageThreadsToCreate = Array.from(messageAccumulatorMap.values())
.map((accumulator) => accumulator.threadToCreate)
.filter(isDefined);
await messageThreadRepository.insert(
messageThreadsToCreate,
transactionManager,
},
);
const messagesToCreate = Array.from(messageAccumulatorMap.values())
.map((accumulator) => accumulator.messageToCreate)
.filter(isDefined);
await messageRepository.insert(messagesToCreate, transactionManager);
const messageChannelMessageAssociationsToCreate = Array.from(
messageAccumulatorMap.values(),
)
.map(
(accumulator) => accumulator.messageChannelMessageAssociationToCreate,
)
.filter(isDefined);
await messageChannelMessageAssociationRepository.insert(
messageChannelMessageAssociationsToCreate,
transactionManager,
);
const messageExternalIdsAndIdsMap = new Map<string, string>();
for (const [externalId, accumulator] of messageAccumulatorMap.entries()) {
if (isDefined(accumulator.messageToCreate)) {
messageExternalIdsAndIdsMap.set(
externalId,
accumulator.messageToCreate.id,
);
}
if (isDefined(accumulator.existingMessageInDB)) {
messageExternalIdsAndIdsMap.set(
externalId,
accumulator.existingMessageInDB.id,
);
}
}
return {
createdMessages: messagesToCreate,
messageExternalIdsAndIdsMap,
};
}
private async enrichMessageAccumulatorWithExistingMessages(
@@ -307,11 +325,6 @@ export class MessagingMessageService {
existingThreadIdInDBIfMessageIsExistingInDB !==
existingThreadIdInDBIfMessageIsReferencedInMessageChannelMessageAssociation
) {
// TODO: this can be handled better
// If we find a messageThreadId different on the existingMessage (found by messageHeaderId which is cross channel)
// And on the the one associatied to the messageThreadExternalId (found by which is channel specific)
// this means that we have to channels that have imported messages separately and that this message is the first connection between the two channels
// we should merge messageThreads
this.logger.warn(
`
WorkspaceId: ${workspaceId} /
@@ -6,7 +6,8 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
@@ -30,6 +31,7 @@ import {
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { filterEmails } from 'src/modules/messaging/message-import-manager/utils/filter-emails.util';
import { MessagingMonitoringService } from 'src/modules/messaging/monitoring/services/messaging-monitoring.service';
@Injectable()
export class MessagingMessagesImportService {
private readonly logger = new Logger(MessagingMessagesImportService.name);
@@ -43,7 +45,7 @@ export class MessagingMessagesImportService {
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly emailAliasManagerService: EmailAliasManagerService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingGetMessagesService: MessagingGetMessagesService,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService,
@@ -56,162 +58,171 @@ export class MessagingMessagesImportService {
) {
let messageIdsToFetch: string[] = [];
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
const authContext = buildSystemAuthContext(workspaceId);
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
},
);
});
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
);
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
);
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccountWithFreshTokens.accountOwnerId,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const allMessages =
await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
);
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const blocklist =
await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccountWithFreshTokens.accountOwnerId,
workspaceId,
);
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases.split(',')],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
);
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (
messageIdsToFetch.length < MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases.split(',')],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
);
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
if (
messageIdsToFetch.length <
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
// TODO: remove this log once we catch better the error codes
this.logger.error(
`Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
},
);
}
private async trackMessageImportCompleted(
@@ -4,7 +4,8 @@ import { isDefined } from 'twenty-shared/utils';
import { In } from 'typeorm';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessageFolderPendingSyncAction,
@@ -19,7 +20,7 @@ export class MessagingProcessFolderActionsService {
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingDeleteFolderMessagesService: MessagingDeleteFolderMessagesService,
) {}
@@ -86,41 +87,48 @@ export class MessagingProcessFolderActionsService {
}
if (processedFolderIds.length > 0 || folderIdsToDelete.length > 0) {
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const authContext = buildSystemAuthContext(workspaceId);
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
'messageFolder',
);
if (processedFolderIds.length > 0) {
await messageFolderRepository.update(
{ id: In(processedFolderIds) },
{ pendingSyncAction: MessageFolderPendingSyncAction.NONE },
transactionManager,
);
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingSyncAction to NONE for ${processedFolderIds.length} folders`,
);
}
if (processedFolderIds.length > 0) {
await messageFolderRepository.update(
{ id: In(processedFolderIds) },
{ pendingSyncAction: MessageFolderPendingSyncAction.NONE },
transactionManager,
);
if (folderIdsToDelete.length > 0) {
await messageFolderRepository.delete(
{ id: In(folderIdsToDelete) },
transactionManager,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingSyncAction to NONE for ${processedFolderIds.length} folders`,
);
}
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Deleted ${folderIdsToDelete.length} folders`,
);
}
if (folderIdsToDelete.length > 0) {
await messageFolderRepository.delete(
{ id: In(folderIdsToDelete) },
transactionManager,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Deleted ${folderIdsToDelete.length} folders`,
);
}
},
);
},
);
}
@@ -3,7 +3,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
MessageChannelPendingGroupEmailsAction,
MessageChannelWorkspaceEntity,
@@ -18,7 +19,7 @@ export class MessagingProcessGroupEmailActionsService {
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingDeleteGroupEmailMessagesService: MessagingDeleteGroupEmailMessagesService,
) {}
@@ -27,19 +28,26 @@ export class MessagingProcessGroupEmailActionsService {
workspaceId: string,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction,
): Promise<void> {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const authContext = buildSystemAuthContext(workspaceId);
await messageChannelRepository.update(
{ id: messageChannel.id },
{ pendingGroupEmailsAction },
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Marked message channel as pending group emails action: ${pendingGroupEmailsAction}`,
await messageChannelRepository.update(
{ id: messageChannel.id },
{ pendingGroupEmailsAction },
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Marked message channel as pending group emails action: ${pendingGroupEmailsAction}`,
);
},
);
}
@@ -60,56 +68,63 @@ export class MessagingProcessGroupEmailActionsService {
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing group email action: ${pendingGroupEmailsAction}`,
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const authContext = buildSystemAuthContext(workspaceId);
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
try {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
switch (pendingGroupEmailsAction) {
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION:
await this.handleGroupEmailsDeletion(
workspaceId,
messageChannel.id,
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
try {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
switch (pendingGroupEmailsAction) {
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION:
await this.handleGroupEmailsDeletion(
workspaceId,
messageChannel.id,
transactionManager,
);
break;
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT:
await this.handleGroupEmailsImport(
workspaceId,
messageChannel.id,
transactionManager,
);
break;
}
await messageChannelRepository.update(
{ id: messageChannel.id },
{
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
transactionManager,
);
break;
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT:
await this.handleGroupEmailsImport(
workspaceId,
messageChannel.id,
transactionManager,
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingGroupEmailsAction to NONE`,
);
break;
}
await messageChannelRepository.update(
{ id: messageChannel.id },
{
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
transactionManager,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingGroupEmailsAction to NONE`,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error processing group email action: ${error.message}`,
error.stack,
);
throw error;
}
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error processing group email action: ${error.message}`,
error.stack,
);
throw error;
}
},
);
},
);
}
@@ -161,7 +176,7 @@ export class MessagingProcessGroupEmailActionsService {
transactionManager: WorkspaceEntityManager;
}) {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
@@ -175,7 +190,7 @@ export class MessagingProcessGroupEmailActionsService {
);
const messageFolderRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
@@ -7,7 +7,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndContactJob } from 'src/modules/contact-creation-manager/jobs/create-company-and-contact.job';
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
@@ -152,11 +152,14 @@ describe('MessagingSaveMessagesAndEnqueueContactCreationService', () => {
},
},
{
provide: TwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: {
getDataSourceForWorkspace: jest
.fn()
.mockResolvedValue(datasourceInstance),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
},
},
],
@@ -6,7 +6,8 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import {
CreateCompanyAndContactJob,
@@ -32,7 +33,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
private readonly messageQueueService: MessageQueueService,
private readonly messageService: MessagingMessageService,
private readonly messageParticipantService: MessagingMessageParticipantService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async saveMessagesAndEnqueueContactCreation(
@@ -42,76 +43,88 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
workspaceId: string,
) {
const handleAliases = connectedAccount.handleAliases?.split(',') || [];
const authContext = buildSystemAuthContext(workspaceId);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
workspaceId,
});
const participantsWithMessageId =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getDataSourceForWorkspace(
workspaceId,
);
const participantsWithMessageId = await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const { messageExternalIdsAndIdsMap } =
await this.messageService.saveMessagesWithinTransaction(
messagesToSave,
messageChannel.id,
transactionManager,
workspaceId,
return workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const { messageExternalIdsAndIdsMap } =
await this.messageService.saveMessagesWithinTransaction(
messagesToSave,
messageChannel.id,
transactionManager,
workspaceId,
);
const participantsWithMessageId: (ParticipantWithMessageId & {
shouldCreateContact: boolean;
})[] = messagesToSave.flatMap((message) => {
const messageId = messageExternalIdsAndIdsMap.get(
message.externalId,
);
return messageId
? message.participants.map((participant: Participant) => {
const fromHandle =
message.participants.find(
(p) => p.role === MessageParticipantRole.FROM,
)?.handle || '';
const isMessageSentByConnectedAccount =
handleAliases.includes(fromHandle) ||
fromHandle === connectedAccount.handle;
const isParticipantConnectedAccount =
handleAliases.includes(participant.handle) ||
participant.handle === connectedAccount.handle;
const isExcludedByNonProfessionalEmails =
messageChannel.excludeNonProfessionalEmails &&
!isWorkEmail(participant.handle);
const shouldCreateContact =
!!participant.handle &&
!isParticipantConnectedAccount &&
!isExcludedByNonProfessionalEmails &&
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED ||
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT &&
isMessageSentByConnectedAccount));
return {
...participant,
messageId,
shouldCreateContact,
};
})
: [];
});
await this.messageParticipantService.saveMessageParticipants(
participantsWithMessageId,
workspaceId,
transactionManager,
);
return participantsWithMessageId;
},
);
},
);
const participantsWithMessageId: (ParticipantWithMessageId & {
shouldCreateContact: boolean;
})[] = messagesToSave.flatMap((message) => {
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
return messageId
? message.participants.map((participant: Participant) => {
const fromHandle =
message.participants.find(
(p) => p.role === MessageParticipantRole.FROM,
)?.handle || '';
const isMessageSentByConnectedAccount =
handleAliases.includes(fromHandle) ||
fromHandle === connectedAccount.handle;
const isParticipantConnectedAccount =
handleAliases.includes(participant.handle) ||
participant.handle === connectedAccount.handle;
const isExcludedByNonProfessionalEmails =
messageChannel.excludeNonProfessionalEmails &&
!isWorkEmail(participant.handle);
const shouldCreateContact =
!!participant.handle &&
!isParticipantConnectedAccount &&
!isExcludedByNonProfessionalEmails &&
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED ||
(messageChannel.contactAutoCreationPolicy ===
MessageChannelContactAutoCreationPolicy.SENT &&
isMessageSentByConnectedAccount));
return {
...participant,
messageId,
shouldCreateContact,
};
})
: [];
});
await this.messageParticipantService.saveMessageParticipants(
participantsWithMessageId,
workspaceId,
transactionManager,
);
return participantsWithMessageId;
},
);
if (messageChannel.isContactAutoCreationEnabled) {
if (
messageChannel.isContactAutoCreationEnabled &&
participantsWithMessageId
) {
const contactsToCreate = participantsWithMessageId.filter(
(participant) => participant.shouldCreateContact,
);
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { MatchParticipantService } from 'src/modules/match-participant/match-participant.service';
import { type MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
import { type ParticipantWithMessageId } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message.type';
@@ -11,7 +12,7 @@ import { type ParticipantWithMessageId } from 'src/modules/messaging/message-imp
@Injectable()
export class MessagingMessageParticipantService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly matchParticipantService: MatchParticipantService<MessageParticipantWorkspaceEntity>,
) {}
@@ -20,55 +21,62 @@ export class MessagingMessageParticipantService {
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
): Promise<void> {
const messageParticipantRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
const authContext = buildSystemAuthContext(workspaceId);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
},
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
}
}
@@ -10,7 +10,8 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMonitoringService } from 'src/modules/messaging/monitoring/services/messaging-monitoring.service';
@@ -23,7 +24,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
private readonly messagingMonitoringService: MessagingMonitoringService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@@ -46,29 +47,36 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
for (const activeWorkspace of activeWorkspaces) {
try {
const messageChannelRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelWorkspaceEntity>(
activeWorkspace.id,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
select: ['id', 'syncStatus', 'connectedAccountId'],
});
const authContext = buildSystemAuthContext(activeWorkspace.id);
for (const messageChannel of messageChannels) {
if (!messageChannel.syncStatus) {
continue;
}
await this.messagingMonitoringService.track({
eventName: `message_channel.monitoring.sync_status.${snakeCase(
messageChannel.syncStatus,
)}`,
workspaceId: activeWorkspace.id,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
message: messageChannel.syncStatus,
});
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
activeWorkspace.id,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
select: ['id', 'syncStatus', 'connectedAccountId'],
});
for (const messageChannel of messageChannels) {
if (!messageChannel.syncStatus) {
continue;
}
await this.messagingMonitoringService.track({
eventName: `message_channel.monitoring.sync_status.${snakeCase(
messageChannel.syncStatus,
)}`,
workspaceId: activeWorkspace.id,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
message: messageChannel.syncStatus,
});
}
},
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
@@ -5,7 +5,8 @@ import { type ObjectRecordNonDestructiveEvent } from 'src/engine/core-modules/ev
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { TimelineActivityService } from 'src/modules/timeline/services/timeline-activity.service';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@@ -14,7 +15,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
export class UpsertTimelineActivityFromInternalEvent {
constructor(
private readonly timelineActivityService: TimelineActivityService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(UpsertTimelineActivityFromInternalEvent.name)
@@ -33,33 +34,40 @@ export class UpsertTimelineActivityFromInternalEvent {
return;
}
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceEventBatch.workspaceId,
WorkspaceMemberWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const authContext = buildSystemAuthContext(workspaceEventBatch.workspaceId);
const userIds = workspaceEventBatch.events
.map((event) => event.userId)
.filter(isDefined);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceEventBatch.workspaceId,
WorkspaceMemberWorkspaceEntity,
{
shouldBypassPermissionChecks: true,
},
);
const workspaceMembers = await workspaceMemberRepository.findBy({
userId: In(userIds),
});
const userIds = workspaceEventBatch.events
.map((event) => event.userId)
.filter(isDefined);
for (const eventData of workspaceEventBatch.events) {
const workspaceMember = workspaceMembers.find(
(workspaceMember) => workspaceMember.userId === eventData.userId,
);
const workspaceMembers = await workspaceMemberRepository.findBy({
userId: In(userIds),
});
if (eventData.userId && workspaceMember) {
eventData.workspaceMemberId = workspaceMember.id;
}
}
for (const eventData of workspaceEventBatch.events) {
const workspaceMember = workspaceMembers.find(
(workspaceMember) => workspaceMember.userId === eventData.userId,
);
await this.timelineActivityService.upsertEvents(workspaceEventBatch);
if (eventData.userId && workspaceMember) {
eventData.workspaceMemberId = workspaceMember.id;
}
}
await this.timelineActivityService.upsertEvents(workspaceEventBatch);
},
);
}
}
@@ -5,7 +5,8 @@ import { type ObjectRecord } from 'twenty-shared/types';
import { In, MoreThan } from 'typeorm';
import { objectRecordDiffMerge } from 'src/engine/core-modules/event-emitter/utils/object-record-diff-merge';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type TimelineActivityPayload } from 'src/modules/timeline/types/timeline-activity-payload';
type TimelineActivityPayloadWorkspaceIdAndObjectSingularName = {
@@ -19,7 +20,7 @@ type TimelineActivityPayloadWorkspaceIdAndObjectSingularName = {
@Injectable()
export class TimelineActivityRepository {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async upsertTimelineActivities({
@@ -27,61 +28,73 @@ export class TimelineActivityRepository {
workspaceId,
payloads,
}: TimelineActivityPayloadWorkspaceIdAndObjectSingularName) {
const recentTimelineActivities = await this.findRecentTimelineActivities({
objectSingularName,
workspaceId,
payloads,
});
const authContext = buildSystemAuthContext(workspaceId);
const payloadsWithDiff = payloads
.filter(({ properties }) => {
const isDiffEmpty =
properties.diff !== null &&
properties.diff &&
Object.keys(properties.diff).length === 0;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const recentTimelineActivities =
await this.findRecentTimelineActivities({
objectSingularName,
workspaceId,
payloads,
});
return !isDiffEmpty;
})
.map(({ properties, ...rest }) => ({
...rest,
properties: isDefined(properties.diff) ? { diff: properties.diff } : {},
}));
const payloadsWithDiff = payloads
.filter(({ properties }) => {
const isDiffEmpty =
properties.diff !== null &&
properties.diff &&
Object.keys(properties.diff).length === 0;
const payloadsToInsert: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'] =
[];
return !isDiffEmpty;
})
.map(({ properties, ...rest }) => ({
...rest,
properties: isDefined(properties.diff)
? { diff: properties.diff }
: {},
}));
for (const payload of payloadsWithDiff) {
const recentTimelineActivity = recentTimelineActivities.find(
(timelineActivity) =>
timelineActivity[`${objectSingularName}Id`] === payload.recordId &&
timelineActivity.workspaceMemberId === payload.workspaceMemberId &&
(!isDefined(payload.linkedRecordId) ||
timelineActivity.linkedRecordId === payload.linkedRecordId) &&
timelineActivity.name === payload.name,
);
const payloadsToInsert: TimelineActivityPayloadWorkspaceIdAndObjectSingularName['payloads'] =
[];
if (recentTimelineActivity) {
const mergedProperties = objectRecordDiffMerge(
recentTimelineActivity.properties,
payload.properties,
);
for (const payload of payloadsWithDiff) {
const recentTimelineActivity = recentTimelineActivities.find(
(timelineActivity) =>
timelineActivity[`${objectSingularName}Id`] ===
payload.recordId &&
timelineActivity.workspaceMemberId ===
payload.workspaceMemberId &&
(!isDefined(payload.linkedRecordId) ||
timelineActivity.linkedRecordId === payload.linkedRecordId) &&
timelineActivity.name === payload.name,
);
await this.updateTimelineActivity({
id: recentTimelineActivity.id,
properties: mergedProperties,
workspaceMemberId: payload.workspaceMemberId,
if (recentTimelineActivity) {
const mergedProperties = objectRecordDiffMerge(
recentTimelineActivity.properties,
payload.properties,
);
await this.updateTimelineActivity({
id: recentTimelineActivity.id,
properties: mergedProperties,
workspaceMemberId: payload.workspaceMemberId,
workspaceId,
});
} else {
payloadsToInsert.push(payload);
}
}
await this.insertTimelineActivities({
objectSingularName,
payloads: payloadsToInsert,
workspaceId,
});
} else {
payloadsToInsert.push(payload);
}
}
await this.insertTimelineActivities({
objectSingularName,
payloads: payloadsToInsert,
workspaceId,
});
},
);
}
private async findRecentTimelineActivities({
@@ -90,7 +103,7 @@ export class TimelineActivityRepository {
payloads,
}: TimelineActivityPayloadWorkspaceIdAndObjectSingularName) {
const timelineActivityTypeORMRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'timelineActivity',
{
@@ -128,7 +141,7 @@ export class TimelineActivityRepository {
}
const timelineActivityTypeORMRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'timelineActivity',
{
@@ -161,7 +174,7 @@ export class TimelineActivityRepository {
workspaceId: string;
}) {
const timelineActivityTypeORMRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'timelineActivity',
{
@@ -8,7 +8,8 @@ import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/work
import { type ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { parseEventNameOrThrow } from 'src/engine/workspace-event-emitter/utils/parse-event-name';
import { NoteWorkspaceEntity } from 'src/modules/note/standard-objects/note.workspace-entity';
@@ -24,7 +25,7 @@ export class TimelineActivityService {
constructor(
@InjectObjectMetadataRepository(TimelineActivityWorkspaceEntity)
private readonly timelineActivityRepository: TimelineActivityRepository,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
) {}
@@ -177,21 +178,29 @@ export class TimelineActivityService {
const { action } = parseEventNameOrThrow(name);
const activityTargetRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
this.targetObjects[activityType],
{
shouldBypassPermissionChecks: true,
const authContext = buildSystemAuthContext(workspaceId);
const activityTargets =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const activityTargetRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
this.targetObjects[activityType],
{
shouldBypassPermissionChecks: true,
},
);
return activityTargetRepository.find({
where: {
[`${activityType}Id`]: In(events.map((event) => event.recordId)),
},
});
},
);
const activityTargets = await activityTargetRepository.find({
where: {
[`${activityType}Id`]: In(events.map((event) => event.recordId)),
},
});
if (activityTargets.length === 0) {
return [];
}
@@ -254,30 +263,38 @@ export class TimelineActivityService {
}): Promise<TimelineActivityPayload[]> {
const { action } = parseEventNameOrThrow(name);
const activityRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
activityType,
{
shouldBypassPermissionChecks: true,
const authContext = buildSystemAuthContext(workspaceId);
const activities =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const activityRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
activityType,
{
shouldBypassPermissionChecks: true,
},
);
return activityRepository.find({
where: {
id: In(
events
.map((event) =>
this.extractActivityIdFromActivityTargetEvent(
event,
activityType,
),
)
.filter(isDefined),
),
},
});
},
);
const activities = await activityRepository.find({
where: {
id: In(
events
.map((event) =>
this.extractActivityIdFromActivityTargetEvent(
event,
activityType,
),
)
.filter(isDefined),
),
},
});
if (activities.length === 0) {
return [];
}
@@ -1,18 +1,19 @@
import { assertIsDefinedOrThrow } from 'twenty-shared/utils';
import { type WorkspacePostQueryHookInstance } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/interfaces/workspace-query-hook.interface';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator';
import { WorkspaceQueryHookType } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/types/workspace-query-hook.type';
import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import {
WorkflowVersionStatus,
type WorkflowVersionWorkspaceEntity,
} from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { type WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
@WorkspaceQueryHook({
key: `workflow.createMany`,
@@ -22,7 +23,7 @@ export class WorkflowCreateManyPostQueryHook
implements WorkspacePostQueryHookInstance
{
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly recordPositionService: RecordPositionService,
) {}
@@ -35,32 +36,37 @@ export class WorkflowCreateManyPostQueryHook
assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError);
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspace.id,
'workflowVersion',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext as WorkspaceAuthContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspace.id,
'workflowVersion',
);
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId: workspace.id,
});
const workflowVersionsToCreate = payload.map((workflow) => ({
workflowId: workflow.id,
status: WorkflowVersionStatus.DRAFT,
name: 'v1',
position,
}));
await Promise.all(
workflowVersionsToCreate.map((workflowVersion) => {
return workflowVersionRepository.insert(workflowVersion);
}),
);
},
workspaceId: workspace.id,
});
const workflowVersionsToCreate = payload.map((workflow) => ({
workflowId: workflow.id,
status: WorkflowVersionStatus.DRAFT,
name: 'v1',
position,
}));
await Promise.all(
workflowVersionsToCreate.map((workflowVersion) => {
return workflowVersionRepository.insert(workflowVersion);
}),
);
}
}
@@ -1,18 +1,19 @@
import { assertIsDefinedOrThrow } from 'twenty-shared/utils';
import { type WorkspacePostQueryHookInstance } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/interfaces/workspace-query-hook.interface';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator';
import { WorkspaceQueryHookType } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/types/workspace-query-hook.type';
import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import {
WorkflowVersionStatus,
type WorkflowVersionWorkspaceEntity,
} from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { type WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
@WorkspaceQueryHook({
key: `workflow.createOne`,
@@ -22,7 +23,7 @@ export class WorkflowCreateOnePostQueryHook
implements WorkspacePostQueryHookInstance
{
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly recordPositionService: RecordPositionService,
) {}
@@ -37,26 +38,31 @@ export class WorkflowCreateOnePostQueryHook
const workflow = payload[0];
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspace.id,
'workflowVersion',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext as WorkspaceAuthContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspace.id,
'workflowVersion',
);
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId: workspace.id,
});
await workflowVersionRepository.insert({
workflowId: workflow.id,
status: WorkflowVersionStatus.DRAFT,
name: 'v1',
position,
});
},
workspaceId: workspace.id,
});
await workflowVersionRepository.insert({
workflowId: workflow.id,
status: WorkflowVersionStatus.DRAFT,
name: 'v1',
position,
});
);
}
}
@@ -2,14 +2,15 @@ import { Injectable } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowCommonException,
WorkflowCommonExceptionCode,
@@ -39,7 +40,7 @@ export type ObjectMetadataInfo = {
@Injectable()
export class WorkflowCommonWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly serverlessFunctionService: ServerlessFunctionService,
private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
) {}
@@ -58,20 +59,27 @@ export class WorkflowCommonWorkspaceService {
);
}
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionId,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const workflowVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionId,
},
});
return this.getValidWorkflowVersionOrFail(workflowVersion);
},
});
return this.getValidWorkflowVersionOrFail(workflowVersion);
);
}
async getValidWorkflowVersionOrFail(
@@ -84,14 +92,6 @@ export class WorkflowCommonWorkspaceService {
);
}
// FIXME: For now we will make the trigger optional. Later, we'll have to ensure the trigger is defined when publishing the flow.
// if (!workflowVersion.trigger) {
// throw new WorkflowTriggerException(
// 'Workflow version does not contains trigger',
// WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
// );
// }
return { ...workflowVersion, trigger: workflowVersion.trigger };
}
@@ -163,73 +163,80 @@ export class WorkflowCommonWorkspaceService {
workspaceId: string;
operation: 'restore' | 'delete' | 'destroy';
}): Promise<void> {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
for (const workflowId of workflowIds) {
switch (operation) {
case 'delete':
await workflowAutomatedTriggerRepository.softDelete({
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
{ shouldBypassPermissionChecks: true },
);
for (const workflowId of workflowIds) {
switch (operation) {
case 'delete':
await workflowAutomatedTriggerRepository.softDelete({
workflowId,
});
await workflowRunRepository.softDelete({
workflowId,
});
await workflowVersionRepository.softDelete({
workflowId,
});
break;
case 'restore':
await workflowAutomatedTriggerRepository.restore({
workflowId,
});
await workflowRunRepository.restore({
workflowId,
});
await workflowVersionRepository.restore({
workflowId,
});
break;
}
await this.deactivateVersionOnDelete({
workflowVersionRepository,
workflowId,
workspaceId,
operation,
});
await workflowRunRepository.softDelete({
await this.handleServerlessFunctionSubEntities({
workflowVersionRepository,
workflowId,
workspaceId,
operation,
});
await workflowVersionRepository.softDelete({
workflowId,
});
break;
case 'restore':
await workflowAutomatedTriggerRepository.restore({
workflowId,
});
await workflowRunRepository.restore({
workflowId,
});
await workflowVersionRepository.restore({
workflowId,
});
break;
}
await this.deactivateVersionOnDelete({
workflowVersionRepository,
workflowId,
workspaceId,
operation,
});
await this.handleServerlessFunctionSubEntities({
workflowVersionRepository,
workflowId,
workspaceId,
operation,
});
}
}
},
);
}
private async deactivateVersionOnDelete({
@@ -248,10 +255,10 @@ export class WorkflowCommonWorkspaceService {
}
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
{ shouldBypassPermissionChecks: true },
);
const workflow = await workflowRepository.findOne({
@@ -9,7 +9,8 @@ import {
type UpdateOneResolverArgs,
} from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowQueryValidationException,
WorkflowQueryValidationExceptionCode,
@@ -25,7 +26,7 @@ import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/work
export class WorkflowVersionValidationWorkspaceService {
constructor(
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async validateWorkflowVersionForCreateOne(
@@ -45,32 +46,38 @@ export class WorkflowVersionValidationWorkspaceService {
);
}
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowAlreadyHasDraftVersion =
await workflowVersionRepository.exists({
where: {
workflowId: payload.data.workflowId,
status: WorkflowVersionStatus.DRAFT,
// FIXME: soft-deleted rows selection will have to be improved globally
deletedAt: IsNull(),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (workflowAlreadyHasDraftVersion) {
throw new WorkflowQueryValidationException(
'Cannot create multiple draft versions for the same workflow',
WorkflowQueryValidationExceptionCode.FORBIDDEN,
{
userFriendlyMessage: msg`Cannot create multiple draft versions for the same workflow`,
},
);
}
const workflowAlreadyHasDraftVersion =
await workflowVersionRepository.exists({
where: {
workflowId: payload.data.workflowId,
status: WorkflowVersionStatus.DRAFT,
deletedAt: IsNull(),
},
});
if (workflowAlreadyHasDraftVersion) {
throw new WorkflowQueryValidationException(
'Cannot create multiple draft versions for the same workflow',
WorkflowQueryValidationExceptionCode.FORBIDDEN,
{
userFriendlyMessage: msg`Cannot create multiple draft versions for the same workflow`,
},
);
}
},
);
}
async validateWorkflowVersionForUpdateOne({
@@ -86,8 +93,6 @@ export class WorkflowVersionValidationWorkspaceService {
workflowVersionId: payload.id,
});
// If the only field updated is the name, we can update the workflow version
// Otherwise, we need to assert that the workflow version is a draft
if (!(Object.keys(payload.data).length === 1 && payload.data.name)) {
assertWorkflowVersionIsDraft(workflowVersion);
}
@@ -123,29 +128,37 @@ export class WorkflowVersionValidationWorkspaceService {
assertWorkflowVersionIsDraft(workflowVersion);
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const otherWorkflowVersionsExist = await workflowVersionRepository.exists({
where: {
workflowId: workflowVersion.workflowId,
deletedAt: IsNull(),
id: Not(workflowVersion.id),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const otherWorkflowVersionsExist =
await workflowVersionRepository.exists({
where: {
workflowId: workflowVersion.workflowId,
deletedAt: IsNull(),
id: Not(workflowVersion.id),
},
});
if (!otherWorkflowVersionsExist) {
throw new WorkflowQueryValidationException(
'The initial version of a workflow can not be deleted',
WorkflowQueryValidationExceptionCode.FORBIDDEN,
{
userFriendlyMessage: msg`The initial version of a workflow can not be deleted`,
},
);
}
},
});
if (!otherWorkflowVersionsExist) {
throw new WorkflowQueryValidationException(
'The initial version of a workflow can not be deleted',
WorkflowQueryValidationExceptionCode.FORBIDDEN,
{
userFriendlyMessage: msg`The initial version of a workflow can not be deleted`,
},
);
}
);
}
}
@@ -2,8 +2,8 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { TRIGGER_STEP_ID } from 'twenty-shared/workflow';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { type WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
import { WorkflowVersionEdgeWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-edge/workflow-version-edge.workspace-service';
@@ -73,7 +73,7 @@ const mockWorkflowVersion = {
} as WorkflowVersionWorkspaceEntity;
describe('WorkflowVersionEdgeWorkspaceService', () => {
let twentyORMGlobalManager: jest.Mocked<TwentyORMGlobalManager>;
let globalWorkspaceOrmManager: jest.Mocked<GlobalWorkspaceOrmManager>;
let workflowCommonWorkspaceService: jest.Mocked<WorkflowCommonWorkspaceService>;
let service: WorkflowVersionEdgeWorkspaceService;
let mockWorkflowVersionWorkspaceRepository: MockWorkspaceRepository;
@@ -88,11 +88,14 @@ describe('WorkflowVersionEdgeWorkspaceService', () => {
mockWorkflowVersion,
);
twentyORMGlobalManager = {
getRepositoryForWorkspace: jest
globalWorkspaceOrmManager = {
executeInWorkspaceContext: jest
.fn()
.mockImplementation(async (_authContext, callback) => callback()),
getRepository: jest
.fn()
.mockResolvedValue(mockWorkflowVersionWorkspaceRepository),
} as unknown as jest.Mocked<TwentyORMGlobalManager>;
} as unknown as jest.Mocked<GlobalWorkspaceOrmManager>;
workflowCommonWorkspaceService = {
getWorkflowVersionOrFail: jest
@@ -104,8 +107,8 @@ describe('WorkflowVersionEdgeWorkspaceService', () => {
providers: [
WorkflowVersionEdgeWorkspaceService,
{
provide: TwentyORMGlobalManager,
useValue: twentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: globalWorkspaceOrmManager,
},
{
provide: WorkflowCommonWorkspaceService,
@@ -4,8 +4,9 @@ import { isDefined } from 'twenty-shared/utils';
import { TRIGGER_STEP_ID } from 'twenty-shared/workflow';
import { type WorkflowVersionStepChangesDTO } from 'src/engine/core-modules/workflow/dtos/workflow-version-step-changes.dto';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowVersionEdgeException,
WorkflowVersionEdgeExceptionCode,
@@ -24,7 +25,7 @@ import { type WorkflowTrigger } from 'src/modules/workflow/workflow-trigger/type
@Injectable()
export class WorkflowVersionEdgeWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
) {}
@@ -41,54 +42,61 @@ export class WorkflowVersionEdgeWorkspaceService {
workspaceId: string;
sourceConnectionOptions?: WorkflowStepConnectionOptions;
}): Promise<WorkflowVersionStepChangesDTO> {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workflowVersionId,
workspaceId,
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
assertWorkflowVersionIsDraft(workflowVersion);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workflowVersionId,
workspaceId,
});
const trigger = workflowVersion.trigger;
const steps = workflowVersion.steps || [];
assertWorkflowVersionIsDraft(workflowVersion);
const targetStep = steps.find((step) => step.id === target);
const trigger = workflowVersion.trigger;
const steps = workflowVersion.steps || [];
if (!isDefined(targetStep)) {
throw new WorkflowVersionEdgeException(
`Target step '${target}' not found in workflowVersion '${workflowVersionId}'`,
WorkflowVersionEdgeExceptionCode.NOT_FOUND,
);
}
const targetStep = steps.find((step) => step.id === target);
const isSourceTrigger = source === TRIGGER_STEP_ID;
if (!isDefined(targetStep)) {
throw new WorkflowVersionEdgeException(
`Target step '${target}' not found in workflowVersion '${workflowVersionId}'`,
WorkflowVersionEdgeExceptionCode.NOT_FOUND,
);
}
if (isSourceTrigger) {
return this.createTriggerEdge({
trigger,
steps,
target,
workflowVersion,
workflowVersionRepository,
});
} else {
return this.createStepEdge({
trigger,
steps,
source,
target,
sourceConnectionOptions,
workflowVersion,
workflowVersionRepository,
});
}
const isSourceTrigger = source === TRIGGER_STEP_ID;
if (isSourceTrigger) {
return this.createTriggerEdge({
trigger,
steps,
target,
workflowVersion,
workflowVersionRepository,
});
} else {
return this.createStepEdge({
trigger,
steps,
source,
target,
sourceConnectionOptions,
workflowVersion,
workflowVersionRepository,
});
}
},
);
}
async deleteWorkflowVersionEdge({
@@ -104,54 +112,61 @@ export class WorkflowVersionEdgeWorkspaceService {
workspaceId: string;
sourceConnectionOptions?: WorkflowStepConnectionOptions;
}): Promise<WorkflowVersionStepChangesDTO> {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workflowVersionId,
workspaceId,
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
assertWorkflowVersionIsDraft(workflowVersion);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workflowVersionId,
workspaceId,
});
const trigger = workflowVersion.trigger;
const steps = workflowVersion.steps || [];
assertWorkflowVersionIsDraft(workflowVersion);
const targetStep = steps.find((step) => step.id === target);
const trigger = workflowVersion.trigger;
const steps = workflowVersion.steps || [];
if (!isDefined(targetStep)) {
throw new WorkflowVersionEdgeException(
`Target step '${target}' not found in workflowVersion '${workflowVersionId}'`,
WorkflowVersionEdgeExceptionCode.NOT_FOUND,
);
}
const targetStep = steps.find((step) => step.id === target);
const isSourceTrigger = source === TRIGGER_STEP_ID;
if (!isDefined(targetStep)) {
throw new WorkflowVersionEdgeException(
`Target step '${target}' not found in workflowVersion '${workflowVersionId}'`,
WorkflowVersionEdgeExceptionCode.NOT_FOUND,
);
}
if (isSourceTrigger) {
return this.deleteTriggerEdge({
trigger,
steps,
target,
workflowVersion,
workflowVersionRepository,
});
} else {
return this.deleteStepEdge({
trigger,
steps,
source,
target,
workflowVersion,
workflowVersionRepository,
sourceConnectionOptions,
});
}
const isSourceTrigger = source === TRIGGER_STEP_ID;
if (isSourceTrigger) {
return this.deleteTriggerEdge({
trigger,
steps,
target,
workflowVersion,
workflowVersionRepository,
});
} else {
return this.deleteStepEdge({
trigger,
steps,
source,
target,
workflowVersion,
workflowVersionRepository,
sourceConnectionOptions,
});
}
},
);
}
private async createTriggerEdge({
@@ -8,7 +8,7 @@ import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-t
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
import { WorkflowVersionStepOperationsWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-operations.workspace-service';
@@ -21,7 +21,7 @@ const mockWorkspaceId = 'workspace-id';
describe('WorkflowVersionStepOperationsWorkspaceService', () => {
let service: WorkflowVersionStepOperationsWorkspaceService;
let twentyORMGlobalManager: jest.Mocked<TwentyORMGlobalManager>;
let globalWorkspaceOrmManager: jest.Mocked<GlobalWorkspaceOrmManager>;
let serverlessFunctionService: jest.Mocked<ServerlessFunctionService>;
let agentRepository: jest.Mocked<any>;
let roleTargetRepository: jest.Mocked<any>;
@@ -67,20 +67,19 @@ describe('WorkflowVersionStepOperationsWorkspaceService', () => {
deleteAgentOnlyRoleIfUnused: jest.fn(),
} as unknown as jest.Mocked<AiAgentRoleService>;
globalWorkspaceOrmManager = {
getRepository: jest.fn(),
} as unknown as jest.Mocked<GlobalWorkspaceOrmManager>;
workspaceCacheService = {
flush: jest.fn(),
} as unknown as jest.Mocked<WorkspaceCacheService>;
twentyORMGlobalManager = {
getRepositoryForWorkspace: jest.fn(),
} as unknown as jest.Mocked<TwentyORMGlobalManager>;
const module: TestingModule = await Test.createTestingModule({
providers: [
WorkflowVersionStepOperationsWorkspaceService,
{
provide: TwentyORMGlobalManager,
useValue: twentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: globalWorkspaceOrmManager,
},
{
provide: ServerlessFunctionService,
@@ -2,17 +2,17 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { TRIGGER_STEP_ID } from 'twenty-shared/workflow';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { type WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
import { WorkflowSchemaWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-schema/workflow-schema.workspace-service';
import { WorkflowVersionStepOperationsWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-operations.workspace-service';
import { WorkflowVersionStepWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step.workspace-service';
import { WorkflowVersionStepHelpersWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-helpers.workspace-service';
import { WorkflowVersionStepCreationWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-creation.workspace-service';
import { WorkflowVersionStepUpdateWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-update.workspace-service';
import { WorkflowVersionStepDeletionWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-deletion.workspace-service';
import { WorkflowVersionStepHelpersWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-helpers.workspace-service';
import { WorkflowVersionStepOperationsWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-operations.workspace-service';
import { WorkflowVersionStepUpdateWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-update.workspace-service';
import { WorkflowVersionStepWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step.workspace-service';
import {
type WorkflowAction,
WorkflowActionType,
@@ -86,7 +86,7 @@ const mockWorkflowVersion = {
} as WorkflowVersionWorkspaceEntity;
describe('WorkflowVersionStepWorkspaceService', () => {
let twentyORMGlobalManager: jest.Mocked<TwentyORMGlobalManager>;
let globalWorkspaceOrmManager: jest.Mocked<GlobalWorkspaceOrmManager>;
let service: WorkflowVersionStepWorkspaceService;
let mockWorkflowVersionWorkspaceRepository: MockWorkspaceRepository;
let mockComputeWorkflowVersionStepChanges: jest.Mock;
@@ -108,11 +108,15 @@ describe('WorkflowVersionStepWorkspaceService', () => {
mockWorkflowVersion,
);
twentyORMGlobalManager = {
getRepositoryForWorkspace: jest
globalWorkspaceOrmManager = {
getRepository: jest
.fn()
.mockResolvedValue(mockWorkflowVersionWorkspaceRepository),
} as unknown as jest.Mocked<TwentyORMGlobalManager>;
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
} as unknown as jest.Mocked<GlobalWorkspaceOrmManager>;
const module: TestingModule = await Test.createTestingModule({
providers: [
@@ -122,8 +126,8 @@ describe('WorkflowVersionStepWorkspaceService', () => {
WorkflowVersionStepUpdateWorkspaceService,
WorkflowVersionStepDeletionWorkspaceService,
{
provide: TwentyORMGlobalManager,
useValue: twentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: globalWorkspaceOrmManager,
},
{
provide: WorkflowSchemaWorkspaceService,
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { assertWorkflowVersionIsDraft } from 'src/modules/workflow/common/utils/assert-workflow-version-is-draft.util';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
@@ -10,7 +11,7 @@ import { type WorkflowTrigger } from 'src/modules/workflow/workflow-trigger/type
@Injectable()
export class WorkflowVersionStepHelpersWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
) {}
@@ -43,23 +44,30 @@ export class WorkflowVersionStepHelpersWorkspaceService {
steps?: WorkflowAction[] | null;
trigger?: WorkflowTrigger | null;
}): Promise<void> {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const updateData: Partial<WorkflowVersionWorkspaceEntity> = {};
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (steps !== undefined) {
updateData.steps = steps;
}
const updateData: Partial<WorkflowVersionWorkspaceEntity> = {};
if (trigger !== undefined) {
updateData.trigger = trigger;
}
if (steps !== undefined) {
updateData.steps = steps;
}
await workflowVersionRepository.update(workflowVersionId, updateData);
if (trigger !== undefined) {
updateData.trigger = trigger;
}
await workflowVersionRepository.update(workflowVersionId, updateData);
},
);
}
}
@@ -15,7 +15,8 @@ import { DEFAULT_SMART_MODEL } from 'src/engine/metadata-modules/ai/ai-models/co
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import {
WorkflowVersionStepException,
@@ -52,7 +53,7 @@ const ITERATOR_EMPTY_STEP_POSITION_OFFSET = {
@Injectable()
export class WorkflowVersionStepOperationsWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly serverlessFunctionService: ServerlessFunctionService,
@InjectRepository(AgentEntity)
private readonly agentRepository: Repository<AgentEntity>,
@@ -359,7 +360,6 @@ export class WorkflowVersionStepOperationsWorkspaceService {
};
}
case WorkflowActionType.AI_AGENT: {
// Get workflow version to use workflow ID and name in agent name
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workflowVersionId,
@@ -465,66 +465,75 @@ export class WorkflowVersionStepOperationsWorkspaceService {
step: WorkflowFormAction;
response: object;
}) {
const responseKeys = Object.keys(response);
const authContext = buildSystemAuthContext(workspaceId);
const enrichedResponses = await Promise.all(
responseKeys.map(async (key) => {
// @ts-expect-error legacy noImplicitAny
if (!isDefined(response[key])) {
// @ts-expect-error legacy noImplicitAny
return { key, value: response[key] };
}
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const responseKeys = Object.keys(response);
const field = step.settings.input.find((field) => field.name === key);
if (
field?.type === 'RECORD' &&
field?.settings?.objectName &&
// @ts-expect-error legacy noImplicitAny
isDefined(response[key].id) &&
// @ts-expect-error legacy noImplicitAny
isValidUuid(response[key].id)
) {
const { flatObjectMetadata, flatFieldMetadataMaps } =
await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
field.settings.objectName,
workspaceId,
);
const relationFieldsNames = getFlatFieldsFromFlatObjectMetadata(
flatObjectMetadata,
flatFieldMetadataMaps,
)
.filter((field) => field.type === FieldMetadataType.RELATION)
.map((field) => field.name);
const repository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
field.settings.objectName,
{ shouldBypassPermissionChecks: true },
);
const record = await repository.findOne({
const enrichedResponses = await Promise.all(
responseKeys.map(async (key) => {
// @ts-expect-error legacy noImplicitAny
where: { id: response[key].id },
relations: relationFieldsNames,
});
if (!isDefined(response[key])) {
// @ts-expect-error legacy noImplicitAny
return { key, value: response[key] };
}
return { key, value: record };
} else {
const field = step.settings.input.find(
(field) => field.name === key,
);
if (
field?.type === 'RECORD' &&
field?.settings?.objectName &&
// @ts-expect-error legacy noImplicitAny
isDefined(response[key].id) &&
// @ts-expect-error legacy noImplicitAny
isValidUuid(response[key].id)
) {
const { flatObjectMetadata, flatFieldMetadataMaps } =
await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
field.settings.objectName,
workspaceId,
);
const relationFieldsNames = getFlatFieldsFromFlatObjectMetadata(
flatObjectMetadata,
flatFieldMetadataMaps,
)
.filter((field) => field.type === FieldMetadataType.RELATION)
.map((field) => field.name);
const repository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
field.settings.objectName,
{ shouldBypassPermissionChecks: true },
);
const record = await repository.findOne({
// @ts-expect-error legacy noImplicitAny
where: { id: response[key].id },
relations: relationFieldsNames,
});
return { key, value: record };
} else {
// @ts-expect-error legacy noImplicitAny
return { key, value: response[key] };
}
}),
);
return enrichedResponses.reduce((acc, { key, value }) => {
// @ts-expect-error legacy noImplicitAny
return { key, value: response[key] };
}
}),
acc[key] = value;
return acc;
}, {});
},
);
return enrichedResponses.reduce((acc, { key, value }) => {
// @ts-expect-error legacy noImplicitAny
acc[key] = value;
return acc;
}, {});
}
async cloneStep({
@@ -650,49 +659,60 @@ export class WorkflowVersionStepOperationsWorkspaceService {
workspaceId: string;
iteratorPosition?: WorkflowStepPositionInput;
}): Promise<WorkflowAction> {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionId,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const workflowVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionId,
},
});
if (!isDefined(workflowVersion)) {
throw new WorkflowVersionStepException(
'WorkflowVersion not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
const existingSteps = workflowVersion.steps ?? [];
const emptyNodeStep: WorkflowEmptyAction = {
id: v4(),
name: 'Add an Action',
type: WorkflowActionType.EMPTY,
valid: true,
nextStepIds: [iteratorStepId],
settings: {
...BASE_STEP_DEFINITION,
input: {},
},
position: {
x:
(iteratorPosition?.x ?? 0) +
ITERATOR_EMPTY_STEP_POSITION_OFFSET.x,
y:
(iteratorPosition?.y ?? 0) +
ITERATOR_EMPTY_STEP_POSITION_OFFSET.y,
},
};
await workflowVersionRepository.update(workflowVersion.id, {
steps: [...existingSteps, emptyNodeStep],
});
return emptyNodeStep;
},
});
if (!isDefined(workflowVersion)) {
throw new WorkflowVersionStepException(
'WorkflowVersion not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
const existingSteps = workflowVersion.steps ?? [];
const emptyNodeStep: WorkflowEmptyAction = {
id: v4(),
name: 'Add an Action',
type: WorkflowActionType.EMPTY,
valid: true,
nextStepIds: [iteratorStepId],
settings: {
...BASE_STEP_DEFINITION,
input: {},
},
position: {
x: (iteratorPosition?.x ?? 0) + ITERATOR_EMPTY_STEP_POSITION_OFFSET.x,
y: (iteratorPosition?.y ?? 0) + ITERATOR_EMPTY_STEP_POSITION_OFFSET.y,
},
};
await workflowVersionRepository.update(workflowVersion.id, {
steps: [...existingSteps, emptyNodeStep],
});
return emptyNodeStep;
);
}
async createDraftStep({
@@ -5,7 +5,8 @@ import { TRIGGER_STEP_ID } from 'twenty-shared/workflow';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { type WorkflowStepPositionUpdateInput } from 'src/engine/core-modules/workflow/dtos/update-workflow-step-position-update-input.dto';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowVersionStepException,
WorkflowVersionStepExceptionCode,
@@ -31,7 +32,7 @@ import {
@Injectable()
export class WorkflowVersionWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowVersionStepWorkspaceService: WorkflowVersionStepWorkspaceService,
private readonly workflowVersionStepOperationsWorkspaceService: WorkflowVersionStepOperationsWorkspaceService,
private readonly recordPositionService: RecordPositionService,
@@ -46,90 +47,99 @@ export class WorkflowVersionWorkspaceService {
workflowId: string;
workflowVersionIdToCopy: string;
}) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersionToCopy = await workflowVersionRepository.findOne({
where: {
id: workflowVersionIdToCopy,
workflowId,
},
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!isDefined(workflowVersionToCopy)) {
throw new WorkflowVersionStepException(
'WorkflowVersion to copy not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
assertWorkflowVersionTriggerIsDefined(workflowVersionToCopy);
assertWorkflowVersionHasSteps(workflowVersionToCopy);
let draftWorkflowVersion = await workflowVersionRepository.findOne({
where: {
workflowId,
status: WorkflowVersionStatus.DRAFT,
},
});
if (!isDefined(draftWorkflowVersion)) {
const workflowVersionsCount = await workflowVersionRepository.count({
where: {
workflowId,
},
});
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId,
});
const insertResult = await workflowVersionRepository.insert({
workflowId,
name: `v${workflowVersionsCount + 1}`,
status: WorkflowVersionStatus.DRAFT,
position,
});
draftWorkflowVersion = insertResult
.generatedMaps[0] as WorkflowVersionWorkspaceEntity;
}
assertWorkflowVersionIsDraft(draftWorkflowVersion);
const newWorkflowVersionTrigger = workflowVersionToCopy.trigger;
const newWorkflowVersionSteps: WorkflowAction[] = [];
for (const step of workflowVersionToCopy.steps) {
const duplicatedStep =
await this.workflowVersionStepWorkspaceService.createDraftStep({
step,
workspaceId,
const workflowVersionToCopy = await workflowVersionRepository.findOne({
where: {
id: workflowVersionIdToCopy,
workflowId,
},
});
newWorkflowVersionSteps.push(duplicatedStep);
}
if (!isDefined(workflowVersionToCopy)) {
throw new WorkflowVersionStepException(
'WorkflowVersion to copy not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
await workflowVersionRepository.update(draftWorkflowVersion.id, {
steps: newWorkflowVersionSteps,
trigger: newWorkflowVersionTrigger,
});
assertWorkflowVersionTriggerIsDefined(workflowVersionToCopy);
assertWorkflowVersionHasSteps(workflowVersionToCopy);
return {
...draftWorkflowVersion,
name: draftWorkflowVersion.name ?? '',
steps: newWorkflowVersionSteps,
trigger: newWorkflowVersionTrigger,
};
let draftWorkflowVersion = await workflowVersionRepository.findOne({
where: {
workflowId,
status: WorkflowVersionStatus.DRAFT,
},
});
if (!isDefined(draftWorkflowVersion)) {
const workflowVersionsCount = await workflowVersionRepository.count({
where: {
workflowId,
},
});
const position = await this.recordPositionService.buildRecordPosition(
{
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId,
},
);
const insertResult = await workflowVersionRepository.insert({
workflowId,
name: `v${workflowVersionsCount + 1}`,
status: WorkflowVersionStatus.DRAFT,
position,
});
draftWorkflowVersion = insertResult
.generatedMaps[0] as WorkflowVersionWorkspaceEntity;
}
assertWorkflowVersionIsDraft(draftWorkflowVersion);
const newWorkflowVersionTrigger = workflowVersionToCopy.trigger;
const newWorkflowVersionSteps: WorkflowAction[] = [];
for (const step of workflowVersionToCopy.steps) {
const duplicatedStep =
await this.workflowVersionStepWorkspaceService.createDraftStep({
step,
workspaceId,
});
newWorkflowVersionSteps.push(duplicatedStep);
}
await workflowVersionRepository.update(draftWorkflowVersion.id, {
steps: newWorkflowVersionSteps,
trigger: newWorkflowVersionTrigger,
});
return {
...draftWorkflowVersion,
name: draftWorkflowVersion.name ?? '',
steps: newWorkflowVersionSteps,
trigger: newWorkflowVersionTrigger,
};
},
);
}
async duplicateWorkflow({
@@ -141,159 +151,167 @@ export class WorkflowVersionWorkspaceService {
workflowIdToDuplicate: string;
workflowVersionIdToCopy: string;
}) {
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const sourceWorkflow = await workflowRepository.findOne({
where: {
id: workflowIdToDuplicate,
},
});
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!isDefined(sourceWorkflow)) {
throw new WorkflowVersionStepException(
'Source workflow not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
const sourceVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionIdToCopy,
workflowId: workflowIdToDuplicate,
},
});
if (!isDefined(sourceVersion)) {
throw new WorkflowVersionStepException(
'WorkflowVersion to copy not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
assertWorkflowVersionTriggerIsDefined(sourceVersion);
assertWorkflowVersionHasSteps(sourceVersion);
const workflowPosition =
await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflow',
},
workspaceId,
});
const insertWorkflowResult = await workflowRepository.insert({
name: `${sourceWorkflow.name} (Duplicate)`,
statuses: [WorkflowStatus.DRAFT],
position: workflowPosition,
});
const newWorkflowId = (
insertWorkflowResult.generatedMaps[0] as WorkflowWorkspaceEntity
).id;
const versionPosition =
await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId,
});
const insertVersionResult = await workflowVersionRepository.insert({
workflowId: newWorkflowId,
name: 'v1',
status: WorkflowVersionStatus.DRAFT,
position: versionPosition,
});
const newDraftVersion = insertVersionResult
.generatedMaps[0] as WorkflowVersionWorkspaceEntity;
const newTrigger = sourceVersion.trigger;
const sourceToClonedPairs: Array<{
source: WorkflowAction;
duplicated: WorkflowAction;
}> = [];
const oldToNewIdMap = new Map<string, string>();
for (const step of sourceVersion.steps ?? []) {
const clonedStep =
await this.workflowVersionStepOperationsWorkspaceService.cloneStep({
step,
workspaceId,
const sourceWorkflow = await workflowRepository.findOne({
where: {
id: workflowIdToDuplicate,
},
});
sourceToClonedPairs.push({
source: step,
duplicated: clonedStep,
});
oldToNewIdMap.set(step.id, clonedStep.id);
}
const remappedTrigger = isDefined(newTrigger)
? {
...newTrigger,
nextStepIds: (newTrigger.nextStepIds ?? []).map(
(oldId) => oldToNewIdMap.get(oldId) ?? oldId,
),
if (!isDefined(sourceWorkflow)) {
throw new WorkflowVersionStepException(
'Source workflow not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
: undefined;
const remappedSteps: WorkflowAction[] = sourceToClonedPairs.map(
({ source, duplicated }) => {
const remappedStep = {
...duplicated,
nextStepIds: (source.nextStepIds ?? []).map(
(oldId) => oldToNewIdMap.get(oldId) ?? oldId,
),
};
const sourceVersion = await workflowVersionRepository.findOne({
where: {
id: workflowVersionIdToCopy,
workflowId: workflowIdToDuplicate,
},
});
if (
source.type === WorkflowActionType.ITERATOR &&
isDefined(source.settings?.input?.initialLoopStepIds)
) {
remappedStep.settings = {
...remappedStep.settings,
input: {
...remappedStep.settings.input,
initialLoopStepIds: source.settings.input.initialLoopStepIds.map(
if (!isDefined(sourceVersion)) {
throw new WorkflowVersionStepException(
'WorkflowVersion to copy not found',
WorkflowVersionStepExceptionCode.NOT_FOUND,
);
}
assertWorkflowVersionTriggerIsDefined(sourceVersion);
assertWorkflowVersionHasSteps(sourceVersion);
const workflowPosition =
await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflow',
},
workspaceId,
});
const insertWorkflowResult = await workflowRepository.insert({
name: `${sourceWorkflow.name} (Duplicate)`,
statuses: [WorkflowStatus.DRAFT],
position: workflowPosition,
});
const newWorkflowId = (
insertWorkflowResult.generatedMaps[0] as WorkflowWorkspaceEntity
).id;
const versionPosition =
await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId,
});
const insertVersionResult = await workflowVersionRepository.insert({
workflowId: newWorkflowId,
name: 'v1',
status: WorkflowVersionStatus.DRAFT,
position: versionPosition,
});
const newDraftVersion = insertVersionResult
.generatedMaps[0] as WorkflowVersionWorkspaceEntity;
const newTrigger = sourceVersion.trigger;
const sourceToClonedPairs: Array<{
source: WorkflowAction;
duplicated: WorkflowAction;
}> = [];
const oldToNewIdMap = new Map<string, string>();
for (const step of sourceVersion.steps ?? []) {
const clonedStep =
await this.workflowVersionStepOperationsWorkspaceService.cloneStep({
step,
workspaceId,
});
sourceToClonedPairs.push({
source: step,
duplicated: clonedStep,
});
oldToNewIdMap.set(step.id, clonedStep.id);
}
const remappedTrigger = isDefined(newTrigger)
? {
...newTrigger,
nextStepIds: (newTrigger.nextStepIds ?? []).map(
(oldId) => oldToNewIdMap.get(oldId) ?? oldId,
),
},
};
}
}
: undefined;
return remappedStep;
const remappedSteps: WorkflowAction[] = sourceToClonedPairs.map(
({ source, duplicated }) => {
const remappedStep = {
...duplicated,
nextStepIds: (source.nextStepIds ?? []).map(
(oldId) => oldToNewIdMap.get(oldId) ?? oldId,
),
};
if (
source.type === WorkflowActionType.ITERATOR &&
isDefined(source.settings?.input?.initialLoopStepIds)
) {
remappedStep.settings = {
...remappedStep.settings,
input: {
...remappedStep.settings.input,
initialLoopStepIds:
source.settings.input.initialLoopStepIds.map(
(oldId) => oldToNewIdMap.get(oldId) ?? oldId,
),
},
};
}
return remappedStep;
},
);
await workflowVersionRepository.update(newDraftVersion.id, {
steps: remappedSteps,
trigger: remappedTrigger,
});
return {
...newDraftVersion,
name: newDraftVersion.name ?? '',
steps: remappedSteps,
trigger: remappedTrigger ?? null,
};
},
);
await workflowVersionRepository.update(newDraftVersion.id, {
steps: remappedSteps,
trigger: remappedTrigger,
});
return {
...newDraftVersion,
name: newDraftVersion.name ?? '',
steps: remappedSteps,
trigger: remappedTrigger ?? null,
};
}
async updateWorkflowVersionPositions({
@@ -305,51 +323,63 @@ export class WorkflowVersionWorkspaceService {
positions: WorkflowStepPositionUpdateInput[];
workspaceId: string;
}) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion = await workflowVersionRepository.findOneOrFail({
where: {
id: workflowVersionId,
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
assertWorkflowVersionIsDraft(workflowVersion);
const workflowVersion = await workflowVersionRepository.findOneOrFail({
where: {
id: workflowVersionId,
},
});
const triggerPosition = positions.find(
(position) => position.id === TRIGGER_STEP_ID,
);
assertWorkflowVersionIsDraft(workflowVersion);
const updatedTrigger =
isDefined(triggerPosition) && isDefined(workflowVersion.trigger)
? {
...workflowVersion.trigger,
position: triggerPosition.position,
const triggerPosition = positions.find(
(position) => position.id === TRIGGER_STEP_ID,
);
const updatedTrigger =
isDefined(triggerPosition) && isDefined(workflowVersion.trigger)
? {
...workflowVersion.trigger,
position: triggerPosition.position,
}
: undefined;
const updatedSteps = workflowVersion.steps?.map((step) => {
const updatedStep = positions.find(
(position) => position.id === step.id,
);
if (updatedStep) {
return {
...step,
position: updatedStep.position,
};
}
: undefined;
const updatedSteps = workflowVersion.steps?.map((step) => {
const updatedStep = positions.find((position) => position.id === step.id);
return step;
});
if (updatedStep) {
return {
...step,
position: updatedStep.position,
const updatePayload = {
...(!isDefined(updatedTrigger) ? {} : { trigger: updatedTrigger }),
...(!isDefined(updatedSteps) ? {} : { steps: updatedSteps }),
};
}
return step;
});
const updatePayload = {
...(!isDefined(updatedTrigger) ? {} : { trigger: updatedTrigger }),
...(!isDefined(updatedSteps) ? {} : { steps: updatedSteps }),
};
await workflowVersionRepository.update(workflowVersionId, updatePayload);
await workflowVersionRepository.update(
workflowVersionId,
updatePayload,
);
},
);
}
}
@@ -7,6 +7,8 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
import { RESUME_DELAYED_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-executor/workflow-actions/delay/contants/resume-delayed-workflow-job-name';
import { isWorkflowDelayAction } from 'src/modules/workflow/workflow-executor/workflow-actions/delay/guards/is-workflow-delay-action.guard';
@@ -28,6 +30,7 @@ export class ResumeDelayedWorkflowJob {
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(RESUME_DELAYED_WORKFLOW_JOB_NAME)
@@ -36,67 +39,74 @@ export class ResumeDelayedWorkflowJob {
workflowRunId,
stepId,
}: ResumeDelayedWorkflowJobData): Promise<void> {
try {
const workflowRun =
await this.workflowRunWorkspaceService.getWorkflowRunOrFail({
workflowRunId,
workspaceId,
});
const authContext = buildSystemAuthContext(workspaceId);
if (workflowRun.status !== WorkflowRunStatus.RUNNING) {
return;
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const workflowRun =
await this.workflowRunWorkspaceService.getWorkflowRunOrFail({
workflowRunId,
workspaceId,
});
const step = workflowRun.state?.flow?.steps?.find(
(step) => step.id === stepId,
);
if (workflowRun.status !== WorkflowRunStatus.RUNNING) {
return;
}
const stepInfo = workflowRun.state?.stepInfos[stepId];
const step = workflowRun.state?.flow?.steps?.find(
(step) => step.id === stepId,
);
if (!step || !isWorkflowDelayAction(step)) {
throw new WorkflowRunException(
'Step not found or is not a delay action',
WorkflowRunExceptionCode.INVALID_INPUT,
);
}
const stepInfo = workflowRun.state?.stepInfos[stepId];
if (stepInfo?.status !== StepStatus.PENDING) {
throw new WorkflowRunException(
'Step is not pending',
WorkflowRunExceptionCode.INVALID_INPUT,
);
}
if (!step || !isWorkflowDelayAction(step)) {
throw new WorkflowRunException(
'Step not found or is not a delay action',
WorkflowRunExceptionCode.INVALID_INPUT,
);
}
await this.workflowRunWorkspaceService.updateWorkflowRunStepInfo({
stepId,
stepInfo: {
status: StepStatus.SUCCESS,
result: {
success: true,
},
},
workspaceId,
workflowRunId,
});
if (stepInfo?.status !== StepStatus.PENDING) {
throw new WorkflowRunException(
'Step is not pending',
WorkflowRunExceptionCode.INVALID_INPUT,
);
}
await this.messageQueueService.add<RunWorkflowJobData>(
RunWorkflowJob.name,
{
workspaceId,
workflowRunId,
lastExecutedStepId: stepId,
},
);
} catch (error) {
await this.workflowRunWorkspaceService.endWorkflowRun({
workflowRunId,
workspaceId,
status: WorkflowRunStatus.FAILED,
error:
error instanceof Error
? error.message
: 'Unknown error during delay resume',
});
}
await this.workflowRunWorkspaceService.updateWorkflowRunStepInfo({
stepId,
stepInfo: {
status: StepStatus.SUCCESS,
result: {
success: true,
},
},
workspaceId,
workflowRunId,
});
await this.messageQueueService.add<RunWorkflowJobData>(
RunWorkflowJob.name,
{
workspaceId,
workflowRunId,
lastExecutedStepId: stepId,
},
);
} catch (error) {
await this.workflowRunWorkspaceService.endWorkflowRun({
workflowRunId,
workspaceId,
status: WorkflowRunStatus.FAILED,
error:
error instanceof Error
? error.message
: 'Unknown error during delay resume',
});
}
},
);
}
}
@@ -7,6 +7,8 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/proc
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service';
@@ -26,6 +28,7 @@ export class RunWorkflowJob {
private readonly workflowExecutorWorkspaceService: WorkflowExecutorWorkspaceService,
private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,
private readonly metricsService: MetricsService,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
@Process(RUN_WORKFLOW_JOB_NAME)
@@ -34,27 +37,34 @@ export class RunWorkflowJob {
lastExecutedStepId,
workspaceId,
}: RunWorkflowJobData): Promise<void> {
try {
if (lastExecutedStepId) {
await this.resumeWorkflowExecution({
workspaceId,
workflowRunId,
lastExecutedStepId,
});
} else {
await this.startWorkflowExecution({
workflowRunId,
workspaceId,
});
}
} catch (error) {
await this.workflowRunWorkspaceService.endWorkflowRun({
workspaceId,
workflowRunId,
status: WorkflowRunStatus.FAILED,
error: error.message,
});
}
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
if (lastExecutedStepId) {
await this.resumeWorkflowExecution({
workspaceId,
workflowRunId,
lastExecutedStepId,
});
} else {
await this.startWorkflowExecution({
workflowRunId,
workspaceId,
});
}
} catch (error) {
await this.workflowRunWorkspaceService.endWorkflowRun({
workspaceId,
workflowRunId,
status: WorkflowRunStatus.FAILED,
error: error.message,
});
}
},
);
}
private async startWorkflowExecution({
@@ -9,7 +9,8 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
import {
WorkflowRunStatus,
@@ -27,7 +28,7 @@ export class WorkflowCleanWorkflowRunsJob {
constructor(
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectDataSource()
private readonly coreDataSource: DataSource,
) {}
@@ -47,37 +48,44 @@ export class WorkflowCleanWorkflowRunsJob {
for (const activeWorkspace of activeWorkspaces) {
const schemaName = getWorkspaceSchemaName(activeWorkspace.id);
const workflowRunsToDelete = await this.coreDataSource.query(
`
WITH ranked_runs AS (
SELECT id,
ROW_NUMBER() OVER (
PARTITION BY "workflowId"
ORDER BY "createdAt" DESC
) AS rn,
"createdAt"
FROM ${schemaName}."workflowRun"
WHERE status IN ('${WorkflowRunStatus.COMPLETED}', '${WorkflowRunStatus.FAILED}')
)
SELECT id, rn FROM ranked_runs
WHERE rn > ${NUMBER_OF_WORKFLOW_RUNS_TO_KEEP}
OR "createdAt" < NOW() - INTERVAL '14 days';
`,
);
const authContext = buildSystemAuthContext(activeWorkspace.id);
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
activeWorkspace.id,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunsToDelete = await this.coreDataSource.query(
`
WITH ranked_runs AS (
SELECT id,
ROW_NUMBER() OVER (
PARTITION BY "workflowId"
ORDER BY "createdAt" DESC
) AS rn,
"createdAt"
FROM ${schemaName}."workflowRun"
WHERE status IN ('${WorkflowRunStatus.COMPLETED}', '${WorkflowRunStatus.FAILED}')
)
SELECT id, rn FROM ranked_runs
WHERE rn > ${NUMBER_OF_WORKFLOW_RUNS_TO_KEEP}
OR "createdAt" < NOW() - INTERVAL '14 days';
`,
);
for (const workflowRunToDelete of workflowRunsToDelete) {
await workflowRunRepository.delete(workflowRunToDelete.id);
}
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository(
activeWorkspace.id,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
this.logger.log(
`Deleted ${workflowRunsToDelete.length} workflow runs for workspace ${activeWorkspace.id} (schema ${schemaName})`,
for (const workflowRunToDelete of workflowRunsToDelete) {
await workflowRunRepository.delete(workflowRunToDelete.id);
}
this.logger.log(
`Deleted ${workflowRunsToDelete.length} workflow runs for workspace ${activeWorkspace.id} (schema ${schemaName})`,
);
},
);
}
}
@@ -2,7 +2,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { IsNull, LessThan, Or } from 'typeorm';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowRunStatus,
WorkflowRunWorkspaceEntity,
@@ -15,43 +16,50 @@ export class WorkflowHandleStaledRunsWorkspaceService {
WorkflowHandleStaledRunsWorkspaceService.name,
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,
) {}
async handleStaledRuns({ workspaceIds }: { workspaceIds: string[] }) {
for (const workspaceId of workspaceIds) {
try {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const staledWorkflowRuns = await workflowRunRepository.find({
where: {
status: WorkflowRunStatus.ENQUEUED,
enqueuedAt: Or(LessThan(oneHourAgo), IsNull()),
const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
const staledWorkflowRuns = await workflowRunRepository.find({
where: {
status: WorkflowRunStatus.ENQUEUED,
enqueuedAt: Or(LessThan(oneHourAgo), IsNull()),
},
});
if (staledWorkflowRuns.length <= 0) {
return;
}
await workflowRunRepository.update(
staledWorkflowRuns.map((workflowRun) => workflowRun.id),
{
enqueuedAt: null,
status: WorkflowRunStatus.NOT_STARTED,
},
);
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
},
});
if (staledWorkflowRuns.length <= 0) {
continue;
}
await workflowRunRepository.update(
staledWorkflowRuns.map((workflowRun) => workflowRun.id),
{
enqueuedAt: null,
status: WorkflowRunStatus.NOT_STARTED,
},
);
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
} catch (error) {
this.logger.error(
@@ -7,7 +7,8 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowRunStatus,
WorkflowRunWorkspaceEntity,
@@ -21,7 +22,7 @@ export class WorkflowRunEnqueueWorkspaceService {
private readonly logger = new Logger(WorkflowRunEnqueueWorkspaceService.name);
constructor(
private readonly workflowThrottlingWorkspaceService: WorkflowThrottlingWorkspaceService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
private readonly metricsService: MetricsService,
@@ -56,100 +57,107 @@ export class WorkflowRunEnqueueWorkspaceService {
}
try {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const notStartedRunsCount = isCacheMode
? await this.workflowThrottlingWorkspaceService.getNotStartedRunsCountFromCache(
workspaceId,
)
: await this.workflowThrottlingWorkspaceService.getNotStartedRunsCountFromDatabase(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const notStartedRunsCount = isCacheMode
? await this.workflowThrottlingWorkspaceService.getNotStartedRunsCountFromCache(
workspaceId,
)
: await this.workflowThrottlingWorkspaceService.getNotStartedRunsCountFromDatabase(
workspaceId,
);
if (notStartedRunsCount <= 0) {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
return;
}
let remainingWorkflowRunToEnqueueCount =
await this.workflowThrottlingWorkspaceService.getRemainingRunsToEnqueueCount(
workspaceId,
);
const workflowRunIdsToEnqueue: string[] = [];
if (remainingWorkflowRunToEnqueueCount > 0) {
const additionalRunsToEnqueue = await workflowRunRepository.find({
where: {
status: WorkflowRunStatus.NOT_STARTED,
...(workflowRunIdsToEnqueue.length > 0
? { id: Not(workflowRunIdsToEnqueue[0]) }
: {}),
},
select: {
id: true,
},
order: {
createdAt: 'ASC',
},
take: remainingWorkflowRunToEnqueueCount,
});
workflowRunIdsToEnqueue.push(
...additionalRunsToEnqueue.map(
(workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id,
),
);
}
if (workflowRunIdsToEnqueue.length <= 0) {
if (!isCacheMode) {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
}
return;
}
await workflowRunRepository.update(workflowRunIdsToEnqueue, {
enqueuedAt: new Date().toISOString(),
status: WorkflowRunStatus.ENQUEUED,
});
await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(
workspaceId,
workflowRunIdsToEnqueue.length,
);
if (notStartedRunsCount <= 0) {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
for (const workflowRunId of workflowRunIdsToEnqueue) {
await this.messageQueueService.add<RunWorkflowJobData>(
RunWorkflowJob.name,
{
workflowRunId,
workspaceId,
},
);
}
return;
}
let remainingWorkflowRunToEnqueueCount =
await this.workflowThrottlingWorkspaceService.getRemainingRunsToEnqueueCount(
workspaceId,
);
const workflowRunIdsToEnqueue: string[] = [];
if (remainingWorkflowRunToEnqueueCount > 0) {
const additionalRunsToEnqueue = await workflowRunRepository.find({
where: {
status: WorkflowRunStatus.NOT_STARTED,
...(workflowRunIdsToEnqueue.length > 0
? { id: Not(workflowRunIdsToEnqueue[0]) }
: {}),
},
select: {
id: true,
},
order: {
createdAt: 'ASC',
},
take: remainingWorkflowRunToEnqueueCount,
});
workflowRunIdsToEnqueue.push(
...additionalRunsToEnqueue.map(
(workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id,
),
);
}
if (workflowRunIdsToEnqueue.length <= 0) {
if (!isCacheMode) {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
}
return;
}
await workflowRunRepository.update(workflowRunIdsToEnqueue, {
enqueuedAt: new Date().toISOString(),
status: WorkflowRunStatus.ENQUEUED,
});
await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(
workspaceId,
workflowRunIdsToEnqueue.length,
if (isCacheMode) {
await this.workflowThrottlingWorkspaceService.decreaseWorkflowRunNotStartedCount(
workspaceId,
workflowRunIdsToEnqueue.length,
);
} else {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
}
},
);
for (const workflowRunId of workflowRunIdsToEnqueue) {
await this.messageQueueService.add<RunWorkflowJobData>(
RunWorkflowJob.name,
{
workflowRunId,
workspaceId,
},
);
}
if (isCacheMode) {
await this.workflowThrottlingWorkspaceService.decreaseWorkflowRunNotStartedCount(
workspaceId,
workflowRunIdsToEnqueue.length,
);
} else {
await this.workflowThrottlingWorkspaceService.recomputeWorkflowRunNotStartedCount(
workspaceId,
);
}
} catch (error) {
this.metricsService.incrementCounter({
key: MetricsKeys.WorkflowRunFailedToEnqueue,
@@ -7,7 +7,8 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowRunStatus,
WorkflowRunWorkspaceEntity,
@@ -18,7 +19,7 @@ export class WorkflowThrottlingWorkspaceService {
constructor(
@InjectCacheStorage(CacheStorageNamespace.ModuleWorkflow)
private readonly cacheStorage: CacheStorageService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly throttlerService: ThrottlerService,
private readonly twentyConfigService: TwentyConfigService,
) {}
@@ -75,19 +76,26 @@ export class WorkflowThrottlingWorkspaceService {
async recomputeWorkflowRunNotStartedCount(
workspaceId: string,
): Promise<void> {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const currentlyNotStartedWorkflowRunCount =
await workflowRunRepository.count({
where: {
status: In([WorkflowRunStatus.NOT_STARTED]),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
return workflowRunRepository.count({
where: {
status: In([WorkflowRunStatus.NOT_STARTED]),
},
});
},
});
);
await this.setWorkflowRunNotStartedCount(
workspaceId,
@@ -102,18 +110,25 @@ export class WorkflowThrottlingWorkspaceService {
async getNotStartedRunsCountFromDatabase(
workspaceId: string,
): Promise<number> {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
return workflowRunRepository.count({
where: {
status: In([WorkflowRunStatus.NOT_STARTED]),
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
WorkflowRunWorkspaceEntity,
{ shouldBypassPermissionChecks: true },
);
return workflowRunRepository.count({
where: {
status: In([WorkflowRunStatus.NOT_STARTED]),
},
});
},
});
);
}
async acquireWorkflowEnqueueLock(
@@ -7,7 +7,8 @@ import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type WorkflowRunWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
@Command({
@@ -20,10 +21,10 @@ export class DeleteWorkflowRunsCommand extends ActiveOrSuspendedWorkspacesMigrat
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService);
}
@Option({
@@ -50,31 +51,40 @@ export class DeleteWorkflowRunsCommand extends ActiveOrSuspendedWorkspacesMigrat
workspaceId,
options,
}: RunOnWorkspaceArgs): Promise<void> {
try {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const createdAtCondition = {
createdAt: LessThan(this.createdBeforeDate || new Date().toISOString()),
};
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const workflowRunCount = await workflowRunRepository.count({
where: createdAtCondition,
});
const createdAtCondition = {
createdAt: LessThan(
this.createdBeforeDate || new Date().toISOString(),
),
};
if (!options.dryRun && workflowRunCount > 0) {
await workflowRunRepository.delete(createdAtCondition);
}
const workflowRunCount = await workflowRunRepository.count({
where: createdAtCondition,
});
this.logger.log(
`${options.dryRun ? ' (DRY RUN): ' : ''}Deleted ${workflowRunCount} workflow runs`,
);
} catch (error) {
this.logger.error('Error while deleting workflowRun', error);
}
if (!options.dryRun && workflowRunCount > 0) {
await workflowRunRepository.delete(createdAtCondition);
}
this.logger.log(
`${options.dryRun ? ' (DRY RUN): ' : ''}Deleted ${workflowRunCount} workflow runs`,
);
} catch (error) {
this.logger.error('Error while deleting workflowRun', error);
}
},
);
}
}
@@ -10,7 +10,8 @@ import { WithLock } from 'src/engine/core-modules/cache-lock/with-lock.decorator
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowRunStatus,
type WorkflowRunState,
@@ -27,7 +28,7 @@ import {
@Injectable()
export class WorkflowRunWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
private readonly recordPositionService: RecordPositionService,
private readonly metricsService: MetricsService,
@@ -53,78 +54,89 @@ export class WorkflowRunWorkspaceService {
error?: string;
workspaceId: string;
}) {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workspaceId,
workflowVersionId,
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const workflowVersion =
await this.workflowCommonWorkspaceService.getWorkflowVersionOrFail({
workspaceId,
workflowVersionId,
});
const workflow = await workflowRepository.findOne({
where: {
id: workflowVersion.workflowId,
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const workflow = await workflowRepository.findOne({
where: {
id: workflowVersion.workflowId,
},
});
if (!workflow) {
throw new WorkflowRunException(
'Workflow id is invalid',
WorkflowRunExceptionCode.WORKFLOW_RUN_INVALID,
);
}
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowRun',
},
workspaceId,
});
const initState = this.getInitState(
workflowVersion,
triggerPayload,
error,
);
const lastWorkflowRun = await workflowRunRepository.findOne({
where: {
workflowId: workflow.id,
},
order: { createdAt: 'desc' },
});
const workflowRunCountMatch = lastWorkflowRun?.name?.match(/#(\d+)/);
const workflowRunCount = workflowRunCountMatch
? parseInt(workflowRunCountMatch[1], 10)
: 0;
const workflowRun = {
id: workflowRunId ?? v4(),
name: `#${workflowRunCount + 1} - ${workflow.name}`,
workflowVersionId,
createdBy,
workflowId: workflow.id,
status,
position,
state: initState,
enqueuedAt: status === WorkflowRunStatus.ENQUEUED ? new Date() : null,
};
await workflowRunRepository.insert(workflowRun);
return workflowRun.id;
},
});
if (!workflow) {
throw new WorkflowRunException(
'Workflow id is invalid',
WorkflowRunExceptionCode.WORKFLOW_RUN_INVALID,
);
}
const position = await this.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowRun',
},
workspaceId,
});
const initState = this.getInitState(workflowVersion, triggerPayload, error);
const lastWorkflowRun = await workflowRunRepository.findOne({
where: {
workflowId: workflow.id,
},
order: { createdAt: 'desc' },
});
const workflowRunCountMatch = lastWorkflowRun?.name?.match(/#(\d+)/);
const workflowRunCount = workflowRunCountMatch
? parseInt(workflowRunCountMatch[1], 10)
: 0;
const workflowRun = {
id: workflowRunId ?? v4(),
name: `#${workflowRunCount + 1} - ${workflow.name}`,
workflowVersionId,
createdBy,
workflowId: workflow.id,
status,
position,
state: initState,
enqueuedAt: status === WorkflowRunStatus.ENQUEUED ? new Date() : null,
};
await workflowRunRepository.insert(workflowRun);
return workflowRun.id;
);
}
@WithLock('workflowRunId')
@@ -328,16 +340,23 @@ export class WorkflowRunWorkspaceService {
workflowRunId: string;
workspaceId: string;
}): Promise<WorkflowRunWorkspaceEntity | null> {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
return await workflowRunRepository.findOne({
where: { id: workflowRunId },
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
return await workflowRunRepository.findOne({
where: { id: workflowRunId },
});
},
);
}
async getWorkflowRunOrFail({
@@ -371,29 +390,36 @@ export class WorkflowRunWorkspaceService {
workspaceId: string;
partialUpdate: QueryDeepPartialEntity<WorkflowRunWorkspaceEntity>;
}) {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowRunToUpdate = await workflowRunRepository.findOneBy({
id: workflowRunId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRunRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowRunWorkspaceEntity>(
workspaceId,
'workflowRun',
{ shouldBypassPermissionChecks: true },
);
if (!workflowRunToUpdate) {
throw new WorkflowRunException(
`workflowRun ${workflowRunId} not found`,
WorkflowRunExceptionCode.WORKFLOW_RUN_NOT_FOUND,
);
}
const workflowRunToUpdate = await workflowRunRepository.findOneBy({
id: workflowRunId,
});
await workflowRunRepository.update(
workflowRunToUpdate.id,
partialUpdate,
undefined,
['id'],
if (!workflowRunToUpdate) {
throw new WorkflowRunException(
`workflowRun ${workflowRunId} not found`,
WorkflowRunExceptionCode.WORKFLOW_RUN_NOT_FOUND,
);
}
await workflowRunRepository.update(
workflowRunToUpdate.id,
partialUpdate,
undefined,
['id'],
);
},
);
}
@@ -4,7 +4,7 @@ import { getRepositoryToken } from '@nestjs/typeorm';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkflowVersionStatus } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { WorkflowStatus } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
import {
@@ -27,8 +27,8 @@ describe('WorkflowStatusesUpdate', () => {
update: jest.fn(),
};
const mockTwentyORMGlobalManager = {
getRepositoryForWorkspace: jest
const mockGlobalWorkspaceOrmManager = {
getRepository: jest
.fn()
.mockImplementation((_workspaceId, entity, options) => {
if (!options?.shouldBypassPermissionChecks) {
@@ -46,6 +46,10 @@ describe('WorkflowStatusesUpdate', () => {
return Promise.resolve(null);
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
};
const mockServerlessFunctionService = {
@@ -58,8 +62,8 @@ describe('WorkflowStatusesUpdate', () => {
providers: [
WorkflowStatusesUpdateJob,
{
provide: TwentyORMGlobalManager,
useValue: mockTwentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: mockGlobalWorkspaceOrmManager,
},
{
provide: ServerlessFunctionService,
@@ -8,8 +8,9 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowVersionStepException,
WorkflowVersionStepExceptionCode,
@@ -69,37 +70,44 @@ export class WorkflowStatusesUpdateJob {
protected readonly logger = new Logger(WorkflowStatusesUpdateJob.name);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly serverlessFunctionService: ServerlessFunctionService,
) {}
@Process(WorkflowStatusesUpdateJob.name)
async handle(event: WorkflowVersionBatchEvent): Promise<void> {
switch (event.type) {
case WorkflowVersionEventType.CREATE:
case WorkflowVersionEventType.DELETE:
await Promise.all(
event.workflowIds.map((workflowId) =>
this.handleWorkflowVersionCreatedOrDeleted({
workflowId,
workspaceId: event.workspaceId,
}),
),
);
break;
case WorkflowVersionEventType.STATUS_UPDATE:
await Promise.all(
event.statusUpdates.map((statusUpdate) =>
this.handleWorkflowVersionStatusUpdated({
statusUpdate,
workspaceId: event.workspaceId,
}),
),
);
break;
default:
break;
}
const authContext = buildSystemAuthContext(event.workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
switch (event.type) {
case WorkflowVersionEventType.CREATE:
case WorkflowVersionEventType.DELETE:
await Promise.all(
event.workflowIds.map((workflowId) =>
this.handleWorkflowVersionCreatedOrDeleted({
workflowId,
workspaceId: event.workspaceId,
}),
),
);
break;
case WorkflowVersionEventType.STATUS_UPDATE:
await Promise.all(
event.statusUpdates.map((statusUpdate) =>
this.handleWorkflowVersionStatusUpdated({
statusUpdate,
workspaceId: event.workspaceId,
}),
),
);
break;
default:
break;
}
},
);
}
private async handleWorkflowVersionCreatedOrDeleted({
@@ -110,14 +118,14 @@ export class WorkflowStatusesUpdateJob {
workspaceId: string;
}): Promise<void> {
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
@@ -212,14 +220,14 @@ export class WorkflowStatusesUpdateJob {
workspaceId: string;
}): Promise<void> {
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
@@ -4,7 +4,7 @@ import { type ToolSet } from 'ai';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { PerObjectToolGeneratorService } from 'src/engine/core-modules/tool-generator/services/per-object-tool-generator.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
import { WorkflowSchemaWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-schema/workflow-schema.workspace-service';
import { WorkflowVersionEdgeWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-edge/workflow-version-edge.workspace-service';
@@ -39,7 +39,7 @@ export class WorkflowToolWorkspaceService {
workflowVersionService: WorkflowVersionWorkspaceService,
workflowTriggerService: WorkflowTriggerWorkspaceService,
workflowSchemaService: WorkflowSchemaWorkspaceService,
twentyORMGlobalManager: TwentyORMGlobalManager,
globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
recordPositionService: RecordPositionService,
private readonly perObjectToolGenerator: PerObjectToolGeneratorService,
) {
@@ -49,7 +49,7 @@ export class WorkflowToolWorkspaceService {
workflowVersionService,
workflowTriggerService,
workflowSchemaService,
twentyORMGlobalManager,
globalWorkspaceOrmManager,
recordPositionService,
};
@@ -6,6 +6,7 @@ import { v4 as uuidv4 } from 'uuid';
import { z } from 'zod';
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkflowVersionStatus } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { WorkflowStatus } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
import { type WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
@@ -63,7 +64,7 @@ type CreateCompleteWorkflowToolDeps = Pick<
| 'workflowVersionService'
| 'workflowVersionEdgeService'
| 'workflowTriggerService'
| 'twentyORMGlobalManager'
| 'globalWorkspaceOrmManager'
| 'recordPositionService'
>;
@@ -196,34 +197,40 @@ const createWorkflow = async ({
context: CreateCompleteWorkflowToolContext;
name: string;
}): Promise<string> => {
const workflowRepository =
await deps.twentyORMGlobalManager.getRepositoryForWorkspace(
context.workspaceId,
'workflow',
context.rolePermissionConfig,
);
const authContext = buildSystemAuthContext(context.workspaceId);
const workflowPosition = await deps.recordPositionService.buildRecordPosition(
{
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflow',
},
workspaceId: context.workspaceId,
return deps.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRepository =
await deps.globalWorkspaceOrmManager.getRepository(
context.workspaceId,
'workflow',
context.rolePermissionConfig,
);
const workflowPosition =
await deps.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflow',
},
workspaceId: context.workspaceId,
});
const workflow = {
id: uuidv4(),
name,
statuses: [WorkflowStatus.DRAFT],
position: workflowPosition,
};
await workflowRepository.insert(workflow);
return workflow.id;
},
);
const workflow = {
id: uuidv4(),
name,
statuses: [WorkflowStatus.DRAFT],
position: workflowPosition,
};
await workflowRepository.insert(workflow);
return workflow.id;
};
const createWorkflowVersion = async ({
@@ -239,35 +246,43 @@ const createWorkflowVersion = async ({
trigger: WorkflowTrigger;
steps: WorkflowAction[];
}): Promise<string> => {
const workflowVersionRepository =
await deps.twentyORMGlobalManager.getRepositoryForWorkspace(
context.workspaceId,
'workflowVersion',
context.rolePermissionConfig,
);
const authContext = buildSystemAuthContext(context.workspaceId);
const versionPosition = await deps.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
return deps.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await deps.globalWorkspaceOrmManager.getRepository(
context.workspaceId,
'workflowVersion',
context.rolePermissionConfig,
);
const versionPosition =
await deps.recordPositionService.buildRecordPosition({
value: 'first',
objectMetadata: {
isCustom: false,
nameSingular: 'workflowVersion',
},
workspaceId: context.workspaceId,
});
const workflowVersion = {
id: uuidv4(),
workflowId,
name: 'v1',
status: WorkflowVersionStatus.DRAFT,
trigger,
steps,
position: versionPosition,
};
await workflowVersionRepository.insert(workflowVersion);
return workflowVersion.id;
},
workspaceId: context.workspaceId,
});
const workflowVersion = {
id: uuidv4(),
workflowId,
name: 'v1',
status: WorkflowVersionStatus.DRAFT,
trigger,
steps,
position: versionPosition,
};
await workflowVersionRepository.insert(workflowVersion);
return workflowVersion.id;
);
};
const updateWorkflowStatus = async ({
@@ -281,15 +296,22 @@ const updateWorkflowStatus = async ({
workflowId: string;
workflowVersionId: string;
}) => {
const workflowRepository =
await deps.twentyORMGlobalManager.getRepositoryForWorkspace(
context.workspaceId,
'workflow',
context.rolePermissionConfig,
);
const authContext = buildSystemAuthContext(context.workspaceId);
await workflowRepository.update(workflowId, {
statuses: [WorkflowStatus.ACTIVE],
lastPublishedVersionId: workflowVersionId,
});
await deps.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowRepository =
await deps.globalWorkspaceOrmManager.getRepository(
context.workspaceId,
'workflow',
context.rolePermissionConfig,
);
await workflowRepository.update(workflowId, {
statuses: [WorkflowStatus.ACTIVE],
lastPublishedVersionId: workflowVersionId,
});
},
);
};
@@ -1,5 +1,5 @@
import type { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import type { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import type { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import type { WorkflowSchemaWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-schema/workflow-schema.workspace-service';
import type { WorkflowVersionEdgeWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-edge/workflow-version-edge.workspace-service';
import type { WorkflowVersionStepWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step.workspace-service';
@@ -12,7 +12,7 @@ export type WorkflowToolDependencies = {
workflowVersionService: WorkflowVersionWorkspaceService;
workflowTriggerService: WorkflowTriggerWorkspaceService;
workflowSchemaService: WorkflowSchemaWorkspaceService;
twentyORMGlobalManager: TwentyORMGlobalManager;
globalWorkspaceOrmManager: GlobalWorkspaceOrmManager;
recordPositionService: RecordPositionService;
};
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
type AutomatedTriggerType,
type WorkflowAutomatedTriggerWorkspaceEntity,
@@ -10,7 +11,7 @@ import { type AutomatedTriggerSettings } from 'src/modules/workflow/workflow-tri
@Injectable()
export class AutomatedTriggerWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async addAutomatedTrigger({
@@ -24,17 +25,24 @@ export class AutomatedTriggerWorkspaceService {
settings: AutomatedTriggerSettings;
workspaceId: string;
}) {
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
const authContext = buildSystemAuthContext(workspaceId);
await workflowAutomatedTriggerRepository.insert({
type,
settings,
workflowId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
await workflowAutomatedTriggerRepository.insert({
type,
settings,
workflowId,
});
},
);
}
async deleteAutomatedTrigger({
@@ -44,12 +52,19 @@ export class AutomatedTriggerWorkspaceService {
workflowId: string;
workspaceId: string;
}) {
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
const authContext = buildSystemAuthContext(workspaceId);
await workflowAutomatedTriggerRepository.delete({ workflowId });
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
await workflowAutomatedTriggerRepository.delete({ workflowId });
},
);
}
}
@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
@@ -11,7 +11,7 @@ import { WorkflowTriggerJob } from 'src/modules/workflow/workflow-trigger/jobs/w
describe('WorkflowDatabaseEventTriggerListener', () => {
let listener: WorkflowDatabaseEventTriggerListener;
let twentyORMGlobalManager: jest.Mocked<TwentyORMGlobalManager>;
let globalWorkspaceOrmManager: jest.Mocked<GlobalWorkspaceOrmManager>;
let messageQueueService: jest.Mocked<MessageQueueService>;
const mockRepository = {
@@ -48,8 +48,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
}) as FlatObjectMetadata;
beforeEach(async () => {
twentyORMGlobalManager = {
getRepositoryForWorkspace: jest.fn().mockResolvedValue(mockRepository),
globalWorkspaceOrmManager = {
getRepository: jest.fn().mockResolvedValue(mockRepository),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
} as any;
messageQueueService = {
@@ -60,8 +63,8 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
providers: [
WorkflowDatabaseEventTriggerListener,
{
provide: TwentyORMGlobalManager,
useValue: twentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: globalWorkspaceOrmManager,
},
{
provide: MessageQueueService,
@@ -20,7 +20,8 @@ import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-m
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import {
AutomatedTriggerType,
@@ -43,7 +44,7 @@ export class WorkflowDatabaseEventTriggerListener {
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
@@ -242,57 +243,66 @@ export class WorkflowDatabaseEventTriggerListener {
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
}) {
const { fieldIdByJoinColumnName } = buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const { fieldIdByJoinColumnName } =
buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
);
for (const [joinColumnName, joinFieldId] of Object.entries(
fieldIdByJoinColumnName,
)) {
const joinField = findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: flatFieldMetadataMaps,
flatEntityId: joinFieldId,
});
const joinRecordIds = records
.map((record) => record[joinColumnName])
.filter(isDefined);
if (joinRecordIds.length === 0) {
continue;
}
const relatedObjectMetadataId =
joinField.relationTargetObjectMetadataId;
if (!isDefined(relatedObjectMetadataId)) {
continue;
}
const relatedObjectMetadataNameSingular =
flatObjectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
if (!isDefined(relatedObjectMetadataNameSingular)) {
continue;
}
const relatedObjectRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
relatedObjectMetadataNameSingular,
{ shouldBypassPermissionChecks: true },
);
const relatedRecords = await relatedObjectRepository.find({
where: { id: In(joinRecordIds) },
});
for (const record of records) {
record[joinField.name] = relatedRecords.find(
(relatedRecord) => relatedRecord.id === record[joinColumnName],
);
}
}
},
);
for (const [joinColumnName, joinFieldId] of Object.entries(
fieldIdByJoinColumnName,
)) {
const joinField = findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: flatFieldMetadataMaps,
flatEntityId: joinFieldId,
});
const joinRecordIds = records
.map((record) => record[joinColumnName])
.filter(isDefined);
if (joinRecordIds.length === 0) {
continue;
}
const relatedObjectMetadataId = joinField.relationTargetObjectMetadataId;
if (!isDefined(relatedObjectMetadataId)) {
continue;
}
const relatedObjectMetadataNameSingular =
flatObjectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
if (!isDefined(relatedObjectMetadataNameSingular)) {
continue;
}
const relatedObjectRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
relatedObjectMetadataNameSingular,
{ shouldBypassPermissionChecks: true },
);
const relatedRecords = await relatedObjectRepository.find({
where: { id: In(joinRecordIds) },
});
for (const record of records) {
record[joinField.name] = relatedRecords.find(
(relatedRecord) => relatedRecord.id === record[joinColumnName],
);
}
}
}
private async shouldIgnoreEvent(
@@ -325,45 +335,52 @@ export class WorkflowDatabaseEventTriggerListener {
const databaseEventName = payload.name;
const automatedTriggerTableName = 'workflowAutomatedTrigger';
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
automatedTriggerTableName,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const eventListeners = await workflowAutomatedTriggerRepository.find({
where: {
type: AutomatedTriggerType.DATABASE_EVENT,
settings: Raw(
() =>
`"${automatedTriggerTableName}"."settings"->>'eventName' = :eventName`,
{ eventName: databaseEventName },
),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
automatedTriggerTableName,
{ shouldBypassPermissionChecks: true },
);
for (const eventListener of eventListeners) {
for (const eventPayload of payload.events) {
const shouldTriggerJob = this.shouldTriggerJob({
eventPayload,
eventListener,
action,
const eventListeners = await workflowAutomatedTriggerRepository.find({
where: {
type: AutomatedTriggerType.DATABASE_EVENT,
settings: Raw(
() =>
`"${automatedTriggerTableName}"."settings"->>'eventName' = :eventName`,
{ eventName: databaseEventName },
),
},
});
if (shouldTriggerJob) {
await this.messageQueueService.add<WorkflowTriggerJobData>(
WorkflowTriggerJob.name,
{
workspaceId,
workflowId: eventListener.workflowId,
payload: eventPayload,
},
{ retryLimit: 3 },
);
for (const eventListener of eventListeners) {
for (const eventPayload of payload.events) {
const shouldTriggerJob = this.shouldTriggerJob({
eventPayload,
eventListener,
action,
});
if (shouldTriggerJob) {
await this.messageQueueService.add<WorkflowTriggerJobData>(
WorkflowTriggerJob.name,
{
workspaceId,
workflowId: eventListener.workflowId,
payload: eventPayload,
},
{ retryLimit: 3 },
);
}
}
}
}
}
},
);
}
private shouldTriggerJob({
@@ -10,7 +10,8 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/proc
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { handleWorkflowTriggerException } from 'src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowVersionStatus,
type WorkflowVersionWorkspaceEntity,
@@ -33,7 +34,7 @@ const DEFAULT_WORKFLOW_NAME = 'Workflow';
@Processor({ queueName: MessageQueue.workflowQueue, scope: Scope.REQUEST })
export class WorkflowTriggerJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService,
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
@@ -41,77 +42,83 @@ export class WorkflowTriggerJob {
@Process(WorkflowTriggerJob.name)
async handle(data: WorkflowTriggerJobData): Promise<void> {
try {
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
data.workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(data.workspaceId);
const workflow = await workflowRepository.findOneBy({
id: data.workflowId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
data.workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
if (!workflow) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
const workflow = await workflowRepository.findOneBy({
id: data.workflowId,
});
if (!workflow.lastPublishedVersionId) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} has no published version in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
if (!workflow) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
data.workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!workflow.lastPublishedVersionId) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} has no published version in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
const workflowVersion = await workflowVersionRepository.findOneBy({
id: workflow.lastPublishedVersionId,
});
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
data.workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!workflowVersion) {
throw new WorkflowTriggerException(
`Workflow version ${workflow.lastPublishedVersionId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
if (workflowVersion.status !== WorkflowVersionStatus.ACTIVE) {
throw new WorkflowTriggerException(
`Workflow version ${workflowVersion.id} is not active in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
const workflowVersion = await workflowVersionRepository.findOneBy({
id: workflow.lastPublishedVersionId,
});
await this.workflowRunnerWorkspaceService.run({
workspaceId: data.workspaceId,
workflowVersionId: workflow.lastPublishedVersionId,
payload: data.payload,
source: {
source: FieldActorSource.WORKFLOW,
name:
isDefined(workflow.name) && !isEmpty(workflow.name)
? workflow.name
: DEFAULT_WORKFLOW_NAME,
context: {},
workspaceMemberId: null,
},
});
} catch (e) {
// We remove cron if it exists when no valid workflowVersion exists
await this.messageQueueService.removeCron({
jobName: WorkflowTriggerJob.name,
jobId: data.workflowId,
});
handleWorkflowTriggerException(e);
}
if (!workflowVersion) {
throw new WorkflowTriggerException(
`Workflow version ${workflow.lastPublishedVersionId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
if (workflowVersion.status !== WorkflowVersionStatus.ACTIVE) {
throw new WorkflowTriggerException(
`Workflow version ${workflowVersion.id} is not active in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
await this.workflowRunnerWorkspaceService.run({
workspaceId: data.workspaceId,
workflowVersionId: workflow.lastPublishedVersionId,
payload: data.payload,
source: {
source: FieldActorSource.WORKFLOW,
name:
isDefined(workflow.name) && !isEmpty(workflow.name)
? workflow.name
: DEFAULT_WORKFLOW_NAME,
context: {},
workspaceMemberId: null,
},
});
} catch (e) {
await this.messageQueueService.removeCron({
jobName: WorkflowTriggerJob.name,
jobId: data.workflowId,
});
handleWorkflowTriggerException(e);
}
},
);
}
}
@@ -3,8 +3,9 @@ import { Injectable } from '@nestjs/common';
import { msg } from '@lingui/core/macro';
import { type ActorMetadata } from 'twenty-shared/types';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
import {
@@ -31,7 +32,7 @@ import { assertNever } from 'src/utils/assert';
@Injectable()
export class WorkflowTriggerWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService,
private readonly automatedTriggerWorkspaceService: AutomatedTriggerWorkspaceService,
@@ -69,71 +70,87 @@ export class WorkflowTriggerWorkspaceService {
workflowVersionId: string,
workspaceId: string,
) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersionNullable = await workflowVersionRepository.findOne({
where: { id: workflowVersionId },
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const workflowVersion =
await this.workflowCommonWorkspaceService.getValidWorkflowVersionOrFail(
workflowVersionNullable,
);
const workflowVersionNullable = await workflowVersionRepository.findOne(
{
where: { id: workflowVersionId },
},
);
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const workflowVersion =
await this.workflowCommonWorkspaceService.getValidWorkflowVersionOrFail(
workflowVersionNullable,
);
const workflow = await workflowRepository.findOne({
where: { id: workflowVersion.workflowId },
});
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
if (!workflow) {
throw new WorkflowTriggerException(
'No workflow found',
WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
);
}
const workflow = await workflowRepository.findOne({
where: { id: workflowVersion.workflowId },
});
assertVersionCanBeActivated(workflowVersion, workflow);
if (!workflow) {
throw new WorkflowTriggerException(
'No workflow found',
WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
);
}
await this.performActivationSteps(
workflow,
workflowVersion,
workflowRepository,
workflowVersionRepository,
workspaceId,
assertVersionCanBeActivated(workflowVersion, workflow);
await this.performActivationSteps(
workflow,
workflowVersion,
workflowRepository,
workflowVersionRepository,
workspaceId,
);
return true;
},
);
return true;
}
async deactivateWorkflowVersion(
workflowVersionId: string,
workspaceId: string,
) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
await this.performDeactivationSteps(
workflowVersionId,
workflowVersionRepository,
workspaceId,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
await this.performDeactivationSteps(
workflowVersionId,
workflowVersionRepository,
workspaceId,
);
return true;
},
);
return true;
}
async stopWorkflowRun(workflowRunId: string, workspaceId: string) {
@@ -4,6 +4,7 @@ import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import { type WorkspacePostQueryHookInstance } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/interfaces/workspace-query-hook.interface';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
import { WorkspaceQueryHook } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator';
import { WorkspaceQueryHookType } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/types/workspace-query-hook.type';
@@ -15,7 +16,7 @@ import {
PermissionsException,
PermissionsExceptionCode,
} from 'src/engine/metadata-modules/permissions/permissions.exception';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceMemberPreQueryHookService } from 'src/modules/workspace-member/query-hooks/workspace-member-pre-query-hook.service';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@@ -27,7 +28,7 @@ export class WorkspaceMemberDeleteOnePostQueryHook
implements WorkspacePostQueryHookInstance
{
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
private readonly workspaceMemberPreQueryHookService: WorkspaceMemberPreQueryHookService,
@@ -60,18 +61,24 @@ export class WorkspaceMemberDeleteOnePostQueryHook
},
);
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspace.id,
'workspaceMember',
);
const workspaceMember =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext as WorkspaceAuthContext,
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspace.id,
'workspaceMember',
);
const workspaceMember = await workspaceMemberRepository.findOne({
where: {
id: targettedWorkspaceMemberId,
},
withDeleted: true,
});
return workspaceMemberRepository.findOne({
where: {
id: targettedWorkspaceMemberId,
},
withDeleted: true,
});
},
);
if (!isDefined(workspaceMember)) {
throw new PermissionsException(