messaging minor perf improvement (#20687)

This PR adds two changes

1. Pass `lite:true` to `ExecuteInWorkspaceContextOptions` introduced in
https://github.com/twentyhq/twenty/pull/18376

2. Remove redundant gmail alias call, it adds 300ms every cron job, we
only do it once now when user connects, realistically I don't see people
changing their aliases every day you only set it up once

actual real diff is small, it's just prettier format contributing to
diff

Objective decrease total time take per job
This commit is contained in:
neo773
2026-05-19 16:08:34 +05:30
committed by GitHub
parent db4a05301a
commit 6cd069ce40
46 changed files with 2685 additions and 2409 deletions
@@ -46,159 +46,164 @@ export class BlocklistItemDeleteMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccountId: In(connectedAccountIds),
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
relations: { connectedAccount: true },
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
const handleAliases = messageChannel.connectedAccount?.handleAliases;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (isDefined(handleAliases)) {
const aliasList: string[] = Array.isArray(handleAliases)
? handleAliases
: (handleAliases as string).split(',');
if (!isDefined(handle)) {
return acc;
}
messageChannelHandles.push(...aliasList);
}
acc.get(workspaceMemberId)?.push(handle);
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return acc;
},
new Map<string, string[]>(),
);
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
if (messageChannelMessageAssociationsToDelete.length === 0) {
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}, authContext);
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccountId: In(connectedAccountIds),
workspaceId,
},
relations: { connectedAccount: true },
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
const handleAliases =
messageChannel.connectedAccount?.handleAliases;
if (isDefined(handleAliases)) {
const aliasList: string[] = Array.isArray(handleAliases)
? handleAliases
: (handleAliases as string).split(',');
messageChannelHandles.push(...aliasList);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
},
authContext,
{ lite: true },
);
}
}
@@ -44,57 +44,63 @@ export class BlocklistReimportMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
connectedAccountId: In(connectedAccountIds),
syncStage: Not(MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
},
});
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
connectedAccountId: In(connectedAccountIds),
syncStage: Not(
MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
),
workspaceId,
},
});
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -157,6 +157,7 @@ export class ApplyMessagesVisibilityRestrictionsService {
return messages;
},
authContext,
{ lite: true },
);
}
}
@@ -53,15 +53,21 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesImportPending(
@@ -75,15 +81,21 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async resetAndMarkAsMessagesListFetchPending(
@@ -102,26 +114,31 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
throttleRetryAfter: null,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
},
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
throttleRetryAfter: null,
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
);
await this.messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds), workspaceId },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
}, authContext);
await this.messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds), workspaceId },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
},
authContext,
{ lite: true },
);
await this.markAsMessagesListFetchPending(messageChannelIds, workspaceId);
}
@@ -136,12 +153,16 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{ syncStageStartedAt: null },
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{ syncStageStartedAt: null },
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesListFetchScheduled(
@@ -154,16 +175,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesListFetchOngoing(
@@ -176,16 +201,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCompletedAndMarkAsMessagesListFetchPending(
@@ -198,19 +227,23 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.MessageChannelSyncJobActive,
@@ -228,14 +261,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesImportOngoing(
@@ -248,16 +285,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsFailed(
@@ -273,50 +314,56 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
throttleRetryAfter: null,
},
);
const metricsKey =
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
if (
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const messageChannels = await this.messageChannelRepository.find({
where: { id: In(messageChannelIds), workspaceId },
});
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
authFailedAt: new Date(),
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
throttleRetryAfter: null,
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
const metricsKey =
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
if (
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const messageChannels = await this.messageChannelRepository.find({
where: { id: In(messageChannelIds), workspaceId },
});
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
private async addToAccountsToReconnect(
@@ -42,44 +42,48 @@ export class MessagingResetChannelCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
const messageChannels = await this.messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
workspaceId,
},
});
if (messageChannels.length === 0) {
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
return;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
workspaceId,
},
});
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
if (messageChannels.length === 0) {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
return;
}
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
}, authContext);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
},
authContext,
{ lite: true },
);
}
@Option({
@@ -29,95 +29,99 @@ export class MessagingMessageCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
const orphanMessages = await messageRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
},
});
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
if (orphanMessages.length <= 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessages = await messageRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
),
messageChannelMessageAssociations: {
id: IsNull(),
messages: {
id: IsNull(),
},
},
},
});
});
if (orphanMessages.length <= 0) {
continue;
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
messages: {
id: IsNull(),
},
},
});
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
async deleteMessageChannelMessageAssociationsByChannelId({
@@ -129,80 +133,20 @@ export class MessagingMessageCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const associations =
await messageChannelMessageAssociationRepository.find(
{
where: { messageChannelId },
take: limit,
skip: offset,
},
transactionManager,
);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`,
);
await messageChannelMessageAssociationRepository.delete(
ids,
transactionManager,
);
},
transactionManager,
);
});
}, authContext);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
@@ -210,72 +154,140 @@ export class MessagingMessageCleanerService {
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
transactionManager?: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
const associations =
await messageChannelMessageAssociationRepository.find(
{
where: { messageChannelId },
take: limit,
skip: offset,
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
await messageChannelMessageAssociationRepository.delete(
ids,
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
}, authContext);
});
},
authContext,
{ lite: true },
);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -188,6 +188,7 @@ export class SyncMessageFoldersService {
return [...updatedExistingFolders, ...createdFolders];
},
authContext,
{ lite: true },
);
}
}
@@ -53,54 +53,58 @@ export class MessagingTriggerMessageListFetchCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(messageChannelId ? { id: messageChannelId } : {}),
workspaceId,
},
});
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channel(s) to process`,
);
for (const messageChannel of messageChannels) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(messageChannelId ? { id: messageChannelId } : {}),
workspaceId,
},
);
});
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Triggered fetch for message channel ${messageChannel.id}`,
`Found ${messageChannels.length} message channel(s) to process`,
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
);
}, authContext);
for (const messageChannel of messageChannels) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
workspaceId,
},
);
this.logger.log(
`Triggered fetch for message channel ${messageChannel.id}`,
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
);
},
authContext,
{ lite: true },
);
}
@Option({
@@ -94,14 +94,18 @@ export class InboundEmailImportService {
);
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
[parsedInboundMessage.message],
messageChannel,
connectedAccount,
workspaceId,
);
}, buildSystemAuthContext(workspaceId));
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
[parsedInboundMessage.message],
messageChannel,
connectedAccount,
workspaceId,
);
},
buildSystemAuthContext(workspaceId),
{ lite: true },
);
await this.inboundEmailStorageService.deleteRawMessage(s3Key);
@@ -48,59 +48,63 @@ export class MessagingMessageListFetchJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
return;
}
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
return;
}
try {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
try {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -43,41 +43,45 @@ export class MessagingMessagesImportJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
return;
}
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (!messageChannel?.isSyncEnabled) {
return;
}
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
}, authContext);
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
},
authContext,
{ lite: true },
);
}
}
@@ -37,52 +37,58 @@ export class MessagingOngoingStaleJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
workspaceId,
},
});
for (const messageChannel of messageChannels) {
if (isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt))) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
workspaceId,
);
},
});
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
for (const messageChannel of messageChannels) {
if (
isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt))
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
);
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
}
}
}
}
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -36,32 +36,36 @@ export class MessagingRelaunchFailedMessageChannelJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
});
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await this.messageChannelRepository.update(
{ id: messageChannelId, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}, authContext);
await this.messageChannelRepository.update(
{ id: messageChannelId, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -269,14 +269,9 @@ describe('MessagingMessagesImportService', () => {
connectedAccountRefreshTokensService.refreshAndSaveTokens,
).toHaveBeenCalledWith(mockConnectedAccount, workspaceId);
expect(emailAliasManagerService.refreshHandleAliases).toHaveBeenCalledWith(
{
...mockConnectedAccount,
accessToken: 'new-access-token',
refreshToken: 'new-refresh-token',
},
workspaceId,
);
expect(
emailAliasManagerService.refreshHandleAliases,
).not.toHaveBeenCalled();
expect(messagingGetMessagesService.getMessages).toHaveBeenCalledWith(
['message-id-1', 'message-id-2'],
{
@@ -26,37 +26,41 @@ export class MessagingCursorService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
if (!folderId) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await this.messageFolderRepository.update(
{ id: folderId, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
if (!folderId) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await this.messageFolderRepository.update(
{ id: folderId, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -37,77 +37,81 @@ export class MessagingDeleteFolderMessagesService {
let totalDeletedCount = 0;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageFolderAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
let hasMoreData = true;
while (hasMoreData) {
const folderAssociations =
await messageFolderAssociationRepository.find({
where: {
messageFolderId: messageFolder.id,
},
take: BATCH_SIZE,
});
if (folderAssociations.length === 0) {
hasMoreData = false;
continue;
}
const folderAssociationIds = folderAssociations.map(
(folderAssociation) => folderAssociation.id,
);
const messageChannelMessageAssociationIds = folderAssociations.map(
(folderAssociation) =>
folderAssociation.messageChannelMessageAssociationId,
);
const associations =
await messageChannelMessageAssociationRepository.find({
where: {
id: In(messageChannelMessageAssociationIds),
messageChannelId: messageChannel.id,
},
});
const messageExternalIds = associations
.map((association) => association.messageExternalId)
.filter(isDefined);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`,
);
if (messageExternalIds.length > 0) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds,
messageChannelId: messageChannel.id,
},
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageFolderAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
totalDeletedCount += messageExternalIds.length;
}
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
await messageFolderAssociationRepository.delete({
id: In(folderAssociationIds),
});
}
}, authContext);
let hasMoreData = true;
while (hasMoreData) {
const folderAssociations =
await messageFolderAssociationRepository.find({
where: {
messageFolderId: messageFolder.id,
},
take: BATCH_SIZE,
});
if (folderAssociations.length === 0) {
hasMoreData = false;
continue;
}
const folderAssociationIds = folderAssociations.map(
(folderAssociation) => folderAssociation.id,
);
const messageChannelMessageAssociationIds = folderAssociations.map(
(folderAssociation) =>
folderAssociation.messageChannelMessageAssociationId,
);
const associations =
await messageChannelMessageAssociationRepository.find({
where: {
id: In(messageChannelMessageAssociationIds),
messageChannelId: messageChannel.id,
},
});
const messageExternalIds = associations
.map((association) => association.messageExternalId)
.filter(isDefined);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`,
);
if (messageExternalIds.length > 0) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds,
messageChannelId: messageChannel.id,
},
);
totalDeletedCount += messageExternalIds.length;
}
await messageFolderAssociationRepository.delete({
id: In(folderAssociationIds),
});
}
},
authContext,
{ lite: true },
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Completed deleting ${totalDeletedCount} messages from folder: ${messageFolder.name}`,
@@ -146,6 +146,7 @@ export class MessagingDeleteGroupEmailMessagesService {
return totalDeletedCount;
},
authContext,
{ lite: true },
);
}
}
@@ -29,57 +29,61 @@ export class MessagingMessageFolderAssociationService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const repository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const repository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
const records = associations.flatMap((association) =>
association.messageFolderIds.map((folderId) => ({
messageChannelMessageAssociationId:
association.messageChannelMessageAssociationId,
messageFolderId: folderId,
})),
);
const records = associations.flatMap((association) =>
association.messageFolderIds.map((folderId) => ({
messageChannelMessageAssociationId:
association.messageChannelMessageAssociationId,
messageFolderId: folderId,
})),
);
if (records.length === 0) {
return;
}
if (records.length === 0) {
return;
}
const associationIds = [
...new Set(
records.map((record) => record.messageChannelMessageAssociationId),
),
];
const existingRecords = await repository.find(
{
where: {
messageChannelMessageAssociationId: In(associationIds),
},
},
transactionManager,
);
const existingKeys = new Set(
existingRecords.map(
(record) =>
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
const recordsToInsert = records.filter(
(record) =>
!existingKeys.has(
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
const associationIds = [
...new Set(
records.map((record) => record.messageChannelMessageAssociationId),
),
);
];
if (recordsToInsert.length > 0) {
await repository.insert(recordsToInsert, transactionManager);
}
}, authContext);
const existingRecords = await repository.find(
{
where: {
messageChannelMessageAssociationId: In(associationIds),
},
},
transactionManager,
);
const existingKeys = new Set(
existingRecords.map(
(record) =>
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
const recordsToInsert = records.filter(
(record) =>
!existingKeys.has(
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
if (recordsToInsert.length > 0) {
await repository.insert(recordsToInsert, transactionManager);
}
},
authContext,
{ lite: true },
);
}
}
@@ -61,241 +61,246 @@ export class MessagingMessageListFetchService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
);
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`,
);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await this.messageChannelRepository.findOne({
where: {
id: messageChannel.id,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
})
: messageChannel;
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
messageChannelId: freshMessageChannel.id,
},
);
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
});
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) => !isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`,
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await this.messageChannelRepository.findOne({
where: {
id: messageChannel.id,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
})
: messageChannel;
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
workspaceId,
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
workspaceId,
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) =>
!isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageExternalIdsToImport.length) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
if (messageExternalIdsToImport.length) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter(
(messageExternalId) => isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
}
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`,
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter((messageExternalId) =>
isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
return;
}
}
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`,
);
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
return;
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
private async processPendingGroupEmailActions(
@@ -311,6 +311,7 @@ export class MessagingMessageService {
};
},
authContext,
{ lite: true },
);
}
@@ -74,58 +74,200 @@ export class MessagingMessagesImportService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
const refreshedHandleAliases =
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
connectedAccountWithFreshTokens.handleAliases = refreshedHandleAliases;
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messagesGetBatchSize,
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
connectedAccountWithFreshTokens.handleAliases =
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
workspaceId,
);
}
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messagesGetBatchSize,
);
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
const allMessages =
await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
messageChannel,
);
// Map external folder IDs to internal folder IDs
const messageFolders = messageChannel.messageFolders ?? [];
const foldersWithExternalId = messageFolders.filter(
(folder): folder is typeof folder & { externalId: string } =>
isDefined(folder.externalId),
);
const folderExternalToInternalMap = new Map<string, string>(
foldersWithExternalId.map((folder) => [
folder.externalId,
folder.id,
]),
);
for (const message of allMessages) {
const externalFolderIds = message.messageFolderExternalIds ?? [];
message.messageFolderIds = externalFolderIds
.map((externalId) => folderExternalToInternalMap.get(externalId))
.filter(isDefined);
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: connectedAccountWithFreshTokens.userWorkspaceId,
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
select: ['id', 'isInternalMessagesImportEnabled'],
});
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
workspace?.isInternalMessagesImportEnabled ?? false,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (messageIdsToFetch.length < messagesGetBatchSize) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
@@ -134,144 +276,10 @@ export class MessagingMessagesImportService {
workspaceId,
);
}
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
messageChannel,
);
// Map external folder IDs to internal folder IDs
const messageFolders = messageChannel.messageFolders ?? [];
const foldersWithExternalId = messageFolders.filter(
(folder): folder is typeof folder & { externalId: string } =>
isDefined(folder.externalId),
);
const folderExternalToInternalMap = new Map<string, string>(
foldersWithExternalId.map((folder) => [folder.externalId, folder.id]),
);
for (const message of allMessages) {
const externalFolderIds = message.messageFolderExternalIds ?? [];
message.messageFolderIds = externalFolderIds
.map((externalId) => folderExternalToInternalMap.get(externalId))
.filter(isDefined);
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: connectedAccountWithFreshTokens.userWorkspaceId,
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
select: ['id', 'isInternalMessagesImportEnabled'],
});
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
workspace?.isInternalMessagesImportEnabled ?? false,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (messageIdsToFetch.length < messagesGetBatchSize) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
private async trackMessageImportCompleted(
@@ -114,6 +114,7 @@ export class MessagingProcessFolderActionsService {
}
},
authContext,
{ lite: true },
);
}
}
@@ -79,6 +79,7 @@ export class MessagingProcessGroupEmailActionsService {
}
},
authContext,
{ lite: true },
);
await this.messageChannelRepository.update(
@@ -157,6 +157,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
);
},
authContext,
{ lite: true },
);
if (
@@ -23,57 +23,61 @@ export class MessagingMessageParticipantService {
): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
});
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -74,6 +74,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
}
},
authContext,
{ lite: true },
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {