Add WorkspaceAuthContextMiddleware (#17487)

## Context
Introduces a middleware that automatically sets the workspace auth
context in AsyncLocalStorage for HTTP requests, making it available
throughout the request lifecycle without explicit parameter passing.

The motivation behind this change is to reduce boilerplate and simplify
the developer experience when working with workspace data in HTTP
request handlers.

The Problem (Before)
Every HTTP request handler that needed to access workspace data had to:
- Extract auth-related info from decorators (@AuthWorkspace(),
@AuthUserWorkspaceId(), etc.) in controller/resolver and pass down to
services
- Build or pass the authContext explicitly (sometimes with type
assertion which was flaky)
Then call executeInWorkspaceContext(authContext, async () => { ... })

## Changes
- Add WorkspaceAuthContextMiddleware that extracts auth context from the
request and stores it in AsyncLocalStorage
- Register middleware for GraphQL, metadata, and REST routes (runs after
hydration middlewares)
- Simplify executeInWorkspaceContext signature: fn is now the first
parameter, authContext is optional second
- If authContext is not provided, it's automatically retrieved from the
storage (set by middleware)
- Update all callers (~120 files) to use the new parameter order


- Fixes a bug in search where system auth context was used, bypassing
RLS feature.
This commit is contained in:
Weiko
2026-01-27 18:24:51 +01:00
committed by GitHub
parent dd98146c99
commit 2daebc6d0f
151 changed files with 3743 additions and 4002 deletions
@@ -36,132 +36,129 @@ export class BlocklistItemDeleteMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const messageChannels = await messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (messageChannel.connectedAccount.handleAliases) {
messageChannelHandles.push(
...messageChannel.connectedAccount.handleAliases.split(','),
);
}
if (!isDefined(handle)) {
return acc;
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
acc.get(workspaceMemberId)?.push(handle);
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
return acc;
},
new Map<string, string[]>(),
);
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const messageChannels = await messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
relations: ['connectedAccount'],
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
if (messageChannel.connectedAccount.handleAliases) {
messageChannelHandles.push(
...messageChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
},
);
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}, authContext);
}
}
@@ -36,37 +36,32 @@ export class BlocklistReimportMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const messageChannels = await messageChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
),
const messageChannels = await messageChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
});
syncStage: Not(MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING),
},
});
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
);
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
}
}
@@ -56,7 +56,7 @@ describe('ApplyMessagesVisibilityRestrictionsService', () => {
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
.mockImplementation((fn: () => any, _authContext?: any) => fn()),
};
beforeEach(async () => {
@@ -28,7 +28,6 @@ export class ApplyMessagesVisibilityRestrictionsService {
const authContext = buildSystemAuthContext(workspaceId);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
@@ -128,6 +127,7 @@ export class ApplyMessagesVisibilityRestrictionsService {
return messages;
},
authContext,
);
}
}
@@ -56,7 +56,6 @@ export class MessageChannelUpdateOnePreQueryHook
const systemAuthContext = buildSystemAuthContext(workspace.id);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
systemAuthContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
@@ -144,6 +143,7 @@ export class MessageChannelUpdateOnePreQueryHook
return payload;
},
systemAuthContext,
);
}
}
@@ -44,21 +44,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
}, authContext);
}
public async markAsMessagesImportPending(
@@ -72,21 +69,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
});
}, authContext);
}
public async resetAndMarkAsMessagesListFetchPending(
@@ -105,37 +99,34 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
await messageChannelRepository.update(messageChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
});
await messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds) },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
},
);
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
await messageChannelRepository.update(messageChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
});
await messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds) },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
}, authContext);
await this.markAsMessagesListFetchPending(messageChannelIds, workspaceId);
}
@@ -150,20 +141,17 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStageStartedAt: null,
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStageStartedAt: null,
});
}, authContext);
}
public async markAsMessagesListFetchScheduled(
@@ -176,22 +164,19 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
}, authContext);
}
public async markAsMessagesListFetchOngoing(
@@ -204,21 +189,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
});
}, authContext);
}
public async markAsCompletedAndMarkAsMessagesListFetchPending(
@@ -231,24 +213,21 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
}, authContext);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.MessageChannelSyncJobActive,
@@ -266,20 +245,17 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
});
}, authContext);
}
public async markAsMessagesImportOngoing(
@@ -292,21 +268,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
},
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
}, authContext);
}
public async markAsFailed(
@@ -322,64 +295,59 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
});
const metricsKey =
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
if (
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'messageChannel',
'connectedAccount',
);
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
const messageChannels = await messageChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(messageChannelIds) },
});
const metricsKey =
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
if (
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const connectedAccountRepository =
await this.globalWorkspaceOrmManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const messageChannels = await messageChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(messageChannelIds) },
});
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await connectedAccountRepository.update(
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
}
private async addToAccountsToReconnect(
@@ -38,52 +38,49 @@ export class MessagingResetChannelCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
},
});
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
if (messageChannels.length === 0) {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
);
return;
}
const messageChannels = await messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
},
});
if (messageChannels.length === 0) {
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
`No message channels found in workspace ${workspaceId}`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
return;
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
},
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
}, authContext);
}
@Option({
@@ -29,197 +29,191 @@ export class MessagingMessageCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
const orphanMessages = await messageRepository.find({
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
if (orphanMessages.length <= 0) {
continue;
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
messages: {
id: IsNull(),
},
},
});
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
},
);
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
const orphanMessages = await messageRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
},
});
if (orphanMessages.length <= 0) {
continue;
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
messages: {
id: IsNull(),
},
},
});
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
}, authContext);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
},
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
}, authContext);
}
}
@@ -126,7 +126,9 @@ describe('SyncMessageFoldersService', () => {
useValue: {
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_, callback) => callback()),
.mockImplementation((callback: () => any, _authContext?: any) =>
callback(),
),
getRepository: jest.fn().mockResolvedValue(mockRepository),
getDataSourceForWorkspace: jest
.fn()
@@ -133,7 +133,6 @@ export class SyncMessageFoldersService {
const authContext = buildSystemAuthContext(workspaceId);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageFolderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
@@ -188,6 +187,7 @@ export class SyncMessageFoldersService {
},
);
},
authContext,
);
}
}
@@ -51,63 +51,60 @@ export class MessagingTriggerMessageListFetchCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const whereCondition: Record<string, unknown> = {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
};
if (messageChannelId) {
whereCondition.id = messageChannelId;
}
const messageChannels =
await messageChannelRepository.find(whereCondition);
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channel(s) to process`,
);
for (const messageChannel of messageChannels) {
await messageChannelRepository.update(messageChannel.id, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
});
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
workspaceId,
'messageChannel',
);
const whereCondition: Record<string, unknown> = {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
};
if (messageChannelId) {
whereCondition.id = messageChannelId;
}
const messageChannels =
await messageChannelRepository.find(whereCondition);
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channel(s) to process`,
},
);
for (const messageChannel of messageChannels) {
await messageChannelRepository.update(messageChannel.id, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
});
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
workspaceId,
},
);
this.logger.log(
`Triggered fetch for message channel ${messageChannel.id}`,
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
`Triggered fetch for message channel ${messageChannel.id}`,
);
},
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
);
}, authContext);
}
@Option({
@@ -48,82 +48,79 @@ export class MessagingMessageListFetchJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount', 'messageFolders'],
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount', 'messageFolders'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
try {
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
true,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
return;
}
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
},
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
}
}
@@ -43,64 +43,61 @@ export class MessagingMessagesImportJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount'],
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: ['connectedAccount'],
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
return;
}
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
true,
);
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
if (
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
true,
);
},
);
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
}, authContext);
}
}
@@ -35,63 +35,60 @@ export class MessagingOngoingStaleJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
},
});
for (const messageChannel of messageChannels) {
if (
messageChannel.syncStageStartedAt &&
isSyncStale(messageChannel.syncStageStartedAt)
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
'messageChannel',
);
const messageChannels = await messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
},
});
for (const messageChannel of messageChannels) {
if (
messageChannel.syncStageStartedAt &&
isSyncStale(messageChannel.syncStageStartedAt)
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
);
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
}
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
}
}
},
);
}
}, authContext);
}
}
@@ -31,40 +31,37 @@ export class MessagingRelaunchFailedMessageChannelJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
{ shouldBypassPermissionChecks: true },
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
{ shouldBypassPermissionChecks: true },
);
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
const messageChannel = await messageChannelRepository.findOne({
where: {
id: messageChannelId,
},
relations: {
connectedAccount: {
accountOwner: true,
},
relations: {
connectedAccount: {
accountOwner: true,
},
},
});
},
});
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await messageChannelRepository.update(messageChannelId, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
});
},
);
await messageChannelRepository.update(messageChannelId, {
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
});
}, authContext);
}
}
@@ -213,7 +213,7 @@ describe('MessagingMessageListFetchService', () => {
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
.mockImplementation((fn: () => any, _authContext?: any) => fn()),
getRepository: jest.fn().mockImplementation((workspaceId, name) => {
if (name === 'messageChannel') {
return {
@@ -113,7 +113,7 @@ describe('MessagingMessagesImportService', () => {
}),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
.mockImplementation((fn: () => any, _authContext?: any) => fn()),
},
},
{
@@ -19,55 +19,52 @@ export class MessagingCursorService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const folderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const folderRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
if (!folderId) {
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await folderRepository.update(
{
id: folderId,
},
{
syncCursor: nextSyncCursor,
},
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
}
},
);
if (!folderId) {
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await folderRepository.update(
{
id: folderId,
},
{
syncCursor: nextSyncCursor,
},
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
}
}, authContext);
}
}
@@ -41,7 +41,6 @@ export class MessagingDeleteGroupEmailMessagesService {
const authContext = buildSystemAuthContext(workspaceId);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
@@ -146,6 +145,7 @@ export class MessagingDeleteGroupEmailMessagesService {
return totalDeletedCount;
},
authContext,
);
}
}
@@ -150,24 +150,21 @@ export class MessageImportExceptionHandlerService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.increment(
{ id: messageChannel.id },
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
},
);
await messageChannelRepository.increment(
{ id: messageChannel.id },
'throttleFailureCount',
1,
undefined,
['throttleFailureCount', 'id'],
);
}, authContext);
switch (syncStep) {
case MessageImportSyncStep.MESSAGE_LIST_FETCH:
@@ -58,254 +58,250 @@ export class MessagingMessageListFetchService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
);
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
);
this.logger.log(
`messageChannelId: ${messageChannel.id} Processing message list fetch`,
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
);
this.logger.log(
`messageChannelId: ${messageChannel.id} Processing message list fetch`,
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await messageChannelRepository.findOne({
where: {
id: messageChannel.id,
},
relations: ['connectedAccount', 'messageFolders'],
})
: messageChannel;
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`error processing message list fetch: messageChannelId: ${messageChannel.id} Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
workspaceId,
'messageChannel',
);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await messageChannelRepository.findOne({
where: {
id: messageChannel.id,
},
relations: ['connectedAccount', 'messageFolders'],
})
: messageChannel;
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`error processing message list fetch: messageChannelId: ${messageChannel.id} Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
workspaceId,
messageChannelId: freshMessageChannel.id,
},
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
messageChannelId: freshMessageChannel.id,
},
};
);
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
workspaceId,
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
workspaceId,
});
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) => !isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
(messageList) => {
messageList.nextSyncCursor;
},
)}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
});
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) =>
!isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
(messageList) => {
messageList.nextSyncCursor;
},
)}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageExternalIdsToImport.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
);
if (allMessageExternalIdsToDelete.length) {
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageExternalIdsToImport.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
totalMessagesToImportCount += messageExternalIdsToImport.length;
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter(
(messageExternalId) => isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
}
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
);
)
: [];
return;
}
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
`messageChannelId: ${freshMessageChannel.id} Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter((messageExternalId) =>
isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
}
}
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
);
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
return;
}
},
);
this.logger.log(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
}
private async processPendingGroupEmailActions(
@@ -55,7 +55,6 @@ export class MessagingMessageService {
const authContext = buildSystemAuthContext(workspaceId);
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
@@ -245,6 +244,7 @@ export class MessagingMessageService {
messageExternalIdsAndIdsMap,
};
},
authContext,
);
}
@@ -60,169 +60,164 @@ export class MessagingMessagesImportService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
workspaceId,
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
);
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
workspaceId,
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
);
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
const allMessages =
await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
);
const blocklist =
await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccountWithFreshTokens.accountOwnerId,
workspaceId,
);
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases.split(',')],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (
messageIdsToFetch.length <
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
},
);
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
);
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
connectedAccountWithFreshTokens.accountOwnerId,
workspaceId,
);
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases.split(',')],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (
messageIdsToFetch.length <
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(
{
id: messageChannel.id,
},
{
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
}, authContext);
}
private async trackMessageImportCompleted(
@@ -90,7 +90,6 @@ export class MessagingProcessFolderActionsService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
@@ -128,6 +127,7 @@ export class MessagingProcessFolderActionsService {
},
);
},
authContext,
);
}
}
@@ -30,25 +30,22 @@ export class MessagingProcessGroupEmailActionsService {
): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
await messageChannelRepository.update(
{ id: messageChannel.id },
{ pendingGroupEmailsAction },
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Marked message channel as pending group emails action: ${pendingGroupEmailsAction}`,
);
},
);
await messageChannelRepository.update(
{ id: messageChannel.id },
{ pendingGroupEmailsAction },
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Marked message channel as pending group emails action: ${pendingGroupEmailsAction}`,
);
}, authContext);
}
async processGroupEmailActions(
@@ -70,61 +67,58 @@ export class MessagingProcessGroupEmailActionsService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
try {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
await workspaceDataSource?.transaction(
async (transactionManager: WorkspaceEntityManager) => {
try {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
switch (pendingGroupEmailsAction) {
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION:
await this.handleGroupEmailsDeletion(
workspaceId,
'messageChannel',
messageChannel.id,
transactionManager,
);
switch (pendingGroupEmailsAction) {
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_DELETION:
await this.handleGroupEmailsDeletion(
workspaceId,
messageChannel.id,
transactionManager,
);
break;
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT:
await this.handleGroupEmailsImport(
workspaceId,
messageChannel.id,
transactionManager,
);
break;
}
await messageChannelRepository.update(
{ id: messageChannel.id },
{
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
transactionManager,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingGroupEmailsAction to NONE`,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error processing group email action: ${error.message}`,
error.stack,
);
throw error;
break;
case MessageChannelPendingGroupEmailsAction.GROUP_EMAILS_IMPORT:
await this.handleGroupEmailsImport(
workspaceId,
messageChannel.id,
transactionManager,
);
break;
}
},
);
},
);
await messageChannelRepository.update(
{ id: messageChannel.id },
{
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
transactionManager,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingGroupEmailsAction to NONE`,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error processing group email action: ${error.message}`,
error.stack,
);
throw error;
}
},
);
}, authContext);
}
private async handleGroupEmailsDeletion(
@@ -159,7 +159,7 @@ describe('MessagingSaveMessagesAndEnqueueContactCreationService', () => {
.mockResolvedValue(datasourceInstance),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
.mockImplementation((fn: () => any, _authContext?: any) => fn()),
},
},
],
@@ -47,7 +47,6 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
const participantsWithMessageId =
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
@@ -117,6 +116,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
},
);
},
authContext,
);
if (
@@ -23,60 +23,57 @@ export class MessagingMessageParticipantService {
): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
});
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
});
},
);
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
}, authContext);
}
}
@@ -50,7 +50,6 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
const authContext = buildSystemAuthContext(activeWorkspace.id);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const messageChannelRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelWorkspaceEntity>(
@@ -76,6 +75,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
});
}
},
authContext,
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {