fix(calendar): ID-first chunked import for Google and CalDAV (#22015)
Google and CalDAV returned full events and imported them inline in the list-fetch job. Large/initial syncs overran BullMQ's lock, the job stalled, the workspace query runner was released mid-import, and TypeORM threw 'Query runner already released'. Mirror the messaging pipeline: every provider now returns event IDs only, cached in Redis; the import job drains them in CALENDAR_EVENT_IMPORT_BATCH_SIZE chunks and re-enqueues until empty, so no single job runs long. Adds Google/CalDAV import-by-id services and a provider dispatcher; removes the full-events inline path. <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/22015?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->
This commit is contained in:
+2
@@ -34,6 +34,7 @@ import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/cal
|
||||
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
|
||||
import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service';
|
||||
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
|
||||
import { CalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service';
|
||||
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
|
||||
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
|
||||
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
|
||||
@@ -73,6 +74,7 @@ import { RefreshTokensManagerModule } from 'src/modules/connected-account/refres
|
||||
CalendarFetchEventsService,
|
||||
CalendarEventImportErrorHandlerService,
|
||||
CalendarGetCalendarEventsService,
|
||||
CalendarImportEventsService,
|
||||
CalendarSaveEventsService,
|
||||
CalendarEventListFetchCronJob,
|
||||
CalendarEventListFetchCronCommand,
|
||||
|
||||
+3
@@ -9,6 +9,7 @@ import { CalDavClientProvider } from 'src/modules/calendar/calendar-event-import
|
||||
import { CalDavClientService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-client.service';
|
||||
import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service';
|
||||
import { CalDavGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service';
|
||||
import { CalDavImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -22,12 +23,14 @@ import { CalDavGetEventsService } from 'src/modules/calendar/calendar-event-impo
|
||||
CalDavClientService,
|
||||
CalDavFetchEventsService,
|
||||
CalDavGetEventsService,
|
||||
CalDavImportEventsService,
|
||||
],
|
||||
exports: [
|
||||
CalDavClientProvider,
|
||||
CalDavClientService,
|
||||
CalDavFetchEventsService,
|
||||
CalDavGetEventsService,
|
||||
CalDavImportEventsService,
|
||||
],
|
||||
})
|
||||
export class CalDavDriverModule {}
|
||||
|
||||
+76
-179
@@ -2,30 +2,26 @@ import { type DAVClient } from 'tsdav';
|
||||
|
||||
import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service';
|
||||
|
||||
const PRIMARY_URL = 'https://caldav.example.com/calendars/user/primary/';
|
||||
const PERSONAL_URL = 'https://caldav.example.com/calendars/user/personal/';
|
||||
const SERVER_URL = 'https://caldav.example.com';
|
||||
const PRIMARY_URL = `${SERVER_URL}/calendars/user/primary/`;
|
||||
const PERSONAL_URL = `${SERVER_URL}/calendars/user/personal/`;
|
||||
const HREF_A = `${PRIMARY_URL}event-a.ics`;
|
||||
const HREF_B = `${PRIMARY_URL}event-b.ics`;
|
||||
|
||||
const buildICal = (uid: string) =>
|
||||
const buildICal = (uid: string, dtStart = '20260601T100000Z') =>
|
||||
[
|
||||
'BEGIN:VCALENDAR',
|
||||
'VERSION:2.0',
|
||||
'BEGIN:VEVENT',
|
||||
`UID:${uid}`,
|
||||
`SUMMARY:${uid}`,
|
||||
'DTSTART:20260601T100000Z',
|
||||
`DTSTART:${dtStart}`,
|
||||
'DTEND:20260601T110000Z',
|
||||
'STATUS:CONFIRMED',
|
||||
'END:VEVENT',
|
||||
'END:VCALENDAR',
|
||||
].join('\r\n');
|
||||
|
||||
const inWindow = {
|
||||
startDate: new Date('2026-01-01'),
|
||||
endDate: new Date('2027-01-01'),
|
||||
};
|
||||
|
||||
const buildClient = () => {
|
||||
const fetchCalendars = jest.fn();
|
||||
const syncCollection = jest.fn();
|
||||
@@ -34,6 +30,7 @@ const buildClient = () => {
|
||||
|
||||
return {
|
||||
client: {
|
||||
serverUrl: SERVER_URL,
|
||||
fetchCalendars,
|
||||
syncCollection,
|
||||
calendarMultiGet,
|
||||
@@ -53,8 +50,8 @@ describe('CalDavFetchEventsService', () => {
|
||||
service = new CalDavFetchEventsService();
|
||||
});
|
||||
|
||||
describe('per-calendar tier dispatch', () => {
|
||||
it('runs Tier-1 on calendars advertising sync-collection and Tier-2/3 on the rest, in parallel', async () => {
|
||||
describe('fetchChangedEventHrefs', () => {
|
||||
it('collects changed hrefs from sync-collection and ctag/etag calendars without fetching bodies', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
@@ -65,54 +62,44 @@ describe('CalDavFetchEventsService', () => {
|
||||
},
|
||||
{ url: PERSONAL_URL, components: ['VEVENT'], reports: [], ctag: 'c-1' },
|
||||
]);
|
||||
|
||||
c.syncCollection.mockResolvedValue([
|
||||
{ href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} },
|
||||
{ href: HREF_A, status: 207, ok: true, props: {} },
|
||||
]);
|
||||
c.propfind.mockResolvedValue([
|
||||
{ href: HREF_B, status: 207, ok: true, props: { getetag: '"etag-b"' } },
|
||||
]);
|
||||
|
||||
c.propfind.mockResolvedValue([
|
||||
const result = await service.fetchChangedEventHrefs(c.client);
|
||||
|
||||
expect(result.changedHrefs.sort()).toEqual([HREF_A, HREF_B].sort());
|
||||
expect(result.cancelledHrefs).toEqual([]);
|
||||
expect(c.calendarMultiGet).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('separates cancelled (404) hrefs from changed ones in a sync-collection delta', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
{
|
||||
href: HREF_B,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { getetag: '"etag-b"' },
|
||||
url: PRIMARY_URL,
|
||||
components: ['VEVENT'],
|
||||
reports: ['syncCollection'],
|
||||
},
|
||||
]);
|
||||
|
||||
c.calendarMultiGet
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
href: HREF_A,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { calendarData: buildICal('uid-a') },
|
||||
},
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
href: HREF_B,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { calendarData: buildICal('uid-b') },
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await service.fetchEvents(c.client, inWindow);
|
||||
|
||||
expect(result.events.map((event) => event.iCalUid).sort()).toEqual([
|
||||
'uid-a',
|
||||
'uid-b',
|
||||
c.syncCollection.mockResolvedValue([
|
||||
{ href: HREF_A, status: 207, ok: true, props: {} },
|
||||
{ href: HREF_B, status: 404, ok: false, props: {} },
|
||||
]);
|
||||
expect(c.syncCollection).toHaveBeenCalledTimes(1);
|
||||
expect(c.propfind).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tier-2/3 ctag short-circuit', () => {
|
||||
it('skips network entirely when the server CTag matches the stored CTag', async () => {
|
||||
const result = await service.fetchChangedEventHrefs(c.client, {
|
||||
syncTokens: { [PRIMARY_URL]: 'token-prior' },
|
||||
});
|
||||
|
||||
expect(result.changedHrefs).toEqual([HREF_A]);
|
||||
expect(result.cancelledHrefs).toEqual([HREF_B]);
|
||||
});
|
||||
|
||||
it('skips network when the server CTag matches the stored CTag', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
@@ -126,24 +113,18 @@ describe('CalDavFetchEventsService', () => {
|
||||
|
||||
const storedEtags = { [HREF_A]: '"etag-a"' };
|
||||
|
||||
const result = await service.fetchEvents(c.client, {
|
||||
...inWindow,
|
||||
syncCursor: {
|
||||
syncTokens: {},
|
||||
ctags: { [PRIMARY_URL]: 'unchanged' },
|
||||
etags: { [PRIMARY_URL]: storedEtags },
|
||||
},
|
||||
const result = await service.fetchChangedEventHrefs(c.client, {
|
||||
syncTokens: {},
|
||||
ctags: { [PRIMARY_URL]: 'unchanged' },
|
||||
etags: { [PRIMARY_URL]: storedEtags },
|
||||
});
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.changedHrefs).toEqual([]);
|
||||
expect(c.propfind).not.toHaveBeenCalled();
|
||||
expect(c.calendarMultiGet).not.toHaveBeenCalled();
|
||||
expect(result.syncCursor.etags).toEqual({ [PRIMARY_URL]: storedEtags });
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tier-2/3 etag diff', () => {
|
||||
it('fetches only changed hrefs and emits cancelled stubs for hrefs vanished from the server', async () => {
|
||||
it('separates changed from vanished hrefs in an etag diff', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
@@ -154,52 +135,28 @@ describe('CalDavFetchEventsService', () => {
|
||||
ctag: 'new-ctag',
|
||||
},
|
||||
]);
|
||||
|
||||
c.propfind.mockResolvedValue([
|
||||
{
|
||||
href: HREF_A,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { getetag: '"etag-a-updated"' },
|
||||
},
|
||||
]);
|
||||
|
||||
c.calendarMultiGet.mockResolvedValue([
|
||||
{
|
||||
href: HREF_A,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { calendarData: buildICal('uid-a') },
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await service.fetchEvents(c.client, {
|
||||
...inWindow,
|
||||
syncCursor: {
|
||||
syncTokens: {},
|
||||
ctags: { [PRIMARY_URL]: 'old-ctag' },
|
||||
etags: {
|
||||
[PRIMARY_URL]: { [HREF_A]: '"etag-a"', [HREF_B]: '"etag-b"' },
|
||||
},
|
||||
const result = await service.fetchChangedEventHrefs(c.client, {
|
||||
syncTokens: {},
|
||||
ctags: { [PRIMARY_URL]: 'old-ctag' },
|
||||
etags: {
|
||||
[PRIMARY_URL]: { [HREF_A]: '"etag-a"', [HREF_B]: '"etag-b"' },
|
||||
},
|
||||
});
|
||||
|
||||
expect(c.calendarMultiGet).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ objectUrls: [HREF_A] }),
|
||||
);
|
||||
|
||||
const live = result.events.filter((event) => !event.isCanceled);
|
||||
const cancelled = result.events.filter((event) => event.isCanceled);
|
||||
|
||||
expect(live.map((event) => event.iCalUid)).toEqual(['uid-a']);
|
||||
expect(cancelled.map((event) => event.id)).toEqual([HREF_B]);
|
||||
expect(result.changedHrefs).toEqual([HREF_A]);
|
||||
expect(result.cancelledHrefs).toEqual([HREF_B]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('per-calendar error isolation', () => {
|
||||
it('preserves the prior cursor entry (token + ctag + etags) for the failing calendar without aborting siblings', async () => {
|
||||
it('preserves the prior cursor entry for a calendar whose sync fails', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
@@ -208,37 +165,19 @@ describe('CalDavFetchEventsService', () => {
|
||||
components: ['VEVENT'],
|
||||
reports: ['syncCollection'],
|
||||
},
|
||||
{
|
||||
url: PERSONAL_URL,
|
||||
components: ['VEVENT'],
|
||||
reports: [],
|
||||
ctag: 'c-new',
|
||||
},
|
||||
]);
|
||||
|
||||
c.syncCollection.mockRejectedValue(new Error('network blip'));
|
||||
c.propfind.mockRejectedValue(new Error('propfind blip'));
|
||||
|
||||
const priorEtags = { [HREF_A]: '"etag-a"' };
|
||||
|
||||
const result = await service.fetchEvents(c.client, {
|
||||
...inWindow,
|
||||
syncCursor: {
|
||||
syncTokens: { [PRIMARY_URL]: 'token-prior' },
|
||||
ctags: { [PERSONAL_URL]: 'c-prior' },
|
||||
etags: { [PERSONAL_URL]: priorEtags },
|
||||
},
|
||||
const result = await service.fetchChangedEventHrefs(c.client, {
|
||||
syncTokens: { [PRIMARY_URL]: 'token-prior' },
|
||||
});
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(result.changedHrefs).toEqual([]);
|
||||
expect(result.cancelledHrefs).toEqual([]);
|
||||
expect(result.syncCursor.syncTokens[PRIMARY_URL]).toBe('token-prior');
|
||||
expect(result.syncCursor.ctags?.[PERSONAL_URL]).toBe('c-prior');
|
||||
expect(result.syncCursor.etags?.[PERSONAL_URL]).toEqual(priorEtags);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cursor shape', () => {
|
||||
it('omits ctags and etags entirely when only sync-collection calendars exist', async () => {
|
||||
it('omits the sync-token on the first run so the server returns a full listing', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
@@ -249,49 +188,10 @@ describe('CalDavFetchEventsService', () => {
|
||||
},
|
||||
]);
|
||||
c.syncCollection.mockResolvedValue([
|
||||
{
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
raw: { multistatus: { syncToken: 'token-fresh' } },
|
||||
},
|
||||
]);
|
||||
c.calendarMultiGet.mockResolvedValue([]);
|
||||
|
||||
const result = await service.fetchEvents(c.client, inWindow);
|
||||
|
||||
expect(result.syncCursor).toEqual({
|
||||
syncTokens: { [PRIMARY_URL]: 'token-fresh' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('initial sync (no stored cursor)', () => {
|
||||
it('omits the sync-token on the first run so the server returns a full listing (RFC 6578 §3.4)', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
{
|
||||
url: PRIMARY_URL,
|
||||
components: ['VEVENT'],
|
||||
reports: ['syncCollection'],
|
||||
syncToken: 'server-current-token',
|
||||
},
|
||||
]);
|
||||
c.syncCollection.mockResolvedValue([
|
||||
{ href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} },
|
||||
]);
|
||||
c.calendarMultiGet.mockResolvedValue([
|
||||
{
|
||||
href: HREF_A,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { calendarData: buildICal('uid-a') },
|
||||
},
|
||||
{ href: HREF_A, status: 207, ok: true, props: {} },
|
||||
]);
|
||||
|
||||
await service.fetchEvents(c.client, inWindow);
|
||||
await service.fetchChangedEventHrefs(c.client);
|
||||
|
||||
expect(c.syncCollection).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({ syncToken: expect.anything() }),
|
||||
@@ -299,38 +199,35 @@ describe('CalDavFetchEventsService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('time-window filtering', () => {
|
||||
it('drops events that fall outside the requested [startDate, endDate] window', async () => {
|
||||
describe('fetchEventsByHrefs', () => {
|
||||
it('fetches bodies for the given hrefs grouped by calendar collection', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.fetchCalendars.mockResolvedValue([
|
||||
{
|
||||
url: PRIMARY_URL,
|
||||
components: ['VEVENT'],
|
||||
reports: ['syncCollection'],
|
||||
},
|
||||
c.calendarMultiGet.mockResolvedValue([
|
||||
{ href: HREF_A, props: { calendarData: buildICal('uid-a') } },
|
||||
]);
|
||||
|
||||
c.syncCollection.mockResolvedValue([
|
||||
{ href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} },
|
||||
]);
|
||||
const events = await service.fetchEventsByHrefs(c.client, [HREF_A]);
|
||||
|
||||
expect(c.calendarMultiGet).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ url: PRIMARY_URL, objectUrls: [HREF_A] }),
|
||||
);
|
||||
expect(events.map((event) => event.iCalUid)).toEqual(['uid-a']);
|
||||
});
|
||||
|
||||
it('drops events that fall outside the import time window', async () => {
|
||||
const c = buildClient();
|
||||
|
||||
c.calendarMultiGet.mockResolvedValue([
|
||||
{
|
||||
href: HREF_A,
|
||||
status: 207,
|
||||
statusText: 'OK',
|
||||
ok: true,
|
||||
props: { calendarData: buildICal('uid-a') },
|
||||
props: { calendarData: buildICal('uid-a', '20990101T100000Z') },
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await service.fetchEvents(c.client, {
|
||||
startDate: new Date('2030-01-01'),
|
||||
endDate: new Date('2030-12-31'),
|
||||
});
|
||||
const events = await service.fetchEventsByHrefs(c.client, [HREF_A]);
|
||||
|
||||
expect(result.events).toEqual([]);
|
||||
expect(events).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+95
-80
@@ -10,7 +10,6 @@ import {
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { type CalDavSyncCursor } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/types/caldav-sync-cursor';
|
||||
import { buildCancelledCalDavEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/build-cancelled-event.util';
|
||||
import { extractICalData } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/extract-ical-data.util';
|
||||
import { isEventInTimeRange } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-event-in-time-range.util';
|
||||
import { isInvalidSyncTokenResponse } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-invalid-sync-token-response.util';
|
||||
@@ -20,22 +19,20 @@ import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fet
|
||||
|
||||
type CalendarSyncResult = {
|
||||
calendarUrl: string;
|
||||
events: FetchedCalendarEvent[];
|
||||
changedHrefs: string[];
|
||||
cancelledHrefs: string[];
|
||||
newSyncToken?: string;
|
||||
newCtag?: string;
|
||||
newEtags?: Record<string, string>;
|
||||
};
|
||||
|
||||
type FetchEventsOptions = {
|
||||
startDate: Date;
|
||||
endDate: Date;
|
||||
syncCursor?: CalDavSyncCursor;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CalDavFetchEventsService {
|
||||
private readonly logger = new Logger(CalDavFetchEventsService.name);
|
||||
|
||||
private static readonly PAST_DAYS_WINDOW = 365 * 5;
|
||||
private static readonly FUTURE_DAYS_WINDOW = 365;
|
||||
|
||||
async listEventCalendars(client: DAVClient): Promise<DAVCalendar[]> {
|
||||
const calendars = await client.fetchCalendars();
|
||||
|
||||
@@ -44,53 +41,118 @@ export class CalDavFetchEventsService {
|
||||
);
|
||||
}
|
||||
|
||||
async fetchEvents(
|
||||
async fetchChangedEventHrefs(
|
||||
client: DAVClient,
|
||||
options: FetchEventsOptions,
|
||||
): Promise<{ events: FetchedCalendarEvent[]; syncCursor: CalDavSyncCursor }> {
|
||||
syncCursor?: CalDavSyncCursor,
|
||||
): Promise<{
|
||||
changedHrefs: string[];
|
||||
cancelledHrefs: string[];
|
||||
syncCursor: CalDavSyncCursor;
|
||||
}> {
|
||||
const calendars = await this.listEventCalendars(client);
|
||||
|
||||
const results = await Promise.all(
|
||||
calendars.map((calendar) => this.syncCalendar(client, calendar, options)),
|
||||
calendars.map((calendar) =>
|
||||
this.syncCalendar(client, calendar, syncCursor),
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
events: results.flatMap((result) => result.events),
|
||||
changedHrefs: results.flatMap((result) => result.changedHrefs),
|
||||
cancelledHrefs: results.flatMap((result) => result.cancelledHrefs),
|
||||
syncCursor: this.mergeSyncCursor(results),
|
||||
};
|
||||
}
|
||||
|
||||
async fetchEventsByHrefs(
|
||||
client: DAVClient,
|
||||
eventHrefs: string[],
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
if (eventHrefs.length === 0) return [];
|
||||
|
||||
const startDate = new Date(
|
||||
Date.now() -
|
||||
CalDavFetchEventsService.PAST_DAYS_WINDOW * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
const endDate = new Date(
|
||||
Date.now() +
|
||||
CalDavFetchEventsService.FUTURE_DAYS_WINDOW * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
|
||||
const collectionUrls = [
|
||||
...new Set(
|
||||
eventHrefs.map((href) => this.resolveCollectionUrl(client, href)),
|
||||
),
|
||||
];
|
||||
|
||||
const calendarObjects = (
|
||||
await Promise.all(
|
||||
collectionUrls.map((collectionUrl) =>
|
||||
client.calendarMultiGet({
|
||||
url: collectionUrl,
|
||||
props: {
|
||||
[`${DAVNamespaceShort.DAV}:getetag`]: {},
|
||||
[`${DAVNamespaceShort.CALDAV}:calendar-data`]: {},
|
||||
},
|
||||
objectUrls: eventHrefs.filter(
|
||||
(href) =>
|
||||
this.resolveCollectionUrl(client, href) === collectionUrl,
|
||||
),
|
||||
depth: '1',
|
||||
}),
|
||||
),
|
||||
)
|
||||
).flat();
|
||||
|
||||
return calendarObjects.flatMap((calendarObject) => {
|
||||
const iCalData = extractICalData(calendarObject.props?.calendarData);
|
||||
|
||||
if (!isNonEmptyString(calendarObject.href) || !iCalData) return [];
|
||||
|
||||
return parseICalEvents(iCalData, calendarObject.href).filter((event) =>
|
||||
isEventInTimeRange(event, startDate, endDate),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private resolveCollectionUrl(client: DAVClient, href: string): string {
|
||||
const collectionPath = href.slice(0, href.lastIndexOf('/') + 1);
|
||||
|
||||
return new URL(collectionPath, client.serverUrl).href;
|
||||
}
|
||||
|
||||
private async syncCalendar(
|
||||
client: DAVClient,
|
||||
calendar: DAVCalendar,
|
||||
options: FetchEventsOptions,
|
||||
syncCursor?: CalDavSyncCursor,
|
||||
): Promise<CalendarSyncResult> {
|
||||
const supportsSyncCollection =
|
||||
calendar.reports?.includes('syncCollection') ?? false;
|
||||
|
||||
try {
|
||||
return supportsSyncCollection
|
||||
? await this.fetchEventsViaSyncCollection(client, calendar, options)
|
||||
: await this.fetchEventsViaCtagEtag(client, calendar, options);
|
||||
? await this.fetchHrefsViaSyncCollection(client, calendar, syncCursor)
|
||||
: await this.fetchHrefsViaCtagEtag(client, calendar, syncCursor);
|
||||
} catch (error) {
|
||||
this.logger.error(`Per-calendar sync failed for ${calendar.url}`, error);
|
||||
|
||||
return {
|
||||
calendarUrl: calendar.url,
|
||||
events: [],
|
||||
newSyncToken: options.syncCursor?.syncTokens[calendar.url],
|
||||
newCtag: options.syncCursor?.ctags?.[calendar.url],
|
||||
newEtags: options.syncCursor?.etags?.[calendar.url],
|
||||
changedHrefs: [],
|
||||
cancelledHrefs: [],
|
||||
newSyncToken: syncCursor?.syncTokens[calendar.url],
|
||||
newCtag: syncCursor?.ctags?.[calendar.url],
|
||||
newEtags: syncCursor?.etags?.[calendar.url],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchEventsViaSyncCollection(
|
||||
private async fetchHrefsViaSyncCollection(
|
||||
client: DAVClient,
|
||||
calendar: DAVCalendar,
|
||||
options: FetchEventsOptions,
|
||||
syncCursor?: CalDavSyncCursor,
|
||||
): Promise<CalendarSyncResult> {
|
||||
const previousSyncToken = options.syncCursor?.syncTokens[calendar.url];
|
||||
const previousSyncToken = syncCursor?.syncTokens[calendar.url];
|
||||
|
||||
const syncResult = await this.runSyncCollection(
|
||||
client,
|
||||
@@ -110,13 +172,6 @@ export class CalDavFetchEventsService {
|
||||
.filter((entry) => entry.status === 404)
|
||||
.map((entry) => entry.href);
|
||||
|
||||
const fetchedEvents = await this.fetchAndParseEvents(
|
||||
client,
|
||||
calendar.url,
|
||||
changedHrefs,
|
||||
options,
|
||||
);
|
||||
|
||||
const rawSyncToken = syncResult[0]?.raw?.multistatus?.syncToken;
|
||||
const newSyncToken = isNonEmptyString(rawSyncToken)
|
||||
? rawSyncToken
|
||||
@@ -124,10 +179,8 @@ export class CalDavFetchEventsService {
|
||||
|
||||
return {
|
||||
calendarUrl: calendar.url,
|
||||
events: [
|
||||
...fetchedEvents,
|
||||
...cancelledHrefs.map(buildCancelledCalDavEvent),
|
||||
],
|
||||
changedHrefs,
|
||||
cancelledHrefs,
|
||||
newSyncToken,
|
||||
};
|
||||
}
|
||||
@@ -164,21 +217,22 @@ export class CalDavFetchEventsService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private async fetchEventsViaCtagEtag(
|
||||
private async fetchHrefsViaCtagEtag(
|
||||
client: DAVClient,
|
||||
calendar: DAVCalendar,
|
||||
options: FetchEventsOptions,
|
||||
syncCursor?: CalDavSyncCursor,
|
||||
): Promise<CalendarSyncResult> {
|
||||
const storedEtags = options.syncCursor?.etags?.[calendar.url] ?? {};
|
||||
const storedEtags = syncCursor?.etags?.[calendar.url] ?? {};
|
||||
const newCtag = isDefined(calendar.ctag)
|
||||
? String(calendar.ctag)
|
||||
: undefined;
|
||||
const storedCtag = options.syncCursor?.ctags?.[calendar.url];
|
||||
const storedCtag = syncCursor?.ctags?.[calendar.url];
|
||||
|
||||
if (isDefined(newCtag) && isDefined(storedCtag) && newCtag === storedCtag) {
|
||||
return {
|
||||
calendarUrl: calendar.url,
|
||||
events: [],
|
||||
changedHrefs: [],
|
||||
cancelledHrefs: [],
|
||||
newCtag,
|
||||
newEtags: storedEtags,
|
||||
};
|
||||
@@ -193,19 +247,10 @@ export class CalDavFetchEventsService {
|
||||
(href) => !(href in currentEtags),
|
||||
);
|
||||
|
||||
const fetchedEvents = await this.fetchAndParseEvents(
|
||||
client,
|
||||
calendar.url,
|
||||
changedHrefs,
|
||||
options,
|
||||
);
|
||||
|
||||
return {
|
||||
calendarUrl: calendar.url,
|
||||
events: [
|
||||
...fetchedEvents,
|
||||
...cancelledHrefs.map(buildCancelledCalDavEvent),
|
||||
],
|
||||
changedHrefs,
|
||||
cancelledHrefs,
|
||||
newCtag,
|
||||
newEtags: currentEtags,
|
||||
};
|
||||
@@ -257,34 +302,4 @@ export class CalDavFetchEventsService {
|
||||
return map;
|
||||
}, {});
|
||||
}
|
||||
|
||||
private async fetchAndParseEvents(
|
||||
client: DAVClient,
|
||||
calendarUrl: string,
|
||||
objectUrls: string[],
|
||||
options: { startDate: Date; endDate: Date },
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
if (objectUrls.length === 0) return [];
|
||||
|
||||
const calendarObjects = await client.calendarMultiGet({
|
||||
url: calendarUrl,
|
||||
props: {
|
||||
[`${DAVNamespaceShort.DAV}:getetag`]: {},
|
||||
[`${DAVNamespaceShort.CALDAV}:calendar-data`]: {},
|
||||
},
|
||||
objectUrls,
|
||||
depth: '1',
|
||||
});
|
||||
|
||||
return calendarObjects.flatMap((calendarObject) => {
|
||||
const iCalData = extractICalData(calendarObject.props?.calendarData);
|
||||
|
||||
if (!iCalData) return [];
|
||||
|
||||
return parseICalEvents(iCalData, calendarObject.href || '').filter(
|
||||
(event) =>
|
||||
isEventInTimeRange(event, options.startDate, options.endDate),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+6
-21
@@ -10,9 +10,6 @@ import { type GetCalendarEventsResponse } from 'src/modules/calendar/calendar-ev
|
||||
export class CalDavGetEventsService {
|
||||
private readonly logger = new Logger(CalDavGetEventsService.name);
|
||||
|
||||
private static readonly PAST_DAYS_WINDOW = 365 * 5;
|
||||
private static readonly FUTURE_DAYS_WINDOW = 365;
|
||||
|
||||
constructor(
|
||||
private readonly calDavClientProvider: CalDavClientProvider,
|
||||
private readonly fetchEventsService: CalDavFetchEventsService,
|
||||
@@ -28,30 +25,18 @@ export class CalDavGetEventsService {
|
||||
const client =
|
||||
await this.calDavClientProvider.getClient(connectedAccountId);
|
||||
|
||||
const startDate = new Date(
|
||||
Date.now() -
|
||||
CalDavGetEventsService.PAST_DAYS_WINDOW * 24 * 60 * 60 * 1000,
|
||||
const result = await this.fetchEventsService.fetchChangedEventHrefs(
|
||||
client,
|
||||
syncCursor ? (JSON.parse(syncCursor) as CalDavSyncCursor) : undefined,
|
||||
);
|
||||
const endDate = new Date(
|
||||
Date.now() +
|
||||
CalDavGetEventsService.FUTURE_DAYS_WINDOW * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
|
||||
const result = await this.fetchEventsService.fetchEvents(client, {
|
||||
startDate,
|
||||
endDate,
|
||||
syncCursor: syncCursor
|
||||
? (JSON.parse(syncCursor) as CalDavSyncCursor)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
this.logger.debug(
|
||||
`Found ${result.events.length} calendar events for ${connectedAccountId}`,
|
||||
`Found ${result.changedHrefs.length} changed and ${result.cancelledHrefs.length} cancelled calendar events for ${connectedAccountId}`,
|
||||
);
|
||||
|
||||
return {
|
||||
fullEvents: true,
|
||||
calendarEvents: result.events,
|
||||
calendarEventIds: result.changedHrefs,
|
||||
calendarEventIdsToDelete: result.cancelledHrefs,
|
||||
nextSyncCursor: JSON.stringify(result.syncCursor),
|
||||
};
|
||||
} catch (error) {
|
||||
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { CalDavClientProvider } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/providers/caldav-client.provider';
|
||||
import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service';
|
||||
import { parseCalDAVError } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/parse-caldav-error.util';
|
||||
import { CalendarEventImportDriverException } from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
|
||||
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
|
||||
|
||||
@Injectable()
|
||||
export class CalDavImportEventsService {
|
||||
private readonly logger = new Logger(CalDavImportEventsService.name);
|
||||
|
||||
constructor(
|
||||
private readonly calDavClientProvider: CalDavClientProvider,
|
||||
private readonly fetchEventsService: CalDavFetchEventsService,
|
||||
) {}
|
||||
|
||||
async getCalendarEvents(
|
||||
connectedAccountId: string,
|
||||
eventExternalIds: string[],
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
this.logger.debug(
|
||||
`Importing ${eventExternalIds.length} calendar events for ${connectedAccountId}`,
|
||||
);
|
||||
|
||||
try {
|
||||
const client =
|
||||
await this.calDavClientProvider.getClient(connectedAccountId);
|
||||
|
||||
return await this.fetchEventsService.fetchEventsByHrefs(
|
||||
client,
|
||||
eventExternalIds,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error in ${CalDavImportEventsService.name} - getCalendarEvents`,
|
||||
error,
|
||||
);
|
||||
|
||||
if (error instanceof CalendarEventImportDriverException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw parseCalDAVError(error as Error);
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-2
@@ -2,11 +2,15 @@ import { Module } from '@nestjs/common';
|
||||
|
||||
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
|
||||
import { GoogleCalendarGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service';
|
||||
import { GoogleCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service';
|
||||
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
|
||||
|
||||
@Module({
|
||||
imports: [TwentyConfigModule, OAuth2ClientManagerModule],
|
||||
providers: [GoogleCalendarGetEventsService],
|
||||
exports: [GoogleCalendarGetEventsService],
|
||||
providers: [
|
||||
GoogleCalendarGetEventsService,
|
||||
GoogleCalendarImportEventsService,
|
||||
],
|
||||
exports: [GoogleCalendarGetEventsService, GoogleCalendarImportEventsService],
|
||||
})
|
||||
export class GoogleCalendarDriverModule {}
|
||||
|
||||
+16
-6
@@ -2,9 +2,8 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { isString } from '@sniptt/guards';
|
||||
import { type GaxiosError } from 'gaxios';
|
||||
import { google, type calendar_v3 as calendarV3 } from 'googleapis';
|
||||
import { google } from 'googleapis';
|
||||
|
||||
import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util';
|
||||
import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util';
|
||||
import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util';
|
||||
import { type GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
|
||||
@@ -34,7 +33,8 @@ export class GoogleCalendarGetEventsService {
|
||||
|
||||
let nextSyncToken: string | null | undefined;
|
||||
let nextPageToken: string | undefined;
|
||||
const events: calendarV3.Schema$Event[] = [];
|
||||
const calendarEventIds: string[] = [];
|
||||
const calendarEventIdsToDelete: string[] = [];
|
||||
|
||||
let hasMoreEvents = true;
|
||||
|
||||
@@ -69,7 +69,17 @@ export class GoogleCalendarGetEventsService {
|
||||
break;
|
||||
}
|
||||
|
||||
events.push(...items);
|
||||
for (const item of items) {
|
||||
if (!isString(item.id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.status === 'cancelled') {
|
||||
calendarEventIdsToDelete.push(item.id);
|
||||
} else {
|
||||
calendarEventIds.push(item.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (!nextPageToken) {
|
||||
hasMoreEvents = false;
|
||||
@@ -77,8 +87,8 @@ export class GoogleCalendarGetEventsService {
|
||||
}
|
||||
|
||||
return {
|
||||
fullEvents: true,
|
||||
calendarEvents: formatGoogleCalendarEvents(events),
|
||||
calendarEventIds,
|
||||
calendarEventIdsToDelete,
|
||||
nextSyncCursor: nextSyncToken || '',
|
||||
};
|
||||
}
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { type GaxiosError } from 'gaxios';
|
||||
import { google } from 'googleapis';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util';
|
||||
import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util';
|
||||
import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util';
|
||||
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
|
||||
import { GoogleOAuth2ClientProvider } from 'src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client.provider';
|
||||
import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
|
||||
|
||||
@Injectable()
|
||||
export class GoogleCalendarImportEventsService {
|
||||
constructor(
|
||||
private readonly googleOAuth2ClientProvider: GoogleOAuth2ClientProvider,
|
||||
) {}
|
||||
|
||||
public async getCalendarEvents(
|
||||
connectedAccount: Pick<ConnectedAccountEntity, 'provider' | 'id'>,
|
||||
eventExternalIds: string[],
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
const oAuth2Client = await this.googleOAuth2ClientProvider.getClient(
|
||||
connectedAccount.id,
|
||||
);
|
||||
|
||||
const googleCalendarClient = google.calendar({
|
||||
version: 'v3',
|
||||
auth: oAuth2Client,
|
||||
});
|
||||
|
||||
const fetchedEvents = await Promise.all(
|
||||
eventExternalIds.map((eventExternalId) =>
|
||||
googleCalendarClient.events
|
||||
.get({ calendarId: 'primary', eventId: eventExternalId })
|
||||
.then((response) => response.data)
|
||||
.catch((error: GaxiosError) => {
|
||||
const status = error.response?.status;
|
||||
|
||||
if (status === 404 || status === 410) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isDefined(status)) {
|
||||
throw parseGaxiosError(error);
|
||||
}
|
||||
|
||||
throw parseGoogleCalendarError({
|
||||
code: status,
|
||||
reason: error.response?.data?.error?.errors?.[0].reason || '',
|
||||
message: error.response?.data?.error?.errors?.[0].message || '',
|
||||
});
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
return formatGoogleCalendarEvents(fetchedEvents.filter(isDefined));
|
||||
}
|
||||
}
|
||||
+7
-2
@@ -25,6 +25,7 @@ export class MicrosoftCalendarGetEventsService {
|
||||
const microsoftClient =
|
||||
await this.microsoftOAuth2ClientProvider.getClient(connectedAccount.id);
|
||||
const eventIds: string[] = [];
|
||||
const eventIdsToDelete: string[] = [];
|
||||
|
||||
const response: PageCollection = await microsoftClient
|
||||
.api(syncCursor || '/me/calendar/events/delta')
|
||||
@@ -32,7 +33,11 @@ export class MicrosoftCalendarGetEventsService {
|
||||
.get();
|
||||
|
||||
const callback: PageIteratorCallback = (data) => {
|
||||
eventIds.push(data.id);
|
||||
if (data['@removed']) {
|
||||
eventIdsToDelete.push(data.id);
|
||||
} else {
|
||||
eventIds.push(data.id);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
@@ -46,8 +51,8 @@ export class MicrosoftCalendarGetEventsService {
|
||||
await pageIterator.iterate();
|
||||
|
||||
return {
|
||||
fullEvents: false,
|
||||
calendarEventIds: eventIds,
|
||||
calendarEventIdsToDelete: eventIdsToDelete,
|
||||
nextSyncCursor: pageIterator.getDeltaLink() || '',
|
||||
};
|
||||
} catch (error) {
|
||||
|
||||
+3
-3
@@ -16,7 +16,7 @@ export class MicrosoftCalendarImportEventsService {
|
||||
|
||||
public async getCalendarEvents(
|
||||
connectedAccount: Pick<ConnectedAccountEntity, 'provider' | 'id'>,
|
||||
changedEventIds: string[],
|
||||
eventExternalIds: string[],
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
try {
|
||||
const microsoftClient =
|
||||
@@ -24,9 +24,9 @@ export class MicrosoftCalendarImportEventsService {
|
||||
|
||||
const events: Event[] = [];
|
||||
|
||||
for (const changedEventId of changedEventIds) {
|
||||
for (const eventExternalId of eventExternalIds) {
|
||||
const event = await microsoftClient
|
||||
.api(`/me/calendar/events/${changedEventId}`)
|
||||
.api(`/me/calendar/events/${eventExternalId}`)
|
||||
.get();
|
||||
|
||||
events.push(event);
|
||||
|
||||
+11
-11
@@ -16,10 +16,6 @@ const mockCalendarChannelRepository = {
|
||||
update: jest.fn(),
|
||||
};
|
||||
|
||||
const mockCalendarEventsImportService = {
|
||||
processCalendarEventsImport: jest.fn(),
|
||||
};
|
||||
|
||||
const mockCalendarEventImportErrorHandlerService = {
|
||||
handleDriverException: jest.fn(),
|
||||
};
|
||||
@@ -34,6 +30,10 @@ const mockGlobalWorkspaceOrmManager = {
|
||||
}),
|
||||
};
|
||||
|
||||
const mockCalendarEventCleanerService = {
|
||||
cleanWorkspaceCalendarEvents: jest.fn(),
|
||||
};
|
||||
|
||||
const workspaceId = 'workspace-123';
|
||||
|
||||
const baseConnectedAccount = {
|
||||
@@ -60,8 +60,8 @@ describe('CalendarFetchEventsService', () => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({
|
||||
fullEvents: true,
|
||||
calendarEvents: [{ id: 'event-1' }],
|
||||
calendarEventIds: ['event-1'],
|
||||
calendarEventIdsToDelete: [],
|
||||
nextSyncCursor: 'new-cursor-abc',
|
||||
});
|
||||
|
||||
@@ -72,7 +72,7 @@ describe('CalendarFetchEventsService', () => {
|
||||
mockCalendarChannelSyncStatusService as any,
|
||||
mockGetCalendarEventsService as any,
|
||||
mockCalendarEventImportErrorHandlerService as any,
|
||||
mockCalendarEventsImportService as any,
|
||||
mockCalendarEventCleanerService as any,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -157,8 +157,8 @@ describe('CalendarFetchEventsService', () => {
|
||||
|
||||
jest.clearAllMocks();
|
||||
mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({
|
||||
fullEvents: true,
|
||||
calendarEvents: [{ id: 'event-2' }],
|
||||
calendarEventIds: ['event-2'],
|
||||
calendarEventIdsToDelete: [],
|
||||
nextSyncCursor: 'new-cursor-def',
|
||||
});
|
||||
|
||||
@@ -194,8 +194,8 @@ describe('CalendarFetchEventsService', () => {
|
||||
|
||||
jest.clearAllMocks();
|
||||
mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({
|
||||
fullEvents: true,
|
||||
calendarEvents: [{ id: 'event-3' }],
|
||||
calendarEventIds: ['event-3'],
|
||||
calendarEventIdsToDelete: [],
|
||||
nextSyncCursor: 'newer-cursor',
|
||||
});
|
||||
|
||||
|
||||
+27
-41
@@ -21,16 +21,15 @@ import {
|
||||
CalendarEventImportDriverException,
|
||||
CalendarEventImportDriverExceptionCode,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
|
||||
import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service';
|
||||
import {
|
||||
CalendarEventImportErrorHandlerService,
|
||||
CalendarEventImportSyncStep,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
|
||||
import { CalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service';
|
||||
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
|
||||
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
|
||||
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
|
||||
import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
|
||||
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
|
||||
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
|
||||
import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
|
||||
|
||||
@@ -46,7 +45,7 @@ export class CalendarEventsImportService {
|
||||
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
|
||||
private readonly calendarSaveEventsService: CalendarSaveEventsService,
|
||||
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
|
||||
private readonly microsoftCalendarImportEventService: MicrosoftCalendarImportEventsService,
|
||||
private readonly calendarImportEventsService: CalendarImportEventsService,
|
||||
private readonly emailAliasManagerService: EmailAliasManagerService,
|
||||
@InjectRepository(UserWorkspaceEntity)
|
||||
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
|
||||
@@ -56,7 +55,6 @@ export class CalendarEventsImportService {
|
||||
calendarChannel: CalendarChannelEntity,
|
||||
connectedAccount: ConnectedAccountEntity,
|
||||
workspaceId: string,
|
||||
fetchedCalendarEvents?: FetchedCalendarEvent[],
|
||||
): Promise<void> {
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportOngoing(
|
||||
[calendarChannel.id],
|
||||
@@ -67,46 +65,27 @@ export class CalendarEventsImportService {
|
||||
|
||||
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
async () => {
|
||||
let calendarEvents: FetchedCalendarEvent[] = [];
|
||||
|
||||
try {
|
||||
if (fetchedCalendarEvents) {
|
||||
calendarEvents = fetchedCalendarEvents;
|
||||
} else {
|
||||
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
|
||||
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
|
||||
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
|
||||
);
|
||||
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
|
||||
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
|
||||
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
|
||||
);
|
||||
|
||||
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
|
||||
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (connectedAccount.provider) {
|
||||
case 'microsoft':
|
||||
calendarEvents =
|
||||
await this.microsoftCalendarImportEventService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
eventIdsToFetch,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!calendarEvents || calendarEvents?.length === 0) {
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
|
||||
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
|
||||
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const calendarEvents =
|
||||
await this.calendarImportEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
eventIdsToFetch,
|
||||
);
|
||||
|
||||
const userWorkspace = await this.userWorkspaceRepository.findOne({
|
||||
where: {
|
||||
id: (connectedAccount as unknown as { userWorkspaceId: string })
|
||||
@@ -190,10 +169,17 @@ export class CalendarEventsImportService {
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
if (eventIdsToFetch.length < CALENDAR_EVENT_IMPORT_BATCH_SIZE) {
|
||||
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await this.calendarEventImportErrorHandlerService.handleDriverException(
|
||||
error,
|
||||
|
||||
+29
-44
@@ -1,24 +1,21 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
import { Any, Repository } from 'typeorm';
|
||||
|
||||
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
|
||||
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
|
||||
import {
|
||||
CalendarEventImportDriverException,
|
||||
CalendarEventImportDriverExceptionCode,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
|
||||
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
|
||||
import {
|
||||
CalendarEventImportErrorHandlerService,
|
||||
CalendarEventImportSyncStep,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
|
||||
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
|
||||
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
|
||||
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
|
||||
import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
|
||||
import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
|
||||
import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
|
||||
|
||||
@@ -34,7 +31,7 @@ export class CalendarFetchEventsService {
|
||||
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
|
||||
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
|
||||
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
|
||||
private readonly calendarEventsImportService: CalendarEventsImportService,
|
||||
private readonly calendarEventCleanerService: CalendarEventCleanerService,
|
||||
) {}
|
||||
|
||||
public async fetchCalendarEvents(
|
||||
@@ -56,28 +53,40 @@ export class CalendarFetchEventsService {
|
||||
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
async () => {
|
||||
try {
|
||||
const getCalendarEventsResponse =
|
||||
const { calendarEventIds, calendarEventIdsToDelete, nextSyncCursor } =
|
||||
await this.getCalendarEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
calendarChannel.syncCursor || undefined,
|
||||
);
|
||||
|
||||
const hasFullEvents = getCalendarEventsResponse.fullEvents;
|
||||
if (calendarEventIdsToDelete.length > 0) {
|
||||
const calendarChannelEventAssociationRepository =
|
||||
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'calendarChannelEventAssociation',
|
||||
);
|
||||
|
||||
const calendarEvents = hasFullEvents
|
||||
? getCalendarEventsResponse.calendarEvents
|
||||
: null;
|
||||
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
|
||||
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
|
||||
await calendarChannelEventAssociationRepository.delete({
|
||||
eventExternalId: Any(calendarEventIdsToDelete),
|
||||
calendarChannelId: calendarChannel.id,
|
||||
});
|
||||
|
||||
if (!calendarEvents || calendarEvents?.length === 0) {
|
||||
await this.calendarChannelRepository.update(
|
||||
{ id: calendarChannel.id, workspaceId },
|
||||
{
|
||||
syncCursor: nextSyncCursor,
|
||||
},
|
||||
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
if (calendarEventIds.length > 0) {
|
||||
await this.cacheStorage.setAdd(
|
||||
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
|
||||
calendarEventIds,
|
||||
);
|
||||
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
@@ -90,30 +99,6 @@ export class CalendarFetchEventsService {
|
||||
syncCursor: nextSyncCursor,
|
||||
},
|
||||
);
|
||||
|
||||
if (hasFullEvents && calendarEvents) {
|
||||
await this.calendarEventsImportService.processCalendarEventsImport(
|
||||
calendarChannel,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
calendarEvents,
|
||||
);
|
||||
} else if (!hasFullEvents && calendarEventIds) {
|
||||
await this.cacheStorage.setAdd(
|
||||
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
|
||||
calendarEventIds,
|
||||
);
|
||||
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
|
||||
[calendarChannel.id],
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
throw new CalendarEventImportDriverException(
|
||||
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
|
||||
CalendarEventImportDriverExceptionCode.UNKNOWN,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`,
|
||||
|
||||
+2
-4
@@ -9,13 +9,11 @@ import {
|
||||
CalendarEventImportException,
|
||||
CalendarEventImportExceptionCode,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
|
||||
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
|
||||
import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
|
||||
|
||||
export type GetCalendarEventsResponse = {
|
||||
fullEvents: boolean;
|
||||
calendarEvents?: FetchedCalendarEvent[];
|
||||
calendarEventIds?: string[];
|
||||
calendarEventIds: string[];
|
||||
calendarEventIdsToDelete: string[];
|
||||
nextSyncCursor: string;
|
||||
};
|
||||
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { CalDavImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service';
|
||||
import { GoogleCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service';
|
||||
import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service';
|
||||
import {
|
||||
CalendarEventImportException,
|
||||
CalendarEventImportExceptionCode,
|
||||
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
|
||||
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
|
||||
import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
|
||||
|
||||
@Injectable()
|
||||
export class CalendarImportEventsService {
|
||||
constructor(
|
||||
private readonly googleCalendarImportEventsService: GoogleCalendarImportEventsService,
|
||||
private readonly microsoftCalendarImportEventsService: MicrosoftCalendarImportEventsService,
|
||||
private readonly caldavCalendarImportEventsService: CalDavImportEventsService,
|
||||
) {}
|
||||
|
||||
public async getCalendarEvents(
|
||||
connectedAccount: Pick<ConnectedAccountEntity, 'provider' | 'id'>,
|
||||
eventExternalIds: string[],
|
||||
): Promise<FetchedCalendarEvent[]> {
|
||||
switch (connectedAccount.provider) {
|
||||
case ConnectedAccountProvider.GOOGLE:
|
||||
return this.googleCalendarImportEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
eventExternalIds,
|
||||
);
|
||||
case ConnectedAccountProvider.MICROSOFT:
|
||||
return this.microsoftCalendarImportEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
eventExternalIds,
|
||||
);
|
||||
case ConnectedAccountProvider.IMAP_SMTP_CALDAV:
|
||||
return this.caldavCalendarImportEventsService.getCalendarEvents(
|
||||
connectedAccount.id,
|
||||
eventExternalIds,
|
||||
);
|
||||
default:
|
||||
throw new CalendarEventImportException(
|
||||
`Provider ${connectedAccount.provider} is not supported`,
|
||||
CalendarEventImportExceptionCode.PROVIDER_NOT_SUPPORTED,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user