From c6aca3f0ea2330d59a1b20f1bb4a8d2a96999485 Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Tue, 23 Jun 2026 20:08:38 +0530 Subject: [PATCH] fix(calendar): ID-first chunked import for Google and CalDAV (#22015) Google and CalDAV returned full events and imported them inline in the list-fetch job. Large/initial syncs overran BullMQ's lock, the job stalled, the workspace query runner was released mid-import, and TypeORM threw 'Query runner already released'. Mirror the messaging pipeline: every provider now returns event IDs only, cached in Redis; the import job drains them in CALENDAR_EVENT_IMPORT_BATCH_SIZE chunks and re-enqueues until empty, so no single job runs long. Adds Google/CalDAV import-by-id services and a provider dispatcher; removes the full-events inline path. Review in cubic --- .../calendar-event-import-manager.module.ts | 2 + .../drivers/caldav/caldav-driver.module.ts | 3 + .../caldav-fetch-events.service.spec.ts | 255 ++++++------------ .../services/caldav-fetch-events.service.ts | 175 ++++++------ .../services/caldav-get-events.service.ts | 27 +- .../services/caldav-import-events.service.ts | 47 ++++ .../google-calendar-driver.module.ts | 8 +- .../google-calendar-get-events.service.ts | 22 +- .../google-calendar-import-events.service.ts | 60 +++++ .../microsoft-calendar-get-events.service.ts | 9 +- ...icrosoft-calendar-import-events.service.ts | 6 +- .../calendar-fetch-events.service.spec.ts | 22 +- .../calendar-events-import.service.ts | 68 ++--- .../services/calendar-fetch-events.service.ts | 73 ++--- .../services/calendar-get-events.service.ts | 6 +- .../calendar-import-events.service.ts | 50 ++++ 16 files changed, 440 insertions(+), 393 deletions(-) create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service.ts create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service.ts create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service.ts diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts index 77a20b3898..66c9914dfc 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts @@ -34,6 +34,7 @@ import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/cal import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service'; import { CalendarFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service'; import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; +import { CalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module'; import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module'; @@ -73,6 +74,7 @@ import { RefreshTokensManagerModule } from 'src/modules/connected-account/refres CalendarFetchEventsService, CalendarEventImportErrorHandlerService, CalendarGetCalendarEventsService, + CalendarImportEventsService, CalendarSaveEventsService, CalendarEventListFetchCronJob, CalendarEventListFetchCronCommand, diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/caldav-driver.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/caldav-driver.module.ts index f74deaebd0..4499dd174b 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/caldav-driver.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/caldav-driver.module.ts @@ -9,6 +9,7 @@ import { CalDavClientProvider } from 'src/modules/calendar/calendar-event-import import { CalDavClientService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-client.service'; import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service'; import { CalDavGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service'; +import { CalDavImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service'; @Module({ imports: [ @@ -22,12 +23,14 @@ import { CalDavGetEventsService } from 'src/modules/calendar/calendar-event-impo CalDavClientService, CalDavFetchEventsService, CalDavGetEventsService, + CalDavImportEventsService, ], exports: [ CalDavClientProvider, CalDavClientService, CalDavFetchEventsService, CalDavGetEventsService, + CalDavImportEventsService, ], }) export class CalDavDriverModule {} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.spec.ts index ae515ccd68..5d3b2a39ac 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.spec.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.spec.ts @@ -2,30 +2,26 @@ import { type DAVClient } from 'tsdav'; import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service'; -const PRIMARY_URL = 'https://caldav.example.com/calendars/user/primary/'; -const PERSONAL_URL = 'https://caldav.example.com/calendars/user/personal/'; +const SERVER_URL = 'https://caldav.example.com'; +const PRIMARY_URL = `${SERVER_URL}/calendars/user/primary/`; +const PERSONAL_URL = `${SERVER_URL}/calendars/user/personal/`; const HREF_A = `${PRIMARY_URL}event-a.ics`; const HREF_B = `${PRIMARY_URL}event-b.ics`; -const buildICal = (uid: string) => +const buildICal = (uid: string, dtStart = '20260601T100000Z') => [ 'BEGIN:VCALENDAR', 'VERSION:2.0', 'BEGIN:VEVENT', `UID:${uid}`, `SUMMARY:${uid}`, - 'DTSTART:20260601T100000Z', + `DTSTART:${dtStart}`, 'DTEND:20260601T110000Z', 'STATUS:CONFIRMED', 'END:VEVENT', 'END:VCALENDAR', ].join('\r\n'); -const inWindow = { - startDate: new Date('2026-01-01'), - endDate: new Date('2027-01-01'), -}; - const buildClient = () => { const fetchCalendars = jest.fn(); const syncCollection = jest.fn(); @@ -34,6 +30,7 @@ const buildClient = () => { return { client: { + serverUrl: SERVER_URL, fetchCalendars, syncCollection, calendarMultiGet, @@ -53,8 +50,8 @@ describe('CalDavFetchEventsService', () => { service = new CalDavFetchEventsService(); }); - describe('per-calendar tier dispatch', () => { - it('runs Tier-1 on calendars advertising sync-collection and Tier-2/3 on the rest, in parallel', async () => { + describe('fetchChangedEventHrefs', () => { + it('collects changed hrefs from sync-collection and ctag/etag calendars without fetching bodies', async () => { const c = buildClient(); c.fetchCalendars.mockResolvedValue([ @@ -65,54 +62,44 @@ describe('CalDavFetchEventsService', () => { }, { url: PERSONAL_URL, components: ['VEVENT'], reports: [], ctag: 'c-1' }, ]); - c.syncCollection.mockResolvedValue([ - { href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} }, + { href: HREF_A, status: 207, ok: true, props: {} }, + ]); + c.propfind.mockResolvedValue([ + { href: HREF_B, status: 207, ok: true, props: { getetag: '"etag-b"' } }, ]); - c.propfind.mockResolvedValue([ + const result = await service.fetchChangedEventHrefs(c.client); + + expect(result.changedHrefs.sort()).toEqual([HREF_A, HREF_B].sort()); + expect(result.cancelledHrefs).toEqual([]); + expect(c.calendarMultiGet).not.toHaveBeenCalled(); + }); + + it('separates cancelled (404) hrefs from changed ones in a sync-collection delta', async () => { + const c = buildClient(); + + c.fetchCalendars.mockResolvedValue([ { - href: HREF_B, - status: 207, - statusText: 'OK', - ok: true, - props: { getetag: '"etag-b"' }, + url: PRIMARY_URL, + components: ['VEVENT'], + reports: ['syncCollection'], }, ]); - - c.calendarMultiGet - .mockResolvedValueOnce([ - { - href: HREF_A, - status: 207, - statusText: 'OK', - ok: true, - props: { calendarData: buildICal('uid-a') }, - }, - ]) - .mockResolvedValueOnce([ - { - href: HREF_B, - status: 207, - statusText: 'OK', - ok: true, - props: { calendarData: buildICal('uid-b') }, - }, - ]); - - const result = await service.fetchEvents(c.client, inWindow); - - expect(result.events.map((event) => event.iCalUid).sort()).toEqual([ - 'uid-a', - 'uid-b', + c.syncCollection.mockResolvedValue([ + { href: HREF_A, status: 207, ok: true, props: {} }, + { href: HREF_B, status: 404, ok: false, props: {} }, ]); - expect(c.syncCollection).toHaveBeenCalledTimes(1); - expect(c.propfind).toHaveBeenCalledTimes(1); - }); - }); - describe('Tier-2/3 ctag short-circuit', () => { - it('skips network entirely when the server CTag matches the stored CTag', async () => { + const result = await service.fetchChangedEventHrefs(c.client, { + syncTokens: { [PRIMARY_URL]: 'token-prior' }, + }); + + expect(result.changedHrefs).toEqual([HREF_A]); + expect(result.cancelledHrefs).toEqual([HREF_B]); + }); + + it('skips network when the server CTag matches the stored CTag', async () => { const c = buildClient(); c.fetchCalendars.mockResolvedValue([ @@ -126,24 +113,18 @@ describe('CalDavFetchEventsService', () => { const storedEtags = { [HREF_A]: '"etag-a"' }; - const result = await service.fetchEvents(c.client, { - ...inWindow, - syncCursor: { - syncTokens: {}, - ctags: { [PRIMARY_URL]: 'unchanged' }, - etags: { [PRIMARY_URL]: storedEtags }, - }, + const result = await service.fetchChangedEventHrefs(c.client, { + syncTokens: {}, + ctags: { [PRIMARY_URL]: 'unchanged' }, + etags: { [PRIMARY_URL]: storedEtags }, }); - expect(result.events).toEqual([]); + expect(result.changedHrefs).toEqual([]); expect(c.propfind).not.toHaveBeenCalled(); - expect(c.calendarMultiGet).not.toHaveBeenCalled(); expect(result.syncCursor.etags).toEqual({ [PRIMARY_URL]: storedEtags }); }); - }); - describe('Tier-2/3 etag diff', () => { - it('fetches only changed hrefs and emits cancelled stubs for hrefs vanished from the server', async () => { + it('separates changed from vanished hrefs in an etag diff', async () => { const c = buildClient(); c.fetchCalendars.mockResolvedValue([ @@ -154,52 +135,28 @@ describe('CalDavFetchEventsService', () => { ctag: 'new-ctag', }, ]); - c.propfind.mockResolvedValue([ { href: HREF_A, status: 207, - statusText: 'OK', ok: true, props: { getetag: '"etag-a-updated"' }, }, ]); - c.calendarMultiGet.mockResolvedValue([ - { - href: HREF_A, - status: 207, - statusText: 'OK', - ok: true, - props: { calendarData: buildICal('uid-a') }, - }, - ]); - - const result = await service.fetchEvents(c.client, { - ...inWindow, - syncCursor: { - syncTokens: {}, - ctags: { [PRIMARY_URL]: 'old-ctag' }, - etags: { - [PRIMARY_URL]: { [HREF_A]: '"etag-a"', [HREF_B]: '"etag-b"' }, - }, + const result = await service.fetchChangedEventHrefs(c.client, { + syncTokens: {}, + ctags: { [PRIMARY_URL]: 'old-ctag' }, + etags: { + [PRIMARY_URL]: { [HREF_A]: '"etag-a"', [HREF_B]: '"etag-b"' }, }, }); - expect(c.calendarMultiGet).toHaveBeenCalledWith( - expect.objectContaining({ objectUrls: [HREF_A] }), - ); - - const live = result.events.filter((event) => !event.isCanceled); - const cancelled = result.events.filter((event) => event.isCanceled); - - expect(live.map((event) => event.iCalUid)).toEqual(['uid-a']); - expect(cancelled.map((event) => event.id)).toEqual([HREF_B]); + expect(result.changedHrefs).toEqual([HREF_A]); + expect(result.cancelledHrefs).toEqual([HREF_B]); }); - }); - describe('per-calendar error isolation', () => { - it('preserves the prior cursor entry (token + ctag + etags) for the failing calendar without aborting siblings', async () => { + it('preserves the prior cursor entry for a calendar whose sync fails', async () => { const c = buildClient(); c.fetchCalendars.mockResolvedValue([ @@ -208,37 +165,19 @@ describe('CalDavFetchEventsService', () => { components: ['VEVENT'], reports: ['syncCollection'], }, - { - url: PERSONAL_URL, - components: ['VEVENT'], - reports: [], - ctag: 'c-new', - }, ]); - c.syncCollection.mockRejectedValue(new Error('network blip')); - c.propfind.mockRejectedValue(new Error('propfind blip')); - const priorEtags = { [HREF_A]: '"etag-a"' }; - - const result = await service.fetchEvents(c.client, { - ...inWindow, - syncCursor: { - syncTokens: { [PRIMARY_URL]: 'token-prior' }, - ctags: { [PERSONAL_URL]: 'c-prior' }, - etags: { [PERSONAL_URL]: priorEtags }, - }, + const result = await service.fetchChangedEventHrefs(c.client, { + syncTokens: { [PRIMARY_URL]: 'token-prior' }, }); - expect(result.events).toEqual([]); + expect(result.changedHrefs).toEqual([]); + expect(result.cancelledHrefs).toEqual([]); expect(result.syncCursor.syncTokens[PRIMARY_URL]).toBe('token-prior'); - expect(result.syncCursor.ctags?.[PERSONAL_URL]).toBe('c-prior'); - expect(result.syncCursor.etags?.[PERSONAL_URL]).toEqual(priorEtags); }); - }); - describe('cursor shape', () => { - it('omits ctags and etags entirely when only sync-collection calendars exist', async () => { + it('omits the sync-token on the first run so the server returns a full listing', async () => { const c = buildClient(); c.fetchCalendars.mockResolvedValue([ @@ -249,49 +188,10 @@ describe('CalDavFetchEventsService', () => { }, ]); c.syncCollection.mockResolvedValue([ - { - status: 207, - statusText: 'OK', - ok: true, - raw: { multistatus: { syncToken: 'token-fresh' } }, - }, - ]); - c.calendarMultiGet.mockResolvedValue([]); - - const result = await service.fetchEvents(c.client, inWindow); - - expect(result.syncCursor).toEqual({ - syncTokens: { [PRIMARY_URL]: 'token-fresh' }, - }); - }); - }); - - describe('initial sync (no stored cursor)', () => { - it('omits the sync-token on the first run so the server returns a full listing (RFC 6578 ยง3.4)', async () => { - const c = buildClient(); - - c.fetchCalendars.mockResolvedValue([ - { - url: PRIMARY_URL, - components: ['VEVENT'], - reports: ['syncCollection'], - syncToken: 'server-current-token', - }, - ]); - c.syncCollection.mockResolvedValue([ - { href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} }, - ]); - c.calendarMultiGet.mockResolvedValue([ - { - href: HREF_A, - status: 207, - statusText: 'OK', - ok: true, - props: { calendarData: buildICal('uid-a') }, - }, + { href: HREF_A, status: 207, ok: true, props: {} }, ]); - await service.fetchEvents(c.client, inWindow); + await service.fetchChangedEventHrefs(c.client); expect(c.syncCollection).toHaveBeenCalledWith( expect.not.objectContaining({ syncToken: expect.anything() }), @@ -299,38 +199,35 @@ describe('CalDavFetchEventsService', () => { }); }); - describe('time-window filtering', () => { - it('drops events that fall outside the requested [startDate, endDate] window', async () => { + describe('fetchEventsByHrefs', () => { + it('fetches bodies for the given hrefs grouped by calendar collection', async () => { const c = buildClient(); - c.fetchCalendars.mockResolvedValue([ - { - url: PRIMARY_URL, - components: ['VEVENT'], - reports: ['syncCollection'], - }, + c.calendarMultiGet.mockResolvedValue([ + { href: HREF_A, props: { calendarData: buildICal('uid-a') } }, ]); - c.syncCollection.mockResolvedValue([ - { href: HREF_A, status: 207, statusText: 'OK', ok: true, props: {} }, - ]); + const events = await service.fetchEventsByHrefs(c.client, [HREF_A]); + + expect(c.calendarMultiGet).toHaveBeenCalledWith( + expect.objectContaining({ url: PRIMARY_URL, objectUrls: [HREF_A] }), + ); + expect(events.map((event) => event.iCalUid)).toEqual(['uid-a']); + }); + + it('drops events that fall outside the import time window', async () => { + const c = buildClient(); c.calendarMultiGet.mockResolvedValue([ { href: HREF_A, - status: 207, - statusText: 'OK', - ok: true, - props: { calendarData: buildICal('uid-a') }, + props: { calendarData: buildICal('uid-a', '20990101T100000Z') }, }, ]); - const result = await service.fetchEvents(c.client, { - startDate: new Date('2030-01-01'), - endDate: new Date('2030-12-31'), - }); + const events = await service.fetchEventsByHrefs(c.client, [HREF_A]); - expect(result.events).toEqual([]); + expect(events).toEqual([]); }); }); }); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts index c8b6545835..e3ba15fd6d 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts @@ -10,7 +10,6 @@ import { import { isDefined } from 'twenty-shared/utils'; import { type CalDavSyncCursor } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/types/caldav-sync-cursor'; -import { buildCancelledCalDavEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/build-cancelled-event.util'; import { extractICalData } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/extract-ical-data.util'; import { isEventInTimeRange } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-event-in-time-range.util'; import { isInvalidSyncTokenResponse } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-invalid-sync-token-response.util'; @@ -20,22 +19,20 @@ import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fet type CalendarSyncResult = { calendarUrl: string; - events: FetchedCalendarEvent[]; + changedHrefs: string[]; + cancelledHrefs: string[]; newSyncToken?: string; newCtag?: string; newEtags?: Record; }; -type FetchEventsOptions = { - startDate: Date; - endDate: Date; - syncCursor?: CalDavSyncCursor; -}; - @Injectable() export class CalDavFetchEventsService { private readonly logger = new Logger(CalDavFetchEventsService.name); + private static readonly PAST_DAYS_WINDOW = 365 * 5; + private static readonly FUTURE_DAYS_WINDOW = 365; + async listEventCalendars(client: DAVClient): Promise { const calendars = await client.fetchCalendars(); @@ -44,53 +41,118 @@ export class CalDavFetchEventsService { ); } - async fetchEvents( + async fetchChangedEventHrefs( client: DAVClient, - options: FetchEventsOptions, - ): Promise<{ events: FetchedCalendarEvent[]; syncCursor: CalDavSyncCursor }> { + syncCursor?: CalDavSyncCursor, + ): Promise<{ + changedHrefs: string[]; + cancelledHrefs: string[]; + syncCursor: CalDavSyncCursor; + }> { const calendars = await this.listEventCalendars(client); const results = await Promise.all( - calendars.map((calendar) => this.syncCalendar(client, calendar, options)), + calendars.map((calendar) => + this.syncCalendar(client, calendar, syncCursor), + ), ); return { - events: results.flatMap((result) => result.events), + changedHrefs: results.flatMap((result) => result.changedHrefs), + cancelledHrefs: results.flatMap((result) => result.cancelledHrefs), syncCursor: this.mergeSyncCursor(results), }; } + async fetchEventsByHrefs( + client: DAVClient, + eventHrefs: string[], + ): Promise { + if (eventHrefs.length === 0) return []; + + const startDate = new Date( + Date.now() - + CalDavFetchEventsService.PAST_DAYS_WINDOW * 24 * 60 * 60 * 1000, + ); + const endDate = new Date( + Date.now() + + CalDavFetchEventsService.FUTURE_DAYS_WINDOW * 24 * 60 * 60 * 1000, + ); + + const collectionUrls = [ + ...new Set( + eventHrefs.map((href) => this.resolveCollectionUrl(client, href)), + ), + ]; + + const calendarObjects = ( + await Promise.all( + collectionUrls.map((collectionUrl) => + client.calendarMultiGet({ + url: collectionUrl, + props: { + [`${DAVNamespaceShort.DAV}:getetag`]: {}, + [`${DAVNamespaceShort.CALDAV}:calendar-data`]: {}, + }, + objectUrls: eventHrefs.filter( + (href) => + this.resolveCollectionUrl(client, href) === collectionUrl, + ), + depth: '1', + }), + ), + ) + ).flat(); + + return calendarObjects.flatMap((calendarObject) => { + const iCalData = extractICalData(calendarObject.props?.calendarData); + + if (!isNonEmptyString(calendarObject.href) || !iCalData) return []; + + return parseICalEvents(iCalData, calendarObject.href).filter((event) => + isEventInTimeRange(event, startDate, endDate), + ); + }); + } + + private resolveCollectionUrl(client: DAVClient, href: string): string { + const collectionPath = href.slice(0, href.lastIndexOf('/') + 1); + + return new URL(collectionPath, client.serverUrl).href; + } + private async syncCalendar( client: DAVClient, calendar: DAVCalendar, - options: FetchEventsOptions, + syncCursor?: CalDavSyncCursor, ): Promise { const supportsSyncCollection = calendar.reports?.includes('syncCollection') ?? false; try { return supportsSyncCollection - ? await this.fetchEventsViaSyncCollection(client, calendar, options) - : await this.fetchEventsViaCtagEtag(client, calendar, options); + ? await this.fetchHrefsViaSyncCollection(client, calendar, syncCursor) + : await this.fetchHrefsViaCtagEtag(client, calendar, syncCursor); } catch (error) { this.logger.error(`Per-calendar sync failed for ${calendar.url}`, error); return { calendarUrl: calendar.url, - events: [], - newSyncToken: options.syncCursor?.syncTokens[calendar.url], - newCtag: options.syncCursor?.ctags?.[calendar.url], - newEtags: options.syncCursor?.etags?.[calendar.url], + changedHrefs: [], + cancelledHrefs: [], + newSyncToken: syncCursor?.syncTokens[calendar.url], + newCtag: syncCursor?.ctags?.[calendar.url], + newEtags: syncCursor?.etags?.[calendar.url], }; } } - private async fetchEventsViaSyncCollection( + private async fetchHrefsViaSyncCollection( client: DAVClient, calendar: DAVCalendar, - options: FetchEventsOptions, + syncCursor?: CalDavSyncCursor, ): Promise { - const previousSyncToken = options.syncCursor?.syncTokens[calendar.url]; + const previousSyncToken = syncCursor?.syncTokens[calendar.url]; const syncResult = await this.runSyncCollection( client, @@ -110,13 +172,6 @@ export class CalDavFetchEventsService { .filter((entry) => entry.status === 404) .map((entry) => entry.href); - const fetchedEvents = await this.fetchAndParseEvents( - client, - calendar.url, - changedHrefs, - options, - ); - const rawSyncToken = syncResult[0]?.raw?.multistatus?.syncToken; const newSyncToken = isNonEmptyString(rawSyncToken) ? rawSyncToken @@ -124,10 +179,8 @@ export class CalDavFetchEventsService { return { calendarUrl: calendar.url, - events: [ - ...fetchedEvents, - ...cancelledHrefs.map(buildCancelledCalDavEvent), - ], + changedHrefs, + cancelledHrefs, newSyncToken, }; } @@ -164,21 +217,22 @@ export class CalDavFetchEventsService { return result; } - private async fetchEventsViaCtagEtag( + private async fetchHrefsViaCtagEtag( client: DAVClient, calendar: DAVCalendar, - options: FetchEventsOptions, + syncCursor?: CalDavSyncCursor, ): Promise { - const storedEtags = options.syncCursor?.etags?.[calendar.url] ?? {}; + const storedEtags = syncCursor?.etags?.[calendar.url] ?? {}; const newCtag = isDefined(calendar.ctag) ? String(calendar.ctag) : undefined; - const storedCtag = options.syncCursor?.ctags?.[calendar.url]; + const storedCtag = syncCursor?.ctags?.[calendar.url]; if (isDefined(newCtag) && isDefined(storedCtag) && newCtag === storedCtag) { return { calendarUrl: calendar.url, - events: [], + changedHrefs: [], + cancelledHrefs: [], newCtag, newEtags: storedEtags, }; @@ -193,19 +247,10 @@ export class CalDavFetchEventsService { (href) => !(href in currentEtags), ); - const fetchedEvents = await this.fetchAndParseEvents( - client, - calendar.url, - changedHrefs, - options, - ); - return { calendarUrl: calendar.url, - events: [ - ...fetchedEvents, - ...cancelledHrefs.map(buildCancelledCalDavEvent), - ], + changedHrefs, + cancelledHrefs, newCtag, newEtags: currentEtags, }; @@ -257,34 +302,4 @@ export class CalDavFetchEventsService { return map; }, {}); } - - private async fetchAndParseEvents( - client: DAVClient, - calendarUrl: string, - objectUrls: string[], - options: { startDate: Date; endDate: Date }, - ): Promise { - if (objectUrls.length === 0) return []; - - const calendarObjects = await client.calendarMultiGet({ - url: calendarUrl, - props: { - [`${DAVNamespaceShort.DAV}:getetag`]: {}, - [`${DAVNamespaceShort.CALDAV}:calendar-data`]: {}, - }, - objectUrls, - depth: '1', - }); - - return calendarObjects.flatMap((calendarObject) => { - const iCalData = extractICalData(calendarObject.props?.calendarData); - - if (!iCalData) return []; - - return parseICalEvents(iCalData, calendarObject.href || '').filter( - (event) => - isEventInTimeRange(event, options.startDate, options.endDate), - ); - }); - } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service.ts index ba4f3b530e..fa95b02f91 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-get-events.service.ts @@ -10,9 +10,6 @@ import { type GetCalendarEventsResponse } from 'src/modules/calendar/calendar-ev export class CalDavGetEventsService { private readonly logger = new Logger(CalDavGetEventsService.name); - private static readonly PAST_DAYS_WINDOW = 365 * 5; - private static readonly FUTURE_DAYS_WINDOW = 365; - constructor( private readonly calDavClientProvider: CalDavClientProvider, private readonly fetchEventsService: CalDavFetchEventsService, @@ -28,30 +25,18 @@ export class CalDavGetEventsService { const client = await this.calDavClientProvider.getClient(connectedAccountId); - const startDate = new Date( - Date.now() - - CalDavGetEventsService.PAST_DAYS_WINDOW * 24 * 60 * 60 * 1000, + const result = await this.fetchEventsService.fetchChangedEventHrefs( + client, + syncCursor ? (JSON.parse(syncCursor) as CalDavSyncCursor) : undefined, ); - const endDate = new Date( - Date.now() + - CalDavGetEventsService.FUTURE_DAYS_WINDOW * 24 * 60 * 60 * 1000, - ); - - const result = await this.fetchEventsService.fetchEvents(client, { - startDate, - endDate, - syncCursor: syncCursor - ? (JSON.parse(syncCursor) as CalDavSyncCursor) - : undefined, - }); this.logger.debug( - `Found ${result.events.length} calendar events for ${connectedAccountId}`, + `Found ${result.changedHrefs.length} changed and ${result.cancelledHrefs.length} cancelled calendar events for ${connectedAccountId}`, ); return { - fullEvents: true, - calendarEvents: result.events, + calendarEventIds: result.changedHrefs, + calendarEventIdsToDelete: result.cancelledHrefs, nextSyncCursor: JSON.stringify(result.syncCursor), }; } catch (error) { diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service.ts new file mode 100644 index 0000000000..e9fa61b8ff --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service.ts @@ -0,0 +1,47 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { CalDavClientProvider } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/providers/caldav-client.provider'; +import { CalDavFetchEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service'; +import { parseCalDAVError } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/parse-caldav-error.util'; +import { CalendarEventImportDriverException } from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception'; +import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; + +@Injectable() +export class CalDavImportEventsService { + private readonly logger = new Logger(CalDavImportEventsService.name); + + constructor( + private readonly calDavClientProvider: CalDavClientProvider, + private readonly fetchEventsService: CalDavFetchEventsService, + ) {} + + async getCalendarEvents( + connectedAccountId: string, + eventExternalIds: string[], + ): Promise { + this.logger.debug( + `Importing ${eventExternalIds.length} calendar events for ${connectedAccountId}`, + ); + + try { + const client = + await this.calDavClientProvider.getClient(connectedAccountId); + + return await this.fetchEventsService.fetchEventsByHrefs( + client, + eventExternalIds, + ); + } catch (error) { + this.logger.error( + `Error in ${CalDavImportEventsService.name} - getCalendarEvents`, + error, + ); + + if (error instanceof CalendarEventImportDriverException) { + throw error; + } + + throw parseCalDAVError(error as Error); + } + } +} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module.ts index 3f515a547b..70ff870df5 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module.ts @@ -2,11 +2,15 @@ import { Module } from '@nestjs/common'; import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module'; import { GoogleCalendarGetEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service'; +import { GoogleCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service'; import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module'; @Module({ imports: [TwentyConfigModule, OAuth2ClientManagerModule], - providers: [GoogleCalendarGetEventsService], - exports: [GoogleCalendarGetEventsService], + providers: [ + GoogleCalendarGetEventsService, + GoogleCalendarImportEventsService, + ], + exports: [GoogleCalendarGetEventsService, GoogleCalendarImportEventsService], }) export class GoogleCalendarDriverModule {} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts index f576f45524..3463bd1702 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts @@ -2,9 +2,8 @@ import { Injectable, Logger } from '@nestjs/common'; import { isString } from '@sniptt/guards'; import { type GaxiosError } from 'gaxios'; -import { google, type calendar_v3 as calendarV3 } from 'googleapis'; +import { google } from 'googleapis'; -import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util'; import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util'; import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util'; import { type GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; @@ -34,7 +33,8 @@ export class GoogleCalendarGetEventsService { let nextSyncToken: string | null | undefined; let nextPageToken: string | undefined; - const events: calendarV3.Schema$Event[] = []; + const calendarEventIds: string[] = []; + const calendarEventIdsToDelete: string[] = []; let hasMoreEvents = true; @@ -69,7 +69,17 @@ export class GoogleCalendarGetEventsService { break; } - events.push(...items); + for (const item of items) { + if (!isString(item.id)) { + continue; + } + + if (item.status === 'cancelled') { + calendarEventIdsToDelete.push(item.id); + } else { + calendarEventIds.push(item.id); + } + } if (!nextPageToken) { hasMoreEvents = false; @@ -77,8 +87,8 @@ export class GoogleCalendarGetEventsService { } return { - fullEvents: true, - calendarEvents: formatGoogleCalendarEvents(events), + calendarEventIds, + calendarEventIdsToDelete, nextSyncCursor: nextSyncToken || '', }; } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service.ts new file mode 100644 index 0000000000..149ff8ae3b --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service.ts @@ -0,0 +1,60 @@ +import { Injectable } from '@nestjs/common'; + +import { type GaxiosError } from 'gaxios'; +import { google } from 'googleapis'; +import { isDefined } from 'twenty-shared/utils'; + +import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util'; +import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util'; +import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util'; +import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; +import { GoogleOAuth2ClientProvider } from 'src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client.provider'; +import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity'; + +@Injectable() +export class GoogleCalendarImportEventsService { + constructor( + private readonly googleOAuth2ClientProvider: GoogleOAuth2ClientProvider, + ) {} + + public async getCalendarEvents( + connectedAccount: Pick, + eventExternalIds: string[], + ): Promise { + const oAuth2Client = await this.googleOAuth2ClientProvider.getClient( + connectedAccount.id, + ); + + const googleCalendarClient = google.calendar({ + version: 'v3', + auth: oAuth2Client, + }); + + const fetchedEvents = await Promise.all( + eventExternalIds.map((eventExternalId) => + googleCalendarClient.events + .get({ calendarId: 'primary', eventId: eventExternalId }) + .then((response) => response.data) + .catch((error: GaxiosError) => { + const status = error.response?.status; + + if (status === 404 || status === 410) { + return null; + } + + if (!isDefined(status)) { + throw parseGaxiosError(error); + } + + throw parseGoogleCalendarError({ + code: status, + reason: error.response?.data?.error?.errors?.[0].reason || '', + message: error.response?.data?.error?.errors?.[0].message || '', + }); + }), + ), + ); + + return formatGoogleCalendarEvents(fetchedEvents.filter(isDefined)); + } +} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts index 28d15bcc8d..90e00142c2 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts @@ -25,6 +25,7 @@ export class MicrosoftCalendarGetEventsService { const microsoftClient = await this.microsoftOAuth2ClientProvider.getClient(connectedAccount.id); const eventIds: string[] = []; + const eventIdsToDelete: string[] = []; const response: PageCollection = await microsoftClient .api(syncCursor || '/me/calendar/events/delta') @@ -32,7 +33,11 @@ export class MicrosoftCalendarGetEventsService { .get(); const callback: PageIteratorCallback = (data) => { - eventIds.push(data.id); + if (data['@removed']) { + eventIdsToDelete.push(data.id); + } else { + eventIds.push(data.id); + } return true; }; @@ -46,8 +51,8 @@ export class MicrosoftCalendarGetEventsService { await pageIterator.iterate(); return { - fullEvents: false, calendarEventIds: eventIds, + calendarEventIdsToDelete: eventIdsToDelete, nextSyncCursor: pageIterator.getDeltaLink() || '', }; } catch (error) { diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts index 05ab092689..7baacced00 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts @@ -16,7 +16,7 @@ export class MicrosoftCalendarImportEventsService { public async getCalendarEvents( connectedAccount: Pick, - changedEventIds: string[], + eventExternalIds: string[], ): Promise { try { const microsoftClient = @@ -24,9 +24,9 @@ export class MicrosoftCalendarImportEventsService { const events: Event[] = []; - for (const changedEventId of changedEventIds) { + for (const eventExternalId of eventExternalIds) { const event = await microsoftClient - .api(`/me/calendar/events/${changedEventId}`) + .api(`/me/calendar/events/${eventExternalId}`) .get(); events.push(event); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/__tests__/calendar-fetch-events.service.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/__tests__/calendar-fetch-events.service.spec.ts index 080e741808..a8ee330854 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/__tests__/calendar-fetch-events.service.spec.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/__tests__/calendar-fetch-events.service.spec.ts @@ -16,10 +16,6 @@ const mockCalendarChannelRepository = { update: jest.fn(), }; -const mockCalendarEventsImportService = { - processCalendarEventsImport: jest.fn(), -}; - const mockCalendarEventImportErrorHandlerService = { handleDriverException: jest.fn(), }; @@ -34,6 +30,10 @@ const mockGlobalWorkspaceOrmManager = { }), }; +const mockCalendarEventCleanerService = { + cleanWorkspaceCalendarEvents: jest.fn(), +}; + const workspaceId = 'workspace-123'; const baseConnectedAccount = { @@ -60,8 +60,8 @@ describe('CalendarFetchEventsService', () => { jest.clearAllMocks(); mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({ - fullEvents: true, - calendarEvents: [{ id: 'event-1' }], + calendarEventIds: ['event-1'], + calendarEventIdsToDelete: [], nextSyncCursor: 'new-cursor-abc', }); @@ -72,7 +72,7 @@ describe('CalendarFetchEventsService', () => { mockCalendarChannelSyncStatusService as any, mockGetCalendarEventsService as any, mockCalendarEventImportErrorHandlerService as any, - mockCalendarEventsImportService as any, + mockCalendarEventCleanerService as any, ); }); @@ -157,8 +157,8 @@ describe('CalendarFetchEventsService', () => { jest.clearAllMocks(); mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({ - fullEvents: true, - calendarEvents: [{ id: 'event-2' }], + calendarEventIds: ['event-2'], + calendarEventIdsToDelete: [], nextSyncCursor: 'new-cursor-def', }); @@ -194,8 +194,8 @@ describe('CalendarFetchEventsService', () => { jest.clearAllMocks(); mockGetCalendarEventsService.getCalendarEvents.mockResolvedValue({ - fullEvents: true, - calendarEvents: [{ id: 'event-3' }], + calendarEventIds: ['event-3'], + calendarEventIdsToDelete: [], nextSyncCursor: 'newer-cursor', }); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts index d8c053742a..6eb0a48162 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts @@ -21,16 +21,15 @@ import { CalendarEventImportDriverException, CalendarEventImportDriverExceptionCode, } from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception'; -import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service'; import { CalendarEventImportErrorHandlerService, CalendarEventImportSyncStep, } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service'; +import { CalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity'; -import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service'; import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -46,7 +45,7 @@ export class CalendarEventsImportService { private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService, private readonly calendarSaveEventsService: CalendarSaveEventsService, private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService, - private readonly microsoftCalendarImportEventService: MicrosoftCalendarImportEventsService, + private readonly calendarImportEventsService: CalendarImportEventsService, private readonly emailAliasManagerService: EmailAliasManagerService, @InjectRepository(UserWorkspaceEntity) private readonly userWorkspaceRepository: Repository, @@ -56,7 +55,6 @@ export class CalendarEventsImportService { calendarChannel: CalendarChannelEntity, connectedAccount: ConnectedAccountEntity, workspaceId: string, - fetchedCalendarEvents?: FetchedCalendarEvent[], ): Promise { await this.calendarChannelSyncStatusService.markAsCalendarEventsImportOngoing( [calendarChannel.id], @@ -67,46 +65,27 @@ export class CalendarEventsImportService { await this.globalWorkspaceOrmManager.executeInWorkspaceContext( async () => { - let calendarEvents: FetchedCalendarEvent[] = []; - try { - if (fetchedCalendarEvents) { - calendarEvents = fetchedCalendarEvents; - } else { - const eventIdsToFetch: string[] = await this.cacheStorage.setPop( - `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, - CALENDAR_EVENT_IMPORT_BATCH_SIZE, - ); + const eventIdsToFetch: string[] = await this.cacheStorage.setPop( + `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, + CALENDAR_EVENT_IMPORT_BATCH_SIZE, + ); - if (!eventIdsToFetch || eventIdsToFetch.length === 0) { - await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( - [calendarChannel.id], - workspaceId, - ); - - return; - } - - switch (connectedAccount.provider) { - case 'microsoft': - calendarEvents = - await this.microsoftCalendarImportEventService.getCalendarEvents( - connectedAccount, - eventIdsToFetch, - ); - break; - default: - break; - } - } - - if (!calendarEvents || calendarEvents?.length === 0) { - await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( + if (!eventIdsToFetch || eventIdsToFetch.length === 0) { + await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( [calendarChannel.id], workspaceId, ); + + return; } + const calendarEvents = + await this.calendarImportEventsService.getCalendarEvents( + connectedAccount, + eventIdsToFetch, + ); + const userWorkspace = await this.userWorkspaceRepository.findOne({ where: { id: (connectedAccount as unknown as { userWorkspaceId: string }) @@ -190,10 +169,17 @@ export class CalendarEventsImportService { workspaceId, ); - await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( - [calendarChannel.id], - workspaceId, - ); + if (eventIdsToFetch.length < CALENDAR_EVENT_IMPORT_BATCH_SIZE) { + await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( + [calendarChannel.id], + workspaceId, + ); + } else { + await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( + [calendarChannel.id], + workspaceId, + ); + } } catch (error) { await this.calendarEventImportErrorHandlerService.handleDriverException( error, diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts index d30092bf5a..50ae990e4b 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts @@ -1,24 +1,21 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { Any, Repository } from 'typeorm'; import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util'; -import { - CalendarEventImportDriverException, - CalendarEventImportDriverExceptionCode, -} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception'; +import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service'; import { CalendarEventImportErrorHandlerService, CalendarEventImportSyncStep, } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service'; -import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service'; import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity'; import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity'; import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity'; @@ -34,7 +31,7 @@ export class CalendarFetchEventsService { private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService, private readonly getCalendarEventsService: CalendarGetCalendarEventsService, private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService, - private readonly calendarEventsImportService: CalendarEventsImportService, + private readonly calendarEventCleanerService: CalendarEventCleanerService, ) {} public async fetchCalendarEvents( @@ -56,28 +53,40 @@ export class CalendarFetchEventsService { await this.globalWorkspaceOrmManager.executeInWorkspaceContext( async () => { try { - const getCalendarEventsResponse = + const { calendarEventIds, calendarEventIdsToDelete, nextSyncCursor } = await this.getCalendarEventsService.getCalendarEvents( connectedAccount, calendarChannel.syncCursor || undefined, ); - const hasFullEvents = getCalendarEventsResponse.fullEvents; + if (calendarEventIdsToDelete.length > 0) { + const calendarChannelEventAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarChannelEventAssociation', + ); - const calendarEvents = hasFullEvents - ? getCalendarEventsResponse.calendarEvents - : null; - const calendarEventIds = getCalendarEventsResponse.calendarEventIds; - const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor; + await calendarChannelEventAssociationRepository.delete({ + eventExternalId: Any(calendarEventIdsToDelete), + calendarChannelId: calendarChannel.id, + }); - if (!calendarEvents || calendarEvents?.length === 0) { - await this.calendarChannelRepository.update( - { id: calendarChannel.id, workspaceId }, - { - syncCursor: nextSyncCursor, - }, + await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( + workspaceId, + ); + } + + if (calendarEventIds.length > 0) { + await this.cacheStorage.setAdd( + `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, + calendarEventIds, ); + await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( + [calendarChannel.id], + workspaceId, + ); + } else { await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( [calendarChannel.id], workspaceId, @@ -90,30 +99,6 @@ export class CalendarFetchEventsService { syncCursor: nextSyncCursor, }, ); - - if (hasFullEvents && calendarEvents) { - await this.calendarEventsImportService.processCalendarEventsImport( - calendarChannel, - connectedAccount, - workspaceId, - calendarEvents, - ); - } else if (!hasFullEvents && calendarEventIds) { - await this.cacheStorage.setAdd( - `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, - calendarEventIds, - ); - - await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( - [calendarChannel.id], - workspaceId, - ); - } else { - throw new CalendarEventImportDriverException( - "Expected 'calendarEvents' or 'calendarEventIds' to be present", - CalendarEventImportDriverExceptionCode.UNKNOWN, - ); - } } catch (error) { this.logger.error( `WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`, diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service.ts index e430f98cdf..2b011dfca9 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service.ts @@ -9,13 +9,11 @@ import { CalendarEventImportException, CalendarEventImportExceptionCode, } from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception'; -import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity'; export type GetCalendarEventsResponse = { - fullEvents: boolean; - calendarEvents?: FetchedCalendarEvent[]; - calendarEventIds?: string[]; + calendarEventIds: string[]; + calendarEventIdsToDelete: string[]; nextSyncCursor: string; }; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service.ts new file mode 100644 index 0000000000..3f33bae2ef --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-import-events.service.ts @@ -0,0 +1,50 @@ +import { Injectable } from '@nestjs/common'; + +import { ConnectedAccountProvider } from 'twenty-shared/types'; + +import { CalDavImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-import-events.service'; +import { GoogleCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-import-events.service'; +import { MicrosoftCalendarImportEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service'; +import { + CalendarEventImportException, + CalendarEventImportExceptionCode, +} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception'; +import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; +import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity'; + +@Injectable() +export class CalendarImportEventsService { + constructor( + private readonly googleCalendarImportEventsService: GoogleCalendarImportEventsService, + private readonly microsoftCalendarImportEventsService: MicrosoftCalendarImportEventsService, + private readonly caldavCalendarImportEventsService: CalDavImportEventsService, + ) {} + + public async getCalendarEvents( + connectedAccount: Pick, + eventExternalIds: string[], + ): Promise { + switch (connectedAccount.provider) { + case ConnectedAccountProvider.GOOGLE: + return this.googleCalendarImportEventsService.getCalendarEvents( + connectedAccount, + eventExternalIds, + ); + case ConnectedAccountProvider.MICROSOFT: + return this.microsoftCalendarImportEventsService.getCalendarEvents( + connectedAccount, + eventExternalIds, + ); + case ConnectedAccountProvider.IMAP_SMTP_CALDAV: + return this.caldavCalendarImportEventsService.getCalendarEvents( + connectedAccount.id, + eventExternalIds, + ); + default: + throw new CalendarEventImportException( + `Provider ${connectedAccount.provider} is not supported`, + CalendarEventImportExceptionCode.PROVIDER_NOT_SUPPORTED, + ); + } + } +}