diff --git a/packages/twenty-server/nest-cli.json b/packages/twenty-server/nest-cli.json index 047cfbae88..ee2fe2a5a5 100644 --- a/packages/twenty-server/nest-cli.json +++ b/packages/twenty-server/nest-cli.json @@ -29,6 +29,10 @@ { "include": "**/database/clickHouse/migrations/*.sql", "outDir": "dist/src" + }, + { + "include": "**/dev-seeder/data/sample-files/**", + "outDir": "dist/assets" } ], "watchAssets": true diff --git a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts index d2cc311a8f..3fb78e9b96 100644 --- a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts +++ b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts @@ -7,16 +7,32 @@ config({ override: true, }); -const getLoggingConfig = (): LogLevel[] => { - if (process.env.NODE_ENV === 'development') { - return ['query', 'error']; - } +const isRunningCommand = (): boolean => { + const scriptPath = process.argv[1] || ''; + return scriptPath.includes('/command/command.'); +}; + +const getLoggingConfig = (): LogLevel[] => { if (process.env.NODE_ENV === 'test') { return []; } + const ormQueryLogging = process.env.ORM_QUERY_LOGGING || 'disabled'; - return ['error']; + switch (ormQueryLogging) { + case 'disabled': + return ['error']; + case 'server-only': + if (isRunningCommand()) { + return ['error']; + } + + return ['query', 'error']; + case 'always': + return ['query', 'error']; + default: + return ['error']; + } }; const isJest = process.argv.some((arg) => arg.includes('jest')); diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant.ts new file mode 100644 index 0000000000..5ae6002e08 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant.ts @@ -0,0 +1,190 @@ +import { COMPANY_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/company-data-seeds.constant'; +import { NOTE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-data-seeds.constant'; +import { OPPORTUNITY_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/opportunity-data-seeds.constant'; +import { PERSON_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant'; +import { TASK_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-data-seeds.constant'; +import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant'; + +type AttachmentDataSeed = { + id: string; + name: string; + fullPath: string; + type: string; + authorId: string | null; + // createdBySource: string; + // createdByWorkspaceMemberId: string; + // createdByName: string; + personId: string | null; + companyId: string | null; + noteId: string | null; + taskId: string | null; + opportunityId: string | null; +}; + +export const ATTACHMENT_DATA_SEED_COLUMNS: (keyof AttachmentDataSeed)[] = [ + 'id', + 'name', + 'fullPath', + 'type', + 'authorId', + // 'createdBySource', + // 'createdByWorkspaceMemberId', + // 'createdByName', + 'personId', + 'companyId', + 'noteId', + 'taskId', + 'opportunityId', +]; + +const GENERATE_ATTACHMENT_IDS = (): Record => { + const ATTACHMENT_IDS: Record = {}; + + for (let INDEX = 1; INDEX <= 400; INDEX++) { + const HEX_INDEX = INDEX.toString(16).padStart(4, '0'); + + ATTACHMENT_IDS[`ID_${INDEX}`] = + `20202020-${HEX_INDEX}-4a7c-8001-123456789aba`; + } + + return ATTACHMENT_IDS; +}; + +export const ATTACHMENT_DATA_SEED_IDS = GENERATE_ATTACHMENT_IDS(); + +// Pool of 5 reusable file templates +const FILE_TEMPLATES = [ + { + name: 'Contract Agreement.pdf', + fullPath: 'attachment/sample-contract.pdf', + type: 'TextDocument', + }, + { + name: 'Budget 2024.xlsx', + fullPath: 'attachment/budget-2024.xlsx', + type: 'Spreadsheet', + }, + { + name: 'Product Presentation.pptx', + fullPath: 'attachment/presentation.pptx', + type: 'Presentation', + }, + { + name: 'Screenshot.png', + fullPath: 'attachment/screenshot.png', + type: 'Image', + }, + { + name: 'Archive.zip', + fullPath: 'attachment/archive.zip', + type: 'Archive', + }, +]; + +// Additional name variations for more realistic variety +const FILE_NAME_VARIATIONS = [ + // Documents + { name: 'Service Agreement.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'NDA Document.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'Project Proposal.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'Invoice Q1 2024.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'Meeting Notes.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'Report Final.pdf', type: 'TextDocument', pathIndex: 0 }, + { name: 'Contract Signed.pdf', type: 'TextDocument', pathIndex: 0 }, + // Spreadsheets + { name: 'Financial Forecast.xlsx', type: 'Spreadsheet', pathIndex: 1 }, + { name: 'Sales Report Q4.xlsx', type: 'Spreadsheet', pathIndex: 1 }, + { name: 'Team Roster.xlsx', type: 'Spreadsheet', pathIndex: 1 }, + { name: 'Expense Report.xlsx', type: 'Spreadsheet', pathIndex: 1 }, + { name: 'Inventory List.xlsx', type: 'Spreadsheet', pathIndex: 1 }, + { name: 'Data Export.csv', type: 'Spreadsheet', pathIndex: 1 }, + // Presentations + { name: 'Pitch Deck.pptx', type: 'Presentation', pathIndex: 2 }, + { name: 'Q4 Results.pptx', type: 'Presentation', pathIndex: 2 }, + { name: 'Roadmap 2024.pptx', type: 'Presentation', pathIndex: 2 }, + { name: 'Company Overview.pptx', type: 'Presentation', pathIndex: 2 }, + { name: 'Training Materials.pptx', type: 'Presentation', pathIndex: 2 }, + // Images + { name: 'Company Logo.png', type: 'Image', pathIndex: 3 }, + { name: 'Product Photo.jpg', type: 'Image', pathIndex: 3 }, + { name: 'Diagram.png', type: 'Image', pathIndex: 3 }, + { name: 'Wireframe.png', type: 'Image', pathIndex: 3 }, + { name: 'Mockup Design.png', type: 'Image', pathIndex: 3 }, + { name: 'Headshot.jpg', type: 'Image', pathIndex: 3 }, + // Archives + { name: 'Project Files.zip', type: 'Archive', pathIndex: 4 }, + { name: 'Backup Data.zip', type: 'Archive', pathIndex: 4 }, + { name: 'Source Code.zip', type: 'Archive', pathIndex: 4 }, +]; + +const GENERATE_ATTACHMENT_SEEDS = (): AttachmentDataSeed[] => { + const ATTACHMENT_SEEDS: AttachmentDataSeed[] = []; + + // Get available entity IDs + const PERSON_IDS = Object.values(PERSON_DATA_SEED_IDS).slice(0, 120); // Use first 120 persons + const COMPANY_IDS = Object.values(COMPANY_DATA_SEED_IDS).slice(0, 120); // Use first 120 companies + const NOTE_IDS = Object.values(NOTE_DATA_SEED_IDS).slice(0, 80); // Use first 80 notes + const TASK_IDS = Object.values(TASK_DATA_SEED_IDS).slice(0, 60); // Use first 60 tasks + const OPPORTUNITY_IDS = Object.values(OPPORTUNITY_DATA_SEED_IDS).slice(0, 20); // Use first 20 opportunities + + let entityIndex = 0; + + for (let INDEX = 1; INDEX <= 400; INDEX++) { + // Cycle through file name variations + const NAME_VARIATION_INDEX = INDEX % FILE_NAME_VARIATIONS.length; + const NAME_VARIATION = FILE_NAME_VARIATIONS[NAME_VARIATION_INDEX]; + const FILE_TEMPLATE = FILE_TEMPLATES[NAME_VARIATION.pathIndex]; + + // Determine which entity this attachment belongs to + // Distribution: ~30% person, ~30% company, ~20% note, ~15% task, ~5% opportunity + let personId: string | null = null; + let companyId: string | null = null; + let noteId: string | null = null; + let taskId: string | null = null; + let opportunityId: string | null = null; + + const DISTRIBUTION_VALUE = INDEX % 100; + + if (DISTRIBUTION_VALUE < 30) { + // 30% Person attachments + personId = PERSON_IDS[entityIndex % PERSON_IDS.length]; + entityIndex++; + } else if (DISTRIBUTION_VALUE < 60) { + // 30% Company attachments + companyId = COMPANY_IDS[entityIndex % COMPANY_IDS.length]; + entityIndex++; + } else if (DISTRIBUTION_VALUE < 80) { + // 20% Note attachments + noteId = NOTE_IDS[entityIndex % NOTE_IDS.length]; + entityIndex++; + } else if (DISTRIBUTION_VALUE < 95) { + // 15% Task attachments + taskId = TASK_IDS[entityIndex % TASK_IDS.length]; + entityIndex++; + } else { + // 5% Opportunity attachments + opportunityId = OPPORTUNITY_IDS[entityIndex % OPPORTUNITY_IDS.length]; + entityIndex++; + } + + ATTACHMENT_SEEDS.push({ + id: ATTACHMENT_DATA_SEED_IDS[`ID_${INDEX}`], + name: NAME_VARIATION.name, + fullPath: FILE_TEMPLATE.fullPath, + type: NAME_VARIATION.type, + authorId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM, + // createdBySource: 'MANUAL', + //createdByWorkspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM, + //createdByName: 'Tim A', + personId, + companyId, + noteId, + taskId, + opportunityId, + }); + } + + return ATTACHMENT_SEEDS; +}; + +export const ATTACHMENT_DATA_SEEDS = GENERATE_ATTACHMENT_SEEDS(); diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant.ts index e5de787cec..746eaa4885 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant.ts @@ -75,3 +75,10 @@ const GENERATE_NOTE_TARGET_SEEDS = (): NoteTargetDataSeed[] => { }; export const NOTE_TARGET_DATA_SEEDS = GENERATE_NOTE_TARGET_SEEDS(); + +// Map for O(1) lookups by note ID +export const NOTE_TARGET_DATA_SEEDS_MAP = new Map( + NOTE_TARGET_DATA_SEEDS.filter((target) => target.noteId !== null).map( + (target) => [target.noteId!, target], + ), +); diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant.ts index 3ff5792c7e..a2109736bb 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant.ts @@ -21653,3 +21653,8 @@ export const PERSON_DATA_SEEDS: PersonDataSeed[] = PERSON_DATA_SEEDS_RAW.map( position: index + 1, }), ); + +// Map for O(1) lookups by person ID +export const PERSON_DATA_SEEDS_MAP = new Map( + PERSON_DATA_SEEDS.map((p) => [p.id, p]), +); diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant.ts index 3c715483c5..2648ef9562 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant.ts @@ -81,3 +81,10 @@ const GENERATE_TASK_TARGET_SEEDS = (): TaskTargetDataSeed[] => { }; export const TASK_TARGET_DATA_SEEDS = GENERATE_TASK_TARGET_SEEDS(); + +// Map for O(1) lookups by task ID +export const TASK_TARGET_DATA_SEEDS_MAP = new Map( + TASK_TARGET_DATA_SEEDS.filter((target) => target.taskId !== null).map( + (target) => [target.taskId!, target], + ), +); diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/archive.zip b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/archive.zip new file mode 100644 index 0000000000..a1c180d206 Binary files /dev/null and b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/archive.zip differ diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/budget-2024.xlsx b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/budget-2024.xlsx new file mode 100644 index 0000000000..93806cc3e0 Binary files /dev/null and b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/budget-2024.xlsx differ diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/presentation.pptx b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/presentation.pptx new file mode 100644 index 0000000000..437da68b45 Binary files /dev/null and b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/presentation.pptx differ diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/sample-contract.pdf b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/sample-contract.pdf new file mode 100644 index 0000000000..49638b2b80 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/sample-contract.pdf @@ -0,0 +1,62 @@ +%PDF-1.4 +1 0 obj +<< +/Type /Catalog +/Pages 2 0 R +>> +endobj +2 0 obj +<< +/Type /Pages +/Kids [3 0 R] +/Count 1 +>> +endobj +3 0 obj +<< +/Type /Page +/Parent 2 0 R +/MediaBox [0 0 612 792] +/Contents 4 0 R +/Resources << +/Font << +/F1 5 0 R +>> +>> +>> +endobj +4 0 obj +<< +/Length 44 +>> +stream +BT +/F1 12 Tf +100 700 Td +(Sample Contract Document) Tj +ET +endstream +endobj +5 0 obj +<< +/Type /Font +/Subtype /Type1 +/BaseFont /Helvetica +>> +endobj +xref +0 6 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000274 00000 n +0000000367 00000 n +trailer +<< +/Size 6 +/Root 1 0 R +>> +startxref +445 +%%EOF diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/screenshot.png b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/screenshot.png new file mode 100644 index 0000000000..08cd6f2bfd Binary files /dev/null and b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/screenshot.png differ diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts index b6a4c8487d..7a6b99eb94 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts @@ -1,12 +1,21 @@ import { Injectable } from '@nestjs/common'; import { InjectDataSource } from '@nestjs/typeorm'; +import { readFile } from 'fs/promises'; +import { join } from 'path'; + import { DataSource } from 'typeorm'; import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; +import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { + ATTACHMENT_DATA_SEED_COLUMNS, + ATTACHMENT_DATA_SEEDS, +} from 'src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant'; import { CALENDAR_CHANNEL_DATA_SEED_COLUMNS, CALENDAR_CHANNEL_DATA_SEEDS, @@ -94,120 +103,157 @@ import { import { TimelineActivitySeederService } from 'src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service'; import { prefillWorkflows } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-workflows'; -const getRecordSeedsConfigs = ( +type RecordSeedConfig = { + tableName: string; + pgColumns: string[]; + recordSeeds: Record[]; +}; + +// Organize seeds into dependency batches for parallel insertion +const getRecordSeedsBatches = ( workspaceId: string, featureFlags?: Record, -) => [ - { - tableName: 'workspaceMember', - pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS, - recordSeeds: getWorkspaceMemberDataSeeds(workspaceId), - }, - { - tableName: 'company', - pgColumns: COMPANY_DATA_SEED_COLUMNS, - recordSeeds: COMPANY_DATA_SEEDS, - }, - ...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED] - ? [ - { - tableName: 'dashboard', - pgColumns: DASHBOARD_DATA_SEED_COLUMNS, - recordSeeds: getDashboardDataSeeds(workspaceId), - }, - ] - : []), - { - tableName: 'person', - pgColumns: PERSON_DATA_SEED_COLUMNS, - recordSeeds: PERSON_DATA_SEEDS, - }, - { - tableName: 'note', - pgColumns: NOTE_DATA_SEED_COLUMNS, - recordSeeds: NOTE_DATA_SEEDS, - }, - { - tableName: 'noteTarget', - pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS, - recordSeeds: NOTE_TARGET_DATA_SEEDS, - }, - { - tableName: 'opportunity', - pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS, - recordSeeds: OPPORTUNITY_DATA_SEEDS, - }, - { - tableName: 'connectedAccount', - pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS, - recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS, - }, - { - tableName: 'calendarChannel', - pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS, - recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS, - }, - { - tableName: 'calendarEvent', - pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS, - recordSeeds: CALENDAR_EVENT_DATA_SEEDS, - }, - { - tableName: 'calendarChannelEventAssociation', - pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS, - recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS, - }, - { - tableName: 'calendarEventParticipant', - pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS, - recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId), - }, - { - tableName: 'messageChannel', - pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS, - recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS, - }, - { - tableName: 'messageThread', - pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS, - recordSeeds: MESSAGE_THREAD_DATA_SEEDS, - }, - { - tableName: 'message', - pgColumns: MESSAGE_DATA_SEED_COLUMNS, - recordSeeds: MESSAGE_DATA_SEEDS, - }, - { - tableName: 'messageChannelMessageAssociation', - pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS, - recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS, - }, - { - tableName: 'messageParticipant', - pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS, - recordSeeds: getMessageParticipantDataSeeds(workspaceId), - }, - { - tableName: '_pet', - pgColumns: PET_DATA_SEED_COLUMNS, - recordSeeds: PET_DATA_SEEDS, - }, - { - tableName: '_surveyResult', - pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS, - recordSeeds: SURVEY_RESULT_DATA_SEEDS, - }, - { - tableName: 'task', - pgColumns: TASK_DATA_SEED_COLUMNS, - recordSeeds: TASK_DATA_SEEDS, - }, - { - tableName: 'taskTarget', - pgColumns: TASK_TARGET_DATA_SEED_COLUMNS, - recordSeeds: TASK_TARGET_DATA_SEEDS, - }, -]; +): RecordSeedConfig[][] => { + // Batch 1: No dependencies + const batch1: RecordSeedConfig[] = [ + { + tableName: 'workspaceMember', + pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS, + recordSeeds: getWorkspaceMemberDataSeeds(workspaceId), + }, + { + tableName: '_surveyResult', + pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS, + recordSeeds: SURVEY_RESULT_DATA_SEEDS, + }, + ]; + + // Batch 2: Depends on workspaceMember + const batch2: RecordSeedConfig[] = [ + { + tableName: 'company', + pgColumns: COMPANY_DATA_SEED_COLUMNS, + recordSeeds: COMPANY_DATA_SEEDS, + }, + { + tableName: 'connectedAccount', + pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS, + recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS, + }, + ...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED] + ? [ + { + tableName: 'dashboard', + pgColumns: DASHBOARD_DATA_SEED_COLUMNS, + recordSeeds: getDashboardDataSeeds(workspaceId), + }, + ] + : []), + ]; + + // Batch 3: Depends on company and connectedAccount + const batch3: RecordSeedConfig[] = [ + { + tableName: 'person', + pgColumns: PERSON_DATA_SEED_COLUMNS, + recordSeeds: PERSON_DATA_SEEDS, + }, + { + tableName: '_pet', + pgColumns: PET_DATA_SEED_COLUMNS, + recordSeeds: PET_DATA_SEEDS, + }, + { + tableName: 'calendarChannel', + pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS, + recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS, + }, + { + tableName: 'messageChannel', + pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS, + recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS, + }, + ]; + + // Batch 4: Depends on person/company or independent + const batch4: RecordSeedConfig[] = [ + { + tableName: 'opportunity', + pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS, + recordSeeds: OPPORTUNITY_DATA_SEEDS, + }, + { + tableName: 'note', + pgColumns: NOTE_DATA_SEED_COLUMNS, + recordSeeds: NOTE_DATA_SEEDS, + }, + { + tableName: 'task', + pgColumns: TASK_DATA_SEED_COLUMNS, + recordSeeds: TASK_DATA_SEEDS, + }, + { + tableName: 'calendarEvent', + pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS, + recordSeeds: CALENDAR_EVENT_DATA_SEEDS, + }, + { + tableName: 'messageThread', + pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS, + recordSeeds: MESSAGE_THREAD_DATA_SEEDS, + }, + ]; + + // Batch 5: Depends on batch 4 entities + const batch5: RecordSeedConfig[] = [ + { + tableName: 'noteTarget', + pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS, + recordSeeds: NOTE_TARGET_DATA_SEEDS, + }, + { + tableName: 'taskTarget', + pgColumns: TASK_TARGET_DATA_SEED_COLUMNS, + recordSeeds: TASK_TARGET_DATA_SEEDS, + }, + { + tableName: 'calendarChannelEventAssociation', + pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS, + recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS, + }, + { + tableName: 'calendarEventParticipant', + pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS, + recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId), + }, + { + tableName: 'message', + pgColumns: MESSAGE_DATA_SEED_COLUMNS, + recordSeeds: MESSAGE_DATA_SEEDS, + }, + ]; + + // Batch 6: Depends on batch 5 entities + const batch6: RecordSeedConfig[] = [ + { + tableName: 'messageChannelMessageAssociation', + pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS, + recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS, + }, + { + tableName: 'messageParticipant', + pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS, + recordSeeds: getMessageParticipantDataSeeds(workspaceId), + }, + { + tableName: 'attachment', + pgColumns: ATTACHMENT_DATA_SEED_COLUMNS, + recordSeeds: ATTACHMENT_DATA_SEEDS, + }, + ]; + + return [batch1, batch2, batch3, batch4, batch5, batch6]; +}; @Injectable() export class DevSeederDataService { @@ -216,6 +262,7 @@ export class DevSeederDataService { private readonly coreDataSource: DataSource, private readonly objectMetadataService: ObjectMetadataService, private readonly timelineActivitySeederService: TimelineActivitySeederService, + private readonly fileStorageService: FileStorageService, ) {} public async seed({ @@ -232,10 +279,47 @@ export class DevSeederDataService { await this.coreDataSource.transaction( async (entityManager: WorkspaceEntityManager) => { - for (const recordSeedsConfig of getRecordSeedsConfigs( + await this.seedRecordsInBatches({ + entityManager, + schemaName, workspaceId, featureFlags, - )) { + objectMetadataItems, + }); + + await this.timelineActivitySeederService.seedTimelineActivities({ + entityManager, + schemaName, + workspaceId, + }); + + await this.seedAttachmentFiles(workspaceId); + + await prefillWorkflows(entityManager, schemaName, objectMetadataItems); + }, + ); + } + + private async seedRecordsInBatches({ + entityManager, + schemaName, + workspaceId, + featureFlags, + objectMetadataItems, + }: { + entityManager: WorkspaceEntityManager; + schemaName: string; + workspaceId: string; + featureFlags?: Record; + objectMetadataItems: ObjectMetadataEntity[]; + }) { + const batches = getRecordSeedsBatches(workspaceId, featureFlags); + + // Process batches sequentially (respecting dependencies) + // but entities within each batch in parallel + for (const batch of batches) { + await Promise.all( + batch.map(async (recordSeedsConfig) => { const objectMetadata = objectMetadataItems.find( (item) => computeTableName(item.nameSingular, item.isCustom) === @@ -244,7 +328,7 @@ export class DevSeederDataService { if (!objectMetadata) { // TODO this continue is hacky, we should have a record seed config per workspace - continue; + return; } await this.seedRecords({ @@ -254,17 +338,9 @@ export class DevSeederDataService { pgColumns: recordSeedsConfig.pgColumns, recordSeeds: recordSeedsConfig.recordSeeds, }); - } - - await this.timelineActivitySeederService.seedTimelineActivities({ - entityManager, - schemaName, - workspaceId, - }); - - await prefillWorkflows(entityManager, schemaName, objectMetadataItems); - }, - ); + }), + ); + } } private async seedRecords({ @@ -288,7 +364,51 @@ export class DevSeederDataService { .into(`${schemaName}.${tableName}`, pgColumns) .orIgnore() .values(recordSeeds) - .returning('*') .execute(); } + + private async seedAttachmentFiles(workspaceId: string): Promise { + // Files are copied to dist/assets during build via nest-cli.json + // The pattern **/dev-seeder/data/sample-files/** preserves the full path + const IS_BUILT = __dirname.includes('/dist/'); + const sampleFilesDir = IS_BUILT + ? join( + __dirname, + '../../../../../../assets/engine/workspace-manager/dev-seeder/data/sample-files', + ) + : join(__dirname, '../sample-files'); + + const filesToCreate = [ + 'sample-contract.pdf', + 'budget-2024.xlsx', + 'presentation.pptx', + 'screenshot.png', + 'archive.zip', + ]; + + for (const filename of filesToCreate) { + const filePath = join(sampleFilesDir, filename); + const fileBuffer = await readFile(filePath); + + await this.fileStorageService.write({ + file: fileBuffer, + name: filename, + folder: `workspace-${workspaceId}/attachment`, + mimeType: this.getMimeType(filename), + }); + } + } + + private getMimeType(filename: string): string { + const ext = filename.split('.').pop()?.toLowerCase(); + const mimeTypes: Record = { + pdf: 'application/pdf', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + png: 'image/png', + zip: 'application/zip', + }; + + return mimeTypes[ext || ''] || 'application/octet-stream'; + } } diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts index 1ea4204f46..4d3a297287 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import chunk from 'lodash.chunk'; +import { ObjectRecord } from 'twenty-shared/types'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; @@ -16,14 +17,19 @@ import { MessageParticipantDataSeed, } from 'src/engine/workspace-manager/dev-seeder/data/constants/message-participant-data-seeds.constant'; import { NOTE_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-data-seeds.constant'; -import { NOTE_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant'; +import { NOTE_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant'; import { OPPORTUNITY_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/opportunity-data-seeds.constant'; -import { PERSON_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant'; +import { + PERSON_DATA_SEEDS, + PERSON_DATA_SEEDS_MAP, +} from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant'; import { TASK_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-data-seeds.constant'; -import { TASK_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant'; +import { TASK_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant'; import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant'; import { type TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; +type RecordSeedWithId = Pick & Record; + type TimelineActivitySeedData = Pick< TimelineActivityWorkspaceEntity, | 'id' @@ -51,13 +57,13 @@ type ActivityTargetInfo = { type CreateTimelineActivityParams = { entityType: string; - recordSeed: Record; + recordSeed: RecordSeedWithId; index: number; }; type CreateLinkedActivityParams = { activityType: 'note' | 'task' | 'calendarEvent' | 'message'; - recordSeed: Record; + recordSeed: RecordSeedWithId; index: number; activityIndex: number; linkedObjectMetadataId: string; @@ -66,7 +72,7 @@ type CreateLinkedActivityParams = { type EntityConfig = { type: string; - seeds: Array>; + seeds: Array; }; type ObjectMetadataIds = { @@ -230,7 +236,7 @@ export class TimelineActivitySeederService { return; } - const batchSize = 100; + const batchSize = 1000; const timelineActivityBatches = chunk(timelineActivities, batchSize); for (const batch of timelineActivityBatches) { @@ -272,7 +278,7 @@ export class TimelineActivitySeederService { index + 1, ); const creationDate = new Date().toISOString(); - const recordId = String(recordSeed.id || ''); + const recordId = recordSeed.id; const timelineActivity: TimelineActivitySeedData = { id: timelineActivityId, @@ -314,7 +320,7 @@ export class TimelineActivitySeederService { private getEventAfterRecordProperties( type: string, - recordSeed: Record, + recordSeed: RecordSeedWithId, ): Record { const commonProperties = { id: recordSeed.id }; @@ -385,7 +391,7 @@ export class TimelineActivitySeederService { messageParticipants, }: { activityType: 'note' | 'task' | 'calendarEvent' | 'message'; - recordSeed: Record; + recordSeed: RecordSeedWithId; index: number; activityIndex: number; linkedObjectMetadataId: string; @@ -417,7 +423,7 @@ export class TimelineActivitySeederService { private getActivityTargetInfos( activityType: 'note' | 'task' | 'calendarEvent' | 'message', - recordSeed: Record, + recordSeed: RecordSeedWithId, calendarEventParticipants: CalendarEventParticipantDataSeed[], messageParticipants: MessageParticipantDataSeed[], ): ActivityTargetInfo[] { @@ -436,11 +442,9 @@ export class TimelineActivitySeederService { } private getNoteTargetInfos( - recordSeed: Record, + recordSeed: RecordSeedWithId, ): ActivityTargetInfo[] { - const noteTargetSeed = NOTE_TARGET_DATA_SEEDS.find( - (target) => target.noteId === recordSeed.id, - ); + const noteTargetSeed = NOTE_TARGET_DATA_SEEDS_MAP.get(recordSeed.id); if (!noteTargetSeed) { return []; @@ -462,11 +466,9 @@ export class TimelineActivitySeederService { } private getTaskTargetInfos( - recordSeed: Record, + recordSeed: RecordSeedWithId, ): ActivityTargetInfo[] { - const taskTargetSeed = TASK_TARGET_DATA_SEEDS.find( - (target) => target.taskId === recordSeed.id, - ); + const taskTargetSeed = TASK_TARGET_DATA_SEEDS_MAP.get(recordSeed.id); if (!taskTargetSeed) { return []; @@ -488,7 +490,7 @@ export class TimelineActivitySeederService { } private getCalendarEventTargetInfos( - recordSeed: Record, + recordSeed: RecordSeedWithId, calendarEventParticipants: CalendarEventParticipantDataSeed[], ): ActivityTargetInfo[] { const eventParticipants = calendarEventParticipants.filter( @@ -504,9 +506,7 @@ export class TimelineActivitySeederService { targetId: participant.personId, }); - const person = PERSON_DATA_SEEDS.find( - (p) => p.id === participant.personId, - ); + const person = PERSON_DATA_SEEDS_MAP.get(participant.personId); if (person?.companyId) { targetInfos.push({ @@ -521,7 +521,7 @@ export class TimelineActivitySeederService { } private getMessageTargetInfos( - recordSeed: Record, + recordSeed: RecordSeedWithId, messageParticipants: MessageParticipantDataSeed[], ): ActivityTargetInfo[] { const filteredMessageParticipants = messageParticipants.filter( @@ -537,9 +537,7 @@ export class TimelineActivitySeederService { targetId: participant.personId, }); - const person = PERSON_DATA_SEEDS.find( - (p) => p.id === participant.personId, - ); + const person = PERSON_DATA_SEEDS_MAP.get(participant.personId); if (person?.companyId) { targetInfos.push({ @@ -575,7 +573,7 @@ export class TimelineActivitySeederService { name: this.getLinkedActivityName(activityType), properties: JSON.stringify({ after: linkedProperties }), linkedRecordCachedName, - linkedRecordId: String(recordSeed.id || ''), + linkedRecordId: recordSeed.id, linkedObjectMetadataId, workspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM, companyId: null, @@ -609,7 +607,7 @@ export class TimelineActivitySeederService { private getLinkedRecordData( activityType: string, - recordSeed: Record, + recordSeed: RecordSeedWithId, index: number, ): { linkedRecordCachedName: string; diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts index 8fbca9772f..de766560ac 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts @@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; @@ -36,6 +37,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp UserRoleModule, ApiKeyModule, FeatureFlagModule, + FileStorageModule, WorkspaceSyncMetadataModule, TypeOrmModule.forFeature([Workspace, ObjectMetadataEntity]), ObjectPermissionModule, diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts index 42628d8b3a..799fd5c3d2 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts @@ -124,7 +124,11 @@ export class DevSeederMetadataService { }); } - await this.seedCoreViews({ workspaceId, dataSourceMetadata, featureFlags }); + await this.seedCoreViews({ + workspaceId, + dataSourceMetadata, + featureFlags, + }); } private async seedCustomObject({ diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index b5860e248b..20391452af 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -61,6 +61,7 @@ export class WorkspaceSyncMetadataService { }> { let workspaceMigrations: WorkspaceMigrationEntity[] = []; const storage = new WorkspaceSyncStorage(); + const queryRunner = this.coreDataSource.createQueryRunner(); this.logger.log('Syncing standard objects and fields metadata');