houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Seeding Attachments, Disable ORM Logs, Seeding Parallelization (#15174)

Browse Source 
Improvements to database seeding performance and developer experience.

**Changes:**

1. **Attachment seeding**: Add sample files (PDF, XLSX, PPTX, PNG, ZIP)
to dev seeds with proper file storage
2. **Seeding parallelization**: Process entities within batches in
parallel while respecting dependencies
3. **ORM query logging**: Replace manual logger toggling with
`ORM_QUERY_LOGGING` env var
- Values: `disabled` (default), `server-only` (for local dev), `always`
- Configured once in `core.datasource.ts`, removed from all seeder
services

**For .env:**
```bash
ORM_QUERY_LOGGING=server-only
```

Net result: Faster seeding, cleaner code (-68 lines), better local dev
experience.
This commit is contained in:
Félix Malfait
2025-10-17 16:50:19 +02:00
committed by GitHub
parent 81bc04d16e
commit 4f1623ff76
16 changed files with 578 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/nest-cli.json
+4
View File
@@ -29,6 +29,10 @@
{
"include": "**/database/clickHouse/migrations/*.sql",
"outDir": "dist/src"
},
{
"include": "**/dev-seeder/data/sample-files/**",
"outDir": "dist/assets"
}
],
"watchAssets": true
packages/twenty-server/src/database/typeorm/core/core.datasource.ts
+21 -5
View File
@@ -7,16 +7,32 @@ config({
override: true,
});
const getLoggingConfig = (): LogLevel[] => {
if (process.env.NODE_ENV === 'development') {
return ['query', 'error'];
}
const isRunningCommand = (): boolean => {
const scriptPath = process.argv[1] || '';
return scriptPath.includes('/command/command.');
};
const getLoggingConfig = (): LogLevel[] => {
if (process.env.NODE_ENV === 'test') {
return [];
}
const ormQueryLogging = process.env.ORM_QUERY_LOGGING || 'disabled';
return ['error'];
switch (ormQueryLogging) {
case 'disabled':
return ['error'];
case 'server-only':
if (isRunningCommand()) {
return ['error'];
}
return ['query', 'error'];
case 'always':
return ['query', 'error'];
default:
return ['error'];
}
};
const isJest = process.argv.some((arg) => arg.includes('jest'));
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant.ts
+190
View File
@@ -0,0 +1,190 @@
import { COMPANY_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/company-data-seeds.constant';
import { NOTE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-data-seeds.constant';
import { OPPORTUNITY_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/opportunity-data-seeds.constant';
import { PERSON_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant';
import { TASK_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-data-seeds.constant';
import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant';
type AttachmentDataSeed = {
id: string;
name: string;
fullPath: string;
type: string;
authorId: string | null;
// createdBySource: string;
// createdByWorkspaceMemberId: string;
// createdByName: string;
personId: string | null;
companyId: string | null;
noteId: string | null;
taskId: string | null;
opportunityId: string | null;
};
export const ATTACHMENT_DATA_SEED_COLUMNS: (keyof AttachmentDataSeed)[] = [
'id',
'name',
'fullPath',
'type',
'authorId',
// 'createdBySource',
// 'createdByWorkspaceMemberId',
// 'createdByName',
'personId',
'companyId',
'noteId',
'taskId',
'opportunityId',
];
const GENERATE_ATTACHMENT_IDS = (): Record<string, string> => {
const ATTACHMENT_IDS: Record<string, string> = {};
for (let INDEX = 1; INDEX <= 400; INDEX++) {
const HEX_INDEX = INDEX.toString(16).padStart(4, '0');
ATTACHMENT_IDS[`ID_${INDEX}`] =
`20202020-${HEX_INDEX}-4a7c-8001-123456789aba`;
}
return ATTACHMENT_IDS;
};
export const ATTACHMENT_DATA_SEED_IDS = GENERATE_ATTACHMENT_IDS();
// Pool of 5 reusable file templates
const FILE_TEMPLATES = [
{
name: 'Contract Agreement.pdf',
fullPath: 'attachment/sample-contract.pdf',
type: 'TextDocument',
},
{
name: 'Budget 2024.xlsx',
fullPath: 'attachment/budget-2024.xlsx',
type: 'Spreadsheet',
},
{
name: 'Product Presentation.pptx',
fullPath: 'attachment/presentation.pptx',
type: 'Presentation',
},
{
name: 'Screenshot.png',
fullPath: 'attachment/screenshot.png',
type: 'Image',
},
{
name: 'Archive.zip',
fullPath: 'attachment/archive.zip',
type: 'Archive',
},
];
// Additional name variations for more realistic variety
const FILE_NAME_VARIATIONS = [
// Documents
{ name: 'Service Agreement.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'NDA Document.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'Project Proposal.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'Invoice Q1 2024.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'Meeting Notes.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'Report Final.pdf', type: 'TextDocument', pathIndex: 0 },
{ name: 'Contract Signed.pdf', type: 'TextDocument', pathIndex: 0 },
// Spreadsheets
{ name: 'Financial Forecast.xlsx', type: 'Spreadsheet', pathIndex: 1 },
{ name: 'Sales Report Q4.xlsx', type: 'Spreadsheet', pathIndex: 1 },
{ name: 'Team Roster.xlsx', type: 'Spreadsheet', pathIndex: 1 },
{ name: 'Expense Report.xlsx', type: 'Spreadsheet', pathIndex: 1 },
{ name: 'Inventory List.xlsx', type: 'Spreadsheet', pathIndex: 1 },
{ name: 'Data Export.csv', type: 'Spreadsheet', pathIndex: 1 },
// Presentations
{ name: 'Pitch Deck.pptx', type: 'Presentation', pathIndex: 2 },
{ name: 'Q4 Results.pptx', type: 'Presentation', pathIndex: 2 },
{ name: 'Roadmap 2024.pptx', type: 'Presentation', pathIndex: 2 },
{ name: 'Company Overview.pptx', type: 'Presentation', pathIndex: 2 },
{ name: 'Training Materials.pptx', type: 'Presentation', pathIndex: 2 },
// Images
{ name: 'Company Logo.png', type: 'Image', pathIndex: 3 },
{ name: 'Product Photo.jpg', type: 'Image', pathIndex: 3 },
{ name: 'Diagram.png', type: 'Image', pathIndex: 3 },
{ name: 'Wireframe.png', type: 'Image', pathIndex: 3 },
{ name: 'Mockup Design.png', type: 'Image', pathIndex: 3 },
{ name: 'Headshot.jpg', type: 'Image', pathIndex: 3 },
// Archives
{ name: 'Project Files.zip', type: 'Archive', pathIndex: 4 },
{ name: 'Backup Data.zip', type: 'Archive', pathIndex: 4 },
{ name: 'Source Code.zip', type: 'Archive', pathIndex: 4 },
];
const GENERATE_ATTACHMENT_SEEDS = (): AttachmentDataSeed[] => {
const ATTACHMENT_SEEDS: AttachmentDataSeed[] = [];
// Get available entity IDs
const PERSON_IDS = Object.values(PERSON_DATA_SEED_IDS).slice(0, 120); // Use first 120 persons
const COMPANY_IDS = Object.values(COMPANY_DATA_SEED_IDS).slice(0, 120); // Use first 120 companies
const NOTE_IDS = Object.values(NOTE_DATA_SEED_IDS).slice(0, 80); // Use first 80 notes
const TASK_IDS = Object.values(TASK_DATA_SEED_IDS).slice(0, 60); // Use first 60 tasks
const OPPORTUNITY_IDS = Object.values(OPPORTUNITY_DATA_SEED_IDS).slice(0, 20); // Use first 20 opportunities
let entityIndex = 0;
for (let INDEX = 1; INDEX <= 400; INDEX++) {
// Cycle through file name variations
const NAME_VARIATION_INDEX = INDEX % FILE_NAME_VARIATIONS.length;
const NAME_VARIATION = FILE_NAME_VARIATIONS[NAME_VARIATION_INDEX];
const FILE_TEMPLATE = FILE_TEMPLATES[NAME_VARIATION.pathIndex];
// Determine which entity this attachment belongs to
// Distribution: ~30% person, ~30% company, ~20% note, ~15% task, ~5% opportunity
let personId: string | null = null;
let companyId: string | null = null;
let noteId: string | null = null;
let taskId: string | null = null;
let opportunityId: string | null = null;
const DISTRIBUTION_VALUE = INDEX % 100;
if (DISTRIBUTION_VALUE < 30) {
// 30% Person attachments
personId = PERSON_IDS[entityIndex % PERSON_IDS.length];
entityIndex++;
} else if (DISTRIBUTION_VALUE < 60) {
// 30% Company attachments
companyId = COMPANY_IDS[entityIndex % COMPANY_IDS.length];
entityIndex++;
} else if (DISTRIBUTION_VALUE < 80) {
// 20% Note attachments
noteId = NOTE_IDS[entityIndex % NOTE_IDS.length];
entityIndex++;
} else if (DISTRIBUTION_VALUE < 95) {
// 15% Task attachments
taskId = TASK_IDS[entityIndex % TASK_IDS.length];
entityIndex++;
} else {
// 5% Opportunity attachments
opportunityId = OPPORTUNITY_IDS[entityIndex % OPPORTUNITY_IDS.length];
entityIndex++;
}
ATTACHMENT_SEEDS.push({
id: ATTACHMENT_DATA_SEED_IDS[`ID_${INDEX}`],
name: NAME_VARIATION.name,
fullPath: FILE_TEMPLATE.fullPath,
type: NAME_VARIATION.type,
authorId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM,
// createdBySource: 'MANUAL',
//createdByWorkspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM,
//createdByName: 'Tim A',
personId,
companyId,
noteId,
taskId,
opportunityId,
});
}
return ATTACHMENT_SEEDS;
};
export const ATTACHMENT_DATA_SEEDS = GENERATE_ATTACHMENT_SEEDS();
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant.ts
+7
View File
@@ -75,3 +75,10 @@ const GENERATE_NOTE_TARGET_SEEDS = (): NoteTargetDataSeed[] => {
};
export const NOTE_TARGET_DATA_SEEDS = GENERATE_NOTE_TARGET_SEEDS();
// Map for O(1) lookups by note ID
export const NOTE_TARGET_DATA_SEEDS_MAP = new Map<string, NoteTargetDataSeed>(
NOTE_TARGET_DATA_SEEDS.filter((target) => target.noteId !== null).map(
(target) => [target.noteId!, target],
),
);
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant.ts
+5
View File
@@ -21653,3 +21653,8 @@ export const PERSON_DATA_SEEDS: PersonDataSeed[] = PERSON_DATA_SEEDS_RAW.map(
position: index + 1,
}),
);
// Map for O(1) lookups by person ID
export const PERSON_DATA_SEEDS_MAP = new Map<string, PersonDataSeed>(
PERSON_DATA_SEEDS.map((p) => [p.id, p]),
);
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant.ts
+7
View File
@@ -81,3 +81,10 @@ const GENERATE_TASK_TARGET_SEEDS = (): TaskTargetDataSeed[] => {
};
export const TASK_TARGET_DATA_SEEDS = GENERATE_TASK_TARGET_SEEDS();
// Map for O(1) lookups by task ID
export const TASK_TARGET_DATA_SEEDS_MAP = new Map<string, TaskTargetDataSeed>(
TASK_TARGET_DATA_SEEDS.filter((target) => target.taskId !== null).map(
(target) => [target.taskId!, target],
),
);
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/archive.zip
BIN
View File
Binary file not shown.
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/budget-2024.xlsx
BIN
View File
Binary file not shown.
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/presentation.pptx
BIN
View File
Binary file not shown.
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/sample-contract.pdf
+62
View File
@@ -0,0 +1,62 @@
%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
>>
>>
>>
endobj
4 0 obj
<<
/Length 44
>>
stream
BT
/F1 12 Tf
100 700 Td
(Sample Contract Document) Tj
ET
endstream
endobj
5 0 obj
<<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
endobj
xref
0 6
0000000000 65535 f
0000000009 00000 n
0000000058 00000 n
0000000115 00000 n
0000000274 00000 n
0000000367 00000 n
trailer
<<
/Size 6
/Root 1 0 R
>>
startxref
445
%%EOF
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/sample-files/screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 70 B

packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts
+247 -127
View File
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { DataSource } from 'typeorm';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import {
ATTACHMENT_DATA_SEED_COLUMNS,
ATTACHMENT_DATA_SEEDS,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant';
import {
CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
CALENDAR_CHANNEL_DATA_SEEDS,
@@ -94,120 +103,157 @@ import {
import { TimelineActivitySeederService } from 'src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service';
import { prefillWorkflows } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-workflows';
const getRecordSeedsConfigs = (
type RecordSeedConfig = {
tableName: string;
pgColumns: string[];
recordSeeds: Record<string, unknown>[];
};
// Organize seeds into dependency batches for parallel insertion
const getRecordSeedsBatches = (
workspaceId: string,
featureFlags?: Record<FeatureFlagKey, boolean>,
) => [
{
tableName: 'workspaceMember',
pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS,
recordSeeds: getWorkspaceMemberDataSeeds(workspaceId),
},
{
tableName: 'company',
pgColumns: COMPANY_DATA_SEED_COLUMNS,
recordSeeds: COMPANY_DATA_SEEDS,
},
...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED]
? [
{
tableName: 'dashboard',
pgColumns: DASHBOARD_DATA_SEED_COLUMNS,
recordSeeds: getDashboardDataSeeds(workspaceId),
},
]
: []),
{
tableName: 'person',
pgColumns: PERSON_DATA_SEED_COLUMNS,
recordSeeds: PERSON_DATA_SEEDS,
},
{
tableName: 'note',
pgColumns: NOTE_DATA_SEED_COLUMNS,
recordSeeds: NOTE_DATA_SEEDS,
},
{
tableName: 'noteTarget',
pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS,
recordSeeds: NOTE_TARGET_DATA_SEEDS,
},
{
tableName: 'opportunity',
pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS,
recordSeeds: OPPORTUNITY_DATA_SEEDS,
},
{
tableName: 'connectedAccount',
pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS,
recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS,
},
{
tableName: 'calendarChannel',
pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS,
},
{
tableName: 'calendarEvent',
pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_EVENT_DATA_SEEDS,
},
{
tableName: 'calendarChannelEventAssociation',
pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'calendarEventParticipant',
pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId),
},
{
tableName: 'messageChannel',
pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS,
},
{
tableName: 'messageThread',
pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_THREAD_DATA_SEEDS,
},
{
tableName: 'message',
pgColumns: MESSAGE_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_DATA_SEEDS,
},
{
tableName: 'messageChannelMessageAssociation',
pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'messageParticipant',
pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getMessageParticipantDataSeeds(workspaceId),
},
{
tableName: '_pet',
pgColumns: PET_DATA_SEED_COLUMNS,
recordSeeds: PET_DATA_SEEDS,
},
{
tableName: '_surveyResult',
pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS,
recordSeeds: SURVEY_RESULT_DATA_SEEDS,
},
{
tableName: 'task',
pgColumns: TASK_DATA_SEED_COLUMNS,
recordSeeds: TASK_DATA_SEEDS,
},
{
tableName: 'taskTarget',
pgColumns: TASK_TARGET_DATA_SEED_COLUMNS,
recordSeeds: TASK_TARGET_DATA_SEEDS,
},
];
): RecordSeedConfig[][] => {
// Batch 1: No dependencies
const batch1: RecordSeedConfig[] = [
{
tableName: 'workspaceMember',
pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS,
recordSeeds: getWorkspaceMemberDataSeeds(workspaceId),
},
{
tableName: '_surveyResult',
pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS,
recordSeeds: SURVEY_RESULT_DATA_SEEDS,
},
];
// Batch 2: Depends on workspaceMember
const batch2: RecordSeedConfig[] = [
{
tableName: 'company',
pgColumns: COMPANY_DATA_SEED_COLUMNS,
recordSeeds: COMPANY_DATA_SEEDS,
},
{
tableName: 'connectedAccount',
pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS,
recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS,
},
...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED]
? [
{
tableName: 'dashboard',
pgColumns: DASHBOARD_DATA_SEED_COLUMNS,
recordSeeds: getDashboardDataSeeds(workspaceId),
},
]
: []),
];
// Batch 3: Depends on company and connectedAccount
const batch3: RecordSeedConfig[] = [
{
tableName: 'person',
pgColumns: PERSON_DATA_SEED_COLUMNS,
recordSeeds: PERSON_DATA_SEEDS,
},
{
tableName: '_pet',
pgColumns: PET_DATA_SEED_COLUMNS,
recordSeeds: PET_DATA_SEEDS,
},
{
tableName: 'calendarChannel',
pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS,
},
{
tableName: 'messageChannel',
pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS,
},
];
// Batch 4: Depends on person/company or independent
const batch4: RecordSeedConfig[] = [
{
tableName: 'opportunity',
pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS,
recordSeeds: OPPORTUNITY_DATA_SEEDS,
},
{
tableName: 'note',
pgColumns: NOTE_DATA_SEED_COLUMNS,
recordSeeds: NOTE_DATA_SEEDS,
},
{
tableName: 'task',
pgColumns: TASK_DATA_SEED_COLUMNS,
recordSeeds: TASK_DATA_SEEDS,
},
{
tableName: 'calendarEvent',
pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_EVENT_DATA_SEEDS,
},
{
tableName: 'messageThread',
pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_THREAD_DATA_SEEDS,
},
];
// Batch 5: Depends on batch 4 entities
const batch5: RecordSeedConfig[] = [
{
tableName: 'noteTarget',
pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS,
recordSeeds: NOTE_TARGET_DATA_SEEDS,
},
{
tableName: 'taskTarget',
pgColumns: TASK_TARGET_DATA_SEED_COLUMNS,
recordSeeds: TASK_TARGET_DATA_SEEDS,
},
{
tableName: 'calendarChannelEventAssociation',
pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'calendarEventParticipant',
pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId),
},
{
tableName: 'message',
pgColumns: MESSAGE_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_DATA_SEEDS,
},
];
// Batch 6: Depends on batch 5 entities
const batch6: RecordSeedConfig[] = [
{
tableName: 'messageChannelMessageAssociation',
pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'messageParticipant',
pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getMessageParticipantDataSeeds(workspaceId),
},
{
tableName: 'attachment',
pgColumns: ATTACHMENT_DATA_SEED_COLUMNS,
recordSeeds: ATTACHMENT_DATA_SEEDS,
},
];
return [batch1, batch2, batch3, batch4, batch5, batch6];
};
@Injectable()
export class DevSeederDataService {
@@ -216,6 +262,7 @@ export class DevSeederDataService {
private readonly coreDataSource: DataSource,
private readonly objectMetadataService: ObjectMetadataService,
private readonly timelineActivitySeederService: TimelineActivitySeederService,
private readonly fileStorageService: FileStorageService,
) {}
public async seed({
@@ -232,10 +279,47 @@ export class DevSeederDataService {
await this.coreDataSource.transaction(
async (entityManager: WorkspaceEntityManager) => {
for (const recordSeedsConfig of getRecordSeedsConfigs(
await this.seedRecordsInBatches({
entityManager,
schemaName,
workspaceId,
featureFlags,
)) {
objectMetadataItems,
});
await this.timelineActivitySeederService.seedTimelineActivities({
entityManager,
schemaName,
workspaceId,
});
await this.seedAttachmentFiles(workspaceId);
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
},
);
}
private async seedRecordsInBatches({
entityManager,
schemaName,
workspaceId,
featureFlags,
objectMetadataItems,
}: {
entityManager: WorkspaceEntityManager;
schemaName: string;
workspaceId: string;
featureFlags?: Record<FeatureFlagKey, boolean>;
objectMetadataItems: ObjectMetadataEntity[];
}) {
const batches = getRecordSeedsBatches(workspaceId, featureFlags);
// Process batches sequentially (respecting dependencies)
// but entities within each batch in parallel
for (const batch of batches) {
await Promise.all(
batch.map(async (recordSeedsConfig) => {
const objectMetadata = objectMetadataItems.find(
(item) =>
computeTableName(item.nameSingular, item.isCustom) ===
@@ -244,7 +328,7 @@ export class DevSeederDataService {
if (!objectMetadata) {
// TODO this continue is hacky, we should have a record seed config per workspace
continue;
return;
}
await this.seedRecords({
@@ -254,17 +338,9 @@ export class DevSeederDataService {
pgColumns: recordSeedsConfig.pgColumns,
recordSeeds: recordSeedsConfig.recordSeeds,
});
}
await this.timelineActivitySeederService.seedTimelineActivities({
entityManager,
schemaName,
workspaceId,
});
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
},
);
}),
);
}
}
private async seedRecords({
@@ -288,7 +364,51 @@ export class DevSeederDataService {
.into(`${schemaName}.${tableName}`, pgColumns)
.orIgnore()
.values(recordSeeds)
.returning('*')
.execute();
}
private async seedAttachmentFiles(workspaceId: string): Promise<void> {
// Files are copied to dist/assets during build via nest-cli.json
// The pattern **/dev-seeder/data/sample-files/** preserves the full path
const IS_BUILT = __dirname.includes('/dist/');
const sampleFilesDir = IS_BUILT
? join(
__dirname,
'../../../../../../assets/engine/workspace-manager/dev-seeder/data/sample-files',
)
: join(__dirname, '../sample-files');
const filesToCreate = [
'sample-contract.pdf',
'budget-2024.xlsx',
'presentation.pptx',
'screenshot.png',
'archive.zip',
];
for (const filename of filesToCreate) {
const filePath = join(sampleFilesDir, filename);
const fileBuffer = await readFile(filePath);
await this.fileStorageService.write({
file: fileBuffer,
name: filename,
folder: `workspace-${workspaceId}/attachment`,
mimeType: this.getMimeType(filename),
});
}
}
private getMimeType(filename: string): string {
const ext = filename.split('.').pop()?.toLowerCase();
const mimeTypes: Record<string, string> = {
pdf: 'application/pdf',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
png: 'image/png',
zip: 'application/zip',
};
return mimeTypes[ext || ''] || 'application/octet-stream';
}
}
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts
+27 -29
View File
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import chunk from 'lodash.chunk';
import { ObjectRecord } from 'twenty-shared/types';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
@@ -16,14 +17,19 @@ import {
MessageParticipantDataSeed,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/message-participant-data-seeds.constant';
import { NOTE_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-data-seeds.constant';
import { NOTE_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant';
import { NOTE_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant';
import { OPPORTUNITY_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/opportunity-data-seeds.constant';
import { PERSON_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant';
import {
PERSON_DATA_SEEDS,
PERSON_DATA_SEEDS_MAP,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant';
import { TASK_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-data-seeds.constant';
import { TASK_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant';
import { TASK_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant';
import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant';
import { type TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity';
type RecordSeedWithId = Pick<ObjectRecord, 'id'> & Record<string, unknown>;
type TimelineActivitySeedData = Pick<
TimelineActivityWorkspaceEntity,
| 'id'
@@ -51,13 +57,13 @@ type ActivityTargetInfo = {
type CreateTimelineActivityParams = {
entityType: string;
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
};
type CreateLinkedActivityParams = {
activityType: 'note' | 'task' | 'calendarEvent' | 'message';
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
activityIndex: number;
linkedObjectMetadataId: string;
@@ -66,7 +72,7 @@ type CreateLinkedActivityParams = {
type EntityConfig = {
type: string;
seeds: Array<Record<string, unknown>>;
seeds: Array<RecordSeedWithId>;
};
type ObjectMetadataIds = {
@@ -230,7 +236,7 @@ export class TimelineActivitySeederService {
return;
}
const batchSize = 100;
const batchSize = 1000;
const timelineActivityBatches = chunk(timelineActivities, batchSize);
for (const batch of timelineActivityBatches) {
@@ -272,7 +278,7 @@ export class TimelineActivitySeederService {
index + 1,
);
const creationDate = new Date().toISOString();
const recordId = String(recordSeed.id || '');
const recordId = recordSeed.id;
const timelineActivity: TimelineActivitySeedData = {
id: timelineActivityId,
@@ -314,7 +320,7 @@ export class TimelineActivitySeederService {
private getEventAfterRecordProperties(
type: string,
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): Record<string, unknown> {
const commonProperties = { id: recordSeed.id };
@@ -385,7 +391,7 @@ export class TimelineActivitySeederService {
messageParticipants,
}: {
activityType: 'note' | 'task' | 'calendarEvent' | 'message';
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
activityIndex: number;
linkedObjectMetadataId: string;
@@ -417,7 +423,7 @@ export class TimelineActivitySeederService {
private getActivityTargetInfos(
activityType: 'note' | 'task' | 'calendarEvent' | 'message',
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
calendarEventParticipants: CalendarEventParticipantDataSeed[],
messageParticipants: MessageParticipantDataSeed[],
): ActivityTargetInfo[] {
@@ -436,11 +442,9 @@ export class TimelineActivitySeederService {
}
private getNoteTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): ActivityTargetInfo[] {
const noteTargetSeed = NOTE_TARGET_DATA_SEEDS.find(
(target) => target.noteId === recordSeed.id,
);
const noteTargetSeed = NOTE_TARGET_DATA_SEEDS_MAP.get(recordSeed.id);
if (!noteTargetSeed) {
return [];
@@ -462,11 +466,9 @@ export class TimelineActivitySeederService {
}
private getTaskTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): ActivityTargetInfo[] {
const taskTargetSeed = TASK_TARGET_DATA_SEEDS.find(
(target) => target.taskId === recordSeed.id,
);
const taskTargetSeed = TASK_TARGET_DATA_SEEDS_MAP.get(recordSeed.id);
if (!taskTargetSeed) {
return [];
@@ -488,7 +490,7 @@ export class TimelineActivitySeederService {
}
private getCalendarEventTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
calendarEventParticipants: CalendarEventParticipantDataSeed[],
): ActivityTargetInfo[] {
const eventParticipants = calendarEventParticipants.filter(
@@ -504,9 +506,7 @@ export class TimelineActivitySeederService {
targetId: participant.personId,
});
const person = PERSON_DATA_SEEDS.find(
(p) => p.id === participant.personId,
);
const person = PERSON_DATA_SEEDS_MAP.get(participant.personId);
if (person?.companyId) {
targetInfos.push({
@@ -521,7 +521,7 @@ export class TimelineActivitySeederService {
}
private getMessageTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
messageParticipants: MessageParticipantDataSeed[],
): ActivityTargetInfo[] {
const filteredMessageParticipants = messageParticipants.filter(
@@ -537,9 +537,7 @@ export class TimelineActivitySeederService {
targetId: participant.personId,
});
const person = PERSON_DATA_SEEDS.find(
(p) => p.id === participant.personId,
);
const person = PERSON_DATA_SEEDS_MAP.get(participant.personId);
if (person?.companyId) {
targetInfos.push({
@@ -575,7 +573,7 @@ export class TimelineActivitySeederService {
name: this.getLinkedActivityName(activityType),
properties: JSON.stringify({ after: linkedProperties }),
linkedRecordCachedName,
linkedRecordId: String(recordSeed.id || ''),
linkedRecordId: recordSeed.id,
linkedObjectMetadataId,
workspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM,
companyId: null,
@@ -609,7 +607,7 @@ export class TimelineActivitySeederService {
private getLinkedRecordData(
activityType: string,
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
index: number,
): {
linkedRecordCachedName: string;
packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts
+2
View File
@@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
@@ -36,6 +37,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
UserRoleModule,
ApiKeyModule,
FeatureFlagModule,
FileStorageModule,
WorkspaceSyncMetadataModule,
TypeOrmModule.forFeature([Workspace, ObjectMetadataEntity]),
ObjectPermissionModule,
packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts
+5 -1
View File
@@ -124,7 +124,11 @@ export class DevSeederMetadataService {
});
}
await this.seedCoreViews({ workspaceId, dataSourceMetadata, featureFlags });
await this.seedCoreViews({
workspaceId,
dataSourceMetadata,
featureFlags,
});
}
private async seedCustomObject({
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts
+1
View File
@@ -61,6 +61,7 @@ export class WorkspaceSyncMetadataService {
}> {
let workspaceMigrations: WorkspaceMigrationEntity[] = [];
const storage = new WorkspaceSyncStorage();
const queryRunner = this.coreDataSource.createQueryRunner();
this.logger.log('Syncing standard objects and fields metadata');