houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Seeding Attachments, Disable ORM Logs, Seeding Parallelization (#15174)

Browse Source 
Improvements to database seeding performance and developer experience.

**Changes:**

1. **Attachment seeding**: Add sample files (PDF, XLSX, PPTX, PNG, ZIP)
to dev seeds with proper file storage
2. **Seeding parallelization**: Process entities within batches in
parallel while respecting dependencies
3. **ORM query logging**: Replace manual logger toggling with
`ORM_QUERY_LOGGING` env var
- Values: `disabled` (default), `server-only` (for local dev), `always`
- Configured once in `core.datasource.ts`, removed from all seeder
services

**For .env:**
```bash
ORM_QUERY_LOGGING=server-only
```

Net result: Faster seeding, cleaner code (-68 lines), better local dev
experience.
This commit is contained in:
Félix Malfait
2025-10-17 16:50:19 +02:00
committed by GitHub
parent 81bc04d16e
commit 4f1623ff76
16 changed files with 578 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/dev-seeder-data.service.ts
+247 -127
View File
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { DataSource } from 'typeorm';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import {
ATTACHMENT_DATA_SEED_COLUMNS,
ATTACHMENT_DATA_SEEDS,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/attachment-data-seeds.constant';
import {
CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
CALENDAR_CHANNEL_DATA_SEEDS,
@@ -94,120 +103,157 @@ import {
import { TimelineActivitySeederService } from 'src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service';
import { prefillWorkflows } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-workflows';
const getRecordSeedsConfigs = (
type RecordSeedConfig = {
tableName: string;
pgColumns: string[];
recordSeeds: Record<string, unknown>[];
};
// Organize seeds into dependency batches for parallel insertion
const getRecordSeedsBatches = (
workspaceId: string,
featureFlags?: Record<FeatureFlagKey, boolean>,
) => [
{
tableName: 'workspaceMember',
pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS,
recordSeeds: getWorkspaceMemberDataSeeds(workspaceId),
},
{
tableName: 'company',
pgColumns: COMPANY_DATA_SEED_COLUMNS,
recordSeeds: COMPANY_DATA_SEEDS,
},
...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED]
? [
{
tableName: 'dashboard',
pgColumns: DASHBOARD_DATA_SEED_COLUMNS,
recordSeeds: getDashboardDataSeeds(workspaceId),
},
]
: []),
{
tableName: 'person',
pgColumns: PERSON_DATA_SEED_COLUMNS,
recordSeeds: PERSON_DATA_SEEDS,
},
{
tableName: 'note',
pgColumns: NOTE_DATA_SEED_COLUMNS,
recordSeeds: NOTE_DATA_SEEDS,
},
{
tableName: 'noteTarget',
pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS,
recordSeeds: NOTE_TARGET_DATA_SEEDS,
},
{
tableName: 'opportunity',
pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS,
recordSeeds: OPPORTUNITY_DATA_SEEDS,
},
{
tableName: 'connectedAccount',
pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS,
recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS,
},
{
tableName: 'calendarChannel',
pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS,
},
{
tableName: 'calendarEvent',
pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_EVENT_DATA_SEEDS,
},
{
tableName: 'calendarChannelEventAssociation',
pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'calendarEventParticipant',
pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId),
},
{
tableName: 'messageChannel',
pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS,
},
{
tableName: 'messageThread',
pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_THREAD_DATA_SEEDS,
},
{
tableName: 'message',
pgColumns: MESSAGE_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_DATA_SEEDS,
},
{
tableName: 'messageChannelMessageAssociation',
pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'messageParticipant',
pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getMessageParticipantDataSeeds(workspaceId),
},
{
tableName: '_pet',
pgColumns: PET_DATA_SEED_COLUMNS,
recordSeeds: PET_DATA_SEEDS,
},
{
tableName: '_surveyResult',
pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS,
recordSeeds: SURVEY_RESULT_DATA_SEEDS,
},
{
tableName: 'task',
pgColumns: TASK_DATA_SEED_COLUMNS,
recordSeeds: TASK_DATA_SEEDS,
},
{
tableName: 'taskTarget',
pgColumns: TASK_TARGET_DATA_SEED_COLUMNS,
recordSeeds: TASK_TARGET_DATA_SEEDS,
},
];
): RecordSeedConfig[][] => {
// Batch 1: No dependencies
const batch1: RecordSeedConfig[] = [
{
tableName: 'workspaceMember',
pgColumns: WORKSPACE_MEMBER_DATA_SEED_COLUMNS,
recordSeeds: getWorkspaceMemberDataSeeds(workspaceId),
},
{
tableName: '_surveyResult',
pgColumns: SURVEY_RESULT_DATA_SEED_COLUMNS,
recordSeeds: SURVEY_RESULT_DATA_SEEDS,
},
];
// Batch 2: Depends on workspaceMember
const batch2: RecordSeedConfig[] = [
{
tableName: 'company',
pgColumns: COMPANY_DATA_SEED_COLUMNS,
recordSeeds: COMPANY_DATA_SEEDS,
},
{
tableName: 'connectedAccount',
pgColumns: CONNECTED_ACCOUNT_DATA_SEED_COLUMNS,
recordSeeds: CONNECTED_ACCOUNT_DATA_SEEDS,
},
...(featureFlags?.[FeatureFlagKey.IS_PAGE_LAYOUT_ENABLED]
? [
{
tableName: 'dashboard',
pgColumns: DASHBOARD_DATA_SEED_COLUMNS,
recordSeeds: getDashboardDataSeeds(workspaceId),
},
]
: []),
];
// Batch 3: Depends on company and connectedAccount
const batch3: RecordSeedConfig[] = [
{
tableName: 'person',
pgColumns: PERSON_DATA_SEED_COLUMNS,
recordSeeds: PERSON_DATA_SEEDS,
},
{
tableName: '_pet',
pgColumns: PET_DATA_SEED_COLUMNS,
recordSeeds: PET_DATA_SEEDS,
},
{
tableName: 'calendarChannel',
pgColumns: CALENDAR_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_DATA_SEEDS,
},
{
tableName: 'messageChannel',
pgColumns: MESSAGE_CHANNEL_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_DATA_SEEDS,
},
];
// Batch 4: Depends on person/company or independent
const batch4: RecordSeedConfig[] = [
{
tableName: 'opportunity',
pgColumns: OPPORTUNITY_DATA_SEED_COLUMNS,
recordSeeds: OPPORTUNITY_DATA_SEEDS,
},
{
tableName: 'note',
pgColumns: NOTE_DATA_SEED_COLUMNS,
recordSeeds: NOTE_DATA_SEEDS,
},
{
tableName: 'task',
pgColumns: TASK_DATA_SEED_COLUMNS,
recordSeeds: TASK_DATA_SEEDS,
},
{
tableName: 'calendarEvent',
pgColumns: CALENDAR_EVENT_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_EVENT_DATA_SEEDS,
},
{
tableName: 'messageThread',
pgColumns: MESSAGE_THREAD_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_THREAD_DATA_SEEDS,
},
];
// Batch 5: Depends on batch 4 entities
const batch5: RecordSeedConfig[] = [
{
tableName: 'noteTarget',
pgColumns: NOTE_TARGET_DATA_SEED_COLUMNS,
recordSeeds: NOTE_TARGET_DATA_SEEDS,
},
{
tableName: 'taskTarget',
pgColumns: TASK_TARGET_DATA_SEED_COLUMNS,
recordSeeds: TASK_TARGET_DATA_SEEDS,
},
{
tableName: 'calendarChannelEventAssociation',
pgColumns: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: CALENDAR_CHANNEL_EVENT_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'calendarEventParticipant',
pgColumns: CALENDAR_EVENT_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getCalendarEventParticipantDataSeeds(workspaceId),
},
{
tableName: 'message',
pgColumns: MESSAGE_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_DATA_SEEDS,
},
];
// Batch 6: Depends on batch 5 entities
const batch6: RecordSeedConfig[] = [
{
tableName: 'messageChannelMessageAssociation',
pgColumns: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEED_COLUMNS,
recordSeeds: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_DATA_SEEDS,
},
{
tableName: 'messageParticipant',
pgColumns: MESSAGE_PARTICIPANT_DATA_SEED_COLUMNS,
recordSeeds: getMessageParticipantDataSeeds(workspaceId),
},
{
tableName: 'attachment',
pgColumns: ATTACHMENT_DATA_SEED_COLUMNS,
recordSeeds: ATTACHMENT_DATA_SEEDS,
},
];
return [batch1, batch2, batch3, batch4, batch5, batch6];
};
@Injectable()
export class DevSeederDataService {
@@ -216,6 +262,7 @@ export class DevSeederDataService {
private readonly coreDataSource: DataSource,
private readonly objectMetadataService: ObjectMetadataService,
private readonly timelineActivitySeederService: TimelineActivitySeederService,
private readonly fileStorageService: FileStorageService,
) {}
public async seed({
@@ -232,10 +279,47 @@ export class DevSeederDataService {
await this.coreDataSource.transaction(
async (entityManager: WorkspaceEntityManager) => {
for (const recordSeedsConfig of getRecordSeedsConfigs(
await this.seedRecordsInBatches({
entityManager,
schemaName,
workspaceId,
featureFlags,
)) {
objectMetadataItems,
});
await this.timelineActivitySeederService.seedTimelineActivities({
entityManager,
schemaName,
workspaceId,
});
await this.seedAttachmentFiles(workspaceId);
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
},
);
}
private async seedRecordsInBatches({
entityManager,
schemaName,
workspaceId,
featureFlags,
objectMetadataItems,
}: {
entityManager: WorkspaceEntityManager;
schemaName: string;
workspaceId: string;
featureFlags?: Record<FeatureFlagKey, boolean>;
objectMetadataItems: ObjectMetadataEntity[];
}) {
const batches = getRecordSeedsBatches(workspaceId, featureFlags);
// Process batches sequentially (respecting dependencies)
// but entities within each batch in parallel
for (const batch of batches) {
await Promise.all(
batch.map(async (recordSeedsConfig) => {
const objectMetadata = objectMetadataItems.find(
(item) =>
computeTableName(item.nameSingular, item.isCustom) ===
@@ -244,7 +328,7 @@ export class DevSeederDataService {
if (!objectMetadata) {
// TODO this continue is hacky, we should have a record seed config per workspace
continue;
return;
}
await this.seedRecords({
@@ -254,17 +338,9 @@ export class DevSeederDataService {
pgColumns: recordSeedsConfig.pgColumns,
recordSeeds: recordSeedsConfig.recordSeeds,
});
}
await this.timelineActivitySeederService.seedTimelineActivities({
entityManager,
schemaName,
workspaceId,
});
await prefillWorkflows(entityManager, schemaName, objectMetadataItems);
},
);
}),
);
}
}
private async seedRecords({
@@ -288,7 +364,51 @@ export class DevSeederDataService {
.into(`${schemaName}.${tableName}`, pgColumns)
.orIgnore()
.values(recordSeeds)
.returning('*')
.execute();
}
private async seedAttachmentFiles(workspaceId: string): Promise<void> {
// Files are copied to dist/assets during build via nest-cli.json
// The pattern **/dev-seeder/data/sample-files/** preserves the full path
const IS_BUILT = __dirname.includes('/dist/');
const sampleFilesDir = IS_BUILT
? join(
__dirname,
'../../../../../../assets/engine/workspace-manager/dev-seeder/data/sample-files',
)
: join(__dirname, '../sample-files');
const filesToCreate = [
'sample-contract.pdf',
'budget-2024.xlsx',
'presentation.pptx',
'screenshot.png',
'archive.zip',
];
for (const filename of filesToCreate) {
const filePath = join(sampleFilesDir, filename);
const fileBuffer = await readFile(filePath);
await this.fileStorageService.write({
file: fileBuffer,
name: filename,
folder: `workspace-${workspaceId}/attachment`,
mimeType: this.getMimeType(filename),
});
}
}
private getMimeType(filename: string): string {
const ext = filename.split('.').pop()?.toLowerCase();
const mimeTypes: Record<string, string> = {
pdf: 'application/pdf',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
png: 'image/png',
zip: 'application/zip',
};
return mimeTypes[ext || ''] || 'application/octet-stream';
}
}
packages/twenty-server/src/engine/workspace-manager/dev-seeder/data/services/timeline-activity-seeder.service.ts
+27 -29
View File
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import chunk from 'lodash.chunk';
import { ObjectRecord } from 'twenty-shared/types';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
@@ -16,14 +17,19 @@ import {
MessageParticipantDataSeed,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/message-participant-data-seeds.constant';
import { NOTE_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-data-seeds.constant';
import { NOTE_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant';
import { NOTE_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/note-target-data-seeds.constant';
import { OPPORTUNITY_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/opportunity-data-seeds.constant';
import { PERSON_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant';
import {
PERSON_DATA_SEEDS,
PERSON_DATA_SEEDS_MAP,
} from 'src/engine/workspace-manager/dev-seeder/data/constants/person-data-seeds.constant';
import { TASK_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-data-seeds.constant';
import { TASK_TARGET_DATA_SEEDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant';
import { TASK_TARGET_DATA_SEEDS_MAP } from 'src/engine/workspace-manager/dev-seeder/data/constants/task-target-data-seeds.constant';
import { WORKSPACE_MEMBER_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/data/constants/workspace-member-data-seeds.constant';
import { type TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity';
type RecordSeedWithId = Pick<ObjectRecord, 'id'> & Record<string, unknown>;
type TimelineActivitySeedData = Pick<
TimelineActivityWorkspaceEntity,
| 'id'
@@ -51,13 +57,13 @@ type ActivityTargetInfo = {
type CreateTimelineActivityParams = {
entityType: string;
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
};
type CreateLinkedActivityParams = {
activityType: 'note' | 'task' | 'calendarEvent' | 'message';
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
activityIndex: number;
linkedObjectMetadataId: string;
@@ -66,7 +72,7 @@ type CreateLinkedActivityParams = {
type EntityConfig = {
type: string;
seeds: Array<Record<string, unknown>>;
seeds: Array<RecordSeedWithId>;
};
type ObjectMetadataIds = {
@@ -230,7 +236,7 @@ export class TimelineActivitySeederService {
return;
}
const batchSize = 100;
const batchSize = 1000;
const timelineActivityBatches = chunk(timelineActivities, batchSize);
for (const batch of timelineActivityBatches) {
@@ -272,7 +278,7 @@ export class TimelineActivitySeederService {
index + 1,
);
const creationDate = new Date().toISOString();
const recordId = String(recordSeed.id || '');
const recordId = recordSeed.id;
const timelineActivity: TimelineActivitySeedData = {
id: timelineActivityId,
@@ -314,7 +320,7 @@ export class TimelineActivitySeederService {
private getEventAfterRecordProperties(
type: string,
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): Record<string, unknown> {
const commonProperties = { id: recordSeed.id };
@@ -385,7 +391,7 @@ export class TimelineActivitySeederService {
messageParticipants,
}: {
activityType: 'note' | 'task' | 'calendarEvent' | 'message';
recordSeed: Record<string, unknown>;
recordSeed: RecordSeedWithId;
index: number;
activityIndex: number;
linkedObjectMetadataId: string;
@@ -417,7 +423,7 @@ export class TimelineActivitySeederService {
private getActivityTargetInfos(
activityType: 'note' | 'task' | 'calendarEvent' | 'message',
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
calendarEventParticipants: CalendarEventParticipantDataSeed[],
messageParticipants: MessageParticipantDataSeed[],
): ActivityTargetInfo[] {
@@ -436,11 +442,9 @@ export class TimelineActivitySeederService {
}
private getNoteTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): ActivityTargetInfo[] {
const noteTargetSeed = NOTE_TARGET_DATA_SEEDS.find(
(target) => target.noteId === recordSeed.id,
);
const noteTargetSeed = NOTE_TARGET_DATA_SEEDS_MAP.get(recordSeed.id);
if (!noteTargetSeed) {
return [];
@@ -462,11 +466,9 @@ export class TimelineActivitySeederService {
}
private getTaskTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
): ActivityTargetInfo[] {
const taskTargetSeed = TASK_TARGET_DATA_SEEDS.find(
(target) => target.taskId === recordSeed.id,
);
const taskTargetSeed = TASK_TARGET_DATA_SEEDS_MAP.get(recordSeed.id);
if (!taskTargetSeed) {
return [];
@@ -488,7 +490,7 @@ export class TimelineActivitySeederService {
}
private getCalendarEventTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
calendarEventParticipants: CalendarEventParticipantDataSeed[],
): ActivityTargetInfo[] {
const eventParticipants = calendarEventParticipants.filter(
@@ -504,9 +506,7 @@ export class TimelineActivitySeederService {
targetId: participant.personId,
});
const person = PERSON_DATA_SEEDS.find(
(p) => p.id === participant.personId,
);
const person = PERSON_DATA_SEEDS_MAP.get(participant.personId);
if (person?.companyId) {
targetInfos.push({
@@ -521,7 +521,7 @@ export class TimelineActivitySeederService {
}
private getMessageTargetInfos(
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
messageParticipants: MessageParticipantDataSeed[],
): ActivityTargetInfo[] {
const filteredMessageParticipants = messageParticipants.filter(
@@ -537,9 +537,7 @@ export class TimelineActivitySeederService {
targetId: participant.personId,
});
const person = PERSON_DATA_SEEDS.find(
(p) => p.id === participant.personId,
);
const person = PERSON_DATA_SEEDS_MAP.get(participant.personId);
if (person?.companyId) {
targetInfos.push({
@@ -575,7 +573,7 @@ export class TimelineActivitySeederService {
name: this.getLinkedActivityName(activityType),
properties: JSON.stringify({ after: linkedProperties }),
linkedRecordCachedName,
linkedRecordId: String(recordSeed.id || ''),
linkedRecordId: recordSeed.id,
linkedObjectMetadataId,
workspaceMemberId: WORKSPACE_MEMBER_DATA_SEED_IDS.TIM,
companyId: null,
@@ -609,7 +607,7 @@ export class TimelineActivitySeederService {
private getLinkedRecordData(
activityType: string,
recordSeed: Record<string, unknown>,
recordSeed: RecordSeedWithId,
index: number,
): {
linkedRecordCachedName: string;