Refactor global datasource part 3 (#16447)

## Context
Following https://github.com/twentyhq/twenty/pull/16399
Now using the new global orm manager everywhere and returning a
GlobalDatasource/WorkspaceDatasource based on a feature flag.
This means we now need to wrap all our ORM calls within
executeInWorkspaceContext callback (at least for now) so the global
datasource can dynamically hydrate its context via the new store (the
global datasource does not store anything related to workspaces as it is
now a unique singleton). If feature flag is off it still uses local data
stored in the workspace datasource.
This commit is contained in:
Weiko
2025-12-10 17:17:33 +01:00
committed by GitHub
parent 4f13022774
commit 9bd8f94b3a
203 changed files with 8887 additions and 7237 deletions
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
type AutomatedTriggerType,
type WorkflowAutomatedTriggerWorkspaceEntity,
@@ -10,7 +11,7 @@ import { type AutomatedTriggerSettings } from 'src/modules/workflow/workflow-tri
@Injectable()
export class AutomatedTriggerWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
) {}
async addAutomatedTrigger({
@@ -24,17 +25,24 @@ export class AutomatedTriggerWorkspaceService {
settings: AutomatedTriggerSettings;
workspaceId: string;
}) {
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
const authContext = buildSystemAuthContext(workspaceId);
await workflowAutomatedTriggerRepository.insert({
type,
settings,
workflowId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
await workflowAutomatedTriggerRepository.insert({
type,
settings,
workflowId,
});
},
);
}
async deleteAutomatedTrigger({
@@ -44,12 +52,19 @@ export class AutomatedTriggerWorkspaceService {
workflowId: string;
workspaceId: string;
}) {
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
const authContext = buildSystemAuthContext(workspaceId);
await workflowAutomatedTriggerRepository.delete({ workflowId });
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
'workflowAutomatedTrigger',
);
await workflowAutomatedTriggerRepository.delete({ workflowId });
},
);
}
}
@@ -2,7 +2,7 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
@@ -11,7 +11,7 @@ import { WorkflowTriggerJob } from 'src/modules/workflow/workflow-trigger/jobs/w
describe('WorkflowDatabaseEventTriggerListener', () => {
let listener: WorkflowDatabaseEventTriggerListener;
let twentyORMGlobalManager: jest.Mocked<TwentyORMGlobalManager>;
let globalWorkspaceOrmManager: jest.Mocked<GlobalWorkspaceOrmManager>;
let messageQueueService: jest.Mocked<MessageQueueService>;
const mockRepository = {
@@ -48,8 +48,11 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
}) as FlatObjectMetadata;
beforeEach(async () => {
twentyORMGlobalManager = {
getRepositoryForWorkspace: jest.fn().mockResolvedValue(mockRepository),
globalWorkspaceOrmManager = {
getRepository: jest.fn().mockResolvedValue(mockRepository),
executeInWorkspaceContext: jest
.fn()
.mockImplementation((_authContext: any, fn: () => any) => fn()),
} as any;
messageQueueService = {
@@ -60,8 +63,8 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
providers: [
WorkflowDatabaseEventTriggerListener,
{
provide: TwentyORMGlobalManager,
useValue: twentyORMGlobalManager,
provide: GlobalWorkspaceOrmManager,
useValue: globalWorkspaceOrmManager,
},
{
provide: MessageQueueService,
@@ -20,7 +20,8 @@ import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-m
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import {
AutomatedTriggerType,
@@ -43,7 +44,7 @@ export class WorkflowDatabaseEventTriggerListener {
);
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
@@ -242,57 +243,66 @@ export class WorkflowDatabaseEventTriggerListener {
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
}) {
const { fieldIdByJoinColumnName } = buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const { fieldIdByJoinColumnName } =
buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
);
for (const [joinColumnName, joinFieldId] of Object.entries(
fieldIdByJoinColumnName,
)) {
const joinField = findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: flatFieldMetadataMaps,
flatEntityId: joinFieldId,
});
const joinRecordIds = records
.map((record) => record[joinColumnName])
.filter(isDefined);
if (joinRecordIds.length === 0) {
continue;
}
const relatedObjectMetadataId =
joinField.relationTargetObjectMetadataId;
if (!isDefined(relatedObjectMetadataId)) {
continue;
}
const relatedObjectMetadataNameSingular =
flatObjectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
if (!isDefined(relatedObjectMetadataNameSingular)) {
continue;
}
const relatedObjectRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
relatedObjectMetadataNameSingular,
{ shouldBypassPermissionChecks: true },
);
const relatedRecords = await relatedObjectRepository.find({
where: { id: In(joinRecordIds) },
});
for (const record of records) {
record[joinField.name] = relatedRecords.find(
(relatedRecord) => relatedRecord.id === record[joinColumnName],
);
}
}
},
);
for (const [joinColumnName, joinFieldId] of Object.entries(
fieldIdByJoinColumnName,
)) {
const joinField = findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: flatFieldMetadataMaps,
flatEntityId: joinFieldId,
});
const joinRecordIds = records
.map((record) => record[joinColumnName])
.filter(isDefined);
if (joinRecordIds.length === 0) {
continue;
}
const relatedObjectMetadataId = joinField.relationTargetObjectMetadataId;
if (!isDefined(relatedObjectMetadataId)) {
continue;
}
const relatedObjectMetadataNameSingular =
flatObjectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
if (!isDefined(relatedObjectMetadataNameSingular)) {
continue;
}
const relatedObjectRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
relatedObjectMetadataNameSingular,
{ shouldBypassPermissionChecks: true },
);
const relatedRecords = await relatedObjectRepository.find({
where: { id: In(joinRecordIds) },
});
for (const record of records) {
record[joinField.name] = relatedRecords.find(
(relatedRecord) => relatedRecord.id === record[joinColumnName],
);
}
}
}
private async shouldIgnoreEvent(
@@ -325,45 +335,52 @@ export class WorkflowDatabaseEventTriggerListener {
const databaseEventName = payload.name;
const automatedTriggerTableName = 'workflowAutomatedTrigger';
const workflowAutomatedTriggerRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
automatedTriggerTableName,
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
const eventListeners = await workflowAutomatedTriggerRepository.find({
where: {
type: AutomatedTriggerType.DATABASE_EVENT,
settings: Raw(
() =>
`"${automatedTriggerTableName}"."settings"->>'eventName' = :eventName`,
{ eventName: databaseEventName },
),
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowAutomatedTriggerRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowAutomatedTriggerWorkspaceEntity>(
workspaceId,
automatedTriggerTableName,
{ shouldBypassPermissionChecks: true },
);
for (const eventListener of eventListeners) {
for (const eventPayload of payload.events) {
const shouldTriggerJob = this.shouldTriggerJob({
eventPayload,
eventListener,
action,
const eventListeners = await workflowAutomatedTriggerRepository.find({
where: {
type: AutomatedTriggerType.DATABASE_EVENT,
settings: Raw(
() =>
`"${automatedTriggerTableName}"."settings"->>'eventName' = :eventName`,
{ eventName: databaseEventName },
),
},
});
if (shouldTriggerJob) {
await this.messageQueueService.add<WorkflowTriggerJobData>(
WorkflowTriggerJob.name,
{
workspaceId,
workflowId: eventListener.workflowId,
payload: eventPayload,
},
{ retryLimit: 3 },
);
for (const eventListener of eventListeners) {
for (const eventPayload of payload.events) {
const shouldTriggerJob = this.shouldTriggerJob({
eventPayload,
eventListener,
action,
});
if (shouldTriggerJob) {
await this.messageQueueService.add<WorkflowTriggerJobData>(
WorkflowTriggerJob.name,
{
workspaceId,
workflowId: eventListener.workflowId,
payload: eventPayload,
},
{ retryLimit: 3 },
);
}
}
}
}
}
},
);
}
private shouldTriggerJob({
@@ -10,7 +10,8 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/proc
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { handleWorkflowTriggerException } from 'src/engine/core-modules/workflow/filters/workflow-trigger-graphql-api-exception.filter';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import {
WorkflowVersionStatus,
type WorkflowVersionWorkspaceEntity,
@@ -33,7 +34,7 @@ const DEFAULT_WORKFLOW_NAME = 'Workflow';
@Processor({ queueName: MessageQueue.workflowQueue, scope: Scope.REQUEST })
export class WorkflowTriggerJob {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService,
@InjectMessageQueue(MessageQueue.workflowQueue)
private readonly messageQueueService: MessageQueueService,
@@ -41,77 +42,83 @@ export class WorkflowTriggerJob {
@Process(WorkflowTriggerJob.name)
async handle(data: WorkflowTriggerJobData): Promise<void> {
try {
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
data.workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(data.workspaceId);
const workflow = await workflowRepository.findOneBy({
id: data.workflowId,
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
try {
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
data.workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
if (!workflow) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
const workflow = await workflowRepository.findOneBy({
id: data.workflowId,
});
if (!workflow.lastPublishedVersionId) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} has no published version in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
if (!workflow) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
data.workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!workflow.lastPublishedVersionId) {
throw new WorkflowTriggerException(
`Workflow ${data.workflowId} has no published version in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
const workflowVersion = await workflowVersionRepository.findOneBy({
id: workflow.lastPublishedVersionId,
});
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
data.workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
if (!workflowVersion) {
throw new WorkflowTriggerException(
`Workflow version ${workflow.lastPublishedVersionId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
if (workflowVersion.status !== WorkflowVersionStatus.ACTIVE) {
throw new WorkflowTriggerException(
`Workflow version ${workflowVersion.id} is not active in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
const workflowVersion = await workflowVersionRepository.findOneBy({
id: workflow.lastPublishedVersionId,
});
await this.workflowRunnerWorkspaceService.run({
workspaceId: data.workspaceId,
workflowVersionId: workflow.lastPublishedVersionId,
payload: data.payload,
source: {
source: FieldActorSource.WORKFLOW,
name:
isDefined(workflow.name) && !isEmpty(workflow.name)
? workflow.name
: DEFAULT_WORKFLOW_NAME,
context: {},
workspaceMemberId: null,
},
});
} catch (e) {
// We remove cron if it exists when no valid workflowVersion exists
await this.messageQueueService.removeCron({
jobName: WorkflowTriggerJob.name,
jobId: data.workflowId,
});
handleWorkflowTriggerException(e);
}
if (!workflowVersion) {
throw new WorkflowTriggerException(
`Workflow version ${workflow.lastPublishedVersionId} not found in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.NOT_FOUND,
);
}
if (workflowVersion.status !== WorkflowVersionStatus.ACTIVE) {
throw new WorkflowTriggerException(
`Workflow version ${workflowVersion.id} is not active in workspace ${data.workspaceId}`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
await this.workflowRunnerWorkspaceService.run({
workspaceId: data.workspaceId,
workflowVersionId: workflow.lastPublishedVersionId,
payload: data.payload,
source: {
source: FieldActorSource.WORKFLOW,
name:
isDefined(workflow.name) && !isEmpty(workflow.name)
? workflow.name
: DEFAULT_WORKFLOW_NAME,
context: {},
workspaceMemberId: null,
},
});
} catch (e) {
await this.messageQueueService.removeCron({
jobName: WorkflowTriggerJob.name,
jobId: data.workflowId,
});
handleWorkflowTriggerException(e);
}
},
);
}
}
@@ -3,8 +3,9 @@ import { Injectable } from '@nestjs/common';
import { msg } from '@lingui/core/macro';
import { type ActorMetadata } from 'twenty-shared/types';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
import {
@@ -31,7 +32,7 @@ import { assertNever } from 'src/utils/assert';
@Injectable()
export class WorkflowTriggerWorkspaceService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly workflowCommonWorkspaceService: WorkflowCommonWorkspaceService,
private readonly workflowRunnerWorkspaceService: WorkflowRunnerWorkspaceService,
private readonly automatedTriggerWorkspaceService: AutomatedTriggerWorkspaceService,
@@ -69,71 +70,87 @@ export class WorkflowTriggerWorkspaceService {
workflowVersionId: string,
workspaceId: string,
) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const authContext = buildSystemAuthContext(workspaceId);
const workflowVersionNullable = await workflowVersionRepository.findOne({
where: { id: workflowVersionId },
});
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const workflowVersion =
await this.workflowCommonWorkspaceService.getValidWorkflowVersionOrFail(
workflowVersionNullable,
);
const workflowVersionNullable = await workflowVersionRepository.findOne(
{
where: { id: workflowVersionId },
},
);
const workflowRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true }, // settings permissions are checked at resolver-level
);
const workflowVersion =
await this.workflowCommonWorkspaceService.getValidWorkflowVersionOrFail(
workflowVersionNullable,
);
const workflow = await workflowRepository.findOne({
where: { id: workflowVersion.workflowId },
});
const workflowRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowWorkspaceEntity>(
workspaceId,
'workflow',
{ shouldBypassPermissionChecks: true },
);
if (!workflow) {
throw new WorkflowTriggerException(
'No workflow found',
WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
);
}
const workflow = await workflowRepository.findOne({
where: { id: workflowVersion.workflowId },
});
assertVersionCanBeActivated(workflowVersion, workflow);
if (!workflow) {
throw new WorkflowTriggerException(
'No workflow found',
WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
);
}
await this.performActivationSteps(
workflow,
workflowVersion,
workflowRepository,
workflowVersionRepository,
workspaceId,
assertVersionCanBeActivated(workflowVersion, workflow);
await this.performActivationSteps(
workflow,
workflowVersion,
workflowRepository,
workflowVersionRepository,
workspaceId,
);
return true;
},
);
return true;
}
async deactivateWorkflowVersion(
workflowVersionId: string,
workspaceId: string,
) {
const workflowVersionRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
const authContext = buildSystemAuthContext(workspaceId);
await this.performDeactivationSteps(
workflowVersionId,
workflowVersionRepository,
workspaceId,
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
authContext,
async () => {
const workflowVersionRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkflowVersionWorkspaceEntity>(
workspaceId,
'workflowVersion',
{ shouldBypassPermissionChecks: true },
);
await this.performDeactivationSteps(
workflowVersionId,
workflowVersionRepository,
workspaceId,
);
return true;
},
);
return true;
}
async stopWorkflowRun(workflowRunId: string, workspaceId: string) {