feat: emit metadata events for schema changes with actor context for webhooks (#17622)

## Summary

This PR adds **metadata eventing**: when schema metadata
(objectMetadata, fieldMetadata, view, viewField, etc.) is created,
updated, or deleted, we now emit events that can trigger webhooks and
future audit logs. It also adds **actor context** (`userId`,
`workspaceMemberId`) to those events so subscribers can attribute
changes to a user or API key.

## What changed

### 1. Metadata eventing (first commit)

- **MetadataEventEmitter**  
New service that emits batch events after successful workspace
migrations. Event names follow `metadata.{entity}.{action}` (e.g.
`metadata.objectMetadata.created`, `metadata.fieldMetadata.updated`).
- **MetadataEventsToDbListener**  
Listens for metadata events and enqueues webhook delivery via
`CallWebhookJobsForMetadataJob`.
- **Event types** (twenty-shared)  
`MetadataEventAction`, `MetadataEventBatch`, and record event types for
create/update/delete.
- **WorkspaceMigrationValidateBuildAndRunService**  
Calls the metadata event emitter after running migrations so all
metadata changes (from any module) emit events from a single place.
- **Create events**  
Sourced from the create action payload (`flatEntity` /
`flatFieldMetadatas`) because `fromToAllFlatEntityMaps` does not provide
a before/after diff for creates. Update/delete events still use the
fromToAllFlatEntityMaps comparison.

### 2. Actor context (second commit)

- **MetadataEventEmitter**  
Accepts optional `actorContext` (`userId`, `workspaceMemberId`) and
includes it on emitted batch events.
- **WorkspaceMigrationValidateBuildAndRunService**  
Passes `actorContext` from the request into the metadata event emitter.
- **Metadata resolvers & services**  
All metadata modules resolve `@AuthUser({ allowUndefined: true })` and
`@AuthUserWorkspaceId()` and pass `userId` and `workspaceMemberId`
through to the migration/event pipeline. Both are optional so
API-key–authenticated requests (no user) still emit events without a
user identity.
  
Shared some questions on Discord about the PR.

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: prastoin <paul@twenty.com>
This commit is contained in:
Abdullah.
2026-02-12 22:14:25 +05:00
committed by GitHub
parent f6b7ab2251
commit 13c2234856
30 changed files with 731 additions and 82 deletions
@@ -1,16 +1,10 @@
import { useObjectMetadataItems } from '@/object-metadata/hooks/useObjectMetadataItems';
import { WebhookEntitySelect } from '@/settings/developers/components/WebhookEntitySelect';
import { Select } from '@/ui/input/components/Select';
import { useIsMobile } from '@/ui/utilities/responsive/hooks/useIsMobile';
import styled from '@emotion/styled';
import { t } from '@lingui/core/macro';
import { isDefined } from 'twenty-shared/utils';
import {
IconBox,
IconNorthStar,
IconPlus,
IconTrash,
useIcons,
} from 'twenty-ui/display';
import { IconBox, IconNorthStar, IconPlus, IconTrash } from 'twenty-ui/display';
import { IconButton, type SelectOption } from 'twenty-ui/input';
const OBJECT_DROPDOWN_WIDTH = 240;
@@ -51,19 +45,6 @@ export const SettingsDatabaseEventsForm = ({
}) => {
const isMobile = useIsMobile();
const { objectMetadataItems } = useObjectMetadataItems();
const { getIcon } = useIcons();
const objectOptions: SelectOption<string>[] = [
{ label: t`All Objects`, value: '*', Icon: IconNorthStar },
...objectMetadataItems.map((item) => ({
label: item.labelPlural,
value: item.nameSingular,
Icon: getIcon(item.icon),
})),
];
const getActionOptions = (
updatedFields?: string[],
): SelectOption<string>[] => {
@@ -86,15 +67,12 @@ export const SettingsDatabaseEventsForm = ({
<>
{events.map((operation, index) => (
<StyledFilterRow key={index} isMobile={isMobile}>
<Select
<WebhookEntitySelect
dropdownId={`object-webhook-type-select-${index}`}
value={operation.object}
options={objectOptions}
onChange={(newValue) =>
updateOperation?.(index, 'object', newValue)
}
fullWidth
emptyOption={{ label: t`Object`, value: null }}
disabled={disabled}
/>
<Select
@@ -0,0 +1,236 @@
import { useObjectMetadataItems } from '@/object-metadata/hooks/useObjectMetadataItems';
import { Dropdown } from '@/ui/layout/dropdown/components/Dropdown';
import { DropdownContent } from '@/ui/layout/dropdown/components/DropdownContent';
import { DropdownMenuItemsContainer } from '@/ui/layout/dropdown/components/DropdownMenuItemsContainer';
import { DropdownMenuSearchInput } from '@/ui/layout/dropdown/components/DropdownMenuSearchInput';
import { DropdownMenuSectionLabel } from '@/ui/layout/dropdown/components/DropdownMenuSectionLabel';
import { DropdownMenuSeparator } from '@/ui/layout/dropdown/components/DropdownMenuSeparator';
import { GenericDropdownContentWidth } from '@/ui/layout/dropdown/constants/GenericDropdownContentWidth';
import { useCloseDropdown } from '@/ui/layout/dropdown/hooks/useCloseDropdown';
import { SelectableList } from '@/ui/layout/selectable-list/components/SelectableList';
import { SelectableListItem } from '@/ui/layout/selectable-list/components/SelectableListItem';
import { selectedItemIdComponentState } from '@/ui/layout/selectable-list/states/selectedItemIdComponentState';
import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValue';
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { t } from '@lingui/core/macro';
import { useState } from 'react';
import { isDefined } from 'twenty-shared/utils';
import {
IconBox,
IconChevronDown,
IconCode,
IconEye,
IconNorthStar,
IconSettings,
IconTable,
useIcons,
} from 'twenty-ui/display';
import { MenuItemSelect } from 'twenty-ui/navigation';
const WEBHOOK_ENTITY_DROPDOWN_ID = 'webhook-entity-select';
const StyledControlContainer = styled.div<{ disabled?: boolean }>`
align-items: center;
background-color: ${({ theme }) => theme.background.transparent.lighter};
border: 1px solid ${({ theme }) => theme.border.color.medium};
border-radius: ${({ theme }) => theme.border.radius.sm};
box-sizing: border-box;
cursor: ${({ disabled }) => (disabled ? 'not-allowed' : 'pointer')};
display: flex;
gap: ${({ theme }) => theme.spacing(1)};
height: ${({ theme }) => theme.spacing(8)};
justify-content: space-between;
padding: 0 ${({ theme }) => theme.spacing(2)};
width: 100%;
&:hover {
background-color: ${({ theme, disabled }) =>
disabled
? theme.background.transparent.lighter
: theme.background.transparent.light};
}
`;
const StyledControlLabel = styled.span`
color: ${({ theme }) => theme.font.color.primary};
font-size: ${({ theme }) => theme.font.size.md};
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
`;
const StyledControlIconChevronDown = styled(IconChevronDown)<{
disabled?: boolean;
}>`
color: ${({ disabled, theme }) =>
disabled ? theme.font.color.extraLight : theme.font.color.tertiary};
`;
type WebhookEntitySelectProps = {
value: string | null;
onChange: (value: string | null) => void;
disabled?: boolean;
dropdownId?: string;
};
export const WebhookEntitySelect = ({
value,
onChange,
disabled = false,
dropdownId = WEBHOOK_ENTITY_DROPDOWN_ID,
}: WebhookEntitySelectProps) => {
const theme = useTheme();
const [searchInput, setSearchInput] = useState('');
const { objectMetadataItems } = useObjectMetadataItems();
const { getIcon } = useIcons();
const { closeDropdown } = useCloseDropdown();
const selectedItemId = useRecoilComponentValue(
selectedItemIdComponentState,
dropdownId,
);
const metadataOptions = [
{ label: t`All Metadata`, value: 'metadata.*', icon: IconNorthStar },
{ label: t`Object`, value: 'metadata.objectMetadata', icon: IconTable },
{ label: t`Field`, value: 'metadata.fieldMetadata', icon: IconBox },
{ label: t`View`, value: 'metadata.view', icon: IconEye },
{ label: t`View Field`, value: 'metadata.viewField', icon: IconEye },
{ label: t`Index`, value: 'metadata.index', icon: IconSettings },
{ label: t`Webhook`, value: 'metadata.webhook', icon: IconCode },
];
const objectOptions = [
{ label: t`All Objects`, value: '*', Icon: IconNorthStar },
...objectMetadataItems.map((item) => ({
label: item.labelPlural,
value: item.nameSingular,
Icon: getIcon(item.icon),
})),
];
const filteredObjectOptions = objectOptions.filter((option) =>
option.label.toLowerCase().includes(searchInput.toLowerCase()),
);
const filteredMetadataOptions = metadataOptions.filter((option) =>
option.label.toLowerCase().includes(searchInput.toLowerCase()),
);
// Find selected option label for display
const getSelectedLabel = () => {
if (!isDefined(value)) {
return t`Select entity`;
}
if (value === '*') {
return t`All Objects`;
}
const metadataOption = metadataOptions.find((opt) => opt.value === value);
if (isDefined(metadataOption)) {
return metadataOption.label;
}
const objectOption = objectOptions.find((opt) => opt.value === value);
if (isDefined(objectOption)) {
return objectOption.label;
}
return value;
};
const handleSelect = (selectedValue: string) => {
if (disabled) return;
onChange(selectedValue);
closeDropdown(dropdownId);
setSearchInput('');
};
const shouldShowObjects = filteredObjectOptions.length > 0;
const shouldShowMetadata = filteredMetadataOptions.length > 0;
const shouldShowSeparator = shouldShowObjects && shouldShowMetadata;
const selectableItemIds = [
...filteredObjectOptions.map((opt) => opt.value),
...filteredMetadataOptions.map((opt) => opt.value),
];
return (
<Dropdown
dropdownId={dropdownId}
dropdownPlacement="bottom-start"
disableClickForClickableComponent={disabled}
onClose={() => setSearchInput('')}
clickableComponent={
<StyledControlContainer disabled={disabled}>
<StyledControlLabel>{getSelectedLabel()}</StyledControlLabel>
<StyledControlIconChevronDown
disabled={disabled}
size={theme.icon.size.md}
/>
</StyledControlContainer>
}
dropdownComponents={
<DropdownContent widthInPixels={GenericDropdownContentWidth.Medium}>
<DropdownMenuSearchInput
autoFocus
value={searchInput}
placeholder={t`Search...`}
onChange={(event) => setSearchInput(event.target.value)}
/>
<DropdownMenuSeparator />
<SelectableList
selectableListInstanceId={dropdownId}
selectableItemIdArray={selectableItemIds}
focusId={dropdownId}
>
<DropdownMenuItemsContainer hasMaxHeight>
{shouldShowObjects && (
<>
<DropdownMenuSectionLabel label={t`Core Objects`} />
{filteredObjectOptions.map((option) => (
<SelectableListItem
key={option.value}
itemId={option.value}
onEnter={() => handleSelect(option.value)}
>
<MenuItemSelect
LeftIcon={option.Icon}
text={option.label}
selected={value === option.value}
focused={selectedItemId === option.value}
onClick={() => handleSelect(option.value)}
/>
</SelectableListItem>
))}
</>
)}
{shouldShowSeparator && <DropdownMenuSeparator />}
{shouldShowMetadata && (
<>
<DropdownMenuSectionLabel label={t`Metadata`} />
{filteredMetadataOptions.map((option) => (
<SelectableListItem
key={option.value}
itemId={option.value}
onEnter={() => handleSelect(option.value)}
>
<MenuItemSelect
LeftIcon={option.icon}
text={option.label}
selected={value === option.value}
focused={selectedItemId === option.value}
onClick={() => handleSelect(option.value)}
/>
</SelectableListItem>
))}
</>
)}
</DropdownMenuItemsContainer>
</SelectableList>
</DropdownContent>
}
/>
);
};
@@ -10,6 +10,7 @@ import { DELETE_WEBHOOK } from '@/settings/developers/graphql/mutations/deleteWe
import { UPDATE_WEBHOOK } from '@/settings/developers/graphql/mutations/updateWebhook';
import { GET_WEBHOOK } from '@/settings/developers/graphql/queries/getWebhook';
import { useWebhookForm } from '@/settings/developers/hooks/useWebhookForm';
import { WEBHOOK_EMPTY_OPERATION } from '~/pages/settings/developers/webhooks/constants/WebhookEmptyOperation';
const mockNavigateSettings = jest.fn();
const mockEnqueueSuccessSnackBar = jest.fn();
@@ -146,7 +147,7 @@ describe('useWebhookForm', () => {
expect(result.current.formConfig.getValues()).toEqual({
targetUrl: '',
description: '',
operations: [{ object: '*', action: '*' }],
operations: [{ object: '*', action: '*' }, WEBHOOK_EMPTY_OPERATION],
secret: '',
});
});
@@ -32,7 +32,7 @@ type UseWebhookFormProps = {
const DEFAULT_FORM_VALUES: WebhookFormValues = {
targetUrl: '',
description: '',
operations: [{ object: '*', action: '*' }],
operations: addEmptyOperationIfNecessary([{ object: '*', action: '*' }]),
secret: '',
};
@@ -18,7 +18,7 @@ describe('addEmptyOperationIfNecessary', () => {
]);
});
it('should not add empty operation when wildcard operation exists', () => {
it('should add empty operation when only record wildcard operation exists', () => {
const operations: WebhookOperationType[] = [
{ object: '*', action: '*' },
{ object: 'person', action: 'created' },
@@ -29,6 +29,36 @@ describe('addEmptyOperationIfNecessary', () => {
expect(result).toEqual([
{ object: '*', action: '*' },
{ object: 'person', action: 'created' },
WEBHOOK_EMPTY_OPERATION,
]);
});
it('should add empty operation when only metadata wildcard operation exists', () => {
const operations: WebhookOperationType[] = [
{ object: 'metadata.*', action: '*' },
{ object: 'person', action: 'created' },
];
const result = addEmptyOperationIfNecessary(operations);
expect(result).toEqual([
{ object: 'metadata.*', action: '*' },
{ object: 'person', action: 'created' },
WEBHOOK_EMPTY_OPERATION,
]);
});
it('should not add empty operation when both record and metadata wildcard operations exist', () => {
const operations: WebhookOperationType[] = [
{ object: '*', action: '*' },
{ object: 'metadata.*', action: '*' },
];
const result = addEmptyOperationIfNecessary(operations);
expect(result).toEqual([
{ object: '*', action: '*' },
{ object: 'metadata.*', action: '*' },
]);
});
@@ -13,14 +13,36 @@ describe('parseOperationsFromStrings', () => {
]);
});
it('should handle wildcard operations', () => {
const operations = ['*.*', 'person.created'];
it('should handle wildcard operations across objects and metadata', () => {
const operations = ['*.*', 'metadata.*.*'];
const result = parseOperationsFromStrings(operations);
expect(result).toEqual([
{ object: '*', action: '*' },
{ object: 'metadata.*', action: '*' },
]);
});
it('should handle mixed object and metadata entity operations', () => {
const operations = ['person.created', 'metadata.objectMetadata.created'];
const result = parseOperationsFromStrings(operations);
expect(result).toEqual([
{ object: 'person', action: 'created' },
{ object: 'metadata.objectMetadata', action: 'created' },
]);
});
it('should handle wildcard and specific operations together', () => {
const operations = ['*.*', 'metadata.objectMetadata.updated'];
const result = parseOperationsFromStrings(operations);
expect(result).toEqual([
{ object: '*', action: '*' },
{ object: 'metadata.objectMetadata', action: 'updated' },
]);
});
@@ -4,11 +4,26 @@ import { type WebhookOperationType } from '~/pages/settings/developers/webhooks/
export const addEmptyOperationIfNecessary = (
newOperations: WebhookOperationType[],
): WebhookOperationType[] => {
if (
!newOperations.some((op) => op.object === '*' && op.action === '*') &&
!newOperations.some((op) => op.object === null)
) {
return [...newOperations, WEBHOOK_EMPTY_OPERATION];
const emptyOperationIndex = newOperations.findIndex(
(op) => op.object === null,
);
const hasEmptyOperation = emptyOperationIndex !== -1;
const nonEmptyOperations = newOperations.filter((op) => op.object !== null);
const hasRecordCatchAll = nonEmptyOperations.some(
(op) => op.object === '*' && op.action === '*',
);
const hasMetadataCatchAll = nonEmptyOperations.some(
(op) => op.object === 'metadata.*' && op.action === '*',
);
if (hasRecordCatchAll && hasMetadataCatchAll) {
return nonEmptyOperations;
}
return newOperations;
if (hasEmptyOperation) {
const emptyOperation = newOperations[emptyOperationIndex];
return [...nonEmptyOperations, emptyOperation];
}
return [...nonEmptyOperations, WEBHOOK_EMPTY_OPERATION];
};
@@ -4,7 +4,17 @@ export const parseOperationsFromStrings = (
operations: string[],
): WebhookOperationType[] => {
return operations.map((op: string) => {
const [object, action] = op.split('.');
const parts = op.split('.');
if (parts[0] === 'metadata' && parts.length === 3) {
return {
object: `${parts[0]}.${parts[1]}`,
action: parts[2],
};
}
const [object, action] = parts;
return { object, action };
});
};
@@ -63,6 +63,7 @@ import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { RowLevelPermissionModule } from 'src/engine/metadata-modules/row-level-permission-predicate/row-level-permission.module';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
import { MetadataEventEmitterModule } from 'src/engine/metadata-event-emitter/metadata-event-emitter.module';
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
import { ChannelSyncModule } from 'src/modules/connected-account/channel-sync/channel-sync.module';
import { DashboardModule } from 'src/modules/dashboard/dashboard.module';
@@ -104,6 +105,7 @@ import { FileModule } from './file/file.module';
PostgresCredentialsModule,
WorkflowApiModule,
WorkspaceEventEmitterModule,
MetadataEventEmitterModule,
ActorModule,
TelemetryModule,
AdminPanelModule,
@@ -0,0 +1,53 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { AllMetadataName } from 'twenty-shared/metadata';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type MetadataEventBatch } from 'src/engine/metadata-event-emitter/types/metadata-event-batch.type';
import { CallWebhookJobsForMetadataJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job';
import { AllMetadataEventType } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
@Injectable()
export class MetadataEventsToDbListener {
constructor(
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly webhookQueueService: MessageQueueService,
) {}
@OnEvent('metadata.*.created')
async handleCreate(
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'created'>,
): Promise<void> {
return this.handleEvent(metadataEventBatch);
}
@OnEvent('metadata.*.updated')
async handleUpdate(
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'updated'>,
): Promise<void> {
return this.handleEvent(metadataEventBatch);
}
@OnEvent('metadata.*.deleted')
async handleDelete(
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'deleted'>,
): Promise<void> {
return this.handleEvent(metadataEventBatch);
}
private async handleEvent(
metadataEventBatch: MetadataEventBatch<
AllMetadataName,
AllMetadataEventType
>,
): Promise<void> {
await this.webhookQueueService.add<
MetadataEventBatch<AllMetadataName, AllMetadataEventType>
>(CallWebhookJobsForMetadataJob.name, metadataEventBatch, {
retryLimit: 3,
});
}
}
@@ -0,0 +1,11 @@
import { Global, Module } from '@nestjs/common';
import { MetadataEventEmitter } from 'src/engine/metadata-event-emitter/metadata-event-emitter';
import { MetadataEventsToDbListener } from 'src/engine/metadata-event-emitter/listeners/metadata-events-to-db.listener';
@Global()
@Module({
providers: [MetadataEventEmitter, MetadataEventsToDbListener],
exports: [MetadataEventEmitter],
})
export class MetadataEventEmitterModule {}
@@ -0,0 +1,121 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { AllMetadataName } from 'twenty-shared/metadata';
import { isDefined } from 'twenty-shared/utils';
import { getWorkspaceAuthContext } from 'src/engine/core-modules/auth/storage/workspace-auth-context.storage';
import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
import { MetadataEventBatch } from 'src/engine/metadata-event-emitter/types/metadata-event-batch.type';
import { computeMetadataEventName } from 'src/engine/metadata-event-emitter/utils/compute-metadata-event-name.util';
import {
AllMetadataEventName,
AllMetadataEventType,
MetadataEvent,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
type EmitMetadataEventsArgs = {
metadataEvents: MetadataEvent[];
workspaceId: string;
initiatorContext?: WorkspaceAuthContext;
};
@Injectable()
export class MetadataEventEmitter {
constructor(private readonly eventEmitter: EventEmitter2) {}
public emitMetadataEvents({
metadataEvents,
workspaceId,
initiatorContext,
}: EmitMetadataEventsArgs): void {
if (metadataEvents.length === 0) {
return;
}
const resolvedInitiatorContext =
this.resolveInitiatorContext(initiatorContext);
const userId =
resolvedInitiatorContext?.type === 'user' ||
resolvedInitiatorContext?.type === 'pendingActivationUser'
? resolvedInitiatorContext.user.id
: undefined;
const apiKeyId =
resolvedInitiatorContext?.type === 'apiKey'
? resolvedInitiatorContext.apiKey.id
: undefined;
const grouped = this.groupByMetadataNameAndAction(metadataEvents);
for (const { eventName, events, metadataName, type } of grouped.values()) {
if (metadataEvents.length === 0) {
continue;
}
const metadataEventBatch: MetadataEventBatch = {
name: eventName,
workspaceId,
metadataName,
type,
events,
userId,
apiKeyId,
};
this.eventEmitter.emit(eventName, metadataEventBatch);
}
}
private resolveInitiatorContext(
initiatorContext?: WorkspaceAuthContext,
): WorkspaceAuthContext | undefined {
if (isDefined(initiatorContext)) {
return initiatorContext;
}
try {
return getWorkspaceAuthContext();
} catch {
return undefined;
}
}
private groupByMetadataNameAndAction(metadataEvents: MetadataEvent[]) {
const grouped = new Map<
AllMetadataEventName,
{
metadataName: AllMetadataName;
type: AllMetadataEventType;
eventName: AllMetadataEventName;
events: MetadataEvent[];
}
>();
for (const metadataEvent of metadataEvents) {
const { metadataName, type } = metadataEvent;
const eventName = computeMetadataEventName({
metadataName,
type,
});
const occurence = grouped.get(eventName);
if (!isDefined(occurence)) {
grouped.set(eventName, {
eventName,
metadataName,
type,
events: [metadataEvent],
});
continue;
}
grouped.set(eventName, {
...occurence,
events: [...occurence.events, metadataEvent],
});
}
return grouped;
}
}
@@ -0,0 +1,19 @@
import type { AllMetadataName } from 'twenty-shared/metadata';
import {
type AllMetadataEventType,
type MetadataEvent,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
export type MetadataEventBatch<
TMetadataName extends AllMetadataName = AllMetadataName,
TType extends AllMetadataEventType = AllMetadataEventType,
> = {
name: `metadata.${TMetadataName}.${TType}`;
workspaceId: string;
metadataName: TMetadataName;
type: TType;
events: MetadataEvent[];
userId?: string;
apiKeyId?: string;
};
@@ -0,0 +1,7 @@
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
export const computeMetadataEventName = ({
metadataName,
type,
}: Pick<MetadataEvent, 'metadataName' | 'type'>) =>
`metadata.${metadataName}.${type}` as const;
@@ -1,8 +1,8 @@
import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
import { Args, Context, Mutation, Parent, ResolveField } from '@nestjs/graphql';
import { isDefined } from 'twenty-shared/utils';
import { PermissionFlagType } from 'twenty-shared/constants';
import { isDefined } from 'twenty-shared/utils';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
@@ -0,0 +1,99 @@
import chunk from 'lodash.chunk';
import { isDefined } from 'twenty-shared/utils';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { type MetadataEventBatch } from 'src/engine/metadata-event-emitter/types/metadata-event-batch.type';
import { type FlatWebhook } from 'src/engine/metadata-modules/flat-webhook/types/flat-webhook.type';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { type CallMetadataWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
const WEBHOOK_JOBS_CHUNK_SIZE = 20;
@Processor(MessageQueue.webhookQueue)
export class CallWebhookJobsForMetadataJob {
constructor(
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workspaceCacheService: WorkspaceCacheService,
) {}
@Process(CallWebhookJobsForMetadataJob.name)
async handle(metadataEventBatch: MetadataEventBatch): Promise<void> {
const eventName = metadataEventBatch.name;
const metadataName = metadataEventBatch.metadataName;
const operation = metadataEventBatch.type;
const operationsToMatch = [
eventName,
`metadata.${metadataName}.*`,
`metadata.*.${operation}`,
`metadata.*.*`,
'*.*',
];
const { flatWebhookMaps } = await this.workspaceCacheService.getOrRecompute(
metadataEventBatch.workspaceId,
['flatWebhookMaps'],
);
const webhooks = Object.values(flatWebhookMaps.byUniversalIdentifier)
.filter(isDefined)
.filter((webhook) =>
operationsToMatch.some((operationToMatch) =>
webhook.operations.includes(operationToMatch),
),
);
if (webhooks.length === 0) {
return;
}
const webhookEvents = this.transformMetadataEventBatchToWebhookEvents({
metadataEventBatch,
webhooks,
});
const webhookEventsChunks = chunk(webhookEvents, WEBHOOK_JOBS_CHUNK_SIZE);
for (const webhookEventsChunk of webhookEventsChunks) {
await this.messageQueueService.add<CallMetadataWebhookJobData[]>(
CallWebhookJob.name,
webhookEventsChunk,
{ retryLimit: 3 },
);
}
}
private transformMetadataEventBatchToWebhookEvents({
metadataEventBatch,
webhooks,
}: {
metadataEventBatch: MetadataEventBatch;
webhooks: FlatWebhook[];
}): CallMetadataWebhookJobData[] {
const result: CallMetadataWebhookJobData[] = [];
for (const webhook of webhooks) {
for (const event of metadataEventBatch.events) {
result.push({
targetUrl: webhook.targetUrl,
eventName: metadataEventBatch.name,
workspaceId: metadataEventBatch.workspaceId,
webhookId: webhook.id,
eventDate: new Date(),
userId: metadataEventBatch.userId,
apiKeyId: metadataEventBatch.apiKeyId,
secret: webhook.secret,
event,
});
}
}
return result;
}
}
@@ -10,10 +10,8 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import {
CallWebhookJob,
type CallWebhookJobData,
} from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { type CallWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
import { transformEventBatchToWebhookEvents } from 'src/engine/metadata-modules/webhook/utils/transform-event-batch-to-webhook-events';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
@@ -10,21 +10,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';
export type CallWebhookJobData = {
targetUrl: string;
eventName: string;
objectMetadata: { id: string; nameSingular: string };
workspaceId: string;
webhookId: string;
eventDate: Date;
userId?: string;
workspaceMemberId?: string;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
record: any;
updatedFields?: string[];
secret?: string;
};
import { type WebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
@Processor(MessageQueue.webhookQueue)
export class CallWebhookJob {
@@ -35,7 +21,7 @@ export class CallWebhookJob {
) {}
private generateSignature(
payload: CallWebhookJobData,
payload: Record<string, unknown>,
secret: string,
timestamp: string,
): string {
@@ -46,7 +32,7 @@ export class CallWebhookJob {
}
@Process(CallWebhookJob.name)
async handle(webhookJobEvents: CallWebhookJobData[]): Promise<void> {
async handle(webhookJobEvents: WebhookJobData[]): Promise<void> {
await Promise.all(
webhookJobEvents.map(
async (webhookJobEvent) => await this.callWebhook(webhookJobEvent),
@@ -54,7 +40,7 @@ export class CallWebhookJob {
);
}
private async callWebhook(data: CallWebhookJobData): Promise<void> {
private async callWebhook(data: WebhookJobData): Promise<void> {
const commonPayload = {
url: data.targetUrl,
webhookId: data.webhookId,
@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';
import { CallWebhookJobsForMetadataJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job';
import { CallWebhookJobsJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@@ -14,6 +15,10 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
SecureHttpClientModule,
WorkspaceCacheModule,
],
providers: [CallWebhookJobsJob, CallWebhookJob],
providers: [
CallWebhookJobsJob,
CallWebhookJobsForMetadataJob,
CallWebhookJob,
],
})
export class WebhookJobModule {}
@@ -0,0 +1,27 @@
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
type WebhookJobBase = {
targetUrl: string;
eventName: string;
workspaceId: string;
webhookId: string;
eventDate: Date;
userId?: string;
apiKeyId?: string;
secret?: string;
};
export type CallWebhookJobData = WebhookJobBase & {
objectMetadata: { id: string; nameSingular: string };
workspaceMemberId?: string;
applicationId?: string;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
record: any;
updatedFields?: string[];
};
export type CallMetadataWebhookJobData = WebhookJobBase & {
event: MetadataEvent;
};
export type WebhookJobData = CallWebhookJobData | CallMetadataWebhookJobData;
@@ -1,7 +1,7 @@
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { type WebhookEntity } from 'src/engine/metadata-modules/webhook/entities/webhook.entity';
import { type CallWebhookJobData } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { type CallWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
import { transformEventToWebhookEvent } from 'src/engine/metadata-modules/webhook/utils/transform-event-to-webhook-event';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
@@ -4,8 +4,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'twenty-shared/utils';
import { IsNull, Repository } from 'typeorm';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
import { fromCreateWebhookInputToFlatWebhookToCreate } from 'src/engine/metadata-modules/flat-webhook/utils/from-create-webhook-input-to-flat-webhook-to-create.util';
@@ -7,6 +7,7 @@ import {
import { isDefined } from 'twenty-shared/utils';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { MetadataEventEmitter } from 'src/engine/metadata-event-emitter/metadata-event-emitter';
import { ALL_METADATA_REQUIRED_METADATA_FOR_VALIDATION } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-required-metadata-for-validation.constant';
import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';
import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
@@ -53,6 +54,7 @@ export class WorkspaceMigrationValidateBuildAndRunService {
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceMigrationBuildOrchestratorService: WorkspaceMigrationBuildOrchestratorService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly metadataEventEmitter: MetadataEventEmitter,
twentyConfigService: TwentyConfigService,
) {
const logLevels = twentyConfigService.get('LOG_LEVELS');
@@ -248,7 +250,13 @@ export class WorkspaceMigrationValidateBuildAndRunService {
})
: validateAndBuildResult.workspaceMigration;
await this.workspaceMigrationRunnerService.run(workspaceMigration);
const { metadataEvents } =
await this.workspaceMigrationRunnerService.run(workspaceMigration);
this.metadataEventEmitter.emitMetadataEvents({
metadataEvents,
workspaceId: args.workspaceId,
});
}
public async validateBuildAndRunWorkspaceMigration({
@@ -9,12 +9,14 @@ import {
export type AllUniversalWorkspaceMigrationAction<
TActionType extends
WorkspaceMigrationActionType = WorkspaceMigrationActionType,
> = MetadataUniversalWorkspaceMigrationAction<AllMetadataName, TActionType>;
TMetadataName extends AllMetadataName = AllMetadataName,
> = MetadataUniversalWorkspaceMigrationAction<TMetadataName, TActionType>;
export type AllFlatWorkspaceMigrationAction<
TActionType extends
WorkspaceMigrationActionType = WorkspaceMigrationActionType,
> = MetadataFlatWorkspaceMigrationAction<AllMetadataName, TActionType>;
TMetadataName extends AllMetadataName = AllMetadataName,
> = MetadataFlatWorkspaceMigrationAction<TMetadataName, TActionType>;
export { WorkspaceMigrationActionType };
@@ -6,7 +6,6 @@ import { QueryRunner } from 'typeorm';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant';
import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
import { AllFlatEntityTypesByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name';
import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { MetadataFlatEntity } from 'src/engine/metadata-modules/flat-entity/types/metadata-flat-entity.type';
import { MetadataRelatedFlatEntityMapsKeys } from 'src/engine/metadata-modules/flat-entity/types/metadata-related-flat-entity-maps-keys.type';
@@ -61,9 +60,15 @@ export abstract class BaseWorkspaceMigrationRunnerActionHandlerService<
TActionType extends WorkspaceMigrationActionType,
TMetadataName extends AllMetadataName,
TUniversalAction extends // TODO create abstracted type utils
AllUniversalWorkspaceMigrationAction = AllFlatEntityTypesByMetadataName[TMetadataName]['universalActions'][TActionType],
AllUniversalWorkspaceMigrationAction = AllUniversalWorkspaceMigrationAction<
TActionType,
TMetadataName
>,
TFlatAction extends
AllFlatWorkspaceMigrationAction = AllFlatEntityTypesByMetadataName[TMetadataName]['flatActions'][TActionType],
AllFlatWorkspaceMigrationAction = AllFlatWorkspaceMigrationAction<
TActionType,
TMetadataName
>,
> {
public actionType: TActionType;
public metadataName: TMetadataName;
@@ -8,7 +8,6 @@ import { DataSource } from 'typeorm';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';
import { getMetadataSerializedRelationNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-serialized-relation-names.util';
@@ -22,6 +21,7 @@ import {
WorkspaceMigrationRunnerExceptionCode,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/exceptions/workspace-migration-runner.exception';
import { WorkspaceMigrationRunnerActionHandlerRegistryService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/registry/workspace-migration-runner-action-handler-registry.service';
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
@Injectable()
export class WorkspaceMigrationRunnerService {
@@ -5,6 +5,7 @@ import { type MetadataUniversalFlatEntityPropertiesToCompare } from 'src/engine/
type BaseMetadataEvent<T extends AllMetadataName, TPayload extends object> = {
metadataName: T;
recordId: string;
properties: TPayload;
};
@@ -12,7 +13,7 @@ export type DeleteMetadataEvent<T extends AllMetadataName> = BaseMetadataEvent<
T,
{ before: MetadataFlatEntity<T> }
> & {
type: 'delete';
type: 'deleted';
};
export type UpdateMetadataEventDiff<
@@ -37,16 +38,21 @@ export type UpdateMetadataEvent<
before: MetadataFlatEntity<T>;
after: MetadataFlatEntity<T>;
}
> & { type: 'update' };
> & { type: 'updated' };
export type CreateMetadataEvent<T extends AllMetadataName> = BaseMetadataEvent<
T,
{
after: MetadataFlatEntity<T>;
}
> & { type: 'create' };
> & { type: 'created' };
export type MetadataEvent<T extends AllMetadataName = AllMetadataName> =
| DeleteMetadataEvent<T>
| UpdateMetadataEvent<T>
| CreateMetadataEvent<T>;
export type AllMetadataEventType = MetadataEvent['type'];
export type AllMetadataEventName =
`metadata.${AllMetadataName}.${MetadataEvent['type']}`;
@@ -13,7 +13,8 @@ export const deriveMetadataEventsFromCreateAction = (
case 'fieldMetadata': {
return flatAction.flatFieldMetadatas.map(
(flatFieldMetadata): CreateMetadataEvent<'fieldMetadata'> => ({
type: 'create',
type: 'created',
recordId: flatFieldMetadata.id,
metadataName: 'fieldMetadata',
properties: {
after: flatFieldMetadata,
@@ -23,8 +24,9 @@ export const deriveMetadataEventsFromCreateAction = (
}
case 'objectMetadata': {
const objectEvent: CreateMetadataEvent<'objectMetadata'> = {
type: 'create',
type: 'created',
metadataName: 'objectMetadata',
recordId: flatAction.flatEntity.id,
properties: {
after: flatAction.flatEntity,
},
@@ -32,7 +34,8 @@ export const deriveMetadataEventsFromCreateAction = (
const fieldEvents: MetadataEvent[] = flatAction.flatFieldMetadatas.map(
(flatFieldMetadata): CreateMetadataEvent<'fieldMetadata'> => ({
type: 'create',
type: 'created',
recordId: flatFieldMetadata.id,
metadataName: 'fieldMetadata',
properties: {
after: flatFieldMetadata,
@@ -65,7 +68,8 @@ export const deriveMetadataEventsFromCreateAction = (
case 'webhook': {
return [
{
type: 'create',
type: 'created',
recordId: flatAction.flatEntity.id,
metadataName: flatAction.metadataName,
properties: {
after: flatAction.flatEntity,
@@ -52,8 +52,9 @@ export const deriveMetadataEventsFromDeleteAction = ({
return [
{
type: 'delete',
type: 'deleted',
metadataName: flatAction.metadataName,
recordId: flatAction.entityId,
properties: {
before: flatEntityToDelete,
},
@@ -8,11 +8,11 @@ import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-e
import { type MetadataUniversalFlatEntityPropertiesToCompare } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/metadata-universal-flat-entity-properties-to-compare.type';
import { type AllFlatWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/workspace-migration-action-common';
import {
type UpdateMetadataEventDiff,
type CreateMetadataEvent,
type DeleteMetadataEvent,
type MetadataEvent,
type UpdateMetadataEvent,
type UpdateMetadataEventDiff,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
export type DeriveMetadataEventsFromUpdateActionArgs = {
@@ -42,8 +42,9 @@ const buildUpdateMetadataEvent = <TMetadataName extends AllMetadataName>({
) as UpdateMetadataEventDiff<TMetadataName, (typeof updatedFields)[number]>;
return {
type: 'update',
type: 'updated',
metadataName,
recordId: before.id,
properties: {
updatedFields,
diff,
@@ -68,18 +69,20 @@ export const deriveMetadataEventsFromUpdateAction = ({
const deleteIndexMetadataEvent: DeleteMetadataEvent<'index'> = {
metadataName: 'index',
recordId: fromFlatEntity.id,
properties: {
before: fromFlatEntity,
},
type: 'delete',
type: 'deleted',
};
const createIndexMetadataEvent: CreateMetadataEvent<'index'> = {
metadataName: 'index',
recordId: toFlatEntity.id,
properties: {
after: toFlatEntity,
},
type: 'create',
type: 'created',
};
return [deleteIndexMetadataEvent, createIndexMetadataEvent];