feat: audit Logs (#17660)

This commit is contained in:
Félix Malfait
2026-02-04 13:30:55 +01:00
committed by GitHub
parent 6407474461
commit 177cae8c53
75 changed files with 2950 additions and 49 deletions
File diff suppressed because one or more lines are too long
@@ -472,6 +472,7 @@ export type BillingEntitlement = {
};
export enum BillingEntitlementKey {
AUDIT_LOGS = 'AUDIT_LOGS',
CUSTOM_DOMAIN = 'CUSTOM_DOMAIN',
RLS = 'RLS',
SSO = 'SSO'
@@ -1356,6 +1357,56 @@ export type EmailsConfiguration = {
configurationType: WidgetConfigurationType;
};
export type EventLogDateRangeInput = {
end?: InputMaybe<Scalars['DateTime']>;
start?: InputMaybe<Scalars['DateTime']>;
};
export type EventLogFiltersInput = {
dateRange?: InputMaybe<EventLogDateRangeInput>;
eventType?: InputMaybe<Scalars['String']>;
objectMetadataId?: InputMaybe<Scalars['String']>;
recordId?: InputMaybe<Scalars['String']>;
userWorkspaceId?: InputMaybe<Scalars['String']>;
};
export type EventLogPageInfo = {
__typename?: 'EventLogPageInfo';
endCursor?: Maybe<Scalars['String']>;
hasNextPage: Scalars['Boolean'];
};
export type EventLogQueryInput = {
after?: InputMaybe<Scalars['String']>;
filters?: InputMaybe<EventLogFiltersInput>;
first?: InputMaybe<Scalars['Int']>;
table: EventLogTable;
};
export type EventLogQueryResult = {
__typename?: 'EventLogQueryResult';
pageInfo: EventLogPageInfo;
records: Array<EventLogRecord>;
totalCount: Scalars['Int'];
};
export type EventLogRecord = {
__typename?: 'EventLogRecord';
event: Scalars['String'];
isCustom?: Maybe<Scalars['Boolean']>;
objectMetadataId?: Maybe<Scalars['String']>;
properties?: Maybe<Scalars['JSON']>;
recordId?: Maybe<Scalars['String']>;
timestamp: Scalars['DateTime'];
userWorkspaceId?: Maybe<Scalars['String']>;
};
export enum EventLogTable {
OBJECT_EVENT = 'OBJECT_EVENT',
PAGEVIEW = 'PAGEVIEW',
WORKSPACE_EVENT = 'WORKSPACE_EVENT'
}
export type EventSubscription = {
__typename?: 'EventSubscription';
eventStreamId: Scalars['String'];
@@ -3482,6 +3533,7 @@ export type Query = {
commandMenuItems: Array<CommandMenuItem>;
currentUser: User;
currentWorkspace: Workspace;
eventLogs: EventLogQueryResult;
field: Field;
fields: FieldConnection;
findManyAgents: Array<Agent>;
@@ -3591,6 +3643,11 @@ export type QueryCommandMenuItemArgs = {
};
export type QueryEventLogsArgs = {
input: EventLogQueryInput;
};
export type QueryFindOneAgentArgs = {
input: AgentIdInput;
};
@@ -4711,6 +4768,7 @@ export type UpdateWorkspaceInput = {
defaultRoleId?: InputMaybe<Scalars['UUID']>;
displayName?: InputMaybe<Scalars['String']>;
editableProfileFields?: InputMaybe<Array<Scalars['String']>>;
eventLogRetentionDays?: InputMaybe<Scalars['Float']>;
fastModel?: InputMaybe<Scalars['String']>;
inviteHash?: InputMaybe<Scalars['String']>;
isGoogleAuthBypassEnabled?: InputMaybe<Scalars['Boolean']>;
@@ -5104,6 +5162,7 @@ export type Workspace = {
deletedAt?: Maybe<Scalars['DateTime']>;
displayName?: Maybe<Scalars['String']>;
editableProfileFields?: Maybe<Array<Scalars['String']>>;
eventLogRetentionDays: Scalars['Float'];
fastModel: Scalars['String'];
featureFlags?: Maybe<Array<FeatureFlagDto>>;
hasValidEnterpriseKey: Scalars['Boolean'];
@@ -65,6 +65,7 @@ const mockWorkspace = {
},
isTwoFactorAuthenticationEnforced: false,
trashRetentionDays: 14,
eventLogRetentionDays: 365 * 3,
fastModel: DEFAULT_FAST_MODEL,
smartModel: DEFAULT_SMART_MODEL,
routerModel: 'auto',
@@ -300,6 +300,14 @@ const SettingsSecurityApprovedAccessDomain = lazy(() =>
),
);
const SettingsEventLogs = lazy(() =>
import('~/pages/settings/security/event-logs/SettingsEventLogs').then(
(module) => ({
default: module.SettingsEventLogs,
}),
),
);
const SettingsNewEmailingDomain = lazy(() =>
import('~/pages/settings/emailing-domains/SettingsNewEmailingDomain').then(
(module) => ({
@@ -615,6 +623,7 @@ export const SettingsRoutes = ({ isAdminPageEnabled }: SettingsRoutesProps) => (
path={SettingsPath.NewApprovedAccessDomain}
element={<SettingsSecurityApprovedAccessDomain />}
/>
<Route path={SettingsPath.EventLogs} element={<SettingsEventLogs />} />
</Route>
{isAdminPageEnabled && (
@@ -33,6 +33,7 @@ export type CurrentWorkspace = Pick<
| 'metadataVersion'
| 'isTwoFactorAuthenticationEnforced'
| 'trashRetentionDays'
| 'eventLogRetentionDays'
| 'fastModel'
| 'smartModel'
| 'editableProfileFields'
@@ -11,6 +11,7 @@ import { isAnalyticsEnabledState } from '@/client-config/states/isAnalyticsEnabl
import { isAttachmentPreviewEnabledState } from '@/client-config/states/isAttachmentPreviewEnabledState';
import { isConfigVariablesInDbEnabledState } from '@/client-config/states/isConfigVariablesInDbEnabledState';
import { isDeveloperDefaultSignInPrefilledState } from '@/client-config/states/isDeveloperDefaultSignInPrefilledState';
import { isClickHouseConfiguredState } from '@/client-config/states/isClickHouseConfiguredState';
import { isCloudflareIntegrationEnabledState } from '@/client-config/states/isCloudflareIntegrationEnabledState';
import { isEmailingDomainsEnabledState } from '@/client-config/states/isEmailingDomainsEnabledState';
import { isEmailVerificationRequiredState } from '@/client-config/states/isEmailVerificationRequiredState';
@@ -120,6 +121,10 @@ export const useClientConfig = (): UseClientConfigResult => {
isCloudflareIntegrationEnabledState,
);
const setIsClickHouseConfigured = useSetRecoilState(
isClickHouseConfiguredState,
);
const setAppVersion = useSetRecoilState(appVersionState);
const fetchClientConfig = useCallback(async () => {
@@ -198,6 +203,7 @@ export const useClientConfig = (): UseClientConfigResult => {
setIsCloudflareIntegrationEnabled(
clientConfig?.isCloudflareIntegrationEnabled,
);
setIsClickHouseConfigured(clientConfig?.isClickHouseConfigured ?? false);
} catch (err) {
const error =
err instanceof Error ? err : new Error('Failed to fetch client config');
@@ -231,6 +237,7 @@ export const useClientConfig = (): UseClientConfigResult => {
setIsImapSmtpCaldavEnabled,
setIsMultiWorkspaceEnabled,
setIsEmailingDomainsEnabled,
setIsClickHouseConfigured,
setIsCloudflareIntegrationEnabled,
setLabPublicFeatureFlags,
setMicrosoftCalendarEnabled,
@@ -0,0 +1,6 @@
import { createState } from 'twenty-ui/utilities';
export const isClickHouseConfiguredState = createState<boolean>({
key: 'isClickHouseConfigured',
defaultValue: false,
});
@@ -33,6 +33,7 @@ export type ClientConfig = {
isImapSmtpCaldavEnabled: boolean;
isEmailingDomainsEnabled: boolean;
isCloudflareIntegrationEnabled: boolean;
isClickHouseConfigured: boolean;
publicFeatureFlags: Array<PublicFeatureFlag>;
sentry: Sentry;
signInPrefilled: boolean;
@@ -61,6 +61,7 @@ const Wrapper = getJestMetadataAndApolloMocksAndActionMenuWrapper({
],
isTwoFactorAuthenticationEnforced: false,
trashRetentionDays: 14,
eventLogRetentionDays: 365 * 3,
fastModel: DEFAULT_FAST_MODEL,
smartModel: DEFAULT_SMART_MODEL,
});
@@ -13,7 +13,11 @@ export const useShowFullscreen = () => {
location,
'settings/' + SettingsPath.RestPlayground + '/*',
) ||
isMatchingLocation(location, 'settings/' + SettingsPath.GraphQLPlayground)
isMatchingLocation(
location,
'settings/' + SettingsPath.GraphQLPlayground,
) ||
isMatchingLocation(location, 'settings/' + SettingsPath.EventLogs)
) {
return true;
}
@@ -90,6 +90,7 @@ export const USER_QUERY_FRAGMENT = gql`
smartModel
isTwoFactorAuthenticationEnforced
trashRetentionDays
eventLogRetentionDays
editableProfileFields
}
availableWorkspaces {
@@ -1,10 +1,14 @@
import isPropValid from '@emotion/is-prop-valid';
import styled from '@emotion/styled';
import { Trans, useLingui } from '@lingui/react/macro';
import { Link } from 'react-router-dom';
import { useDebouncedCallback } from 'use-debounce';
import { currentWorkspaceState } from '@/auth/states/currentWorkspaceState';
import { authProvidersState } from '@/client-config/states/authProvidersState';
import { isClickHouseConfiguredState } from '@/client-config/states/isClickHouseConfiguredState';
import { isMultiWorkspaceEnabledState } from '@/client-config/states/isMultiWorkspaceEnabledState';
import { SettingsOptionCardContentButton } from '@/settings/components/SettingsOptions/SettingsOptionCardContentButton';
import { SettingsOptionCardContentCounter } from '@/settings/components/SettingsOptions/SettingsOptionCardContentCounter';
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
import { SettingsSSOIdentitiesProvidersListCard } from '@/settings/security/components/SSO/SettingsSSOIdentitiesProvidersListCard';
@@ -20,7 +24,14 @@ import { useRecoilState, useRecoilValue } from 'recoil';
import { SettingsPath } from 'twenty-shared/types';
import { getSettingsPath } from 'twenty-shared/utils';
import { Tag } from 'twenty-ui/components';
import { H2Title, IconLock, IconTrash } from 'twenty-ui/display';
import {
H2Title,
IconClockHour8,
IconHistory,
IconLock,
IconTrash,
} from 'twenty-ui/display';
import { Button } from 'twenty-ui/input';
import { Card, Section } from 'twenty-ui/layout';
import { useUpdateWorkspaceMutation } from '~/generated-metadata/graphql';
@@ -39,11 +50,19 @@ const StyledSection = styled(Section)`
flex-shrink: 0;
`;
const StyledLink = styled(Link, {
shouldForwardProp: (prop) => isPropValid(prop) && prop !== 'isDisabled',
})<{ isDisabled: boolean }>`
pointer-events: ${({ isDisabled }) => (isDisabled ? 'none' : 'auto')};
text-decoration: none;
`;
export const SettingsSecurity = () => {
const { t } = useLingui();
const { enqueueErrorSnackBar } = useSnackBar();
const isMultiWorkspaceEnabled = useRecoilValue(isMultiWorkspaceEnabledState);
const isClickHouseConfigured = useRecoilValue(isClickHouseConfiguredState);
const authProviders = useRecoilValue(authProvidersState);
const SSOIdentitiesProviders = useRecoilValue(SSOIdentitiesProvidersState);
const [currentWorkspace, setCurrentWorkspace] = useRecoilState(
@@ -51,12 +70,8 @@ export const SettingsSecurity = () => {
);
const [updateWorkspace] = useUpdateWorkspaceMutation();
const saveWorkspace = useDebouncedCallback(async (value: number) => {
const saveTrashRetention = useDebouncedCallback(async (value: number) => {
try {
if (!currentWorkspace?.id) {
throw new Error('User is not logged in');
}
await updateWorkspace({
variables: {
input: {
@@ -71,6 +86,22 @@ export const SettingsSecurity = () => {
}
}, 500);
const saveEventLogRetention = useDebouncedCallback(async (value: number) => {
try {
await updateWorkspace({
variables: {
input: {
eventLogRetentionDays: value,
},
},
});
} catch (err) {
enqueueErrorSnackBar({
apolloError: err instanceof ApolloError ? err : undefined,
});
}
}, 500);
const handleTrashRetentionDaysChange = (value: number) => {
if (!currentWorkspace) {
return;
@@ -85,7 +116,24 @@ export const SettingsSecurity = () => {
trashRetentionDays: value,
});
saveWorkspace(value);
saveTrashRetention(value);
};
const handleEventLogRetentionDaysChange = (value: number) => {
if (!currentWorkspace) {
return;
}
if (value === currentWorkspace.eventLogRetentionDays) {
return;
}
setCurrentWorkspace({
...currentWorkspace,
eventLogRetentionDays: value,
});
saveEventLogRetention(value);
};
const hasSsoIdentityProviders = SSOIdentitiesProviders.length > 0;
@@ -102,6 +150,9 @@ export const SettingsSecurity = () => {
!hasDirectAuthEnabled &&
hasBypassProviderAvailable;
const hasEnterpriseAccess = currentWorkspace?.hasValidEnterpriseKey === true;
const isEventLogsEnabled = hasEnterpriseAccess && isClickHouseConfigured;
return (
<SubMenuTopBarContainer
title={t`Security`}
@@ -169,6 +220,58 @@ export const SettingsSecurity = () => {
<ToggleImpersonate />
</Section>
)}
<Section>
<H2Title
title={t`Audit Logs`}
description={t`View workspace activity logs`}
adornment={
<Tag
text={t`Enterprise`}
color="transparent"
Icon={IconLock}
variant="border"
/>
}
/>
<Card rounded>
<SettingsOptionCardContentButton
Icon={IconHistory}
title={t`Audit Logs`}
description={
!isClickHouseConfigured
? t`ClickHouse is required for audit logs. Contact your administrator.`
: !hasEnterpriseAccess
? t`Upgrade to Enterprise to access audit logs`
: t`View and filter events, page views, object changes`
}
Button={
<StyledLink
to={getSettingsPath(SettingsPath.EventLogs)}
isDisabled={!isEventLogsEnabled}
>
<Button
title={t`View Logs`}
variant="secondary"
size="small"
disabled={!isEventLogsEnabled}
/>
</StyledLink>
}
/>
{isEventLogsEnabled && (
<SettingsOptionCardContentCounter
Icon={IconClockHour8}
title={t`Log retention`}
description={t`Number of days to retain audit logs (30-1095 days)`}
value={currentWorkspace?.eventLogRetentionDays ?? 90}
onChange={handleEventLogRetentionDaysChange}
minValue={30}
maxValue={1095}
showButtons={false}
/>
)}
</Card>
</Section>
<Section>
<H2Title
title={t`Other`}
@@ -0,0 +1,245 @@
import styled from '@emotion/styled';
import { Trans, useLingui } from '@lingui/react/macro';
import { useState } from 'react';
import { FullScreenContainer } from '@/ui/layout/fullscreen/components/FullScreenContainer';
import { SettingsPath } from 'twenty-shared/types';
import { getSettingsPath, isDefined } from 'twenty-shared/utils';
import { IconAlertTriangle, IconRefresh } from 'twenty-ui/display';
import { Button } from 'twenty-ui/input';
import { useNavigateSettings } from '~/hooks/useNavigateSettings';
import { EventLogTable } from '~/generated-metadata/graphql';
import { EventLogFilters } from './components/EventLogFilters';
import { EventLogResultsTable } from './components/EventLogResultsTable';
import { EventLogTableSelector } from './components/EventLogTableSelector';
import { useEventLogs } from './hooks/useQueryEventLogs';
const StyledContainer = styled.div`
background: ${({ theme }) => theme.background.primary};
border: 1px solid ${({ theme }) => theme.border.color.medium};
border-radius: ${({ theme }) => theme.border.radius.md};
display: flex;
flex-direction: column;
height: 100%;
overflow: hidden;
`;
const StyledHeader = styled.div`
border-bottom: 1px solid ${({ theme }) => theme.border.color.light};
display: flex;
flex-direction: column;
gap: ${({ theme }) => theme.spacing(4)};
padding: ${({ theme }) => theme.spacing(4)};
`;
const StyledHeaderRow = styled.div`
align-items: center;
display: flex;
justify-content: space-between;
`;
const StyledRecordCount = styled.span`
color: ${({ theme }) => theme.font.color.secondary};
font-size: ${({ theme }) => theme.font.size.sm};
`;
const StyledTableWrapper = styled.div`
flex: 1;
min-height: 0;
overflow: hidden;
`;
const StyledErrorContainer = styled.div`
align-items: center;
display: flex;
flex-direction: column;
gap: ${({ theme }) => theme.spacing(4)};
justify-content: center;
padding: ${({ theme }) => theme.spacing(8)};
text-align: center;
`;
const StyledErrorIcon = styled.div`
color: ${({ theme }) => theme.color.orange};
`;
const StyledErrorTitle = styled.h3`
color: ${({ theme }) => theme.font.color.primary};
font-size: ${({ theme }) => theme.font.size.lg};
font-weight: ${({ theme }) => theme.font.weight.semiBold};
margin: 0;
`;
const StyledErrorMessage = styled.p`
color: ${({ theme }) => theme.font.color.secondary};
font-size: ${({ theme }) => theme.font.size.md};
margin: 0;
max-width: 400px;
`;
export type EventLogFiltersState = {
eventType?: string;
userWorkspaceId?: string;
dateRange?: {
start?: Date;
end?: Date;
};
recordId?: string;
objectMetadataId?: string;
};
const RECORDS_PER_PAGE = 100;
export const SettingsEventLogs = () => {
const { t } = useLingui();
const navigateSettings = useNavigateSettings();
const [selectedTable, setSelectedTable] = useState<EventLogTable>(
EventLogTable.PAGEVIEW,
);
const [filters, setFilters] = useState<EventLogFiltersState>({});
const {
records,
totalCount,
hasNextPage,
loading,
error,
refetch,
loadMore,
} = useEventLogs({
table: selectedTable,
filters: {
eventType: filters.eventType,
userWorkspaceId: filters.userWorkspaceId,
dateRange: filters.dateRange
? {
start: filters.dateRange.start?.toISOString(),
end: filters.dateRange.end?.toISOString(),
}
: undefined,
recordId: filters.recordId,
objectMetadataId: filters.objectMetadataId,
},
first: RECORDS_PER_PAGE,
});
const handleTableChange = (table: EventLogTable) => {
setSelectedTable(table);
setFilters({});
};
const handleFiltersChange = (newFilters: EventLogFiltersState) => {
setFilters(newFilters);
};
const handleExitFullScreen = () => {
navigateSettings(SettingsPath.Security);
};
const recordCount = records.length;
const getErrorContent = () => {
if (!isDefined(error)) {
return null;
}
const errorMessage = error.message || '';
const isClickHouseError = errorMessage.includes('ClickHouse');
const isEntitlementError =
errorMessage.includes('Enterprise') ||
errorMessage.includes('entitlement');
if (isClickHouseError) {
return {
title: t`ClickHouse Not Configured`,
message: t`Audit logs require ClickHouse to be configured. Please contact your administrator to set up ClickHouse.`,
};
}
if (isEntitlementError) {
return {
title: t`Enterprise Feature`,
message: t`Audit logs are available with an Enterprise subscription. Please upgrade to access this feature.`,
};
}
return {
title: t`Error Loading Audit Logs`,
message:
errorMessage || t`An unexpected error occurred. Please try again.`,
};
};
const errorContent = getErrorContent();
return (
<FullScreenContainer
exitFullScreen={handleExitFullScreen}
links={[
{
children: <Trans>Workspace</Trans>,
href: getSettingsPath(SettingsPath.Workspace),
},
{
children: <Trans>Security</Trans>,
href: getSettingsPath(SettingsPath.Security),
},
{ children: <Trans>Audit Logs</Trans> },
]}
>
<StyledContainer>
{isDefined(errorContent) ? (
<StyledErrorContainer>
<StyledErrorIcon>
<IconAlertTriangle size={48} />
</StyledErrorIcon>
<StyledErrorTitle>{errorContent.title}</StyledErrorTitle>
<StyledErrorMessage>{errorContent.message}</StyledErrorMessage>
<Button
title={t`Go Back`}
variant="secondary"
onClick={handleExitFullScreen}
/>
</StyledErrorContainer>
) : (
<>
<StyledHeader>
<EventLogTableSelector
value={selectedTable}
onChange={handleTableChange}
/>
<EventLogFilters
table={selectedTable}
value={filters}
onChange={handleFiltersChange}
/>
<StyledHeaderRow>
<StyledRecordCount>
{t`${recordCount} of ${totalCount} records`}
</StyledRecordCount>
<Button
Icon={IconRefresh}
variant="tertiary"
size="small"
onClick={() => refetch()}
title={t`Refresh`}
/>
</StyledHeaderRow>
</StyledHeader>
<StyledTableWrapper>
<EventLogResultsTable
records={records}
loading={loading}
hasNextPage={hasNextPage}
onLoadMore={loadMore}
selectedTable={selectedTable}
/>
</StyledTableWrapper>
</>
)}
</StyledContainer>
</FullScreenContainer>
);
};
@@ -0,0 +1,148 @@
import styled from '@emotion/styled';
import { useLingui } from '@lingui/react/macro';
import { useRef, useState } from 'react';
import { Temporal } from 'temporal-polyfill';
import {
DateTimePicker,
MONTH_AND_YEAR_DROPDOWN_MONTH_SELECT_ID,
MONTH_AND_YEAR_DROPDOWN_YEAR_SELECT_ID,
} from '@/ui/input/components/internal/date/components/DateTimePicker';
import { OverlayContainer } from '@/ui/layout/overlay/components/OverlayContainer';
import { useListenClickOutside } from '@/ui/utilities/pointer-event/hooks/useListenClickOutside';
import { isDefined } from 'twenty-shared/utils';
import { IconCalendar } from 'twenty-ui/display';
const StyledInputContainer = styled.div`
position: relative;
`;
const StyledInput = styled.div<{ hasValue: boolean }>`
align-items: center;
background-color: ${({ theme }) => theme.background.transparent.lighter};
border: 1px solid ${({ theme }) => theme.border.color.medium};
border-radius: ${({ theme }) => theme.border.radius.sm};
box-sizing: border-box;
color: ${({ theme, hasValue }) =>
hasValue ? theme.font.color.primary : theme.font.color.light};
cursor: pointer;
display: flex;
gap: ${({ theme }) => theme.spacing(2)};
height: 32px;
padding: 0 ${({ theme }) => theme.spacing(2)};
width: 100%;
&:hover {
border-color: ${({ theme }) => theme.border.color.strong};
}
`;
const StyledLabel = styled.div`
color: ${({ theme }) => theme.font.color.light};
font-size: ${({ theme }) => theme.font.size.xs};
font-weight: ${({ theme }) => theme.font.weight.medium};
margin-bottom: ${({ theme }) => theme.spacing(1)};
`;
const StyledDatePickerContainer = styled.div`
left: 0;
position: absolute;
top: 100%;
z-index: 1000;
`;
const StyledIconContainer = styled.div`
color: ${({ theme }) => theme.font.color.tertiary};
display: flex;
`;
type EventLogDatePickerInputProps = {
label: string;
value: Date | undefined;
onChange: (date: Date | undefined) => void;
placeholder?: string;
};
export const EventLogDatePickerInput = ({
label,
value,
onChange,
placeholder,
}: EventLogDatePickerInputProps) => {
const { t } = useLingui();
const [isOpen, setIsOpen] = useState(false);
const containerRef = useRef<HTMLDivElement>(null);
const handleClose = () => {
setIsOpen(false);
};
useListenClickOutside({
refs: [containerRef],
listenerId: `event-log-date-picker-${label}`,
callback: handleClose,
enabled: isOpen,
excludedClickOutsideIds: [
MONTH_AND_YEAR_DROPDOWN_MONTH_SELECT_ID,
MONTH_AND_YEAR_DROPDOWN_YEAR_SELECT_ID,
],
});
const handleDateTimeSelect = (newDateTime: Temporal.ZonedDateTime | null) => {
if (isDefined(newDateTime)) {
onChange(new Date(newDateTime.epochMilliseconds));
}
handleClose();
};
const handleClear = () => {
onChange(undefined);
handleClose();
};
const formatDisplayValue = (date: Date | undefined): string => {
if (!isDefined(date)) {
return placeholder ?? t`Select date & time`;
}
return date.toLocaleString(undefined, {
year: 'numeric',
month: 'short',
day: 'numeric',
hour: '2-digit',
minute: '2-digit',
});
};
const zonedDateTime = isDefined(value)
? Temporal.Instant.fromEpochMilliseconds(
value.getTime(),
).toZonedDateTimeISO(Temporal.Now.timeZoneId())
: null;
return (
<StyledInputContainer ref={containerRef}>
<StyledLabel>{label}</StyledLabel>
<StyledInput hasValue={isDefined(value)} onClick={() => setIsOpen(true)}>
<StyledIconContainer>
<IconCalendar size={16} />
</StyledIconContainer>
{formatDisplayValue(value)}
</StyledInput>
{isOpen && (
<StyledDatePickerContainer>
<OverlayContainer>
<DateTimePicker
instanceId={`event-log-date-picker-${label}`}
date={zonedDateTime}
onChange={handleDateTimeSelect}
onClose={handleDateTimeSelect}
onClear={handleClear}
clearable
/>
</OverlayContainer>
</StyledDatePickerContainer>
)}
</StyledInputContainer>
);
};
@@ -0,0 +1,173 @@
import styled from '@emotion/styled';
import { useLingui } from '@lingui/react/macro';
import { useRecoilValue } from 'recoil';
import { currentWorkspaceMembersState } from '@/auth/states/currentWorkspaceMembersState';
import { useObjectMetadataItems } from '@/object-metadata/hooks/useObjectMetadataItems';
import { Select } from '@/ui/input/components/Select';
import { TextInput } from '@/ui/input/components/TextInput';
import { IconUser, IconBox, useIcons } from 'twenty-ui/display';
import { type SelectOption } from 'twenty-ui/input';
import { EventLogTable } from '~/generated-metadata/graphql';
import { type EventLogFiltersState } from '~/pages/settings/security/event-logs/SettingsEventLogs';
import { EventLogDatePickerInput } from './EventLogDatePickerInput';
type EventLogFiltersProps = {
table: EventLogTable;
value: EventLogFiltersState;
onChange: (filters: EventLogFiltersState) => void;
};
const StyledFiltersContainer = styled.div`
align-items: flex-end;
display: flex;
flex-wrap: wrap;
gap: ${({ theme }) => theme.spacing(2)};
`;
const StyledFilterItem = styled.div`
flex: 1;
min-width: 200px;
max-width: 300px;
`;
export const EventLogFilters = ({
table,
value,
onChange,
}: EventLogFiltersProps) => {
const { t } = useLingui();
const currentWorkspaceMembers = useRecoilValue(currentWorkspaceMembersState);
const { objectMetadataItems } = useObjectMetadataItems();
const { getIcon } = useIcons();
const handleEventTypeChange = (eventType: string) => {
onChange({ ...value, eventType: eventType || undefined });
};
const handleUserWorkspaceChange = (userWorkspaceId: string | null) => {
onChange({ ...value, userWorkspaceId: userWorkspaceId || undefined });
};
const handleStartDateChange = (date: Date | undefined) => {
onChange({
...value,
dateRange: {
...value.dateRange,
start: date,
},
});
};
const handleEndDateChange = (date: Date | undefined) => {
onChange({
...value,
dateRange: {
...value.dateRange,
end: date,
},
});
};
const handleRecordIdChange = (recordId: string) => {
onChange({ ...value, recordId: recordId || undefined });
};
const handleObjectMetadataIdChange = (objectMetadataId: string | null) => {
onChange({ ...value, objectMetadataId: objectMetadataId || undefined });
};
const eventLabel =
table === EventLogTable.PAGEVIEW ? t`Page Name` : t`Event Type`;
const userWorkspaceOptions: SelectOption<string | null>[] = [
{ label: t`All Members`, value: null, Icon: IconUser },
...currentWorkspaceMembers
.filter((member) => member.userWorkspaceId)
.map((workspaceMember) => ({
label:
`${workspaceMember.name.firstName ?? ''} ${workspaceMember.name.lastName ?? ''}`.trim(),
value: workspaceMember.userWorkspaceId as string,
Icon: IconUser,
})),
];
const objectMetadataOptions: SelectOption<string | null>[] = [
{ label: t`All Objects`, value: null, Icon: IconBox },
...objectMetadataItems.map((item) => ({
label: item.labelPlural,
value: item.id,
Icon: getIcon(item.icon) ?? IconBox,
})),
];
return (
<StyledFiltersContainer>
<StyledFilterItem>
<TextInput
label={eventLabel}
value={value.eventType ?? ''}
onChange={handleEventTypeChange}
placeholder={t`Filter by event`}
fullWidth
/>
</StyledFilterItem>
<StyledFilterItem>
<Select
dropdownId="event-log-user-workspace-filter"
label={t`Workspace Member`}
value={value.userWorkspaceId ?? null}
options={userWorkspaceOptions}
onChange={handleUserWorkspaceChange}
fullWidth
withSearchInput
/>
</StyledFilterItem>
<StyledFilterItem>
<EventLogDatePickerInput
label={t`Start Date`}
value={value.dateRange?.start}
onChange={handleStartDateChange}
/>
</StyledFilterItem>
<StyledFilterItem>
<EventLogDatePickerInput
label={t`End Date`}
value={value.dateRange?.end}
onChange={handleEndDateChange}
/>
</StyledFilterItem>
{table === EventLogTable.OBJECT_EVENT && (
<>
<StyledFilterItem>
<Select
dropdownId="event-log-object-type-filter"
label={t`Object Type`}
value={value.objectMetadataId ?? null}
options={objectMetadataOptions}
onChange={handleObjectMetadataIdChange}
fullWidth
withSearchInput
/>
</StyledFilterItem>
<StyledFilterItem>
<TextInput
label={t`Record ID`}
value={value.recordId ?? ''}
onChange={handleRecordIdChange}
placeholder={t`Filter by record ID`}
fullWidth
/>
</StyledFilterItem>
</>
)}
</StyledFiltersContainer>
);
};
@@ -0,0 +1,65 @@
import styled from '@emotion/styled';
import { t } from '@lingui/core/macro';
import { useRef, useState } from 'react';
import { JsonDisplay } from '@/ui/field/display/components/JsonDisplay';
import { ExpandedFieldDisplay } from '@/ui/layout/expandable-list/components/ExpandedFieldDisplay';
import { type JsonValue } from 'type-fest';
import { isDefined } from 'twenty-shared/utils';
import { isTwoFirstDepths, JsonTree } from 'twenty-ui/json-visualizer';
import { useCopyToClipboard } from '~/hooks/useCopyToClipboard';
type EventLogJsonCellProps = {
value: Record<string, unknown> | null | undefined;
};
const StyledJsonContainer = styled.div`
cursor: pointer;
`;
const StyledEmptyCell = styled.span`
color: ${({ theme }) => theme.font.color.tertiary};
`;
export const EventLogJsonCell = ({ value }: EventLogJsonCellProps) => {
const { copyToClipboard } = useCopyToClipboard();
const [isExpanded, setIsExpanded] = useState(false);
const anchorRef = useRef<HTMLDivElement>(null);
if (!isDefined(value) || Object.keys(value).length === 0) {
return <StyledEmptyCell>-</StyledEmptyCell>;
}
const handleClick = () => {
setIsExpanded(true);
};
const handleClickOutside = () => {
setIsExpanded(false);
};
return (
<>
<StyledJsonContainer ref={anchorRef} onClick={handleClick}>
<JsonDisplay text={JSON.stringify(value)} />
</StyledJsonContainer>
{isExpanded && (
<ExpandedFieldDisplay
anchorElement={anchorRef.current ?? undefined}
onClickOutside={handleClickOutside}
>
<JsonTree
value={value as JsonValue}
shouldExpandNodeInitially={isTwoFirstDepths}
emptyArrayLabel={t`Empty Array`}
emptyObjectLabel={t`Empty Object`}
emptyStringLabel={t`[empty string]`}
arrowButtonCollapsedLabel={t`Expand`}
arrowButtonExpandedLabel={t`Collapse`}
onNodeValueClick={copyToClipboard}
/>
</ExpandedFieldDisplay>
)}
</>
);
};
@@ -0,0 +1,344 @@
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { msg } from '@lingui/core/macro';
import { Trans, useLingui } from '@lingui/react/macro';
import { useCallback, useEffect, useState } from 'react';
import { useInView } from 'react-intersection-observer';
import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
import { Table } from '@/ui/layout/table/components/Table';
import { TableCell } from '@/ui/layout/table/components/TableCell';
import { TableHeader } from '@/ui/layout/table/components/TableHeader';
import { TableRow } from '@/ui/layout/table/components/TableRow';
import { ScrollWrapper } from '@/ui/utilities/scroll/components/ScrollWrapper';
import { useScrollWrapperHTMLElement } from '@/ui/utilities/scroll/hooks/useScrollWrapperHTMLElement';
import { beautifyPastDateRelativeToNow } from '~/utils/date-utils';
import { type MessageDescriptor } from '@lingui/core';
import {
type EventLogRecord,
EventLogTable,
} from '~/generated-metadata/graphql';
import { EventLogJsonCell } from './EventLogJsonCell';
type EventLogResultsTableProps = {
records: EventLogRecord[];
loading: boolean;
hasNextPage: boolean;
onLoadMore: () => void;
selectedTable: EventLogTable;
};
type ColumnConfig = {
id: string;
label: MessageDescriptor;
minWidth: number;
defaultWidth: number;
};
const DEFAULT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 200 },
{ id: 'timestamp', label: msg`Timestamp`, minWidth: 100, defaultWidth: 150 },
{
id: 'userWorkspaceId',
label: msg`User Workspace`,
minWidth: 100,
defaultWidth: 150,
},
{
id: 'properties',
label: msg`Properties`,
minWidth: 200,
defaultWidth: 400,
},
];
const OBJECT_EVENT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 180 },
{ id: 'timestamp', label: msg`Timestamp`, minWidth: 100, defaultWidth: 130 },
{
id: 'userWorkspaceId',
label: msg`User Workspace`,
minWidth: 100,
defaultWidth: 130,
},
{ id: 'recordId', label: msg`Record ID`, minWidth: 100, defaultWidth: 130 },
{
id: 'objectMetadataId',
label: msg`Object ID`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'properties',
label: msg`Properties`,
minWidth: 150,
defaultWidth: 300,
},
];
const StyledScrollWrapper = styled(ScrollWrapper)`
height: 100%;
`;
const StyledTable = styled(Table)`
border-collapse: collapse;
min-width: 100%;
table-layout: fixed;
`;
const StyledHeaderRow = styled(TableRow)<{ gridTemplateColumns: string }>`
display: grid;
grid-template-columns: ${({ gridTemplateColumns }) => gridTemplateColumns};
`;
const StyledDataRow = styled(TableRow)<{ gridTemplateColumns: string }>`
display: grid;
grid-template-columns: ${({ gridTemplateColumns }) => gridTemplateColumns};
`;
const StyledResizableHeader = styled(TableHeader)<{ isResizing?: boolean }>`
position: relative;
user-select: ${({ isResizing }) => (isResizing ? 'none' : 'auto')};
`;
const StyledResizeHandle = styled.div<{ isResizing: boolean }>`
background: ${({ isResizing, theme }) =>
isResizing ? theme.color.blue : 'transparent'};
bottom: 0;
cursor: col-resize;
position: absolute;
right: 0;
top: 0;
width: 4px;
&:hover {
background: ${({ theme }) => theme.color.blue};
}
`;
const StyledTableCell = styled(TableCell)`
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
& > * {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
`;
const StyledEmptyState = styled.div`
color: ${({ theme }) => theme.font.color.tertiary};
padding: ${({ theme }) => theme.spacing(8)};
text-align: center;
`;
const StyledLoadingMore = styled.div`
color: ${({ theme }) => theme.font.color.tertiary};
padding: ${({ theme }) => theme.spacing(4)};
text-align: center;
`;
const StyledSkeletonContainer = styled.div`
padding: ${({ theme }) => theme.spacing(2)};
`;
const StyledIntersectionObserver = styled.div`
height: 1px;
`;
const SKELETON_ROW_COUNT = 8;
const EVENT_LOG_SCROLL_WRAPPER_INSTANCE_ID = 'event-log-results-table';
const buildGridTemplateColumns = (
columns: ColumnConfig[],
widths: Record<string, number>,
): string => {
return columns
.map((col, index) => {
const isLastColumn = index === columns.length - 1;
const width = widths[col.id] ?? col.defaultWidth;
// eslint-disable-next-line lingui/no-unlocalized-strings
return isLastColumn ? `minmax(${width}px, 1fr)` : `${width}px`;
})
.join(' ');
};
export const EventLogResultsTable = ({
records,
loading,
hasNextPage,
onLoadMore,
selectedTable,
}: EventLogResultsTableProps) => {
const { t } = useLingui();
const theme = useTheme();
const showObjectEventColumns = selectedTable === EventLogTable.OBJECT_EVENT;
const baseColumns = showObjectEventColumns
? OBJECT_EVENT_COLUMNS
: DEFAULT_COLUMNS;
const [columnWidths, setColumnWidths] = useState<Record<string, number>>(() =>
Object.fromEntries(baseColumns.map((col) => [col.id, col.defaultWidth])),
);
const [resizingColumn, setResizingColumn] = useState<string | null>(null);
// Reset column widths when switching tables to avoid undefined widths for new columns
useEffect(() => {
setColumnWidths(
Object.fromEntries(baseColumns.map((col) => [col.id, col.defaultWidth])),
);
}, [selectedTable, baseColumns]);
const handleResizeStart = useCallback(
(columnId: string, event: React.PointerEvent) => {
event.preventDefault();
setResizingColumn(columnId);
const startX = event.clientX;
const column = baseColumns.find((col) => col.id === columnId);
const startWidth = columnWidths[columnId] ?? column?.defaultWidth ?? 100;
const handlePointerMove = (moveEvent: PointerEvent) => {
const delta = moveEvent.clientX - startX;
const newWidth = Math.max(column?.minWidth ?? 50, startWidth + delta);
setColumnWidths((prev) => ({ ...prev, [columnId]: newWidth }));
};
const handlePointerUp = () => {
setResizingColumn(null);
document.removeEventListener('pointermove', handlePointerMove);
document.removeEventListener('pointerup', handlePointerUp);
};
document.addEventListener('pointermove', handlePointerMove);
document.addEventListener('pointerup', handlePointerUp);
},
[columnWidths, baseColumns],
);
const gridTemplateColumns = buildGridTemplateColumns(
baseColumns,
columnWidths,
);
const { scrollWrapperHTMLElement } = useScrollWrapperHTMLElement(
EVENT_LOG_SCROLL_WRAPPER_INSTANCE_ID,
);
const [shouldFetchMore, setShouldFetchMore] = useState(false);
const { ref: fetchMoreRef, inView } = useInView({
root: scrollWrapperHTMLElement,
rootMargin: '400px',
});
useEffect(() => {
if (inView && hasNextPage && !loading && !shouldFetchMore) {
setShouldFetchMore(true);
onLoadMore();
}
}, [inView, hasNextPage, loading, shouldFetchMore, onLoadMore]);
useEffect(() => {
if (!loading) {
setShouldFetchMore(false);
}
}, [loading]);
const isInitialLoading = loading && records.length === 0;
if (isInitialLoading) {
return (
<StyledScrollWrapper
componentInstanceId={EVENT_LOG_SCROLL_WRAPPER_INSTANCE_ID}
>
<StyledTable>
<StyledHeaderRow gridTemplateColumns={gridTemplateColumns}>
{baseColumns.map((column) => (
<TableHeader key={column.id}>{t(column.label)}</TableHeader>
))}
</StyledHeaderRow>
</StyledTable>
<SkeletonTheme
baseColor={theme.background.tertiary}
highlightColor={theme.background.transparent.lighter}
borderRadius={4}
>
<StyledSkeletonContainer>
{Array.from({ length: SKELETON_ROW_COUNT }).map((_, index) => (
<Skeleton height={40} key={index} style={{ marginBottom: 4 }} />
))}
</StyledSkeletonContainer>
</SkeletonTheme>
</StyledScrollWrapper>
);
}
if (!loading && records.length === 0) {
return (
<StyledEmptyState>
<Trans>No event logs found</Trans>
</StyledEmptyState>
);
}
return (
<StyledScrollWrapper
componentInstanceId={EVENT_LOG_SCROLL_WRAPPER_INSTANCE_ID}
>
<StyledTable>
<StyledHeaderRow gridTemplateColumns={gridTemplateColumns}>
{baseColumns.map((column) => (
<StyledResizableHeader
key={column.id}
isResizing={resizingColumn === column.id}
>
{t(column.label)}
<StyledResizeHandle
isResizing={resizingColumn === column.id}
onPointerDown={(event) => handleResizeStart(column.id, event)}
/>
</StyledResizableHeader>
))}
</StyledHeaderRow>
{records.map((record, index) => (
<StyledDataRow
key={`${record.timestamp}-${record.event}-${index}`}
gridTemplateColumns={gridTemplateColumns}
>
<StyledTableCell>{record.event}</StyledTableCell>
<StyledTableCell>
{beautifyPastDateRelativeToNow(record.timestamp)}
</StyledTableCell>
<StyledTableCell>{record.userWorkspaceId ?? '-'}</StyledTableCell>
{showObjectEventColumns && (
<>
<StyledTableCell>{record.recordId ?? '-'}</StyledTableCell>
<StyledTableCell>
{record.objectMetadataId ?? '-'}
</StyledTableCell>
</>
)}
<StyledTableCell>
<EventLogJsonCell value={record.properties} />
</StyledTableCell>
</StyledDataRow>
))}
</StyledTable>
<StyledIntersectionObserver ref={fetchMoreRef} />
{loading && records.length > 0 && (
<StyledLoadingMore>
<Trans>Loading more...</Trans>
</StyledLoadingMore>
)}
</StyledScrollWrapper>
);
};
@@ -0,0 +1,42 @@
import { useLingui } from '@lingui/react/macro';
import { Select } from '@/ui/input/components/Select';
import { EventLogTable } from '~/generated-metadata/graphql';
type EventLogTableSelectorProps = {
value: EventLogTable;
onChange: (value: EventLogTable) => void;
};
export const EventLogTableSelector = ({
value,
onChange,
}: EventLogTableSelectorProps) => {
const { t } = useLingui();
const options = [
{
value: EventLogTable.PAGEVIEW,
label: t`Page Views`,
},
{
value: EventLogTable.WORKSPACE_EVENT,
label: t`Workspace Events`,
},
{
value: EventLogTable.OBJECT_EVENT,
label: t`Object Events`,
},
];
return (
<Select
dropdownId="event-log-table-selector"
label={t`Table`}
fullWidth
value={value}
options={options}
onChange={onChange}
/>
);
};
@@ -0,0 +1,22 @@
import { gql } from '@apollo/client';
export const GET_EVENT_LOGS = gql`
query EventLogs($input: EventLogQueryInput!) {
eventLogs(input: $input) {
records {
event
timestamp
userWorkspaceId
properties
recordId
objectMetadataId
isCustom
}
totalCount
pageInfo {
endCursor
hasNextPage
}
}
}
`;
@@ -0,0 +1,71 @@
import { useQuery } from '@apollo/client';
import {
type EventLogQueryInput,
type EventLogQueryResult,
type EventLogRecord,
} from '~/generated-metadata/graphql';
import { GET_EVENT_LOGS } from '~/pages/settings/security/event-logs/graphql/queries/getEventLogs';
type EventLogsData = {
eventLogs: EventLogQueryResult;
};
type EventLogsVariables = {
input: EventLogQueryInput;
};
export const useEventLogs = (input: EventLogQueryInput) => {
const { data, loading, error, refetch, fetchMore } = useQuery<
EventLogsData,
EventLogsVariables
>(GET_EVENT_LOGS, {
variables: { input },
fetchPolicy: 'network-only',
});
const records = data?.eventLogs.records ?? ([] as EventLogRecord[]);
const totalCount = data?.eventLogs.totalCount ?? 0;
const endCursor = data?.eventLogs.pageInfo.endCursor;
const hasNextPage = data?.eventLogs.pageInfo.hasNextPage ?? false;
const loadMore = () => {
if (!hasNextPage || loading || !endCursor) {
return;
}
fetchMore({
variables: {
input: {
...input,
after: endCursor,
},
},
updateQuery: (previousResult, { fetchMoreResult }) => {
if (!fetchMoreResult) {
return previousResult;
}
return {
eventLogs: {
...fetchMoreResult.eventLogs,
records: [
...previousResult.eventLogs.records,
...fetchMoreResult.eventLogs.records,
],
},
};
},
});
};
return {
records,
totalCount,
hasNextPage,
loading,
error,
refetch,
loadMore,
};
};
@@ -58,4 +58,5 @@ export const mockedClientConfig: ClientConfig = {
isEmailingDomainsEnabled: false,
allowRequestsToTwentyIcons: true,
isCloudflareIntegrationEnabled: false,
isClickHouseConfigured: false,
};
@@ -155,6 +155,7 @@ export const mockCurrentWorkspace = {
databaseSchema: '',
databaseUrl: '',
isTwoFactorAuthenticationEnforced: false,
eventLogRetentionDays: 90,
__typename: 'Workspace',
} as const satisfies Workspace;
@@ -229,6 +229,34 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
}
}
public async executeCommand(
query: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
params?: Record<string, any>,
clientId?: string,
): Promise<boolean> {
try {
const client = clientId
? await this.connectToClient(clientId)
: this.mainClient;
if (!client) {
return false;
}
await client.command({
query,
query_params: params,
});
return true;
} catch (err) {
this.logger.error('Error executing command in ClickHouse', err);
return false;
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private async insertInChunks<T extends Record<string, any>>(
client: ClickHouseClient,
@@ -0,0 +1,11 @@
/**
* Formats a Date or ISO string for ClickHouse DateTime64(3) type.
* ClickHouse expects: YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator, no 'Z' suffix)
* JavaScript toISOString() returns: YYYY-MM-DDTHH:mm:ss.SSSZ
*/
export const formatDateForClickHouse = (date: Date | string): string => {
const iso = typeof date === 'string' ? date : date.toISOString();
// Extract date (YYYY-MM-DD) and time with milliseconds (HH:mm:ss.SSS)
return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;
};
@@ -0,0 +1,26 @@
-- Optimize workspaceEvent table for time-series queries
-- ClickHouse doesn't allow changing ORDER BY, so we create a new table and migrate data
-- Step 1: Create new table with optimized structure
CREATE TABLE IF NOT EXISTS workspaceEvent_v2
(
`event` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userWorkspaceId` String DEFAULT '',
`workspaceId` String NOT NULL,
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (workspaceId, timestamp, event, userWorkspaceId)
TTL timestamp + INTERVAL 3 YEAR DELETE;
-- Step 2: Migrate existing data (userWorkspaceId will be empty for historical data)
INSERT INTO workspaceEvent_v2
SELECT event, timestamp, '' as userWorkspaceId, workspaceId, properties
FROM workspaceEvent;
-- Step 3: Atomic swap (EXCHANGE is atomic and instant)
EXCHANGE TABLES workspaceEvent AND workspaceEvent_v2;
-- Step 4: Drop old table (now named workspaceEvent_v2 after swap)
DROP TABLE IF EXISTS workspaceEvent_v2;
@@ -0,0 +1,26 @@
-- Optimize pageview table for time-series queries
-- ClickHouse doesn't allow changing ORDER BY, so we create a new table and migrate data
-- Step 1: Create new table with optimized structure
CREATE TABLE IF NOT EXISTS pageview_v2
(
`name` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userWorkspaceId` String DEFAULT '',
`workspaceId` String DEFAULT '',
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (workspaceId, timestamp, name, userWorkspaceId)
TTL timestamp + INTERVAL 3 YEAR DELETE;
-- Step 2: Migrate existing data (userWorkspaceId will be empty for historical data)
INSERT INTO pageview_v2
SELECT name, timestamp, '' as userWorkspaceId, workspaceId, properties
FROM pageview;
-- Step 3: Atomic swap
EXCHANGE TABLES pageview AND pageview_v2;
-- Step 4: Drop old table
DROP TABLE IF EXISTS pageview_v2;
@@ -0,0 +1,29 @@
-- Optimize objectEvent table for time-series queries and object filtering
-- ClickHouse doesn't allow changing ORDER BY, so we create a new table and migrate data
-- Step 1: Create new table with optimized structure
CREATE TABLE IF NOT EXISTS objectEvent_v2
(
`event` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userWorkspaceId` String DEFAULT '',
`workspaceId` String NOT NULL,
`recordId` String NOT NULL,
`objectMetadataId` String NOT NULL,
`properties` JSON,
`isCustom` Boolean DEFAULT FALSE
)
ENGINE = MergeTree
ORDER BY (workspaceId, timestamp, objectMetadataId, recordId, event, userWorkspaceId)
TTL timestamp + INTERVAL 3 YEAR DELETE;
-- Step 2: Migrate existing data (userWorkspaceId will be empty for historical data)
INSERT INTO objectEvent_v2
SELECT event, timestamp, '' as userWorkspaceId, workspaceId, recordId, objectMetadataId, properties, isCustom
FROM objectEvent;
-- Step 3: Atomic swap
EXCHANGE TABLES objectEvent AND objectEvent_v2;
-- Step 4: Drop old table
DROP TABLE IF EXISTS objectEvent_v2;
@@ -97,7 +97,29 @@ async function runMigrations() {
const sql = fs.readFileSync(path.join(dir, file), 'utf8');
console.log(`⚡ Running ${file}...`);
await client.command({ query: sql });
// Split by semicolons and filter out empty statements/comments
const statements = sql
.split(';')
.map((stmt) => stmt.trim())
.filter(
(stmt) =>
stmt.length > 0 && !stmt.startsWith('--') && !stmt.match(/^[\s-]*$/),
);
for (const statement of statements) {
// Skip comment-only blocks
const cleanedStatement = statement
.split('\n')
.filter((line) => !line.trim().startsWith('--'))
.join('\n')
.trim();
if (cleanedStatement.length > 0) {
await client.command({ query: cleanedStatement });
}
}
await recordMigration(file, client);
}
@@ -11,7 +11,7 @@ export const fixtures: Array<GenericTrackEvent> = [
event: CUSTOM_DOMAIN_ACTIVATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
userWorkspaceId: '20202020-3957-45c9-be39-337dc4d9100a',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
@@ -20,7 +20,7 @@ export const fixtures: Array<GenericTrackEvent> = [
event: CUSTOM_DOMAIN_DEACTIVATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
userWorkspaceId: '20202020-3957-45c9-be39-337dc4d9100a',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
@@ -29,7 +29,7 @@ export const fixtures: Array<GenericTrackEvent> = [
event: OBJECT_RECORD_CREATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
userWorkspaceId: '20202020-3957-45c9-be39-337dc4d9100a',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
@@ -38,7 +38,7 @@ export const fixtures: Array<GenericTrackEvent> = [
event: OBJECT_RECORD_UPDATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
userWorkspaceId: '20202020-3957-45c9-be39-337dc4d9100a',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
@@ -47,7 +47,7 @@ export const fixtures: Array<GenericTrackEvent> = [
event: OBJECT_RECORD_DELETED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
userWorkspaceId: '20202020-3957-45c9-be39-337dc4d9100a',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
@@ -2,6 +2,7 @@ import { Logger } from '@nestjs/common';
import { Command, CommandRunner } from 'nest-commander';
import { EventLogCleanupCronCommand } from 'src/engine/core-modules/event-logs/cleanup/commands/event-log-cleanup.cron.command';
import { CheckPublicDomainsValidRecordsCronCommand } from 'src/engine/core-modules/public-domain/crons/commands/check-public-domains-valid-records.cron.command';
import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/workspace/crons/commands/check-custom-domain-valid-records.cron.command';
import { CronTriggerCronCommand } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.command';
@@ -50,6 +51,7 @@ export class CronRegisterAllCommand extends CommandRunner {
private readonly cleanSuspendedWorkspacesCronCommand: CleanSuspendedWorkspacesCronCommand,
private readonly cleanOnboardingWorkspacesCronCommand: CleanOnboardingWorkspacesCronCommand,
private readonly trashCleanupCronCommand: TrashCleanupCronCommand,
private readonly eventLogCleanupCronCommand: EventLogCleanupCronCommand,
) {
super();
}
@@ -130,6 +132,10 @@ export class CronRegisterAllCommand extends CommandRunner {
name: 'TrashCleanup',
command: this.trashCleanupCronCommand,
},
{
name: 'EventLogCleanup',
command: this.eventLogCleanupCronCommand,
},
];
let successCount = 0;
@@ -8,6 +8,7 @@ import { ConfirmationQuestion } from 'src/database/commands/questions/confirmati
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { EventLogCleanupModule } from 'src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { PublicDomainModule } from 'src/engine/core-modules/public-domain/public-domain.module';
@@ -52,6 +53,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
WorkspaceMigrationModule,
TrashCleanupModule,
PublicDomainModule,
EventLogCleanupModule,
],
providers: [
DataSeedWorkspaceCommand,
@@ -0,0 +1,20 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddWorkspaceEventLogRetention1770051000000
implements MigrationInterface
{
name = 'AddWorkspaceEventLogRetention1770051000000';
public async up(queryRunner: QueryRunner): Promise<void> {
// Default 90 days retention for event logs
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "eventLogRetentionDays" integer NOT NULL DEFAULT '90'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "eventLogRetentionDays"`,
);
}
}
@@ -24,7 +24,7 @@ export class TelemetryListener {
payload.events.map(async (eventPayload) => {
this.auditService
.createContext({
userId: eventPayload.userId,
userWorkspaceId: eventPayload.userWorkspaceId,
workspaceId: payload.workspaceId,
})
.insertWorkspaceEvent(USER_SIGNUP_EVENT, {});
@@ -4,7 +4,6 @@ import {
AuditException,
AuditExceptionCode,
} from 'src/engine/core-modules/audit/audit.exception';
import { type UserEntity } from 'src/engine/core-modules/user/user.entity';
import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuditResolver } from './audit.resolver';
@@ -56,12 +55,12 @@ describe('AuditResolver', () => {
const result = await resolver.trackAnalytics(
input,
{ id: 'workspace-1' } as WorkspaceEntity,
{ id: 'user-1' } as UserEntity,
'user-workspace-1',
);
expect(auditService.createContext).toHaveBeenCalledWith({
workspaceId: 'workspace-1',
userId: 'user-1',
userWorkspaceId: 'user-workspace-1',
});
expect(mockInsertPageviewEvent).toHaveBeenCalledWith('Test Page', {});
expect(result).toBe('Pageview created');
@@ -86,12 +85,12 @@ describe('AuditResolver', () => {
const result = await resolver.trackAnalytics(
input,
{ id: 'workspace-2' } as WorkspaceEntity,
{ id: 'user-2' } as UserEntity,
'user-workspace-2',
);
expect(auditService.createContext).toHaveBeenCalledWith({
workspaceId: 'workspace-2',
userId: 'user-2',
userWorkspaceId: 'user-workspace-2',
});
expect(mockInsertWorkspaceEvent).toHaveBeenCalledWith(
'Custom Domain Activated',
@@ -121,12 +120,12 @@ describe('AuditResolver', () => {
const result = await resolver.createObjectEvent(
input,
{ id: 'workspace-3' } as WorkspaceEntity,
{ id: 'user-3' } as UserEntity,
'user-workspace-3',
);
expect(auditService.createContext).toHaveBeenCalledWith({
workspaceId: 'workspace-3',
userId: 'user-3',
userWorkspaceId: 'user-workspace-3',
});
expect(mockInsertObjectEvent).toHaveBeenCalledWith(
@@ -9,9 +9,8 @@ import {
import { CreateObjectEventInput } from 'src/engine/core-modules/audit/dtos/create-object-event.input';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
import { AuthUserWorkspaceId } from 'src/engine/decorators/auth/auth-user-workspace-id.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
import { PublicEndpointGuard } from 'src/engine/guards/public-endpoint.guard';
@@ -37,9 +36,13 @@ export class AuditResolver {
createAnalyticsInput: CreateAnalyticsInputV2,
@AuthWorkspace({ allowUndefined: true })
workspace: WorkspaceEntity | undefined,
@AuthUser({ allowUndefined: true }) user: UserEntity | undefined,
@AuthUserWorkspaceId() userWorkspaceId: string | undefined,
) {
return this.trackAnalytics(createAnalyticsInput, workspace, user);
return this.trackAnalytics(
createAnalyticsInput,
workspace,
userWorkspaceId,
);
}
@Mutation(() => Analytics)
@@ -48,7 +51,7 @@ export class AuditResolver {
@Args()
createObjectEventInput: CreateObjectEventInput,
@AuthWorkspace() workspace: WorkspaceEntity | undefined,
@AuthUser({ allowUndefined: true }) user: UserEntity | undefined,
@AuthUserWorkspaceId() userWorkspaceId: string | undefined,
) {
if (!workspace) {
throw new AuditException(
@@ -59,7 +62,7 @@ export class AuditResolver {
const analyticsContext = this.auditService.createContext({
workspaceId: workspace.id,
userId: user?.id,
userWorkspaceId,
});
return analyticsContext.createObjectEvent(createObjectEventInput.event, {
@@ -77,11 +80,11 @@ export class AuditResolver {
createAnalyticsInput: CreateAnalyticsInputV2,
@AuthWorkspace({ allowUndefined: true })
workspace: WorkspaceEntity | undefined,
@AuthUser({ allowUndefined: true }) user: UserEntity | undefined,
@AuthUserWorkspaceId() userWorkspaceId: string | undefined,
) {
const analyticsContext = this.auditService.createContext({
workspaceId: workspace?.id,
userId: user?.id,
userWorkspaceId,
});
if (isPageviewAnalyticsInput(createAnalyticsInput)) {
@@ -30,7 +30,7 @@ export class CreateAuditLogFromInternalEvent {
const auditService = this.auditService.createContext({
workspaceId: workspaceEventBatch.workspaceId,
userId: eventData.userId,
userWorkspaceId: eventData.userWorkspaceId,
});
// Since these are object record events, we use createObjectEvent
@@ -25,12 +25,14 @@ export class AuditService {
createContext(context?: {
workspaceId?: string | null | undefined;
userId?: string | null | undefined;
userWorkspaceId?: string | null | undefined;
}) {
const userIdAndWorkspaceId = context
const contextFields = context
? {
...(context.userId ? { userId: context.userId } : {}),
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(context.userWorkspaceId
? { userWorkspaceId: context.userWorkspaceId }
: {}),
}
: {};
@@ -41,7 +43,7 @@ export class AuditService {
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('workspaceEvent', [
{ ...userIdAndWorkspaceId, ...makeTrackEvent(event, properties) },
{ ...contextFields, ...makeTrackEvent(event, properties) },
]),
),
createObjectEvent: <T extends TrackEventName>(
@@ -53,7 +55,7 @@ export class AuditService {
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('objectEvent', [
{ ...userIdAndWorkspaceId, ...makeTrackEvent(event, properties) },
{ ...contextFields, ...makeTrackEvent(event, properties) },
]),
),
createPageviewEvent: (
@@ -62,7 +64,7 @@ export class AuditService {
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('pageview', [
{ ...userIdAndWorkspaceId, ...makePageview(name, properties) },
{ ...contextFields, ...makePageview(name, properties) },
]),
),
};
@@ -15,7 +15,7 @@ export type GenericTrackEvent<E extends string = string> = {
properties: any;
timestamp: string;
version: string;
userId?: string;
userWorkspaceId?: string;
workspaceId?: string;
};
@@ -590,7 +590,7 @@ export class AuthResolver {
workspaceId,
impersonatorUserWorkspaceId,
impersonatedUserWorkspaceId,
impersonatorUserId,
_impersonatorUserId: impersonatorUserId,
impersonatedUserId,
},
);
@@ -703,7 +703,7 @@ export class AuthResolver {
const auditService = this.auditService.createContext({
workspaceId: workspace.id,
userId: impersonatorUserWorkspace.user.id,
userWorkspaceId: impersonatorUserWorkspace.id,
});
auditService.insertWorkspaceEvent(MONITORING_EVENT, {
@@ -397,20 +397,20 @@ export class AuthService {
workspaceId,
impersonatorUserWorkspaceId,
impersonatedUserWorkspaceId,
impersonatorUserId,
_impersonatorUserId,
impersonatedUserId,
}: {
workspaceId: string;
impersonatorUserWorkspaceId: string;
impersonatedUserWorkspaceId: string;
impersonatorUserId: string;
_impersonatorUserId: string;
impersonatedUserId: string;
}): Promise<AuthTokens> {
const correlationId = randomUUID();
const analytics = this.auditService.createContext({
workspaceId,
userId: impersonatorUserId,
userWorkspaceId: impersonatorUserWorkspaceId,
});
analytics.insertWorkspaceEvent('Monitoring', {
@@ -53,6 +53,12 @@ describe('transformStripeEntitlementUpdatedEventToDatabaseEntitlement', () => {
value: false,
workspaceId: 'workspaceId',
},
{
key: BillingEntitlementKey.AUDIT_LOGS,
stripeCustomerId: 'cus_123',
value: false,
workspaceId: 'workspaceId',
},
]);
});
@@ -103,6 +109,12 @@ describe('transformStripeEntitlementUpdatedEventToDatabaseEntitlement', () => {
value: false,
workspaceId: 'workspaceId',
},
{
key: BillingEntitlementKey.AUDIT_LOGS,
stripeCustomerId: 'cus_123',
value: false,
workspaceId: 'workspaceId',
},
]);
});
});
@@ -4,4 +4,5 @@ export enum BillingEntitlementKey {
SSO = 'SSO',
CUSTOM_DOMAIN = 'CUSTOM_DOMAIN',
RLS = 'RLS',
AUDIT_LOGS = 'AUDIT_LOGS',
}
@@ -100,6 +100,7 @@ describe('ClientConfigController', () => {
isTwoFactorAuthenticationEnabled: false,
allowRequestsToTwentyIcons: true,
isCloudflareIntegrationEnabled: false,
isClickHouseConfigured: false,
};
jest
@@ -203,4 +203,7 @@ export class ClientConfig {
@Field(() => Boolean)
isCloudflareIntegrationEnabled: boolean;
@Field(() => Boolean)
isClickHouseConfigured: boolean;
}
@@ -161,6 +161,7 @@ describe('ClientConfigService', () => {
isImapSmtpCaldavEnabled: false,
calendarBookingPageId: 'team/twenty/talk-to-us',
isCloudflareIntegrationEnabled: false,
isClickHouseConfigured: false,
});
});
@@ -202,6 +202,7 @@ export class ClientConfigService {
? calendarBookingPageId
: undefined,
isCloudflareIntegrationEnabled: this.isCloudflareIntegrationEnabled(),
isClickHouseConfigured: !!this.twentyConfigService.get('CLICKHOUSE_URL'),
};
return clientConfig;
@@ -72,6 +72,7 @@ import { DashboardModule } from 'src/modules/dashboard/dashboard.module';
import { AuditModule } from './audit/audit.module';
import { ClientConfigModule } from './client-config/client-config.module';
import { EventLogsModule } from './event-logs/event-logs.module';
import { FileModule } from './file/file.module';
@Module({
@@ -158,6 +159,7 @@ import { FileModule } from './file/file.module';
TrashCleanupModule,
DashboardModule,
RowLevelPermissionModule,
EventLogsModule,
],
exports: [
AuditModule,
@@ -0,0 +1,35 @@
/* @license Enterprise */
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { EVENT_LOG_CLEANUP_CRON_PATTERN } from 'src/engine/core-modules/event-logs/cleanup/constants/event-log-cleanup-cron-pattern.constant';
import { EventLogCleanupCronJob } from 'src/engine/core-modules/event-logs/cleanup/crons/event-log-cleanup.cron.job';
@Command({
name: 'cron:event-log-cleanup',
description:
'Starts a cron job to clean up old event logs based on workspace retention settings',
})
export class EventLogCleanupCronCommand extends CommandRunner {
constructor(
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
async run(): Promise<void> {
await this.messageQueueService.addCron<undefined>({
jobName: EventLogCleanupCronJob.name,
data: undefined,
options: {
repeat: {
pattern: EVENT_LOG_CLEANUP_CRON_PATTERN,
},
},
});
}
}
@@ -0,0 +1,2 @@
// Run daily at 3 AM UTC
export const EVENT_LOG_CLEANUP_CRON_PATTERN = '0 3 * * *';
@@ -0,0 +1,97 @@
/* @license Enterprise */
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { EVENT_LOG_CLEANUP_CRON_PATTERN } from 'src/engine/core-modules/event-logs/cleanup/constants/event-log-cleanup-cron-pattern.constant';
import {
EventLogCleanupJob,
type EventLogCleanupJobData,
} from 'src/engine/core-modules/event-logs/cleanup/jobs/event-log-cleanup.job';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
@Injectable()
@Processor(MessageQueue.cronQueue)
export class EventLogCleanupCronJob {
private readonly logger = new Logger(EventLogCleanupCronJob.name);
constructor(
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectMessageQueue(MessageQueue.workspaceQueue)
private readonly messageQueueService: MessageQueueService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@Process(EventLogCleanupCronJob.name)
@SentryCronMonitor(
EventLogCleanupCronJob.name,
EVENT_LOG_CLEANUP_CRON_PATTERN,
)
async handle(): Promise<void> {
const workspaces = await this.getActiveWorkspaces();
if (workspaces.length === 0) {
this.logger.log('No active workspaces found for event log cleanup');
return;
}
this.logger.log(
`Enqueuing event log cleanup jobs for ${workspaces.length} workspace(s)`,
);
for (const workspace of workspaces) {
try {
await this.messageQueueService.add<EventLogCleanupJobData>(
EventLogCleanupJob.name,
{
workspaceId: workspace.id,
eventLogRetentionDays: workspace.eventLogRetentionDays,
},
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
id: workspace.id,
},
});
}
}
this.logger.log(
`Successfully enqueued ${workspaces.length} event log cleanup job(s)`,
);
}
private async getActiveWorkspaces(): Promise<
Array<{ id: string; eventLogRetentionDays: number }>
> {
const workspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
},
select: ['id', 'eventLogRetentionDays'],
order: { id: 'ASC' },
});
if (workspaces.length === 0) {
return [];
}
return workspaces.map((workspace) => ({
id: workspace.id,
eventLogRetentionDays: workspace.eventLogRetentionDays,
}));
}
}
@@ -0,0 +1,23 @@
/* @license Enterprise */
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { EventLogCleanupCronCommand } from 'src/engine/core-modules/event-logs/cleanup/commands/event-log-cleanup.cron.command';
import { EventLogCleanupCronJob } from 'src/engine/core-modules/event-logs/cleanup/crons/event-log-cleanup.cron.job';
import { EventLogCleanupJob } from 'src/engine/core-modules/event-logs/cleanup/jobs/event-log-cleanup.job';
import { EventLogCleanupService } from 'src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service';
@Module({
imports: [TypeOrmModule.forFeature([WorkspaceEntity]), ClickHouseModule],
providers: [
EventLogCleanupService,
EventLogCleanupJob,
EventLogCleanupCronJob,
EventLogCleanupCronCommand,
],
exports: [EventLogCleanupCronCommand],
})
export class EventLogCleanupModule {}
@@ -0,0 +1,41 @@
/* @license Enterprise */
import { Injectable, Logger } from '@nestjs/common';
import { EventLogCleanupService } from 'src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
export type EventLogCleanupJobData = {
workspaceId: string;
eventLogRetentionDays: number;
};
@Injectable()
@Processor(MessageQueue.workspaceQueue)
export class EventLogCleanupJob {
private readonly logger = new Logger(EventLogCleanupJob.name);
constructor(
private readonly eventLogCleanupService: EventLogCleanupService,
) {}
@Process(EventLogCleanupJob.name)
async handle(data: EventLogCleanupJobData): Promise<void> {
const { workspaceId, eventLogRetentionDays } = data;
try {
await this.eventLogCleanupService.cleanupWorkspaceEventLogs({
workspaceId,
retentionDays: eventLogRetentionDays,
});
} catch (error) {
this.logger.error(
`Event log cleanup failed for workspace ${workspaceId}`,
error instanceof Error ? error.stack : String(error),
);
throw error;
}
}
}
@@ -0,0 +1,74 @@
/* @license Enterprise */
import { Injectable, Logger } from '@nestjs/common';
import { EventLogTable } from 'twenty-shared/types';
import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
const CLICKHOUSE_TABLE_NAMES: Record<EventLogTable, string> = {
[EventLogTable.WORKSPACE_EVENT]: 'workspaceEvent',
[EventLogTable.PAGEVIEW]: 'pageview',
[EventLogTable.OBJECT_EVENT]: 'objectEvent',
};
export type EventLogCleanupParams = {
workspaceId: string;
retentionDays: number;
};
@Injectable()
export class EventLogCleanupService {
private readonly logger = new Logger(EventLogCleanupService.name);
constructor(private readonly clickHouseService: ClickHouseService) {}
async cleanupWorkspaceEventLogs({
workspaceId,
retentionDays,
}: EventLogCleanupParams): Promise<void> {
if (!this.clickHouseService.getMainClient()) {
this.logger.debug(
'ClickHouse not configured, skipping event log cleanup',
);
return;
}
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
for (const table of Object.values(EventLogTable)) {
const tableName = CLICKHOUSE_TABLE_NAMES[table];
try {
// ClickHouse ALTER TABLE DELETE is async by default
// We use lightweight deletes (mutations) which are efficient
const success = await this.clickHouseService.executeCommand(
`ALTER TABLE ${tableName} DELETE WHERE "workspaceId" = {workspaceId:String} AND "timestamp" < {cutoffDate:DateTime64(3)}`,
{
workspaceId,
cutoffDate: formatDateForClickHouse(cutoffDate),
},
);
if (success) {
this.logger.log(
`Scheduled deletion of old ${tableName} events for workspace ${workspaceId} (retention: ${retentionDays} days)`,
);
} else {
this.logger.warn(
`Failed to schedule deletion for ${tableName} in workspace ${workspaceId}`,
);
}
} catch (error) {
this.logger.error(
`Error cleaning up ${tableName} for workspace ${workspaceId}`,
error instanceof Error ? error.stack : String(error),
);
}
}
}
}
@@ -0,0 +1,28 @@
import { Field, InputType } from '@nestjs/graphql';
@InputType()
export class EventLogDateRangeInput {
@Field(() => Date, { nullable: true })
start?: Date;
@Field(() => Date, { nullable: true })
end?: Date;
}
@InputType()
export class EventLogFiltersInput {
@Field(() => String, { nullable: true })
eventType?: string;
@Field(() => String, { nullable: true })
userWorkspaceId?: string;
@Field(() => EventLogDateRangeInput, { nullable: true })
dateRange?: EventLogDateRangeInput;
@Field(() => String, { nullable: true })
recordId?: string;
@Field(() => String, { nullable: true })
objectMetadataId?: string;
}
@@ -0,0 +1,28 @@
import { Field, InputType, Int } from '@nestjs/graphql';
import { IsInt, IsOptional, Max, Min } from 'class-validator';
import { EventLogTable } from 'twenty-shared/types';
import { EventLogFiltersInput } from './event-log-filters.input';
import { registerEventLogTableEnum } from './event-log-table.enum';
registerEventLogTableEnum();
@InputType()
export class EventLogQueryInput {
@Field(() => EventLogTable)
table: EventLogTable;
@Field(() => EventLogFiltersInput, { nullable: true })
filters?: EventLogFiltersInput;
@Field(() => Int, { nullable: true, defaultValue: 100 })
@IsInt()
@Min(1)
@Max(10000)
@IsOptional()
first?: number;
@Field(() => String, { nullable: true })
after?: string;
}
@@ -0,0 +1,48 @@
import { Field, Int, ObjectType } from '@nestjs/graphql';
import GraphQLJSON from 'graphql-type-json';
@ObjectType()
export class EventLogRecord {
@Field(() => String)
event: string;
@Field(() => Date)
timestamp: Date;
@Field(() => String, { nullable: true })
userWorkspaceId?: string;
@Field(() => GraphQLJSON, { nullable: true })
properties?: Record<string, unknown>;
@Field(() => String, { nullable: true })
recordId?: string;
@Field(() => String, { nullable: true })
objectMetadataId?: string;
@Field(() => Boolean, { nullable: true })
isCustom?: boolean;
}
@ObjectType()
export class EventLogPageInfo {
@Field(() => String, { nullable: true })
endCursor?: string;
@Field(() => Boolean)
hasNextPage: boolean;
}
@ObjectType()
export class EventLogQueryResult {
@Field(() => [EventLogRecord])
records: EventLogRecord[];
@Field(() => Int)
totalCount: number;
@Field(() => EventLogPageInfo)
pageInfo: EventLogPageInfo;
}
@@ -0,0 +1,11 @@
import { registerEnumType } from '@nestjs/graphql';
import { EventLogTable } from 'twenty-shared/types';
export const registerEventLogTableEnum = () => {
registerEnumType(EventLogTable, {
name: 'EventLogTable',
});
};
export { EventLogTable };
@@ -0,0 +1,38 @@
/* @license Enterprise */
import { type MessageDescriptor } from '@lingui/core';
import { msg } from '@lingui/core/macro';
import { assertUnreachable } from 'twenty-shared/utils';
import { CustomException } from 'src/utils/custom-exception';
export enum EventLogsExceptionCode {
CLICKHOUSE_NOT_CONFIGURED = 'CLICKHOUSE_NOT_CONFIGURED',
NO_ENTITLEMENT = 'NO_ENTITLEMENT',
}
const getEventLogsExceptionUserFriendlyMessage = (
code: EventLogsExceptionCode,
) => {
switch (code) {
case EventLogsExceptionCode.CLICKHOUSE_NOT_CONFIGURED:
return msg`Audit logs require ClickHouse to be configured.`;
case EventLogsExceptionCode.NO_ENTITLEMENT:
return msg`Audit logs require an Enterprise subscription.`;
default:
assertUnreachable(code);
}
};
export class EventLogsException extends CustomException<EventLogsExceptionCode> {
constructor(
message: string,
code: EventLogsExceptionCode,
{ userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},
) {
super(message, code, {
userFriendlyMessage:
userFriendlyMessage ?? getEventLogsExceptionUserFriendlyMessage(code),
});
}
}
@@ -0,0 +1,23 @@
/* @license Enterprise */
import { Module } from '@nestjs/common';
import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { GuardRedirectModule } from 'src/engine/core-modules/guard-redirect/guard-redirect.module';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { EventLogsResolver } from './event-logs.resolver';
import { EventLogsService } from './event-logs.service';
@Module({
imports: [
ClickHouseModule,
PermissionsModule,
BillingModule,
GuardRedirectModule,
],
providers: [EventLogsResolver, EventLogsService],
exports: [EventLogsService],
})
export class EventLogsModule {}
@@ -0,0 +1,49 @@
/* @license Enterprise */
import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
import { Args, Query, Resolver } from '@nestjs/graphql';
import { PermissionFlagType } from 'twenty-shared/constants';
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
import { EnterpriseFeaturesEnabledGuard } from 'src/engine/core-modules/auth/guards/enterprise-features-enabled.guard';
import { EventLogsGraphqlApiExceptionFilter } from 'src/engine/core-modules/event-logs/filters/event-logs-graphql-api-exception.filter';
import { ForbiddenExceptionGraphqlFilter } from 'src/engine/core-modules/event-logs/filters/forbidden-exception-graphql.filter';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { PermissionsGraphqlApiExceptionFilter } from 'src/engine/metadata-modules/permissions/utils/permissions-graphql-api-exception.filter';
import { EventLogsService } from './event-logs.service';
import { EventLogQueryInput } from './dtos/event-log-query.input';
import { EventLogQueryResult } from './dtos/event-log-result.output';
@Resolver()
@UseFilters(
ForbiddenExceptionGraphqlFilter,
AuthGraphqlApiExceptionFilter,
EventLogsGraphqlApiExceptionFilter,
PermissionsGraphqlApiExceptionFilter,
PreventNestToAutoLogGraphqlErrorsFilter,
)
@UsePipes(ResolverValidationPipe)
export class EventLogsResolver {
constructor(private readonly eventLogsService: EventLogsService) {}
@UseGuards(
WorkspaceAuthGuard,
EnterpriseFeaturesEnabledGuard,
SettingsPermissionGuard(PermissionFlagType.SECURITY),
)
@Query(() => EventLogQueryResult)
async eventLogs(
@AuthWorkspace() workspace: WorkspaceEntity,
@Args('input') input: EventLogQueryInput,
): Promise<EventLogQueryResult> {
return this.eventLogsService.queryEventLogs(workspace.id, input);
}
}
@@ -0,0 +1,233 @@
/* @license Enterprise */
import { BadRequestException, Injectable } from '@nestjs/common';
import { EventLogTable } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
import {
EventLogsException,
EventLogsExceptionCode,
} from './event-logs.exception';
import { EventLogFiltersInput } from './dtos/event-log-filters.input';
import { EventLogQueryInput } from './dtos/event-log-query.input';
import {
EventLogQueryResult,
EventLogRecord,
} from './dtos/event-log-result.output';
type ClickHouseEventRecord = {
event?: string;
name?: string;
timestamp: string;
userWorkspaceId?: string;
properties?: Record<string, unknown>;
recordId?: string;
objectMetadataId?: string;
isCustom?: boolean;
};
const ALLOWED_TABLES = Object.values(EventLogTable);
const MAX_LIMIT = 10000;
const CLICKHOUSE_TABLE_NAMES: Record<EventLogTable, string> = {
[EventLogTable.WORKSPACE_EVENT]: 'workspaceEvent',
[EventLogTable.PAGEVIEW]: 'pageview',
[EventLogTable.OBJECT_EVENT]: 'objectEvent',
};
@Injectable()
export class EventLogsService {
constructor(
private readonly clickHouseService: ClickHouseService,
private readonly billingService: BillingService,
) {}
async queryEventLogs(
workspaceId: string,
input: EventLogQueryInput,
): Promise<EventLogQueryResult> {
await this.validateAccess(workspaceId);
if (!ALLOWED_TABLES.includes(input.table)) {
throw new BadRequestException(`Invalid table: ${input.table}`);
}
const limit = Math.min(input.first ?? 100, MAX_LIMIT);
const tableName = CLICKHOUSE_TABLE_NAMES[input.table];
const eventFieldName =
input.table === EventLogTable.PAGEVIEW ? 'name' : 'event';
const whereClauses: string[] = ['"workspaceId" = {workspaceId:String}'];
const params: Record<string, unknown> = { workspaceId };
this.applyFilters(
whereClauses,
params,
input.filters,
eventFieldName,
input.table,
);
const paginationClauses = [...whereClauses];
if (isDefined(input.after)) {
const cursorMs = this.decodeCursor(input.after);
paginationClauses.push(
'"timestamp" < fromUnixTimestamp64Milli({cursorMs:Int64})',
);
params.cursorMs = cursorMs;
}
const filterWhereClause = whereClauses.join(' AND ');
const paginationWhereClause = paginationClauses.join(' AND ');
const countQuery = `
SELECT count() as totalCount
FROM ${tableName}
WHERE ${filterWhereClause}
`;
const query = `
SELECT *
FROM ${tableName}
WHERE ${paginationWhereClause}
ORDER BY "timestamp" DESC
LIMIT {limit:Int32}
`;
params.limit = limit + 1;
const [records, countResult] = await Promise.all([
this.clickHouseService.select<ClickHouseEventRecord>(query, params),
this.clickHouseService.select<{ totalCount: number }>(countQuery, params),
]);
const totalCount = countResult[0]?.totalCount ?? 0;
const hasNextPage = records.length > limit;
if (hasNextPage) {
records.pop();
}
const normalizedRecords = this.normalizeRecords(records, input.table);
const lastRecord = normalizedRecords[normalizedRecords.length - 1];
const endCursor =
hasNextPage && lastRecord
? this.encodeCursor(lastRecord.timestamp)
: undefined;
return {
records: normalizedRecords,
totalCount,
pageInfo: {
endCursor,
hasNextPage,
},
};
}
private async validateAccess(workspaceId: string): Promise<void> {
if (!this.clickHouseService.getMainClient()) {
throw new EventLogsException(
'Audit logs require ClickHouse to be configured. Please set the CLICKHOUSE_URL environment variable.',
EventLogsExceptionCode.CLICKHOUSE_NOT_CONFIGURED,
);
}
const hasEntitlement = await this.billingService.hasEntitlement(
workspaceId,
BillingEntitlementKey.AUDIT_LOGS,
);
if (!hasEntitlement) {
throw new EventLogsException(
'Audit logs require an Enterprise subscription.',
EventLogsExceptionCode.NO_ENTITLEMENT,
);
}
}
private applyFilters(
whereClauses: string[],
params: Record<string, unknown>,
filters: EventLogFiltersInput | undefined,
eventFieldName: string,
table: EventLogTable,
): void {
if (!isDefined(filters)) {
return;
}
if (isDefined(filters.eventType)) {
whereClauses.push(
`lower("${eventFieldName}") LIKE {eventTypePattern:String}`,
);
params.eventTypePattern = `%${filters.eventType.toLowerCase()}%`;
}
if (isDefined(filters.userWorkspaceId)) {
whereClauses.push('"userWorkspaceId" = {userWorkspaceId:String}');
params.userWorkspaceId = filters.userWorkspaceId;
}
if (isDefined(filters.dateRange?.start)) {
whereClauses.push('"timestamp" >= {startDate:DateTime64(3)}');
params.startDate = formatDateForClickHouse(filters.dateRange.start);
}
if (isDefined(filters.dateRange?.end)) {
whereClauses.push('"timestamp" <= {endDate:DateTime64(3)}');
params.endDate = formatDateForClickHouse(filters.dateRange.end);
}
if (table === EventLogTable.OBJECT_EVENT) {
if (isDefined(filters.recordId)) {
whereClauses.push('"recordId" = {recordId:String}');
params.recordId = filters.recordId;
}
if (isDefined(filters.objectMetadataId)) {
whereClauses.push('"objectMetadataId" = {objectMetadataId:String}');
params.objectMetadataId = filters.objectMetadataId;
}
}
}
private encodeCursor(timestamp: Date): string {
return Buffer.from(String(timestamp.getTime())).toString('base64');
}
private decodeCursor(cursor: string): number {
return parseInt(Buffer.from(cursor, 'base64').toString('utf-8'), 10);
}
private normalizeRecords(
records: ClickHouseEventRecord[],
table: EventLogTable,
): EventLogRecord[] {
return records.map((record) => {
const eventName =
table === EventLogTable.PAGEVIEW
? (record.name ?? '')
: (record.event ?? '');
return {
event: eventName,
timestamp: new Date(record.timestamp),
userWorkspaceId: record.userWorkspaceId,
properties: record.properties,
recordId: record.recordId,
objectMetadataId: record.objectMetadataId,
isCustom: record.isCustom,
};
});
}
}
@@ -0,0 +1,13 @@
/* @license Enterprise */
import { Catch, type ExceptionFilter } from '@nestjs/common';
import { EventLogsException } from 'src/engine/core-modules/event-logs/event-logs.exception';
import { eventLogsGraphqlApiExceptionHandler } from 'src/engine/core-modules/event-logs/utils/event-logs-graphql-api-exception-handler.util';
@Catch(EventLogsException)
export class EventLogsGraphqlApiExceptionFilter implements ExceptionFilter {
catch(exception: EventLogsException) {
return eventLogsGraphqlApiExceptionHandler(exception);
}
}
@@ -0,0 +1,18 @@
import {
Catch,
type ExceptionFilter,
ForbiddenException,
} from '@nestjs/common';
import { msg } from '@lingui/core/macro';
import { AuthenticationError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
@Catch(ForbiddenException)
export class ForbiddenExceptionGraphqlFilter implements ExceptionFilter {
catch(exception: ForbiddenException) {
throw new AuthenticationError(exception.message, {
userFriendlyMessage: msg`Authentication required.`,
});
}
}
@@ -0,0 +1,22 @@
/* @license Enterprise */
import { assertUnreachable } from 'twenty-shared/utils';
import {
type EventLogsException,
EventLogsExceptionCode,
} from 'src/engine/core-modules/event-logs/event-logs.exception';
import { ForbiddenError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
export const eventLogsGraphqlApiExceptionHandler = (
exception: EventLogsException,
) => {
switch (exception.code) {
case EventLogsExceptionCode.CLICKHOUSE_NOT_CONFIGURED:
case EventLogsExceptionCode.NO_ENTITLEMENT:
throw new ForbiddenError(exception);
default: {
assertUnreachable(exception.code);
}
}
};
@@ -141,7 +141,7 @@ export class ImpersonationService {
) {
const auditService = this.auditService.createContext({
workspaceId: impersonatorUserWorkspace.workspace.id,
userId: impersonatorUserWorkspace.user.id,
userWorkspaceId: impersonatorUserWorkspace.id,
});
auditService.insertWorkspaceEvent(MONITORING_EVENT, {
@@ -8,6 +8,7 @@ import {
IsString,
IsUUID,
Matches,
Max,
Min,
} from 'class-validator';
@@ -99,6 +100,13 @@ export class UpdateWorkspaceInput {
@IsOptional()
trashRetentionDays?: number;
@Field({ nullable: true })
@IsInt()
@Min(30) // Minimum 30 days retention for audit compliance
@Max(1095) // Maximum 3 years (matches ClickHouse table-level TTL)
@IsOptional()
eventLogRetentionDays?: number;
@Field({ nullable: true })
@IsString()
@IsOptional()
@@ -72,6 +72,7 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
displayName: PermissionFlagType.WORKSPACE,
logo: PermissionFlagType.WORKSPACE,
trashRetentionDays: PermissionFlagType.WORKSPACE,
eventLogRetentionDays: PermissionFlagType.SECURITY,
inviteHash: PermissionFlagType.WORKSPACE_MEMBERS,
isPublicInviteLinkEnabled: PermissionFlagType.SECURITY,
allowImpersonation: PermissionFlagType.SECURITY,
@@ -104,6 +104,10 @@ export class WorkspaceEntity {
@Column({ type: 'integer', default: 14 })
trashRetentionDays: number;
@Field()
@Column({ type: 'integer', default: 90 })
eventLogRetentionDays: number;
// Relations
@OneToMany(() => AppTokenEntity, (appToken) => appToken.workspace, {
cascade: true,
@@ -7,12 +7,12 @@ import {
ObjectRecordUpsertEvent,
type ObjectRecordDiff,
} from 'twenty-shared/database-events';
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
import {
assertUnreachable,
isDefined,
isNonEmptyArray,
} from 'twenty-shared/utils';
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
import type { ObjectLiteral } from 'typeorm';
@@ -82,6 +82,7 @@ export const formatTwentyOrmEventToDatabaseBatchEvent = <
const event = new ObjectRecordCreateEvent<T>();
event.userId = authContext?.user?.id;
event.userWorkspaceId = authContext?.userWorkspaceId;
event.workspaceMemberId = authContext?.workspaceMemberId;
event.recordId = recordAfter.id;
event.properties = { after: recordAfter };
@@ -143,6 +144,7 @@ export const formatTwentyOrmEventToDatabaseBatchEvent = <
const eventPayload = {
userId: authContext?.user?.id,
userWorkspaceId: authContext?.userWorkspaceId,
workspaceMemberId: authContext?.workspaceMemberId,
recordId: recordAfter.id,
properties: {
@@ -192,6 +194,7 @@ export const formatTwentyOrmEventToDatabaseBatchEvent = <
const event = new ObjectRecordDestroyEvent<T>();
event.userId = authContext?.user?.id;
event.userWorkspaceId = authContext?.userWorkspaceId;
event.workspaceMemberId = authContext?.workspaceMemberId;
event.recordId = recordBefore.id;
event.properties = { before: recordBefore };
@@ -213,6 +216,7 @@ export const formatTwentyOrmEventToDatabaseBatchEvent = <
const event = new ObjectRecordUpsertEvent<T>();
event.userId = authContext?.user?.id;
event.userWorkspaceId = authContext?.userWorkspaceId;
event.workspaceMemberId = authContext?.workspaceMemberId;
event.recordId = recordAfter.id;
@@ -68,8 +68,9 @@ describe('ClickHouse Event Registration (integration)', () => {
expect(rows.length).toEqual(1);
expect(rows[0].properties).toEqual(variables.properties);
expect(rows[0].event).toEqual(variables.event);
expect(rows[0].workspaceId).toEqual('');
expect(rows[0].userId).toEqual('');
// workspaceId and userWorkspaceId are empty/undefined for unauthenticated requests
expect(rows[0].workspaceId ?? '').toEqual('');
expect(rows[0].userWorkspaceId ?? '').toEqual('');
expect(rows[0].timestamp).toHaveLength(23);
});
});
@@ -0,0 +1,532 @@
import process from 'process';
import { type ClickHouseClient, createClient } from '@clickhouse/client';
import request from 'supertest';
const client = request(`http://localhost:${APP_PORT}`);
describe('Event Logs (integration)', () => {
let clickHouseClient: ClickHouseClient;
const testWorkspaceId = '20202020-1c25-4d02-bf25-6aeccf7ea419';
const testUserWorkspaceId = '20202020-3957-45c9-be39-337dc4d9100a';
beforeAll(async () => {
jest.useRealTimers();
clickHouseClient = createClient({
url: process.env.CLICKHOUSE_URL,
clickhouse_settings: {
allow_experimental_json_type: 1,
},
});
await seedTestData();
});
afterAll(async () => {
await cleanupTestData();
if (clickHouseClient) {
await clickHouseClient.close();
}
});
const seedTestData = async () => {
const now = new Date();
const pageviewRecords = Array.from({ length: 25 }, (_, i) => ({
workspaceId: testWorkspaceId,
userWorkspaceId: testUserWorkspaceId,
name: i % 2 === 0 ? 'settings/profile' : 'objects/companies',
timestamp: new Date(now.getTime() - i * 60000)
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
properties: { path: `/settings/${i}` },
}));
const workspaceEventRecords = Array.from({ length: 15 }, (_, i) => ({
workspaceId: testWorkspaceId,
userWorkspaceId: testUserWorkspaceId,
event:
i % 3 === 0
? 'user.login'
: i % 3 === 1
? 'user.logout'
: 'settings.updated',
timestamp: new Date(now.getTime() - i * 120000)
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
properties: { action: `action_${i}` },
}));
const objectEventRecords = Array.from({ length: 20 }, (_, i) => ({
workspaceId: testWorkspaceId,
userWorkspaceId: testUserWorkspaceId,
event: i % 2 === 0 ? 'company.created' : 'company.updated',
timestamp: new Date(now.getTime() - i * 90000)
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
properties: { field: `field_${i}` },
recordId: `record-${i}`,
objectMetadataId: i % 2 === 0 ? 'object-meta-1' : 'object-meta-2',
isCustom: i % 4 === 0,
}));
await clickHouseClient.insert({
table: 'pageview',
values: pageviewRecords,
format: 'JSONEachRow',
});
await clickHouseClient.insert({
table: 'workspaceEvent',
values: workspaceEventRecords,
format: 'JSONEachRow',
});
await clickHouseClient.insert({
table: 'objectEvent',
values: objectEventRecords,
format: 'JSONEachRow',
});
// Wait for ClickHouse async inserts to complete
await new Promise((resolve) => setTimeout(resolve, 2000));
};
const cleanupTestData = async () => {
try {
await clickHouseClient.command({
query: `ALTER TABLE pageview DELETE WHERE workspaceId = '${testWorkspaceId}'`,
});
await clickHouseClient.command({
query: `ALTER TABLE workspaceEvent DELETE WHERE workspaceId = '${testWorkspaceId}'`,
});
await clickHouseClient.command({
query: `ALTER TABLE objectEvent DELETE WHERE workspaceId = '${testWorkspaceId}'`,
});
} catch {
// Ignore cleanup errors
}
};
const makeEventLogsQuery = (
input: Record<string, unknown>,
token = APPLE_JANE_ADMIN_ACCESS_TOKEN,
) => {
return client
.post('/graphql')
.set('Authorization', `Bearer ${token}`)
.send({
query: `
query EventLogs($input: EventLogQueryInput!) {
eventLogs(input: $input) {
records {
event
timestamp
userWorkspaceId
properties
recordId
objectMetadataId
isCustom
}
totalCount
pageInfo {
endCursor
hasNextPage
}
}
}
`,
variables: { input },
});
};
describe('querying different tables', () => {
it('should query PAGEVIEW table', async () => {
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 10,
});
expect(response.status).toBe(200);
expect(response.body.errors).toBeUndefined();
expect(response.body.data.eventLogs).toBeDefined();
expect(response.body.data.eventLogs.records.length).toBeGreaterThan(0);
expect(response.body.data.eventLogs.totalCount).toBeGreaterThanOrEqual(
response.body.data.eventLogs.records.length,
);
});
it('should query WORKSPACE_EVENT table', async () => {
const response = await makeEventLogsQuery({
table: 'WORKSPACE_EVENT',
first: 10,
});
expect(response.status).toBe(200);
expect(response.body.errors).toBeUndefined();
expect(response.body.data.eventLogs).toBeDefined();
expect(response.body.data.eventLogs.records.length).toBeGreaterThan(0);
});
it('should query OBJECT_EVENT table', async () => {
const response = await makeEventLogsQuery({
table: 'OBJECT_EVENT',
first: 10,
});
expect(response.status).toBe(200);
expect(response.body.errors).toBeUndefined();
expect(response.body.data.eventLogs).toBeDefined();
expect(response.body.data.eventLogs.records.length).toBeGreaterThan(0);
const record = response.body.data.eventLogs.records[0];
expect(record.recordId).toBeDefined();
expect(record.objectMetadataId).toBeDefined();
});
});
describe('pagination', () => {
it('should return hasNextPage=true when more records exist', async () => {
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 5,
});
expect(response.status).toBe(200);
expect(response.body.data.eventLogs.records.length).toBe(5);
expect(response.body.data.eventLogs.pageInfo.hasNextPage).toBe(true);
expect(response.body.data.eventLogs.pageInfo.endCursor).toBeDefined();
});
it('should fetch next page using cursor', async () => {
const firstPage = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 5,
});
expect(firstPage.body.data.eventLogs.pageInfo.hasNextPage).toBe(true);
const cursor = firstPage.body.data.eventLogs.pageInfo.endCursor;
const secondPage = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 5,
after: cursor,
});
expect(secondPage.status).toBe(200);
expect(secondPage.body.data.eventLogs.records.length).toBeGreaterThan(0);
const firstPageTimestamps = firstPage.body.data.eventLogs.records.map(
(r: { timestamp: string }) => r.timestamp,
);
const secondPageTimestamps = secondPage.body.data.eventLogs.records.map(
(r: { timestamp: string }) => r.timestamp,
);
const lastFirstPage = new Date(
firstPageTimestamps[firstPageTimestamps.length - 1],
);
const firstSecondPage = new Date(secondPageTimestamps[0]);
expect(lastFirstPage.getTime()).toBeGreaterThanOrEqual(
firstSecondPage.getTime(),
);
});
it('should return correct totalCount regardless of page size', async () => {
const smallPage = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 3,
});
const largePage = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 100,
});
expect(smallPage.body.data.eventLogs.totalCount).toBe(
largePage.body.data.eventLogs.totalCount,
);
});
});
describe('filters', () => {
describe('eventType filter', () => {
it('should filter by event type (partial match)', async () => {
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 50,
filters: {
eventType: 'settings',
},
});
expect(response.status).toBe(200);
expect(response.body.data.eventLogs.records.length).toBeGreaterThan(0);
response.body.data.eventLogs.records.forEach(
(record: { event: string }) => {
expect(record.event.toLowerCase()).toContain('settings');
},
);
});
it('should filter by event type case-insensitively', async () => {
const lowerCase = await makeEventLogsQuery({
table: 'WORKSPACE_EVENT',
first: 50,
filters: { eventType: 'login' },
});
const upperCase = await makeEventLogsQuery({
table: 'WORKSPACE_EVENT',
first: 50,
filters: { eventType: 'LOGIN' },
});
expect(lowerCase.body.data.eventLogs.totalCount).toBe(
upperCase.body.data.eventLogs.totalCount,
);
});
});
describe('dateRange filter', () => {
it('should filter by start date', async () => {
const now = new Date();
const tenMinutesAgo = new Date(now.getTime() - 10 * 60 * 1000);
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 50,
filters: {
dateRange: {
start: tenMinutesAgo.toISOString(),
},
},
});
expect(response.status).toBe(200);
response.body.data.eventLogs.records.forEach(
(record: { timestamp: string }) => {
expect(new Date(record.timestamp).getTime()).toBeGreaterThanOrEqual(
tenMinutesAgo.getTime(),
);
},
);
});
it('should filter by end date', async () => {
const now = new Date();
const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000);
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 50,
filters: {
dateRange: {
end: fiveMinutesAgo.toISOString(),
},
},
});
expect(response.status).toBe(200);
response.body.data.eventLogs.records.forEach(
(record: { timestamp: string }) => {
expect(new Date(record.timestamp).getTime()).toBeLessThanOrEqual(
fiveMinutesAgo.getTime(),
);
},
);
});
it('should filter by date range (start and end)', async () => {
const now = new Date();
const twentyMinutesAgo = new Date(now.getTime() - 20 * 60 * 1000);
const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000);
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 50,
filters: {
dateRange: {
start: twentyMinutesAgo.toISOString(),
end: fiveMinutesAgo.toISOString(),
},
},
});
expect(response.status).toBe(200);
response.body.data.eventLogs.records.forEach(
(record: { timestamp: string }) => {
const timestamp = new Date(record.timestamp).getTime();
expect(timestamp).toBeGreaterThanOrEqual(
twentyMinutesAgo.getTime(),
);
expect(timestamp).toBeLessThanOrEqual(fiveMinutesAgo.getTime());
},
);
});
});
describe('object event specific filters', () => {
it('should filter by recordId', async () => {
const response = await makeEventLogsQuery({
table: 'OBJECT_EVENT',
first: 50,
filters: {
recordId: 'record-0',
},
});
expect(response.status).toBe(200);
if (response.body.data.eventLogs.records.length > 0) {
response.body.data.eventLogs.records.forEach(
(record: { recordId: string }) => {
expect(record.recordId).toBe('record-0');
},
);
}
});
it('should filter by objectMetadataId', async () => {
const response = await makeEventLogsQuery({
table: 'OBJECT_EVENT',
first: 50,
filters: {
objectMetadataId: 'object-meta-1',
},
});
expect(response.status).toBe(200);
if (response.body.data.eventLogs.records.length > 0) {
response.body.data.eventLogs.records.forEach(
(record: { objectMetadataId: string }) => {
expect(record.objectMetadataId).toBe('object-meta-1');
},
);
}
});
});
describe('combined filters', () => {
it('should apply multiple filters together', async () => {
const now = new Date();
const thirtyMinutesAgo = new Date(now.getTime() - 30 * 60 * 1000);
const response = await makeEventLogsQuery({
table: 'WORKSPACE_EVENT',
first: 50,
filters: {
eventType: 'login',
dateRange: {
start: thirtyMinutesAgo.toISOString(),
},
},
});
expect(response.status).toBe(200);
response.body.data.eventLogs.records.forEach(
(record: { event: string; timestamp: string }) => {
expect(record.event.toLowerCase()).toContain('login');
expect(new Date(record.timestamp).getTime()).toBeGreaterThanOrEqual(
thirtyMinutesAgo.getTime(),
);
},
);
});
});
});
describe('record structure', () => {
it('should return properly structured pageview records', async () => {
const response = await makeEventLogsQuery({
table: 'PAGEVIEW',
first: 1,
});
expect(response.status).toBe(200);
const record = response.body.data.eventLogs.records[0];
expect(record).toHaveProperty('event');
expect(record).toHaveProperty('timestamp');
expect(record).toHaveProperty('userWorkspaceId');
expect(record).toHaveProperty('properties');
expect(typeof record.event).toBe('string');
expect(typeof record.timestamp).toBe('string');
});
it('should return properly structured object event records', async () => {
const response = await makeEventLogsQuery({
table: 'OBJECT_EVENT',
first: 1,
});
expect(response.status).toBe(200);
const record = response.body.data.eventLogs.records[0];
expect(record).toHaveProperty('event');
expect(record).toHaveProperty('timestamp');
expect(record).toHaveProperty('recordId');
expect(record).toHaveProperty('objectMetadataId');
expect(record).toHaveProperty('isCustom');
});
});
describe('permissions', () => {
it('should deny access to member role', async () => {
const response = await makeEventLogsQuery(
{ table: 'PAGEVIEW', first: 10 },
APPLE_JONY_MEMBER_ACCESS_TOKEN,
);
expect(response.status).toBe(200);
expect(response.body.errors).toBeDefined();
expect(response.body.errors[0].extensions.code).toBe('FORBIDDEN');
});
it('should deny access without authentication', async () => {
const response = await client.post('/graphql').send({
query: `
query EventLogs($input: EventLogQueryInput!) {
eventLogs(input: $input) {
records { event }
totalCount
pageInfo { hasNextPage }
}
}
`,
variables: { input: { table: 'PAGEVIEW', first: 10 } },
});
expect(response.status).toBe(200);
expect(response.body.errors).toBeDefined();
expect(response.body.errors[0].extensions.code).toBe('UNAUTHENTICATED');
});
});
describe('error handling', () => {
it('should reject invalid table name', async () => {
const response = await makeEventLogsQuery({
table: 'INVALID_TABLE' as 'PAGEVIEW',
first: 10,
});
// Invalid enum values are rejected by GraphQL validation before reaching the resolver
expect(response.status).toBe(400);
expect(response.body.errors).toBeDefined();
});
});
});
@@ -10,6 +10,7 @@ type Properties<T> = {
export class ObjectRecordBaseEvent<T = object> {
recordId: string;
userId?: string;
userWorkspaceId?: string;
workspaceMemberId?: string;
properties: Properties<T>;
}
@@ -0,0 +1,5 @@
export enum EventLogTable {
WORKSPACE_EVENT = 'WORKSPACE_EVENT',
PAGEVIEW = 'PAGEVIEW',
OBJECT_EVENT = 'OBJECT_EVENT',
}
@@ -50,6 +50,7 @@ export enum SettingsPath {
Integrations = 'integrations',
Security = 'security',
NewSSOIdentityProvider = 'security/sso/new',
EventLogs = 'security/event-logs',
AdminPanel = 'admin-panel',
AdminPanelHealthStatus = 'admin-panel#health-status',
@@ -52,6 +52,7 @@ export type { ConfigVariableValue } from './ConfigVariableValue';
export { ConnectedAccountProvider } from './ConnectedAccountProvider';
export { CrudOperationType } from './CrudOperationType';
export type { EnumFieldMetadataType } from './EnumFieldMetadataType';
export { EventLogTable } from './EventLogTable';
export type { ExcludeFunctions } from './ExcludeFunctions';
export type { ExtractPropertiesThatEndsWithId } from './ExtractPropertiesThatEndsWithId';
export type { ExtractPropertiesThatEndsWithIds } from './ExtractPropertiesThatEndsWithIds';