From 5e27e04c0a9dc1b3d6c9caec4ed408fdf53bf643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Fri, 17 Jul 2026 14:03:11 +0200 Subject: [PATCH] Add mostly-empty field hints to data model settings (#22962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Fields that are empty in almost all records now show a subtle `Mostly empty` hint next to their name in the object's Fields settings table (same visual treatment as `Deactivated`), with a tooltip explaining the signal and a matching **Mostly empty** toggle in the search filter dropdown. The goal is to nudge admins to clean up and deactivate fields nobody uses, while keeping the page untouched when the data model is healthy. ## How **No table scans.** Emptiness is read from Postgres planner statistics, so the cost is a catalog lookup regardless of table size: - `pg_class.reltuples` gates the feature on an approximate row count (≥ 100 records; never-analyzed tables mean no hints). Reuses the shared helper extracted from `ObjectRecordCountService`. - `pg_stats.null_frac` plus the sampled frequency of the column type's empty sentinel (`''` for text columns, `'{}'` for arrays, `'{}'`/`'[]'` for json — matched per physical column type) gives a per-column empty fraction. A value dominating ≥ 95% of a column is guaranteed to appear in the most-common-values list, so the approximation is reliable exactly at the threshold we care about. **Decision rules** (pure util, unit-tested): - Flag when every relevant column is ≥ 95% empty and the object has ≥ 100 records. - Skip system fields, the label identifier, relations, booleans, and actor fields (exhaustive switch — a new `FieldMetadataType` fails to compile until classified). - Composite fields must have all their columns empty, with column sets derived from `compositeTypeDefinitions`; only default-bearing code columns (`currencyCode`, phone country/calling codes) are excluded so stamped defaults don't mask emptiness. - Anything unknown (missing stats, new column since last ANALYZE) degrades to silence — no hint is ever shown on missing data. **API:** one `mostlyEmptyFieldMetadataIds(objectMetadataId)` query on the metadata schema, guarded by the `DATA_MODEL` settings permission, fetched lazily when the fields page opens. **UI:** exception-based — no new columns, no persistent controls. The badge and the filter toggle only materialize when at least one field qualifies, and disappear once things are cleaned up. ## Test - Unit tests for the decision util (threshold, system/label-identifier/inactive exclusion, missing statistics, composite all-columns rule, links label/secondary data, currency narrowing, excluded types). - Catalog SQL validated against Postgres 16 with a table mimicking Twenty's column shapes (text `''` defaults, enums, arrays, jsonb, currency pairs), including the type-aware sentinel matching (a text column full of literal `"{}"` strings does not count as empty). - End-to-end on a seeded dev instance: 899 companies with a mix of filled/empty fields — the API returned exactly the five fields predicted by the raw statistics (`annualRevenue`, `employees`, `introVideo`, `tagline`, `workPolicy`) and correctly excluded `address` (city 33% filled), actor/system fields, and the label identifier. - UI driven with Playwright: badge, tooltip copy, filter toggle, and filtered table all verified visually. - `lint:diff-with-main`, `typecheck` (server + front), and all three `graphql:generate` configurations + SDK metadata client regenerated and committed. --- .../src/metadata/generated/schema.graphql | 1 + .../src/metadata/generated/schema.ts | 2 + .../src/metadata/generated/types.ts | 9 + .../src/generated-metadata/graphql.ts | 14 ++ .../object-metadata/graphql/queries.ts | 6 + .../SettingsNameCellSecondaryLabel.tsx | 4 +- .../SettingsObjectFieldItemTableRow.tsx | 17 ++ .../hooks/useMostlyEmptyFieldMetadataIds.ts | 33 ++++ .../data-model/SettingsObjectFieldTable.tsx | 38 +++- ...ostly-empty-fraction-threshold.constant.ts | 3 + ...mostly-empty-minimum-row-count.constant.ts | 3 + .../mostly-empty-fields.service.ts | 118 ++++++++++++ .../object-metadata/object-metadata.module.ts | 2 + .../object-metadata.resolver.ts | 23 +++ .../object-record-count.service.ts | 37 ++-- ...stly-empty-field-metadata-ids.util.spec.ts | 169 ++++++++++++++++++ ...te-mostly-empty-field-metadata-ids.util.ts | 50 ++++++ ...t-emptiness-column-names-for-field.util.ts | 80 +++++++++ ...pty-field-metadata-ids.integration-spec.ts | 143 +++++++++++++++ 19 files changed, 735 insertions(+), 17 deletions(-) create mode 100644 packages/twenty-front/src/modules/settings/data-model/object-details/hooks/useMostlyEmptyFieldMetadataIds.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-fraction-threshold.constant.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-minimum-row-count.constant.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/mostly-empty-fields.service.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/compute-mostly-empty-field-metadata-ids.util.spec.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/get-emptiness-column-names-for-field.util.ts create mode 100644 packages/twenty-server/test/integration/metadata/suites/object-metadata/mostly-empty-field-metadata-ids.integration-spec.ts diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql index afd1d291da..0a291bf23e 100644 --- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql +++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql @@ -3065,6 +3065,7 @@ type Query { findManyAgents: [Agent!]! findOneAgent(input: AgentIdInput!): Agent! objectRecordCounts: [ObjectRecordCount!]! + mostlyEmptyFieldMetadataIds(objectMetadataId: UUID!): [UUID!]! object( """The id of the record to find.""" id: UUID! diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.ts index 0607c1b170..a944f081ef 100644 --- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts +++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts @@ -2723,6 +2723,7 @@ export interface Query { findManyAgents: Agent[] findOneAgent: Agent objectRecordCounts: ObjectRecordCount[] + mostlyEmptyFieldMetadataIds: Scalars['UUID'][] object: Object objects: ObjectConnection findOneLogicFunction: LogicFunction @@ -5897,6 +5898,7 @@ export interface QueryGenqlSelection{ findManyAgents?: AgentGenqlSelection findOneAgent?: (AgentGenqlSelection & { __args: {input: AgentIdInput} }) objectRecordCounts?: ObjectRecordCountGenqlSelection + mostlyEmptyFieldMetadataIds?: { __args: {objectMetadataId: Scalars['UUID']} } object?: (ObjectGenqlSelection & { __args: { /** The id of the record to find. */ id: Scalars['UUID']} }) diff --git a/packages/twenty-client-sdk/src/metadata/generated/types.ts b/packages/twenty-client-sdk/src/metadata/generated/types.ts index 5087965200..0fba7ec1d0 100644 --- a/packages/twenty-client-sdk/src/metadata/generated/types.ts +++ b/packages/twenty-client-sdk/src/metadata/generated/types.ts @@ -6279,6 +6279,15 @@ export default { "objectRecordCounts": [ 202 ], + "mostlyEmptyFieldMetadataIds": [ + 3, + { + "objectMetadataId": [ + 3, + "UUID!" + ] + } + ], "object": [ 47, { diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index 90b4cea3c3..cb21d4bc0c 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -4404,6 +4404,7 @@ export type Query = { lineChartData: LineChartData; listPlans: Array; minimalMetadata: MinimalMetadata; + mostlyEmptyFieldMetadataIds: Array; myCalendarChannels: Array; myConnectedAccounts: Array; myMessageChannels: Array; @@ -4747,6 +4748,11 @@ export type QueryLineChartDataArgs = { }; +export type QueryMostlyEmptyFieldMetadataIdsArgs = { + objectMetadataId: Scalars['UUID']['input']; +}; + + export type QueryMyCalendarChannelsArgs = { connectedAccountId?: InputMaybe; }; @@ -7341,6 +7347,13 @@ export type ObjectRecordCountsQueryVariables = Exact<{ [key: string]: never; }>; export type ObjectRecordCountsQuery = { __typename?: 'Query', objectRecordCounts: Array<{ __typename?: 'ObjectRecordCount', objectNamePlural: string, totalCount: number }> }; +export type MostlyEmptyFieldMetadataIdsQueryVariables = Exact<{ + objectMetadataId: Scalars['UUID']['input']; +}>; + + +export type MostlyEmptyFieldMetadataIdsQuery = { __typename?: 'Query', mostlyEmptyFieldMetadataIds: Array }; + export type SkipSyncEmailOnboardingStepMutationVariables = Exact<{ [key: string]: never; }>; @@ -8925,6 +8938,7 @@ export const CreateOneIndexMetadataItemDocument = {"kind":"Document","definition export const DeleteOneIndexMetadataItemDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteOneIndexMetadataItem"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"idToDelete"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UUID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteOneIndex"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"idToDelete"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]} as unknown as DocumentNode; export const ObjectMetadataItemsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ObjectMetadataItems"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"objects"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"paging"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"first"},"value":{"kind":"IntValue","value":"1000"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"edges"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"node"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ObjectMetadataFields"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"pageInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"hasNextPage"}},{"kind":"Field","name":{"kind":"Name","value":"hasPreviousPage"}},{"kind":"Field","name":{"kind":"Name","value":"startCursor"}},{"kind":"Field","name":{"kind":"Name","value":"endCursor"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ObjectMetadataFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Object"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"universalIdentifier"}},{"kind":"Field","name":{"kind":"Name","value":"nameSingular"}},{"kind":"Field","name":{"kind":"Name","value":"namePlural"}},{"kind":"Field","name":{"kind":"Name","value":"labelSingular"}},{"kind":"Field","name":{"kind":"Name","value":"labelPlural"}},{"kind":"Field","name":{"kind":"Name","value":"color"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"icon"}},{"kind":"Field","name":{"kind":"Name","value":"isRemote"}},{"kind":"Field","name":{"kind":"Name","value":"isActive"}},{"kind":"Field","name":{"kind":"Name","value":"isSystem"}},{"kind":"Field","name":{"kind":"Name","value":"isUIEditable"}},{"kind":"Field","name":{"kind":"Name","value":"isUICreatable"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"labelIdentifierFieldMetadataId"}},{"kind":"Field","name":{"kind":"Name","value":"imageIdentifierFieldMetadataId"}},{"kind":"Field","name":{"kind":"Name","value":"applicationId"}},{"kind":"Field","name":{"kind":"Name","value":"shortcut"}},{"kind":"Field","name":{"kind":"Name","value":"isLabelSyncedWithName"}},{"kind":"Field","name":{"kind":"Name","value":"isSearchable"}},{"kind":"Field","name":{"kind":"Name","value":"duplicateCriteria"}},{"kind":"Field","name":{"kind":"Name","value":"searchFieldMetadataList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"fieldMetadataId"}},{"kind":"Field","name":{"kind":"Name","value":"tsVectorFieldMetadataId"}},{"kind":"Field","name":{"kind":"Name","value":"position"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}},{"kind":"Field","name":{"kind":"Name","value":"indexMetadataList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"indexWhereClause"}},{"kind":"Field","name":{"kind":"Name","value":"indexType"}},{"kind":"Field","name":{"kind":"Name","value":"isUnique"}},{"kind":"Field","name":{"kind":"Name","value":"isCustom"}},{"kind":"Field","name":{"kind":"Name","value":"indexFieldMetadataList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"fieldMetadataId"}},{"kind":"Field","name":{"kind":"Name","value":"subFieldName"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"order"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"fieldsList"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"universalIdentifier"}},{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"icon"}},{"kind":"Field","name":{"kind":"Name","value":"isActive"}},{"kind":"Field","name":{"kind":"Name","value":"isSystem"}},{"kind":"Field","name":{"kind":"Name","value":"isUIEditable"}},{"kind":"Field","name":{"kind":"Name","value":"isNullable"}},{"kind":"Field","name":{"kind":"Name","value":"isUnique"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"defaultValue"}},{"kind":"Field","name":{"kind":"Name","value":"options"}},{"kind":"Field","name":{"kind":"Name","value":"settings"}},{"kind":"Field","name":{"kind":"Name","value":"isLabelSyncedWithName"}},{"kind":"Field","name":{"kind":"Name","value":"morphId"}},{"kind":"Field","name":{"kind":"Name","value":"applicationId"}},{"kind":"Field","name":{"kind":"Name","value":"relation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"sourceObjectMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"nameSingular"}},{"kind":"Field","name":{"kind":"Name","value":"namePlural"}}]}},{"kind":"Field","name":{"kind":"Name","value":"targetObjectMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"nameSingular"}},{"kind":"Field","name":{"kind":"Name","value":"namePlural"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sourceFieldMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"targetFieldMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"morphRelations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"sourceObjectMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"nameSingular"}},{"kind":"Field","name":{"kind":"Name","value":"namePlural"}}]}},{"kind":"Field","name":{"kind":"Name","value":"targetObjectMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"nameSingular"}},{"kind":"Field","name":{"kind":"Name","value":"namePlural"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sourceFieldMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"targetFieldMetadata"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const ObjectRecordCountsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ObjectRecordCounts"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"objectRecordCounts"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"objectNamePlural"}},{"kind":"Field","name":{"kind":"Name","value":"totalCount"}}]}}]}}]} as unknown as DocumentNode; +export const MostlyEmptyFieldMetadataIdsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"MostlyEmptyFieldMetadataIds"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"objectMetadataId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UUID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"mostlyEmptyFieldMetadataIds"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"objectMetadataId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"objectMetadataId"}}}]}]}}]} as unknown as DocumentNode; export const SkipSyncEmailOnboardingStepDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"SkipSyncEmailOnboardingStep"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"skipSyncEmailOnboardingStep"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}}]}}]}}]} as unknown as DocumentNode; export const TriggerInstallAppsOnboardingStepDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"TriggerInstallAppsOnboardingStep"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"universalIdentifiers"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"triggerInstallAppsOnboardingStep"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"universalIdentifiers"},"value":{"kind":"Variable","name":{"kind":"Name","value":"universalIdentifiers"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"success"}}]}}]}}]} as unknown as DocumentNode; export const GetInviteSuggestionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetInviteSuggestions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getInviteSuggestions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"email"}},{"kind":"Field","name":{"kind":"Name","value":"displayName"}}]}}]}}]} as unknown as DocumentNode; diff --git a/packages/twenty-front/src/modules/object-metadata/graphql/queries.ts b/packages/twenty-front/src/modules/object-metadata/graphql/queries.ts index 07e7553966..bbcda19fb9 100644 --- a/packages/twenty-front/src/modules/object-metadata/graphql/queries.ts +++ b/packages/twenty-front/src/modules/object-metadata/graphql/queries.ts @@ -28,3 +28,9 @@ export const OBJECT_RECORD_COUNTS = gql` } } `; + +export const MOSTLY_EMPTY_FIELD_METADATA_IDS = gql` + query MostlyEmptyFieldMetadataIds($objectMetadataId: UUID!) { + mostlyEmptyFieldMetadataIds(objectMetadataId: $objectMetadataId) + } +`; diff --git a/packages/twenty-front/src/modules/settings/components/SettingsNameCellSecondaryLabel.tsx b/packages/twenty-front/src/modules/settings/components/SettingsNameCellSecondaryLabel.tsx index 4d705607cf..c70810e8be 100644 --- a/packages/twenty-front/src/modules/settings/components/SettingsNameCellSecondaryLabel.tsx +++ b/packages/twenty-front/src/modules/settings/components/SettingsNameCellSecondaryLabel.tsx @@ -22,13 +22,15 @@ const StyledSettingsNameCellSecondaryLabel = styled.span` type SettingsNameCellSecondaryLabelProps = { children: ReactNode; title?: string; + id?: string; }; export const SettingsNameCellSecondaryLabel = ({ children, title, + id, }: SettingsNameCellSecondaryLabelProps) => ( - + {children} ); diff --git a/packages/twenty-front/src/modules/settings/data-model/object-details/components/SettingsObjectFieldItemTableRow.tsx b/packages/twenty-front/src/modules/settings/data-model/object-details/components/SettingsObjectFieldItemTableRow.tsx index 6e650fa722..d1494dd609 100644 --- a/packages/twenty-front/src/modules/settings/data-model/object-details/components/SettingsObjectFieldItemTableRow.tsx +++ b/packages/twenty-front/src/modules/settings/data-model/object-details/components/SettingsObjectFieldItemTableRow.tsx @@ -28,6 +28,7 @@ import { } from 'twenty-ui/icon'; import { LightIconButton } from 'twenty-ui/input'; import { UndecoratedLink } from 'twenty-ui/navigation'; +import { AppTooltip, TooltipDelay } from 'twenty-ui/surfaces'; import { ThemeContext, themeCssVariables } from 'twenty-ui/theme-constants'; import { RelationType } from '~/generated-metadata/graphql'; import { useNavigateSettings } from '~/hooks/useNavigateSettings'; @@ -38,6 +39,7 @@ type SettingsObjectFieldItemTableRowProps = { settingsObjectDetailTableItem: SettingsObjectDetailTableItem; status: 'active' | 'disabled'; mode: 'view' | 'new-field'; + isMostlyEmpty?: boolean; }; export const OBJECT_FIELD_TABLE_ROW_GRID_TEMPLATE_COLUMNS = @@ -67,6 +69,7 @@ export const SettingsObjectFieldItemTableRow = ({ settingsObjectDetailTableItem, mode, status, + isMostlyEmpty = false, }: SettingsObjectFieldItemTableRowProps) => { const { theme } = useContext(ThemeContext); const { t } = useLingui(); @@ -106,6 +109,8 @@ export const SettingsObjectFieldItemTableRow = ({ objectMetadataItem, }); + const mostlyEmptyLabelId = `mostly-empty-field-${fieldMetadataItem.id}`; + const canToggleField = !isLabelIdentifier; const linkToNavigate = getSettingsPath(SettingsPath.ObjectFieldEdit, { @@ -189,6 +194,18 @@ export const SettingsObjectFieldItemTableRow = ({ {t`Deactivated`} )} + {fieldMetadataItem.isActive && isMostlyEmpty && ( + <> + + {t`Mostly empty`} + + + + )} diff --git a/packages/twenty-front/src/modules/settings/data-model/object-details/hooks/useMostlyEmptyFieldMetadataIds.ts b/packages/twenty-front/src/modules/settings/data-model/object-details/hooks/useMostlyEmptyFieldMetadataIds.ts new file mode 100644 index 0000000000..1149c5668e --- /dev/null +++ b/packages/twenty-front/src/modules/settings/data-model/object-details/hooks/useMostlyEmptyFieldMetadataIds.ts @@ -0,0 +1,33 @@ +import { useQuery } from '@apollo/client/react'; +import { useMemo } from 'react'; + +import { MOSTLY_EMPTY_FIELD_METADATA_IDS } from '@/object-metadata/graphql/queries'; + +type MostlyEmptyFieldMetadataIdsResult = { + mostlyEmptyFieldMetadataIds: string[]; +}; + +// Approximate, computed server-side from Postgres planner statistics; on any +// error the hint simply doesn't show +export const useMostlyEmptyFieldMetadataIds = ({ + objectMetadataItemId, + skip, +}: { + objectMetadataItemId: string; + skip?: boolean; +}) => { + const { data } = useQuery( + MOSTLY_EMPTY_FIELD_METADATA_IDS, + { + variables: { objectMetadataId: objectMetadataItemId }, + skip, + }, + ); + + const mostlyEmptyFieldMetadataIds = useMemo( + () => new Set(data?.mostlyEmptyFieldMetadataIds ?? []), + [data], + ); + + return { mostlyEmptyFieldMetadataIds }; +}; diff --git a/packages/twenty-front/src/pages/settings/data-model/SettingsObjectFieldTable.tsx b/packages/twenty-front/src/pages/settings/data-model/SettingsObjectFieldTable.tsx index d76ea42459..ae3cf13699 100644 --- a/packages/twenty-front/src/pages/settings/data-model/SettingsObjectFieldTable.tsx +++ b/packages/twenty-front/src/pages/settings/data-model/SettingsObjectFieldTable.tsx @@ -25,10 +25,11 @@ import { useAtomFamilyStateValue } from '@/ui/utilities/state/jotai/hooks/useAto import { useSetAtomFamilyState } from '@/ui/utilities/state/jotai/hooks/useSetAtomFamilyState'; import { useEffect, useMemo, useState } from 'react'; import { FieldMetadataType } from 'twenty-shared/types'; -import { IconArchive, IconSettings } from 'twenty-ui/icon'; +import { IconArchive, IconCircleDashed, IconSettings } from 'twenty-ui/icon'; import { SearchInput } from 'twenty-ui/input'; import { MenuItemToggle } from 'twenty-ui/navigation'; import { themeCssVariables } from 'twenty-ui/theme-constants'; +import { useMostlyEmptyFieldMetadataIds } from '@/settings/data-model/object-details/hooks/useMostlyEmptyFieldMetadataIds'; import { useMapFieldMetadataItemToSettingsObjectDetailTableItem } from '~/pages/settings/data-model/hooks/useMapFieldMetadataItemToSettingsObjectDetailTableItem'; import { type SettingsObjectDetailTableItem } from '~/pages/settings/data-model/types/SettingsObjectDetailTableItem'; import { normalizeSearchText } from '~/utils/normalizeSearchText'; @@ -82,6 +83,12 @@ export const SettingsObjectFieldTable = ({ const [searchTerm, setSearchTerm] = useState(''); const [showInactive, setShowInactive] = useState(true); const [showSystemFields, setShowSystemFields] = useState(false); + const [showOnlyMostlyEmpty, setShowOnlyMostlyEmpty] = useState(false); + + const { mostlyEmptyFieldMetadataIds } = useMostlyEmptyFieldMetadataIds({ + objectMetadataItemId: objectMetadataItem.id, + skip: mode !== 'view', + }); const isAdvancedModeEnabled = useAtomStateValue(isAdvancedModeEnabledState); @@ -142,13 +149,23 @@ export const SettingsObjectFieldTable = ({ const matchesActiveFilter = showInactive || item.fieldMetadataItem.isActive; + const matchesMostlyEmptyFilter = + !showOnlyMostlyEmpty || + mostlyEmptyFieldMetadataIds.has(item.fieldMetadataItem.id); + const matchesSearch = normalizeSearchText(item.label).includes(searchNormalized) || normalizeSearchText(item.dataType).includes(searchNormalized); - return matchesActiveFilter && matchesSearch; + return matchesActiveFilter && matchesMostlyEmptyFilter && matchesSearch; }); - }, [sortedAllObjectSettingsDetailItems, searchTerm, showInactive]); + }, [ + sortedAllObjectSettingsDetailItems, + searchTerm, + showInactive, + showOnlyMostlyEmpty, + mostlyEmptyFieldMetadataIds, + ]); return ( <> @@ -173,6 +190,18 @@ export const SettingsObjectFieldTable = ({ text={t`Inactive`} toggleSize="small" /> + {(mostlyEmptyFieldMetadataIds.size > 0 || + showOnlyMostlyEmpty) && ( + + setShowOnlyMostlyEmpty(!showOnlyMostlyEmpty) + } + toggled={showOnlyMostlyEmpty} + text={t`Mostly empty`} + toggleSize="small" + /> + )} {isAdvancedModeEnabled && ( ); })} diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-fraction-threshold.constant.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-fraction-threshold.constant.ts new file mode 100644 index 0000000000..d9e2c92ad0 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-fraction-threshold.constant.ts @@ -0,0 +1,3 @@ +// A field is hinted as "mostly empty" when at least this fraction of records +// leave it empty, per Postgres planner statistics (pg_stats) +export const MOSTLY_EMPTY_FRACTION_THRESHOLD = 0.95; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-minimum-row-count.constant.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-minimum-row-count.constant.ts new file mode 100644 index 0000000000..a8db2d9dbf --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/mostly-empty-minimum-row-count.constant.ts @@ -0,0 +1,3 @@ +// Below this approximate record count the emptiness signal is meaningless and +// young workspaces would get flooded with hints +export const MOSTLY_EMPTY_MINIMUM_ROW_COUNT = 100; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/mostly-empty-fields.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/mostly-empty-fields.service.ts new file mode 100644 index 0000000000..4672e30b41 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/mostly-empty-fields.service.ts @@ -0,0 +1,118 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util'; +import { findManyFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-many-flat-entity-by-id-in-flat-entity-maps-or-throw.util'; +import { MOSTLY_EMPTY_MINIMUM_ROW_COUNT } from 'src/engine/metadata-modules/object-metadata/constants/mostly-empty-minimum-row-count.constant'; +import { ObjectRecordCountService } from 'src/engine/metadata-modules/object-metadata/object-record-count.service'; +import { computeMostlyEmptyFieldMetadataIds } from 'src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util'; +import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; +import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; +import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; + +// Detects fields that are empty in almost all records of an object, reading +// Postgres planner statistics (pg_class / pg_stats) instead of scanning the +// table: cost is a catalog lookup regardless of table size, at the price of +// approximate results — acceptable for a settings-page hint +@Injectable() +export class MostlyEmptyFieldsService { + constructor( + private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, + private readonly objectRecordCountService: ObjectRecordCountService, + ) {} + + async getMostlyEmptyFieldMetadataIds({ + workspaceId, + objectMetadataId, + }: { + workspaceId: string; + objectMetadataId: string; + }): Promise { + const { flatObjectMetadataMaps, flatFieldMetadataMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'], + }, + ); + + const flatObjectMetadata = findFlatEntityByIdInFlatEntityMapsOrThrow({ + flatEntityId: objectMetadataId, + flatEntityMaps: flatObjectMetadataMaps, + }); + + const schemaName = getWorkspaceSchemaName(workspaceId); + const tableName = computeObjectTargetTable(flatObjectMetadata); + + const approximateRecordCountByTableName = + await this.objectRecordCountService.getApproximateRecordCountByTableName( + workspaceId, + ); + + const approximateRowCount = + approximateRecordCountByTableName.get(tableName) ?? 0; + + if (approximateRowCount < MOSTLY_EMPTY_MINIMUM_ROW_COUNT) { + return []; + } + + const dataSource = + await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); + + // Per-column emptiness: null fraction plus the sampled frequency of the + // column type's empty sentinel — '' for text columns (NOT NULL DEFAULT ''), + // '{}' for arrays, '{}'/'[]' for json. Sentinels are matched per physical + // column type so a text value that happens to be '{}' does not count + const columnStatisticsRows: { + column_name: string; + empty_fraction: number; + }[] = await dataSource.query( + `SELECT s.attname AS column_name, + (s.null_frac + COALESCE(empty_sentinel.frequency, 0))::float AS empty_fraction + FROM pg_stats s + JOIN pg_namespace n ON n.nspname = s.schemaname + JOIN pg_class c ON c.relnamespace = n.oid AND c.relname = s.tablename + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = s.attname + JOIN pg_type t ON t.oid = a.atttypid + LEFT JOIN LATERAL ( + SELECT SUM(most_common_value_frequency) AS frequency + FROM unnest(s.most_common_vals::text::text[], s.most_common_freqs) + AS most_common_value_entry(most_common_value, most_common_value_frequency) + WHERE most_common_value_entry.most_common_value = ANY ( + CASE + WHEN t.typcategory = 'S' THEN ARRAY[''] + WHEN t.typcategory = 'A' THEN ARRAY['{}'] + WHEN t.typname IN ('json', 'jsonb') THEN ARRAY['{}', '[]'] + ELSE ARRAY[]::text[] + END + ) + ) empty_sentinel ON TRUE + WHERE s.schemaname = $1 + AND s.tablename = $2 + AND NOT s.inherited`, + [schemaName, tableName], + undefined, + { shouldBypassPermissionChecks: true }, + ); + + const emptyFractionByColumnName = new Map( + columnStatisticsRows.map((row) => [ + row.column_name, + Number(row.empty_fraction), + ]), + ); + + const flatFieldMetadatas = findManyFlatEntityByIdInFlatEntityMapsOrThrow({ + flatEntityIds: flatObjectMetadata.fieldIds, + flatEntityMaps: flatFieldMetadataMaps, + }); + + return computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: flatFieldMetadatas, + labelIdentifierFieldMetadataId: + flatObjectMetadata.labelIdentifierFieldMetadataId, + emptyFractionByColumnName, + }); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index 63a538ac87..bce3ce92fa 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -24,6 +24,7 @@ import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/d import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; import { UpdateObjectPayload } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/object-metadata/interceptors/object-metadata-graphql-api-exception.interceptor'; +import { MostlyEmptyFieldsService } from 'src/engine/metadata-modules/object-metadata/mostly-empty-fields.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; @@ -103,6 +104,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace ObjectMetadataService, ObjectMetadataResolver, ObjectRecordCountService, + MostlyEmptyFieldsService, ObjectMetadataToolsFactory, ], exports: [ObjectMetadataService, ObjectMetadataToolsFactory], diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.resolver.ts index ab953fb890..abdae6b524 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.resolver.ts @@ -29,6 +29,7 @@ import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/d import { ObjectRecordCountDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-record-count.dto'; import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; import { getEffectiveImageIdentifierFieldMetadataId } from 'src/engine/metadata-modules/object-metadata/utils/get-effective-image-identifier-field-metadata-id.util'; +import { MostlyEmptyFieldsService } from 'src/engine/metadata-modules/object-metadata/mostly-empty-fields.service'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { ObjectRecordCountService } from 'src/engine/metadata-modules/object-metadata/object-record-count.service'; import { objectMetadataGraphqlApiExceptionHandler } from 'src/engine/metadata-modules/object-metadata/utils/object-metadata-graphql-api-exception-handler.util'; @@ -47,6 +48,7 @@ export class ObjectMetadataResolver { constructor( private readonly objectMetadataService: ObjectMetadataService, private readonly objectRecordCountService: ObjectRecordCountService, + private readonly mostlyEmptyFieldsService: MostlyEmptyFieldsService, private readonly i18nService: I18nService, ) {} @@ -67,6 +69,27 @@ export class ObjectMetadataResolver { return this.objectRecordCountService.getRecordCounts(workspaceId); } + @UseGuards(SettingsPermissionGuard(PermissionFlagType.DATA_MODEL)) + @Query(() => [UUIDScalarType]) + async mostlyEmptyFieldMetadataIds( + @Args('objectMetadataId', { type: () => UUIDScalarType }) + objectMetadataId: string, + @AuthWorkspace() { id: workspaceId }: WorkspaceEntity, + ): Promise { + try { + return await this.mostlyEmptyFieldsService.getMostlyEmptyFieldMetadataIds( + { + workspaceId, + objectMetadataId, + }, + ); + } catch (error) { + objectMetadataGraphqlApiExceptionHandler(error); + + return []; + } + } + private async resolveStandardOverride( objectMetadata: ObjectMetadataDTO, labelKey: diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-record-count.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-record-count.service.ts index 2dad95f33f..7a3cbc8567 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-record-count.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-record-count.service.ts @@ -15,19 +15,11 @@ export class ObjectRecordCountService { private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} - async getRecordCounts(workspaceId: string): Promise { - const { flatObjectMetadataMaps } = - await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( - { - workspaceId, - flatMapsKeys: ['flatObjectMetadataMaps'], - }, - ); - - const flatObjectMetadatas = Object.values( - flatObjectMetadataMaps.byUniversalIdentifier, - ).filter(isDefined); - + // reltuples is the planner's row estimate, refreshed by autovacuum's + // ANALYZE; never-analyzed tables report -1, clamped to 0 here + async getApproximateRecordCountByTableName( + workspaceId: string, + ): Promise> { const schemaName = getWorkspaceSchemaName(workspaceId); const dataSource = @@ -54,6 +46,25 @@ export class ObjectRecordCountService { ); } + return countByTableName; + } + + async getRecordCounts(workspaceId: string): Promise { + const { flatObjectMetadataMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps'], + }, + ); + + const flatObjectMetadatas = Object.values( + flatObjectMetadataMaps.byUniversalIdentifier, + ).filter(isDefined); + + const countByTableName = + await this.getApproximateRecordCountByTableName(workspaceId); + return flatObjectMetadatas.map((flatObjectMetadata) => ({ objectNamePlural: flatObjectMetadata.namePlural, totalCount: diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/compute-mostly-empty-field-metadata-ids.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/compute-mostly-empty-field-metadata-ids.util.spec.ts new file mode 100644 index 0000000000..af9b262e8a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/compute-mostly-empty-field-metadata-ids.util.spec.ts @@ -0,0 +1,169 @@ +import { FieldMetadataType } from 'twenty-shared/types'; + +import { + computeMostlyEmptyFieldMetadataIds, + type FieldMetadataForEmptinessCheck, +} from 'src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util'; + +const buildFieldMetadata = ( + overrides: Partial = {}, +): FieldMetadataForEmptinessCheck => ({ + id: 'field-id', + name: 'churnReason', + type: FieldMetadataType.TEXT, + isActive: true, + isSystem: false, + ...overrides, +}); + +describe('computeMostlyEmptyFieldMetadataIds', () => { + it('should flag an active field when its column reaches the emptiness threshold', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [buildFieldMetadata()], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([['churnReason', 0.95]]), + }); + + expect(result).toEqual(['field-id']); + }); + + it('should not flag a field when its column is below the emptiness threshold', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [buildFieldMetadata()], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([['churnReason', 0.5]]), + }); + + expect(result).toEqual([]); + }); + + it('should not flag inactive or system fields', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [ + buildFieldMetadata({ id: 'inactive-field-id', isActive: false }), + buildFieldMetadata({ id: 'system-field-id', isSystem: true }), + ], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([['churnReason', 1]]), + }); + + expect(result).toEqual([]); + }); + + it('should not flag the label identifier field', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [buildFieldMetadata({ id: 'label-identifier-field-id' })], + labelIdentifierFieldMetadataId: 'label-identifier-field-id', + emptyFractionByColumnName: new Map([['churnReason', 1]]), + }); + + expect(result).toEqual([]); + }); + + it('should not flag a field when its column is missing from statistics', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [buildFieldMetadata()], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map(), + }); + + expect(result).toEqual([]); + }); + + it('should flag a composite field only when all of its columns are mostly empty', () => { + const fullNameFieldMetadata = buildFieldMetadata({ + name: 'name', + type: FieldMetadataType.FULL_NAME, + }); + + const resultWithOneFilledColumn = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [fullNameFieldMetadata], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['nameFirstName', 0.99], + ['nameLastName', 0.5], + ]), + }); + + const resultWithAllColumnsEmpty = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [fullNameFieldMetadata], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['nameFirstName', 0.99], + ['nameLastName', 0.99], + ]), + }); + + expect(resultWithOneFilledColumn).toEqual([]); + expect(resultWithAllColumnsEmpty).toEqual(['field-id']); + }); + + it('should not flag a links field whose label or secondary links carry data', () => { + const linksFieldMetadata = buildFieldMetadata({ + name: 'introVideo', + type: FieldMetadataType.LINKS, + }); + + const resultWithFilledLabel = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [linksFieldMetadata], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['introVideoPrimaryLinkLabel', 0.4], + ['introVideoPrimaryLinkUrl', 1], + ['introVideoSecondaryLinks', 1], + ]), + }); + + const resultWithAllColumnsEmpty = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [linksFieldMetadata], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['introVideoPrimaryLinkLabel', 1], + ['introVideoPrimaryLinkUrl', 1], + ['introVideoSecondaryLinks', 1], + ]), + }); + + expect(resultWithFilledLabel).toEqual([]); + expect(resultWithAllColumnsEmpty).toEqual(['field-id']); + }); + + it('should ignore the currency code column when evaluating a currency field', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [ + buildFieldMetadata({ name: 'arr', type: FieldMetadataType.CURRENCY }), + ], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['arrAmountMicros', 0.99], + ['arrCurrencyCode', 0], + ]), + }); + + expect(result).toEqual(['field-id']); + }); + + it('should not flag field types without meaningful emptiness', () => { + const result = computeMostlyEmptyFieldMetadataIds({ + fieldMetadatas: [ + buildFieldMetadata({ + id: 'relation-field-id', + name: 'company', + type: FieldMetadataType.RELATION, + }), + buildFieldMetadata({ + id: 'boolean-field-id', + name: 'idealCustomerProfile', + type: FieldMetadataType.BOOLEAN, + }), + ], + labelIdentifierFieldMetadataId: null, + emptyFractionByColumnName: new Map([ + ['company', 1], + ['idealCustomerProfile', 1], + ]), + }); + + expect(result).toEqual([]); + }); +}); diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util.ts new file mode 100644 index 0000000000..b641ffbb0e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/compute-mostly-empty-field-metadata-ids.util.ts @@ -0,0 +1,50 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { MOSTLY_EMPTY_FRACTION_THRESHOLD } from 'src/engine/metadata-modules/object-metadata/constants/mostly-empty-fraction-threshold.constant'; +import { getEmptinessColumnNamesForField } from 'src/engine/metadata-modules/object-metadata/utils/get-emptiness-column-names-for-field.util'; + +export type FieldMetadataForEmptinessCheck = Pick< + FlatFieldMetadata, + 'id' | 'name' | 'type' | 'isActive' | 'isSystem' +>; + +export const computeMostlyEmptyFieldMetadataIds = ({ + fieldMetadatas, + labelIdentifierFieldMetadataId, + emptyFractionByColumnName, +}: { + fieldMetadatas: FieldMetadataForEmptinessCheck[]; + labelIdentifierFieldMetadataId: string | null; + emptyFractionByColumnName: Map; +}): string[] => { + return fieldMetadatas + .filter((fieldMetadata) => { + if (!fieldMetadata.isActive || fieldMetadata.isSystem) { + return false; + } + + // The label identifier cannot be deactivated, so hinting it is a dead end + if (fieldMetadata.id === labelIdentifierFieldMetadataId) { + return false; + } + + const columnNames = getEmptinessColumnNamesForField(fieldMetadata); + + if (!isDefined(columnNames)) { + return false; + } + + // A column missing from statistics (e.g. added after the last ANALYZE) + // means we don't know, and not knowing means no hint + return columnNames.every((columnName) => { + const emptyFraction = emptyFractionByColumnName.get(columnName); + + return ( + isDefined(emptyFraction) && + emptyFraction >= MOSTLY_EMPTY_FRACTION_THRESHOLD + ); + }); + }) + .map((fieldMetadata) => fieldMetadata.id); +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/get-emptiness-column-names-for-field.util.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/get-emptiness-column-names-for-field.util.ts new file mode 100644 index 0000000000..7ef221bba3 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/get-emptiness-column-names-for-field.util.ts @@ -0,0 +1,80 @@ +import { + compositeTypeDefinitions, + FieldMetadataType, +} from 'twenty-shared/types'; +import { assertUnreachable, isDefined } from 'twenty-shared/utils'; + +import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; + +// Composite properties that carry workspace defaults (currency codes, phone +// country/calling codes): a stamped default would mask emptiness of the value +// users actually fill in +const DEFAULT_BEARING_COMPOSITE_PROPERTIES: Partial< + Record +> = { + [FieldMetadataType.CURRENCY]: ['currencyCode'], + [FieldMetadataType.PHONES]: [ + 'primaryPhoneCountryCode', + 'primaryPhoneCallingCode', + ], +}; + +const getCompositeEmptinessColumnNames = ( + flatFieldMetadata: Pick, +): string[] | null => { + const compositeType = compositeTypeDefinitions.get(flatFieldMetadata.type); + + if (!isDefined(compositeType)) { + return null; + } + + const defaultBearingProperties = + DEFAULT_BEARING_COMPOSITE_PROPERTIES[flatFieldMetadata.type] ?? []; + + return compositeType.properties + .filter((property) => !defaultBearingProperties.includes(property.name)) + .map((property) => + computeCompositeColumnName(flatFieldMetadata.name, property), + ); +}; + +// Returns the physical columns whose emptiness determines whether the field is +// empty, or null for types where emptiness is not meaningful (relations, +// booleans, system-managed types) +export const getEmptinessColumnNamesForField = ( + flatFieldMetadata: Pick, +): string[] | null => { + switch (flatFieldMetadata.type) { + case FieldMetadataType.TEXT: + case FieldMetadataType.NUMBER: + case FieldMetadataType.NUMERIC: + case FieldMetadataType.DATE: + case FieldMetadataType.DATE_TIME: + case FieldMetadataType.RATING: + case FieldMetadataType.SELECT: + case FieldMetadataType.MULTI_SELECT: + case FieldMetadataType.ARRAY: + case FieldMetadataType.RAW_JSON: + case FieldMetadataType.FILES: + return [flatFieldMetadata.name]; + case FieldMetadataType.CURRENCY: + case FieldMetadataType.PHONES: + case FieldMetadataType.FULL_NAME: + case FieldMetadataType.ADDRESS: + case FieldMetadataType.LINKS: + case FieldMetadataType.EMAILS: + case FieldMetadataType.RICH_TEXT: + return getCompositeEmptinessColumnNames(flatFieldMetadata); + case FieldMetadataType.ACTOR: + case FieldMetadataType.BOOLEAN: + case FieldMetadataType.MORPH_RELATION: + case FieldMetadataType.POSITION: + case FieldMetadataType.RELATION: + case FieldMetadataType.TS_VECTOR: + case FieldMetadataType.UUID: + return null; + default: + return assertUnreachable(flatFieldMetadata.type); + } +}; diff --git a/packages/twenty-server/test/integration/metadata/suites/object-metadata/mostly-empty-field-metadata-ids.integration-spec.ts b/packages/twenty-server/test/integration/metadata/suites/object-metadata/mostly-empty-field-metadata-ids.integration-spec.ts new file mode 100644 index 0000000000..0e34269b84 --- /dev/null +++ b/packages/twenty-server/test/integration/metadata/suites/object-metadata/mostly-empty-field-metadata-ids.integration-spec.ts @@ -0,0 +1,143 @@ +import gql from 'graphql-tag'; +import { createOneFieldMetadata } from 'test/integration/metadata/suites/field-metadata/utils/create-one-field-metadata.util'; +import { createOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/create-one-object-metadata.util'; +import { deleteOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/delete-one-object-metadata.util'; +import { updateOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/update-one-object-metadata.util'; +import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util'; + +import { FieldMetadataType } from 'twenty-shared/types'; + +const TEST_TABLE_NAME = '_mostlyEmptyProbe'; + +const MOSTLY_EMPTY_FIELD_METADATA_IDS_QUERY = gql` + query MostlyEmptyFieldMetadataIds($objectMetadataId: UUID!) { + mostlyEmptyFieldMetadataIds(objectMetadataId: $objectMetadataId) + } +`; + +const fetchMostlyEmptyFieldMetadataIds = async ( + objectMetadataId: string, +): Promise => { + const response = await makeMetadataAPIRequest({ + query: MOSTLY_EMPTY_FIELD_METADATA_IDS_QUERY, + variables: { objectMetadataId }, + }); + + expect(response.body.errors).toBeUndefined(); + + return response.body.data.mostlyEmptyFieldMetadataIds; +}; + +describe('mostlyEmptyFieldMetadataIds', () => { + let testObjectMetadataId: string; + let probeNotesFieldMetadataId: string; + let testSchemaName: string; + + const insertProbeRecords = async (count: number) => { + await global.testDataSource.query( + `INSERT INTO "${testSchemaName}"."${TEST_TABLE_NAME}" + (name, "createdByName", "createdBySource", "updatedByName", "updatedBySource", position) + SELECT 'Probe record ' || i, 'Integration Test', 'MANUAL', 'Integration Test', 'MANUAL', i + FROM generate_series(1, ${count}) AS i`, + ); + }; + + // Statistics are normally refreshed by autovacuum; tests need them on demand + const analyzeProbeTable = async () => { + await global.testDataSource.query( + `ANALYZE "${testSchemaName}"."${TEST_TABLE_NAME}"`, + ); + }; + + beforeAll(async () => { + const { + data: { + createOneObject: { id: objectMetadataId }, + }, + } = await createOneObjectMetadata({ + expectToFail: false, + input: { + nameSingular: 'mostlyEmptyProbe', + namePlural: 'mostlyEmptyProbes', + labelSingular: 'Mostly Empty Probe', + labelPlural: 'Mostly Empty Probes', + icon: 'IconChartHistogram', + isLabelSyncedWithName: false, + }, + }); + + testObjectMetadataId = objectMetadataId; + + const { + data: { + createOneField: { id: notesFieldId }, + }, + } = await createOneFieldMetadata({ + expectToFail: false, + input: { + name: 'probeNotes', + label: 'Probe Notes', + type: FieldMetadataType.TEXT, + objectMetadataId: testObjectMetadataId, + isLabelSyncedWithName: false, + }, + gqlFields: `id name`, + }); + + probeNotesFieldMetadataId = notesFieldId; + + const schemaRows: { schema_name: string }[] = + await global.testDataSource.query( + `SELECT w."databaseSchema" AS schema_name + FROM core."objectMetadata" om + JOIN core.workspace w ON w.id = om."workspaceId" + WHERE om.id = $1`, + [testObjectMetadataId], + ); + + testSchemaName = schemaRows[0].schema_name; + }); + + afterAll(async () => { + await updateOneObjectMetadata({ + expectToFail: false, + input: { + idToUpdate: testObjectMetadataId, + updatePayload: { isActive: false }, + }, + }); + await deleteOneObjectMetadata({ + expectToFail: false, + input: { idToDelete: testObjectMetadataId }, + }); + }); + + it('should flag a never-filled field only while the record gate passes and the field stays empty', async () => { + // Below the minimum record count nothing is flagged + await insertProbeRecords(50); + await analyzeProbeTable(); + + expect( + await fetchMostlyEmptyFieldMetadataIds(testObjectMetadataId), + ).toEqual([]); + + // Past the gate, the untouched custom text field is the only expected hit: + // name is the label identifier and system fields are excluded + await insertProbeRecords(100); + await analyzeProbeTable(); + + expect( + await fetchMostlyEmptyFieldMetadataIds(testObjectMetadataId), + ).toEqual([probeNotesFieldMetadataId]); + + // Once the field is backfilled everywhere the hint disappears + await global.testDataSource.query( + `UPDATE "${testSchemaName}"."${TEST_TABLE_NAME}" SET "probeNotes" = 'filled'`, + ); + await analyzeProbeTable(); + + expect( + await fetchMostlyEmptyFieldMetadataIds(testObjectMetadataId), + ).toEqual([]); + }); +});