feat(ai): reliable bulk data import via code-interpreter (#22209)
## Summary Makes AI-assisted bulk data import (CSV/Excel/spreadsheets) reliable and token-efficient by letting an entire import run inside a single code-interpreter call, with a persistent sandbox session and server-side bulk helpers. Also includes supporting improvements to attachment handling, upsert reporting, and field-permission error messages. ## Changes ### Code interpreter - **Persistent per-session kernel** in `LocalDriver`: a long-lived Python process per `sessionId` keeps variables, imports, and files alive across calls (matching E2B behavior). Falls back to the existing ephemeral per-call path when no session is provided. CAN BE REMOVED, INTERESTING FOR DEV X - Idle watchdog that self-terminates the kernel, configurable via the new `CODE_INTERPRETER_IDLE_TIMEOUT_MS` config variable; the process also exits on parent shutdown (EOF on control fd). CAN BE REMOVED, INTERESTING FOR DEV X - New `bulk_upsert` and `lookup_by` helpers on the sandbox `twenty` object for idempotent batched writes (≤200/batch) and bounded relation-ID resolution. ### Records - `upsert_many_*` now reports a `created` / `updated` / `total` split in its result and log line (new `isFreshlyCreatedRecord` util). ### AI chat - `replaceUnsupportedFileParts`: user-attached files whose MIME type the model can't handle natively (and that aren't code-interpreter-supported) are downgraded to a descriptive text note instead of being sent as unsupported file parts. Modality→MIME mapping drives native support detection. - Finalize dangling tool parts before `convertToModelMessages` to avoid malformed model messages. - Extracted shared types/constants for code-interpreter file extraction. ### Permissions - Field permission-denied exceptions now include the field name and entity name for easier debugging. ### Skill docs - Added the bulk-import recipe ## To do in following PR - [ ] Skill command migration ## Test plan - [x] Unit tests for `getNativeMimeTypesForModalities` and `replaceUnsupportedFileParts` pass - [x] Run a bulk import (>50 rows) end-to-end through the code interpreter and verify a single sandbox call handles read → resolve relations → upsert → summary - [x] Verify session persistence: define a variable in one call, use it in the next within the same session - [x] Verify the kernel self-terminates after `CODE_INTERPRETER_IDLE_TIMEOUT_MS` - [x] Verify unsupported attachments are replaced with a text note for models lacking the modality - [x] Verify `upsert_many_*` returns correct created/updated counts - [x] Verify field-restricted role triggers a permission error naming the field and entity - [ ] Test with [hotel_business.xlsx](https://github.com/user-attachments/files/29376307/hotel_business.xlsx) and simple "import record" prompt <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/22209?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->
This commit is contained in:
@@ -144,6 +144,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
allFieldsSelected,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
break;
|
||||
case 'insert':
|
||||
@@ -158,6 +160,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
|
||||
if (updatedColumns.length > 0) {
|
||||
@@ -177,6 +181,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
updatedColumns: updatedColumnsWithoutRlsFields,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -193,6 +199,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
|
||||
if (updatedColumns.length > 0) {
|
||||
@@ -200,6 +208,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
updatedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
}
|
||||
break;
|
||||
@@ -215,6 +225,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
break;
|
||||
case 'restore':
|
||||
@@ -230,6 +242,8 @@ export const validateOperationIsPermittedOrThrow = ({
|
||||
restrictedFields: permissionsForEntity.restrictedFields,
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
});
|
||||
break;
|
||||
default:
|
||||
@@ -397,15 +411,41 @@ const validatePermissionsForJoinsAndReturnSelectsWithoutJoins = ({
|
||||
return { selectsWithoutJoinedAliases };
|
||||
};
|
||||
|
||||
const buildFieldPermissionDeniedMessage = ({
|
||||
action,
|
||||
column,
|
||||
fieldMetadataId,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
}: {
|
||||
action: 'read' | 'write';
|
||||
column: string;
|
||||
fieldMetadataId: string;
|
||||
entityName: string;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
}): string => {
|
||||
const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({
|
||||
flatEntityId: fieldMetadataId,
|
||||
flatEntityMaps: flatFieldMetadataMaps,
|
||||
});
|
||||
const fieldName = fieldMetadata?.name ?? column;
|
||||
|
||||
return `${PermissionsExceptionMessage.PERMISSION_DENIED}: no permission to ${action} field "${fieldName}" on "${entityName}"`;
|
||||
};
|
||||
|
||||
const validateReadFieldPermissionOrThrow = ({
|
||||
restrictedFields,
|
||||
selectedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
allFieldsSelected,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
}: {
|
||||
restrictedFields: RestrictedFieldsPermissions;
|
||||
selectedColumns: string[] | '*';
|
||||
columnNameToFieldMetadataIdMap: Record<string, string>;
|
||||
entityName: string;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
allFieldsSelected?: boolean;
|
||||
}) => {
|
||||
const noReadRestrictions =
|
||||
@@ -434,7 +474,13 @@ const validateReadFieldPermissionOrThrow = ({
|
||||
|
||||
if (restrictedFields[fieldMetadataId]?.canRead === false) {
|
||||
throw new PermissionsException(
|
||||
PermissionsExceptionMessage.PERMISSION_DENIED,
|
||||
buildFieldPermissionDeniedMessage({
|
||||
action: 'read',
|
||||
column,
|
||||
fieldMetadataId,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
}),
|
||||
PermissionsExceptionCode.PERMISSION_DENIED,
|
||||
);
|
||||
}
|
||||
@@ -445,10 +491,14 @@ const validateUpdateFieldPermissionOrThrow = ({
|
||||
restrictedFields,
|
||||
updatedColumns,
|
||||
columnNameToFieldMetadataIdMap,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
}: {
|
||||
restrictedFields: RestrictedFieldsPermissions;
|
||||
updatedColumns: string[];
|
||||
columnNameToFieldMetadataIdMap: Record<string, string>;
|
||||
entityName: string;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
}) => {
|
||||
if (isEmpty(restrictedFields)) {
|
||||
return;
|
||||
@@ -465,7 +515,13 @@ const validateUpdateFieldPermissionOrThrow = ({
|
||||
|
||||
if (restrictedFields[fieldMetadataId]?.canUpdate === false) {
|
||||
throw new PermissionsException(
|
||||
PermissionsExceptionMessage.PERMISSION_DENIED,
|
||||
buildFieldPermissionDeniedMessage({
|
||||
action: 'write',
|
||||
column,
|
||||
fieldMetadataId,
|
||||
entityName,
|
||||
flatFieldMetadataMaps,
|
||||
}),
|
||||
PermissionsExceptionCode.PERMISSION_DENIED,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user