From c025a0c2b810ccafb58c961c3af6c56226aeafdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bosi?= <71827178+bosiraphael@users.noreply.github.com> Date: Thu, 26 Feb 2026 12:14:51 +0100 Subject: [PATCH] Add events and properties to video, audio and iFrame (#18257) - Add media-specific events - Extend `iFrame` with missing properties - Enrich `SerializedEventData` with media-related target fields so serialized events carry the media element state. - Refactor the `remote-elements` code generator to support per-element custom events --- .../generate-remote-dom-elements.ts | 4 +- .../generators/remote-elements.generator.ts | 56 ++++++--- .../scripts/remote-dom/generators/schemas.ts | 1 + .../host/utils/createHtmlHostWrapper.ts | 21 ++++ .../remote/generated/remote-components.ts | 38 ++++++ .../remote/generated/remote-elements.ts | 110 +++++++++++++++++- .../utils/patchRemoteElementSetAttribute.ts | 3 + .../constants/AllowedHtmlElements.ts | 54 ++++++++- .../constants/EventToReact.ts | 19 +++ .../constants/SerializedEventData.ts | 7 ++ 10 files changed, 288 insertions(+), 25 deletions(-) diff --git a/packages/twenty-sdk/scripts/remote-dom/generate-remote-dom-elements.ts b/packages/twenty-sdk/scripts/remote-dom/generate-remote-dom-elements.ts index 08cbffda5e..f93e38ccd5 100644 --- a/packages/twenty-sdk/scripts/remote-dom/generate-remote-dom-elements.ts +++ b/packages/twenty-sdk/scripts/remote-dom/generate-remote-dom-elements.ts @@ -48,7 +48,9 @@ const getHtmlElementSchemas = (): ComponentSchema[] => { ...HTML_COMMON_PROPERTIES, ...element.properties, }, - events: COMMON_HTML_EVENTS, + events: element.events + ? [...COMMON_HTML_EVENTS, ...element.events] + : COMMON_HTML_EVENTS, htmlTag: extractHtmlTag(element.tag), })); }; diff --git a/packages/twenty-sdk/scripts/remote-dom/generators/remote-elements.generator.ts b/packages/twenty-sdk/scripts/remote-dom/generators/remote-elements.generator.ts index 6ea7e5eed0..41ce4a5ad6 100644 --- a/packages/twenty-sdk/scripts/remote-dom/generators/remote-elements.generator.ts +++ b/packages/twenty-sdk/scripts/remote-dom/generators/remote-elements.generator.ts @@ -162,12 +162,15 @@ const generateElementDefinition = ( sourceFile: SourceFile, component: ComponentSchema, specificProperties: Record, - useSharedEvents: boolean, - useSharedPropertiesConfig: boolean, + commonEventNames: Set, + shouldUseCommonHtmlPropertiesConfig: boolean, ): void => { const isHtml = isDefined(component.htmlTag); - const useShared = useSharedEvents && isHtml; - const hasEvents = component.events.length > 0; + const hasCommonHtmlEvents = commonEventNames.size > 0 && isHtml; + const customEvents = component.events.filter( + (event) => !hasCommonHtmlEvents || !commonEventNames.has(event), + ); + const hasEvents = hasCommonHtmlEvents || customEvents.length > 0; const hasSpecificProps = Object.keys(specificProperties).length > 0; const hasProps = Object.keys(component.properties).length > 0; @@ -177,11 +180,19 @@ const generateElementDefinition = ( ? TYPE_NAMES.COMMON_PROPERTIES : TYPE_NAMES.EMPTY_RECORD; - const eventsType = hasEvents - ? useShared - ? TYPE_NAMES.COMMON_EVENTS - : `{ ${component.events.map((event) => `${event}(event: RemoteEvent): void`).join('; ')} }` - : TYPE_NAMES.EMPTY_RECORD; + const customEventsInline = customEvents + .map((event) => `${event}(event: RemoteEvent): void`) + .join('; '); + + let eventsType: string = TYPE_NAMES.EMPTY_RECORD; + + if (hasCommonHtmlEvents && customEvents.length > 0) { + eventsType = `${TYPE_NAMES.COMMON_EVENTS} & { ${customEventsInline} }`; + } else if (hasCommonHtmlEvents) { + eventsType = TYPE_NAMES.COMMON_EVENTS; + } else if (customEvents.length > 0) { + eventsType = `{ ${customEventsInline} }`; + } sourceFile.addVariableStatement({ isExported: true, @@ -220,7 +231,7 @@ const generateElementDefinition = ( }); writer.write(','); writer.newLine(); - } else if (useSharedPropertiesConfig && isHtml) { + } else if (shouldUseCommonHtmlPropertiesConfig && isHtml) { writer.write( `properties: ${TYPE_NAMES.COMMON_PROPERTIES_CONFIG},`, ); @@ -235,10 +246,16 @@ const generateElementDefinition = ( } } if (hasEvents) { + const formattedCustomEvents = customEvents + .map((event) => `'${event}'`) + .join(', '); + writer.write( - useShared - ? `events: [...${TYPE_NAMES.COMMON_EVENTS_ARRAY}],` - : `events: [${component.events.map((event) => `'${event}'`).join(', ')}],`, + hasCommonHtmlEvents && customEvents.length > 0 + ? `events: [...${TYPE_NAMES.COMMON_EVENTS_ARRAY}, ${formattedCustomEvents}],` + : hasCommonHtmlEvents + ? `events: [...${TYPE_NAMES.COMMON_EVENTS_ARRAY}],` + : `events: [${formattedCustomEvents}],`, ); writer.newLine(); } @@ -304,8 +321,9 @@ export const generateRemoteElements = ( overwrite: true, }); - const useSharedEvents = commonEvents.length > 0; - const useSharedPropertiesConfig = Object.keys(commonProperties).length > 0; + const commonEventNames = new Set(commonEvents); + const shouldUseCommonHtmlPropertiesConfig = + Object.keys(commonProperties).length > 0; sourceFile.addImportDeclaration({ moduleSpecifier: '@remote-dom/core/elements', @@ -327,11 +345,11 @@ export const generateRemoteElements = ( generateCommonPropertiesType(sourceFile, commonProperties); - if (useSharedEvents) { + if (commonEventNames.size > 0) { generateCommonEventsType(sourceFile, commonEvents); } - if (useSharedPropertiesConfig) { + if (shouldUseCommonHtmlPropertiesConfig) { generateCommonPropertiesConfig(sourceFile, commonProperties); } @@ -345,8 +363,8 @@ export const generateRemoteElements = ( sourceFile, component, specificProperties, - useSharedEvents, - useSharedPropertiesConfig, + commonEventNames, + shouldUseCommonHtmlPropertiesConfig, ); } diff --git a/packages/twenty-sdk/scripts/remote-dom/generators/schemas.ts b/packages/twenty-sdk/scripts/remote-dom/generators/schemas.ts index 66f2dc8ff2..217dbfa912 100644 --- a/packages/twenty-sdk/scripts/remote-dom/generators/schemas.ts +++ b/packages/twenty-sdk/scripts/remote-dom/generators/schemas.ts @@ -11,6 +11,7 @@ export const HtmlElementConfigZ = z.object({ .string() .regex(/^Html[A-Z]/, 'Name must be PascalCase starting with Html'), properties: z.record(z.string(), PropertySchemaZ), + events: z.array(z.string()).optional(), }); export const HtmlElementConfigArrayZ = z.array(HtmlElementConfigZ); diff --git a/packages/twenty-sdk/src/front-component-renderer/host/utils/createHtmlHostWrapper.ts b/packages/twenty-sdk/src/front-component-renderer/host/utils/createHtmlHostWrapper.ts index 19436bfd53..758b41a735 100644 --- a/packages/twenty-sdk/src/front-component-renderer/host/utils/createHtmlHostWrapper.ts +++ b/packages/twenty-sdk/src/front-component-renderer/host/utils/createHtmlHostWrapper.ts @@ -108,6 +108,27 @@ const serializeEvent = (event: unknown): SerializedEventData => { if ('scrollLeft' in target && typeof target.scrollLeft === 'number') { serialized.scrollLeft = target.scrollLeft; } + if ('currentTime' in target && typeof target.currentTime === 'number') { + serialized.currentTime = target.currentTime; + } + if ('duration' in target && typeof target.duration === 'number') { + serialized.duration = target.duration; + } + if ('paused' in target && typeof target.paused === 'boolean') { + serialized.paused = target.paused; + } + if ('ended' in target && typeof target.ended === 'boolean') { + serialized.ended = target.ended; + } + if ('volume' in target && typeof target.volume === 'number') { + serialized.volume = target.volume; + } + if ('muted' in target && typeof target.muted === 'boolean') { + serialized.muted = target.muted; + } + if ('playbackRate' in target && typeof target.playbackRate === 'number') { + serialized.playbackRate = target.playbackRate; + } } return serialized; diff --git a/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-components.ts b/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-components.ts index 91a3bc4d53..e057085bd5 100644 --- a/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-components.ts +++ b/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-components.ts @@ -1172,6 +1172,25 @@ export const HtmlVideo = createRemoteComponent('html-video', HtmlVideoElement, { onWheel: { event: 'wheel' }, onContextMenu: { event: 'contextmenu' }, onDrag: { event: 'drag' }, + onTimeUpdate: { event: 'timeupdate' }, + onPlay: { event: 'play' }, + onPause: { event: 'pause' }, + onEnded: { event: 'ended' }, + onLoadedMetadata: { event: 'loadedmetadata' }, + onLoadedData: { event: 'loadeddata' }, + onVolumeChange: { event: 'volumechange' }, + onSeeking: { event: 'seeking' }, + onSeeked: { event: 'seeked' }, + onError: { event: 'error' }, + onCanPlay: { event: 'canplay' }, + onCanPlayThrough: { event: 'canplaythrough' }, + onWaiting: { event: 'waiting' }, + onProgress: { event: 'progress' }, + onDurationChange: { event: 'durationchange' }, + onRateChange: { event: 'ratechange' }, + onStalled: { event: 'stalled' }, + onSuspend: { event: 'suspend' }, + onEmptied: { event: 'emptied' }, }, }); export const HtmlAudio = createRemoteComponent('html-audio', HtmlAudioElement, { @@ -1196,6 +1215,25 @@ export const HtmlAudio = createRemoteComponent('html-audio', HtmlAudioElement, { onWheel: { event: 'wheel' }, onContextMenu: { event: 'contextmenu' }, onDrag: { event: 'drag' }, + onTimeUpdate: { event: 'timeupdate' }, + onPlay: { event: 'play' }, + onPause: { event: 'pause' }, + onEnded: { event: 'ended' }, + onLoadedMetadata: { event: 'loadedmetadata' }, + onLoadedData: { event: 'loadeddata' }, + onVolumeChange: { event: 'volumechange' }, + onSeeking: { event: 'seeking' }, + onSeeked: { event: 'seeked' }, + onError: { event: 'error' }, + onCanPlay: { event: 'canplay' }, + onCanPlayThrough: { event: 'canplaythrough' }, + onWaiting: { event: 'waiting' }, + onProgress: { event: 'progress' }, + onDurationChange: { event: 'durationchange' }, + onRateChange: { event: 'ratechange' }, + onStalled: { event: 'stalled' }, + onSuspend: { event: 'suspend' }, + onEmptied: { event: 'emptied' }, }, }); export const HtmlSource = createRemoteComponent( diff --git a/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-elements.ts b/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-elements.ts index 3d448b6856..c6211566bf 100644 --- a/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-elements.ts +++ b/packages/twenty-sdk/src/front-component-renderer/remote/generated/remote-elements.ts @@ -603,11 +603,15 @@ export const HtmlHrElement = createRemoteElement< export type HtmlIframeProperties = HtmlCommonProperties & { src?: string; + name?: string; width?: string; height?: string; allow?: string; sandbox?: string; allowFullScreen?: boolean; + loading?: string; + referrerPolicy?: string; + srcDoc?: string; }; export const HtmlIframeElement = createRemoteElement< @@ -619,11 +623,15 @@ export const HtmlIframeElement = createRemoteElement< properties: { ...HTML_COMMON_PROPERTIES_CONFIG, src: { type: String }, + name: { type: String }, width: { type: String }, height: { type: String }, allow: { type: String }, sandbox: { type: String }, allowFullScreen: { type: Boolean }, + loading: { type: String }, + referrerPolicy: { type: String }, + srcDoc: { type: String }, }, events: [...HTML_COMMON_EVENTS_ARRAY], }); @@ -640,13 +648,35 @@ export type HtmlVideoProperties = HtmlCommonProperties & { height?: string; crossOrigin?: string; playsInline?: boolean; + disablePictureInPicture?: boolean; + disableRemotePlayback?: boolean; }; export const HtmlVideoElement = createRemoteElement< HtmlVideoProperties, Record, Record, - HtmlCommonEvents + HtmlCommonEvents & { + timeupdate(event: RemoteEvent): void; + play(event: RemoteEvent): void; + pause(event: RemoteEvent): void; + ended(event: RemoteEvent): void; + loadedmetadata(event: RemoteEvent): void; + loadeddata(event: RemoteEvent): void; + volumechange(event: RemoteEvent): void; + seeking(event: RemoteEvent): void; + seeked(event: RemoteEvent): void; + error(event: RemoteEvent): void; + canplay(event: RemoteEvent): void; + canplaythrough(event: RemoteEvent): void; + waiting(event: RemoteEvent): void; + progress(event: RemoteEvent): void; + durationchange(event: RemoteEvent): void; + ratechange(event: RemoteEvent): void; + stalled(event: RemoteEvent): void; + suspend(event: RemoteEvent): void; + emptied(event: RemoteEvent): void; + } >({ properties: { ...HTML_COMMON_PROPERTIES_CONFIG, @@ -661,8 +691,31 @@ export const HtmlVideoElement = createRemoteElement< height: { type: String }, crossOrigin: { type: String }, playsInline: { type: Boolean }, + disablePictureInPicture: { type: Boolean }, + disableRemotePlayback: { type: Boolean }, }, - events: [...HTML_COMMON_EVENTS_ARRAY], + events: [ + ...HTML_COMMON_EVENTS_ARRAY, + 'timeupdate', + 'play', + 'pause', + 'ended', + 'loadedmetadata', + 'loadeddata', + 'volumechange', + 'seeking', + 'seeked', + 'error', + 'canplay', + 'canplaythrough', + 'waiting', + 'progress', + 'durationchange', + 'ratechange', + 'stalled', + 'suspend', + 'emptied', + ], }); export type HtmlAudioProperties = HtmlCommonProperties & { @@ -679,7 +732,27 @@ export const HtmlAudioElement = createRemoteElement< HtmlAudioProperties, Record, Record, - HtmlCommonEvents + HtmlCommonEvents & { + timeupdate(event: RemoteEvent): void; + play(event: RemoteEvent): void; + pause(event: RemoteEvent): void; + ended(event: RemoteEvent): void; + loadedmetadata(event: RemoteEvent): void; + loadeddata(event: RemoteEvent): void; + volumechange(event: RemoteEvent): void; + seeking(event: RemoteEvent): void; + seeked(event: RemoteEvent): void; + error(event: RemoteEvent): void; + canplay(event: RemoteEvent): void; + canplaythrough(event: RemoteEvent): void; + waiting(event: RemoteEvent): void; + progress(event: RemoteEvent): void; + durationchange(event: RemoteEvent): void; + ratechange(event: RemoteEvent): void; + stalled(event: RemoteEvent): void; + suspend(event: RemoteEvent): void; + emptied(event: RemoteEvent): void; + } >({ properties: { ...HTML_COMMON_PROPERTIES_CONFIG, @@ -691,13 +764,38 @@ export const HtmlAudioElement = createRemoteElement< preload: { type: String }, crossOrigin: { type: String }, }, - events: [...HTML_COMMON_EVENTS_ARRAY], + events: [ + ...HTML_COMMON_EVENTS_ARRAY, + 'timeupdate', + 'play', + 'pause', + 'ended', + 'loadedmetadata', + 'loadeddata', + 'volumechange', + 'seeking', + 'seeked', + 'error', + 'canplay', + 'canplaythrough', + 'waiting', + 'progress', + 'durationchange', + 'ratechange', + 'stalled', + 'suspend', + 'emptied', + ], }); export type HtmlSourceProperties = HtmlCommonProperties & { src?: string; type?: string; media?: string; + srcSet?: string; + sizes?: string; + width?: number; + height?: number; }; export const HtmlSourceElement = createRemoteElement< @@ -711,6 +809,10 @@ export const HtmlSourceElement = createRemoteElement< src: { type: String }, type: { type: String }, media: { type: String }, + srcSet: { type: String }, + sizes: { type: String }, + width: { type: Number }, + height: { type: Number }, }, events: [...HTML_COMMON_EVENTS_ARRAY], }); diff --git a/packages/twenty-sdk/src/front-component-renderer/remote/utils/patchRemoteElementSetAttribute.ts b/packages/twenty-sdk/src/front-component-renderer/remote/utils/patchRemoteElementSetAttribute.ts index d5b2d35c76..382cc4ff56 100644 --- a/packages/twenty-sdk/src/front-component-renderer/remote/utils/patchRemoteElementSetAttribute.ts +++ b/packages/twenty-sdk/src/front-component-renderer/remote/utils/patchRemoteElementSetAttribute.ts @@ -9,6 +9,9 @@ const ATTRIBUTE_TO_PROPERTY_MAP: Record = { tabIndex: 'tabIndex', tabindex: 'tabIndex', + + srcDoc: 'srcDoc', + srcdoc: 'srcDoc', }; export const patchRemoteElementSetAttribute = (): void => { diff --git a/packages/twenty-sdk/src/sdk/front-component-api/constants/AllowedHtmlElements.ts b/packages/twenty-sdk/src/sdk/front-component-api/constants/AllowedHtmlElements.ts index 4c982cbf0b..af5923bbd6 100644 --- a/packages/twenty-sdk/src/sdk/front-component-api/constants/AllowedHtmlElements.ts +++ b/packages/twenty-sdk/src/sdk/front-component-api/constants/AllowedHtmlElements.ts @@ -7,6 +7,7 @@ export type AllowedHtmlElement = { tag: string; name: string; properties: Record; + events?: string[]; }; export const ALLOWED_HTML_ELEMENTS: AllowedHtmlElement[] = [ @@ -150,12 +151,15 @@ export const ALLOWED_HTML_ELEMENTS: AllowedHtmlElement[] = [ name: 'HtmlIframe', properties: { src: { type: 'string', optional: true }, - title: { type: 'string', optional: true }, + name: { type: 'string', optional: true }, width: { type: 'string', optional: true }, height: { type: 'string', optional: true }, allow: { type: 'string', optional: true }, sandbox: { type: 'string', optional: true }, allowFullScreen: { type: 'boolean', optional: true }, + loading: { type: 'string', optional: true }, + referrerPolicy: { type: 'string', optional: true }, + srcDoc: { type: 'string', optional: true }, }, }, { @@ -173,7 +177,30 @@ export const ALLOWED_HTML_ELEMENTS: AllowedHtmlElement[] = [ height: { type: 'string', optional: true }, crossOrigin: { type: 'string', optional: true }, playsInline: { type: 'boolean', optional: true }, + disablePictureInPicture: { type: 'boolean', optional: true }, + disableRemotePlayback: { type: 'boolean', optional: true }, }, + events: [ + 'timeupdate', + 'play', + 'pause', + 'ended', + 'loadedmetadata', + 'loadeddata', + 'volumechange', + 'seeking', + 'seeked', + 'error', + 'canplay', + 'canplaythrough', + 'waiting', + 'progress', + 'durationchange', + 'ratechange', + 'stalled', + 'suspend', + 'emptied', + ], }, { tag: 'html-audio', @@ -187,6 +214,27 @@ export const ALLOWED_HTML_ELEMENTS: AllowedHtmlElement[] = [ preload: { type: 'string', optional: true }, crossOrigin: { type: 'string', optional: true }, }, + events: [ + 'timeupdate', + 'play', + 'pause', + 'ended', + 'loadedmetadata', + 'loadeddata', + 'volumechange', + 'seeking', + 'seeked', + 'error', + 'canplay', + 'canplaythrough', + 'waiting', + 'progress', + 'durationchange', + 'ratechange', + 'stalled', + 'suspend', + 'emptied', + ], }, { tag: 'html-source', @@ -195,6 +243,10 @@ export const ALLOWED_HTML_ELEMENTS: AllowedHtmlElement[] = [ src: { type: 'string', optional: true }, type: { type: 'string', optional: true }, media: { type: 'string', optional: true }, + srcSet: { type: 'string', optional: true }, + sizes: { type: 'string', optional: true }, + width: { type: 'number', optional: true }, + height: { type: 'number', optional: true }, }, }, ]; diff --git a/packages/twenty-sdk/src/sdk/front-component-api/constants/EventToReact.ts b/packages/twenty-sdk/src/sdk/front-component-api/constants/EventToReact.ts index 6fab2bea0d..621b05306a 100644 --- a/packages/twenty-sdk/src/sdk/front-component-api/constants/EventToReact.ts +++ b/packages/twenty-sdk/src/sdk/front-component-api/constants/EventToReact.ts @@ -19,4 +19,23 @@ export const EVENT_TO_REACT: Record = { wheel: 'onWheel', contextmenu: 'onContextMenu', drag: 'onDrag', + timeupdate: 'onTimeUpdate', + play: 'onPlay', + pause: 'onPause', + ended: 'onEnded', + loadedmetadata: 'onLoadedMetadata', + loadeddata: 'onLoadedData', + volumechange: 'onVolumeChange', + seeking: 'onSeeking', + seeked: 'onSeeked', + error: 'onError', + canplay: 'onCanPlay', + canplaythrough: 'onCanPlayThrough', + waiting: 'onWaiting', + progress: 'onProgress', + durationchange: 'onDurationChange', + ratechange: 'onRateChange', + stalled: 'onStalled', + suspend: 'onSuspend', + emptied: 'onEmptied', }; diff --git a/packages/twenty-sdk/src/sdk/front-component-api/constants/SerializedEventData.ts b/packages/twenty-sdk/src/sdk/front-component-api/constants/SerializedEventData.ts index 7671ed3a1b..ad69807862 100644 --- a/packages/twenty-sdk/src/sdk/front-component-api/constants/SerializedEventData.ts +++ b/packages/twenty-sdk/src/sdk/front-component-api/constants/SerializedEventData.ts @@ -23,4 +23,11 @@ export type SerializedEventData = { deltaY?: number; deltaZ?: number; deltaMode?: number; + currentTime?: number; + duration?: number; + paused?: boolean; + ended?: boolean; + volume?: number; + muted?: boolean; + playbackRate?: number; };