From 3f64be292baed41351bff084f4557acd37533381 Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Wed, 18 Nov 2026 08:13:09 -0400 Subject: [PATCH] feat(canvas): draw NativeScript views into a canvas on Linux HTML-in-Canvas on GTK 4. A content="drawable" canvas holds each child in a slot the canvas widget allocates but never draws: the slot is placed with the transform the child was last drawn with and clipped to the drawn part, so GTK's own picking sends input where the view appears. The canvas snapshots its children into a scratch snapshot to keep their render nodes current; a GtkWidgetPaintable per drawable view reports when that node changes, and the window's GSK renderer renders it into the snapshot. The per-frame work is native (NSCDrawableHost in canvas-napi): JS makes one update() call a frame and no GTK calls. Geometry is sent as a Float64Array, and only when it changed. A press on a hosted view no longer moves focus to the canvas. Node-API canvases now measure their children at the canvas width and stack them, as on iOS. The htmlincanvas spec suite runs on Linux, with a new test that a source rect limits hit testing to the drawn part. --- crates/canvas-napi/src/host/linux.rs | 16 + crates/canvas-napi/src/host/linux/drawable.rs | 486 ++++++++++++++++++ packages/canvas/Canvas/drawable/common.ts | 5 +- .../canvas/Canvas/drawable/index.linux.ts | 163 +++++- packages/canvas/Canvas/index.linux.ts | 5 +- packages/canvas/Canvas/napi-host.ts | 23 +- packages/canvas/README.md | 3 +- tools/demo/canvas/spec/htmlincanvas.ts | 37 ++ tools/demo/canvas/spec/index.ts | 4 +- 9 files changed, 730 insertions(+), 12 deletions(-) create mode 100644 crates/canvas-napi/src/host/linux/drawable.rs diff --git a/crates/canvas-napi/src/host/linux.rs b/crates/canvas-napi/src/host/linux.rs index 5af0f5d30..8284ae816 100644 --- a/crates/canvas-napi/src/host/linux.rs +++ b/crates/canvas-napi/src/host/linux.rs @@ -25,6 +25,8 @@ use napi::bindgen_prelude::ObjectFinalize; use napi::{Env, Error, Result}; use napi_derive::napi; +mod drawable; + mod imp { use super::*; @@ -54,7 +56,12 @@ mod imp { (0, 0, -1, -1) } + fn size_allocate(&self, _: i32, _: i32, _: i32) { + drawable::allocate_children(&self.obj()); + } + fn snapshot(&self, snapshot: >k::Snapshot) { + drawable::snapshot_children(&self.obj()); let Some(frame) = self.frame.borrow().clone() else { return; }; @@ -835,6 +842,15 @@ impl NSCCanvas { self.view.as_ptr() as usize as f64 } + /// Whether a hosted child (`content="drawable"`) takes input at (x, y) DIP. + #[napi] + pub fn picks_child(&self, x: f64, y: f64) -> bool { + self + .view + .pick(x, y, gtk::PickFlags::DEFAULT) + .is_some_and(|picked| picked != *self.view.upcast_ref::()) + } + #[napi(getter)] pub fn surface_width(&self) -> u32 { if !self.offscreen.is_null() { diff --git a/crates/canvas-napi/src/host/linux/drawable.rs b/crates/canvas-napi/src/host/linux/drawable.rs new file mode 100644 index 000000000..841a8c522 --- /dev/null +++ b/crates/canvas-napi/src/host/linux/drawable.rs @@ -0,0 +1,486 @@ +//! `CanvasModule.NSCDrawableHost` on Linux: the children of a `content="drawable"` canvas +//! (HTML-in-Canvas). The canvas widget holds each child in a slot it allocates but never shows. +//! A slot is placed where its child was last drawn and clips to the drawn part, so GTK's own +//! picking sends input there. Snapshots are rendered by the window's GSK renderer from the view's +//! last render node, and a `GtkWidgetPaintable` per view reports when that node changes. + +use std::cell::Cell; +use std::collections::HashMap; +use std::rc::Rc; + +use gtk::prelude::*; +use gtk::subclass::prelude::*; +use gtk::{gdk, glib, graphene, gsk}; +use napi::bindgen_prelude::Float64ArraySlice; +use napi_derive::napi; + +use super::{gtk_ready, CanvasView, NSCCanvas}; +use crate::image_asset::ImageAsset; + +/// Slots with no geometry are allocated this far left of the canvas, where nothing picks them. +const OFFSCREEN: f32 = -1.0e6; + +/// Frames a view may go without a render node before its snapshot is taken as empty. +const MAX_WAIT_FRAMES: u32 = 30; + +#[derive(Clone, Copy, Default, PartialEq)] +pub struct Placement { + /// Layout size of the child (DIP). + pub width: i32, + pub height: i32, + /// Border box to canvas view (DIP), in DOMMatrix order (graphene's too). + pub transform: Option<[f32; 16]>, + /// The drawn part (x, y, width, height) of the border box. + pub clip: Option<[f32; 4]>, +} + +mod imp { + use super::*; + + #[derive(Default)] + pub struct DrawableSlot { + pub placement: Cell, + pub size: Cell<(i32, i32)>, + } + + #[glib::object_subclass] + impl ObjectSubclass for DrawableSlot { + const NAME: &'static str = "NSCDrawableSlot"; + type Type = super::DrawableSlot; + type ParentType = gtk::Widget; + } + + impl ObjectImpl for DrawableSlot { + fn constructed(&self) { + self.parent_constructed(); + // Picking stops at the padding box: the slot is the clip. + self.obj().set_overflow(gtk::Overflow::Hidden); + } + + fn dispose(&self) { + while let Some(child) = self.obj().first_child() { + child.unparent(); + } + } + } + + impl WidgetImpl for DrawableSlot { + fn measure(&self, _: gtk::Orientation, _: i32) -> (i32, i32, i32, i32) { + (0, 0, -1, -1) + } + + fn size_allocate(&self, _: i32, _: i32, _: i32) { + let Some(child) = self.obj().first_child() else { + return; + }; + let (width, height) = self.size.get(); + let (x, y) = self.placement.get().clip.map_or((0., 0.), |c| (c[0], c[1])); + let origin = gsk::Transform::new().translate(&graphene::Point::new(-x, -y)); + child.allocate(width, height, -1, Some(origin)); + } + } +} + +glib::wrapper! { + pub struct DrawableSlot(ObjectSubclass) + @extends gtk::Widget, + @implements gtk::Buildable, gtk::ConstraintTarget; +} + +pub(super) fn allocate_children(view: &CanvasView) { + let mut next = view.first_child(); + while let Some(widget) = next { + next = widget.next_sibling(); + let Ok(slot) = widget.downcast::() else { + continue; + }; + let Some(child) = slot.first_child().filter(|c| c.should_layout()) else { + continue; + }; + let placement = slot.imp().placement.get(); + // GTK needs at least the minimum size; NativeScript may lay out smaller. + let (min_width, ..) = child.measure(gtk::Orientation::Horizontal, -1); + let width = placement.width.max(min_width); + let (min_height, ..) = child.measure(gtk::Orientation::Vertical, width); + let height = placement.height.max(min_height); + slot.imp().size.set((width, height)); + let (transform, clip) = match placement.transform { + Some(m) => { + let clip = placement + .clip + .unwrap_or([0., 0., width as f32, height as f32]); + let t = gsk::Transform::new() + .matrix(&graphene::Matrix::from_float(m)) + .translate(&graphene::Point::new(clip[0], clip[1])); + (t, clip) + } + None => ( + gsk::Transform::new().translate(&graphene::Point::new(OFFSCREEN, 0.)), + [0., 0., width as f32, height as f32], + ), + }; + let size = |v: f32| (v.ceil() as i32).max(1); + slot.allocate(size(clip[2]), size(clip[3]), -1, Some(transform)); + } +} + +/// Keeps each child's render node current without showing it: snapshots come from that node, and +/// a view's paintables only report a change once it is redrawn. +pub(super) fn snapshot_children(view: &CanvasView) { + let Some(first) = view.first_child() else { + return; + }; + let scratch = gtk::Snapshot::new(); + let mut next = Some(first); + while let Some(child) = next { + view.snapshot_child(&child, &scratch); + next = child.next_sibling(); + } + drop(scratch.to_node()); +} + +fn widget_at(address: f64) -> Option { + let pointer = address as usize as *mut gtk::ffi::GtkWidget; + if pointer.is_null() || !gtk_ready() { + return None; + } + Some(unsafe { glib::translate::from_glib_none(pointer) }) +} + +fn key(widget: >k::Widget) -> usize { + widget.as_ptr() as usize +} + +struct Entry { + widget: glib::WeakRef, + paintable: gtk::WidgetPaintable, + handler: Option, + dirty: Rc>, + texture: Option, + waited: u32, +} + +impl Entry { + fn new(widget: >k::Widget) -> Self { + let paintable = gtk::WidgetPaintable::new(Some(widget)); + let dirty = Rc::new(Cell::new(true)); + let flag = Rc::clone(&dirty); + let handler = paintable.connect_invalidate_contents(move |_| flag.set(true)); + Entry { + widget: widget.downgrade(), + paintable, + handler: Some(handler), + dirty, + texture: None, + waited: 0, + } + } +} + +impl Drop for Entry { + fn drop(&mut self) { + if let Some(handler) = self.handler.take() { + self.paintable.disconnect(handler); + } + self.paintable.set_widget(None::<>k::Widget>); + } +} + +/// `paintable` (the view at its CSS size) scaled by `scale` and offset by `-source`, into a +/// `width` x `height` texture. None while the view has never been drawn. +fn render( + widget: >k::Widget, + paintable: &impl IsA, + width: i32, + height: i32, + scale: (f32, f32), + source: (f32, f32), +) -> Option { + let renderer = widget.native()?.renderer()?; + let snapshot = gtk::Snapshot::new(); + snapshot.scale(scale.0, scale.1); + snapshot.translate(&graphene::Point::new(-source.0, -source.1)); + paintable.snapshot(&snapshot, widget.width() as f64, widget.height() as f64); + let node = snapshot.to_node()?; + let viewport = graphene::Rect::new(0., 0., width as f32, height as f32); + Some(renderer.render_texture(node, Some(&viewport))) +} + +fn transparent(width: i32, height: i32) -> gdk::Texture { + let bytes = glib::Bytes::from_owned(vec![0u8; width as usize * height as usize * 4]); + gdk::MemoryTexture::new( + width, + height, + gdk::MemoryFormat::R8g8b8a8Premultiplied, + &bytes, + width as usize * 4, + ) + .upcast() +} + +fn load(asset: &ImageAsset, texture: &gdk::Texture) -> bool { + asset.from_gdk_texture_sync(texture.as_ptr() as usize as f64) +} + +#[napi(js_name = "NSCDrawableHost")] +pub struct DrawableHost { + view: glib::WeakRef, + /// In `setDrawables` order. + entries: Vec, +} + +#[napi] +impl DrawableHost { + #[napi(constructor)] + pub fn new(canvas: &NSCCanvas) -> Self { + DrawableHost { + view: canvas.view.downgrade(), + entries: Vec::new(), + } + } + + /// Parents `child` (a GtkWidget address), in a slot, at `index` among the canvas's children. + #[napi] + pub fn attach(&self, child: f64, index: i32) -> bool { + let (Some(view), Some(child)) = (self.view.upgrade(), widget_at(child)) else { + return false; + }; + if let Some(parent) = child.parent() { + child.unparent(); + if let Ok(old) = parent.downcast::() { + old.unparent(); + } + } + let slot: DrawableSlot = glib::Object::new(); + child.set_parent(&slot); + let mut sibling = view.first_child(); + for _ in 0..index.max(0) { + sibling = sibling.and_then(|s| s.next_sibling()); + } + match sibling { + Some(sibling) if index >= 0 => slot.insert_before(&view, Some(&sibling)), + _ => slot.set_parent(&view), + } + true + } + + #[napi] + pub fn detach(&self, child: f64) { + let Some((slot, child)) = self.slot_of(child) else { + return; + }; + child.unparent(); + slot.unparent(); + } + + fn slot_of(&self, child: f64) -> Option<(DrawableSlot, gtk::Widget)> { + let view = self.view.upgrade()?; + let child = widget_at(child)?; + let slot = child.parent()?.downcast::().ok()?; + (slot.parent().as_ref() == Some(view.upcast_ref())).then_some((slot, child)) + } + + /// Layout sizes (DIP) of the canvas's children: `[widget, width, height, ...]`. + #[napi] + pub fn place_all(&self, sizes: Vec) { + for chunk in sizes.chunks_exact(3) { + let (width, height) = (chunk[1].round() as i32, chunk[2].round() as i32); + self.update_placement(chunk[0], |p| { + p.width = width; + p.height = height; + }); + } + } + + /// `[width, height]` (DIP) of a view. + #[napi] + pub fn size(&self, child: f64) -> Vec { + widget_at(child).map_or(vec![0., 0.], |w| vec![w.width() as f64, w.height() as f64]) + } + + /// `matrix` (DOMMatrix order) maps a child's border box to the canvas view (DIP); the clip is + /// in its border box. `raise` puts it above its siblings for picking. + #[napi] + pub fn set_geometry( + &self, + child: f64, + matrix: Float64ArraySlice, + has_clip: bool, + clip_x: f64, + clip_y: f64, + clip_width: f64, + clip_height: f64, + raise: bool, + ) { + if matrix.len() != 16 { + return; + } + let mut m = [0f32; 16]; + for (out, value) in m.iter_mut().zip(matrix.iter()) { + *out = *value as f32; + } + let clip = has_clip.then(|| { + let (x, width) = if clip_width < 0. { + (clip_x + clip_width, -clip_width) + } else { + (clip_x, clip_width) + }; + let (y, height) = if clip_height < 0. { + (clip_y + clip_height, -clip_height) + } else { + (clip_y, clip_height) + }; + [x as f32, y as f32, width as f32, height as f32] + }); + self.update_placement(child, |p| { + p.transform = Some(m); + p.clip = clip; + }); + if let (true, Some(view), Some((slot, _))) = (raise, self.view.upgrade(), self.slot_of(child)) { + if slot.next_sibling().is_some() { + slot.insert_before(&view, None::<>k::Widget>); + } + } + } + + #[napi] + pub fn clear_geometry(&self, child: f64) { + self.update_placement(child, |p| { + p.transform = None; + p.clip = None; + }); + } + + fn update_placement(&self, child: f64, f: impl FnOnce(&mut Placement)) { + let (Some(view), Some((slot, _))) = (self.view.upgrade(), self.slot_of(child)) else { + return; + }; + let before = slot.imp().placement.get(); + let mut placement = before; + f(&mut placement); + if placement != before { + slot.imp().placement.set(placement); + view.queue_allocate(); + } + } + + /// The views to snapshot (GtkWidget addresses), in tree order. + #[napi] + pub fn set_drawables(&mut self, widgets: Vec) { + let mut old: HashMap = self + .entries + .drain(..) + .filter_map(|e| e.widget.upgrade().map(|w| (key(&w), e))) + .collect(); + for address in widgets { + let Some(widget) = widget_at(address) else { + continue; + }; + let entry = old + .remove(&key(&widget)) + .unwrap_or_else(|| Entry::new(&widget)); + self.entries.push(entry); + } + } + + /// Snapshots the views whose render node changed since the last call, at `scale` pixels per + /// DIP; returns `[index, cssWidth, cssHeight, ...]` for each, index in `setDrawables` order. + #[napi] + pub fn update(&mut self, scale_x: f64, scale_y: f64) -> Option> { + let mut changed: Option> = None; + for (index, entry) in self.entries.iter_mut().enumerate() { + let Some(widget) = entry.widget.upgrade() else { + continue; + }; + let (css_width, css_height) = (widget.width(), widget.height()); + if !widget.is_mapped() || css_width <= 0 || css_height <= 0 { + continue; + } + let width = ((css_width as f64 * scale_x - 1e-3).ceil() as i32).max(1); + let height = ((css_height as f64 * scale_y - 1e-3).ceil() as i32).max(1); + let resized = entry + .texture + .as_ref() + .is_none_or(|t| t.width() != width || t.height() != height); + if !resized && !entry.dirty.get() { + continue; + } + let scale = (scale_x as f32, scale_y as f32); + let texture = match render(&widget, &entry.paintable, width, height, scale, (0., 0.)) { + Some(texture) => texture, + None if entry.waited >= MAX_WAIT_FRAMES => transparent(width, height), + None => { + entry.waited += 1; + continue; + } + }; + entry.waited = 0; + entry.dirty.set(false); + entry.texture = Some(texture); + changed.get_or_insert_with(Vec::new).extend([ + index as f64, + css_width as f64, + css_height as f64, + ]); + } + changed + } + + #[napi] + pub fn load_snapshot(&self, child: f64, asset: &ImageAsset) -> bool { + let Some(child) = widget_at(child) else { + return false; + }; + self + .entries + .iter() + .find(|e| e.widget.upgrade().as_ref() == Some(&child)) + .and_then(|e| e.texture.as_ref()) + .is_some_and(|texture| load(asset, texture)) + } + + /// A fresh snapshot of `child` into `asset`, not shared with the per-frame one: `width` x + /// `height` pixels of the `source` rect (DIP) of its border box, or all of it when the rect is + /// empty. + #[napi] + pub fn capture( + &self, + child: f64, + width: i32, + height: i32, + source_x: f64, + source_y: f64, + source_width: f64, + source_height: f64, + asset: &ImageAsset, + ) -> bool { + let Some(widget) = widget_at(child) else { + return false; + }; + let (source_width, source_height) = if source_width > 0. && source_height > 0. { + (source_width, source_height) + } else { + (widget.width() as f64, widget.height() as f64) + }; + if width <= 0 || height <= 0 || source_width <= 0. || source_height <= 0. { + return false; + } + let (scale_x, scale_y) = (width as f64 / source_width, height as f64 / source_height); + let existing = self + .entries + .iter() + .find(|e| e.widget.upgrade().as_ref() == Some(&widget)) + .map(|e| e.paintable.clone()); + let paintable = existing.unwrap_or_else(|| gtk::WidgetPaintable::new(Some(&widget))); + let scale = (scale_x as f32, scale_y as f32); + let source = (source_x as f32, source_y as f32); + let texture = render(&widget, &paintable, width, height, scale, source) + .unwrap_or_else(|| transparent(width, height)); + load(asset, &texture) + } + + #[napi] + pub fn dispose(&mut self) { + self.entries.clear(); + } +} diff --git a/packages/canvas/Canvas/drawable/common.ts b/packages/canvas/Canvas/drawable/common.ts index 2df0bbafe..d26abab27 100644 --- a/packages/canvas/Canvas/drawable/common.ts +++ b/packages/canvas/Canvas/drawable/common.ts @@ -512,9 +512,8 @@ export abstract class DrawableController { /** Brings snapshots up to date; returns the views that got a new one. */ protected abstract refresh(list: View[]): View[]; - /** Notes that `view` has a new native snapshot, taken at `scale`. */ - protected recordSnapshot(view: View, scale: { x: number; y: number }) { - const css = this.cssSize(view); + /** Notes that `view` has a new native snapshot, taken at `scale`; `css` is its size (DIP). */ + protected recordSnapshot(view: View, scale: { x: number; y: number }, css = this.cssSize(view)) { let rec = this.snapshots.get(view); if (!rec) { rec = { asset: null, width: 0, height: 0, cssWidth: 0, cssHeight: 0, defaultWidth: 0, defaultHeight: 0 }; diff --git a/packages/canvas/Canvas/drawable/index.linux.ts b/packages/canvas/Canvas/drawable/index.linux.ts index 5756f86d9..bb72eb8f6 100644 --- a/packages/canvas/Canvas/drawable/index.linux.ts +++ b/packages/canvas/Canvas/drawable/index.linux.ts @@ -1,8 +1,165 @@ -import type { DrawableController } from './common'; +import { View } from '@nativescript/core'; +import { ImageAsset } from '../../ImageAsset'; +import { DrawableController, ElementGeometry, Rect, SnapshotRecord } from './common'; export * from './common'; -/** Node-API hosts have no native child views to draw yet. */ +const addresses = new WeakMap(); + +function address(view: View): number { + const native: any = view?.nativeViewProtected; + if (!native || typeof native !== 'object') { + return 0; + } + let value = addresses.get(native); + if (value === undefined) { + value = typeof native.toNumber === 'function' ? native.toNumber() : 0; + addresses.set(native, value); + } + return value; +} + +/** + * Parenting, allocation (which places each child where it was drawn and clips it there, for GTK's + * picking), change tracking and capture live in `CanvasModule.NSCDrawableHost` (crates/canvas-napi). + */ +class LinuxDrawableController extends DrawableController { + readonly host: any; + private list: View[] = []; + private scaleX = 0; + private scaleY = 0; + + constructor(canvas: any) { + super(canvas); + this.host = new global.CanvasModule.NSCDrawableHost(canvas._canvas); + } + + attachChild(child: View, index?: number): boolean { + const widget = address(child); + return widget ? this.host.attach(widget, typeof index === 'number' ? index : -1) : false; + } + + detachChild(child: View) { + const widget = address(child); + if (widget) { + this.host.detach(widget); + } + } + + /** Layout units on Linux are DIPs. */ + layoutHost() { + const sizes: number[] = []; + this.canvas?.eachChildView((child: View) => { + const widget = address(child); + if (widget) { + sizes.push(widget, child.getMeasuredWidth(), child.getMeasuredHeight()); + } + return true; + }); + if (sizes.length) { + this.host.placeAll(sizes); + } + } + + cssSize(view: View): { width: number; height: number } { + const widget = address(view); + if (!widget) { + return { width: 0, height: 0 }; + } + const [width, height] = this.host.size(widget); + return { width, height }; + } + + protected refresh(list: View[]): View[] { + const live = list.filter((view) => !!address(view)); + if (live.length !== this.list.length || live.some((view, i) => view !== this.list[i])) { + this.list = live; + this.host.setDrawables(live.map(address)); + } + const scale = this.gridScale(); + // [index, cssWidth, cssHeight, ...] + const updates: number[] | null = this.host.update(scale.x, scale.y); + if (!updates) { + return []; + } + const changed: View[] = []; + for (let i = 0; i + 2 < updates.length; i += 3) { + const view = this.list[updates[i]]; + if (!view) { + continue; + } + this.recordSnapshot(view, scale, { width: updates[i + 1], height: updates[i + 2] }); + changed.push(view); + } + return changed; + } + + assetOf(view: View, rec: SnapshotRecord): ImageAsset | null { + if (rec.asset && rec.assetGeneration === rec.generation) { + return rec.asset; + } + const asset = rec.asset ?? new ImageAsset(); + if (this.host.loadSnapshot(address(view), asset.native)) { + rec.asset = asset; + rec.assetGeneration = rec.generation; + } + return rec.asset; + } + + render(view: View, width: number, height: number, source?: Rect): ImageAsset | null { + const widget = address(view); + width = Math.max(1, Math.round(width)); + height = Math.max(1, Math.round(height)); + if (!widget) { + return null; + } + const asset = new ImageAsset(); + // An empty source rect is the whole border box. + return this.host.capture(widget, width, height, source?.x ?? 0, source?.y ?? 0, source?.width ?? 0, source?.height ?? 0, asset.native) ? asset : null; + } + + protected applyGeometry(view: View, geometry: ElementGeometry | null, moveToTop = false) { + const widget = address(view); + if (!widget) { + return; + } + const m = geometry ? this.viewTransform(geometry) : null; + const clip = geometry?.clip ?? null; + const { send, raise } = this.geometryUpdate(view, widget, m, clip, moveToTop); + if (!send) { + return; + } + if (m) { + this.host.setGeometry(widget, m, !!clip, clip?.x ?? 0, clip?.y ?? 0, clip?.width ?? 0, clip?.height ?? 0, raise); + } else { + this.host.clearGeometry(widget); + } + } + + /** Geometry is in grid units: re-sent only when the grid's scale changes. */ + protected syncGeometry() { + const scale = this.gridScale(); + if (scale.x === this.scaleX && scale.y === this.scaleY) { + return; + } + this.scaleX = scale.x; + this.scaleY = scale.y; + for (const [view, geometry] of this.geometry) { + this.applyGeometry(view, geometry, false); + } + } + + protected releaseRecord(rec: SnapshotRecord) { + super.releaseRecord(rec); + rec.native = null; + } + + dispose() { + super.dispose(); + this.host.dispose(); + } +} + export function createDrawableController(canvas: any): DrawableController | null { - return null; + return global.CanvasModule?.NSCDrawableHost ? new LinuxDrawableController(canvas) : null; } diff --git a/packages/canvas/Canvas/index.linux.ts b/packages/canvas/Canvas/index.linux.ts index 24ce3a7b3..02ad7fcf6 100644 --- a/packages/canvas/Canvas/index.linux.ts +++ b/packages/canvas/Canvas/index.linux.ts @@ -160,7 +160,10 @@ export class Canvas extends NapiCanvas { } start = last = [x, y]; down = true; - gtk_widget_grab_focus(widget); + // A press on a hosted child (content="drawable") leaves focus to it. + if (!canvas._canvas.picksChild(x, y)) { + gtk_widget_grab_focus(widget); + } canvas._handleEvents({ event: 'down', ptrId: 0, x, y, isPrimary: true }); }); on(drag, 'drag-update', ['double', 'double'], (dx: number, dy: number) => { diff --git a/packages/canvas/Canvas/napi-host.ts b/packages/canvas/Canvas/napi-host.ts index 71821eff4..f82924566 100644 --- a/packages/canvas/Canvas/napi-host.ts +++ b/packages/canvas/Canvas/napi-host.ts @@ -14,7 +14,7 @@ import { WebGLRenderingContext } from '../WebGL/WebGLRenderingContext'; import { WebGL2RenderingContext } from '../WebGL2/WebGL2RenderingContext'; import { GPUCanvasContext } from '../WebGPU'; import { ImageBitmapRenderingContext } from '../ImageBitmapRenderingContext'; -import { ImageSource, Screen, Utils, widthProperty, heightProperty } from '@nativescript/core'; +import { ImageSource, Screen, Utils, View, widthProperty, heightProperty } from '@nativescript/core'; import { handleContextOptions, microtask, CanvasContextType, isPercentLength, setParentPercentSize, holdBackAnimationFramesWhileBehind } from './utils'; import { Helpers } from '../helpers'; @@ -374,6 +374,17 @@ export abstract class NapiCanvas extends CanvasBase { onLayout(left: number, top: number, right: number, bottom: number) { super.onLayout(left, top, right, bottom); + if (this.getChildrenCount() > 0) { + const width = right - left; + let y = 0; + this.eachChildView((child) => { + const size = child.getMeasuredHeight(); + View.layoutChild(this, child, 0, y, width, y + size); + y += size; + return true; + }); + } + (this._drawables as any)?.layoutHost?.(); this._shimParent(); } @@ -424,7 +435,15 @@ export abstract class NapiCanvas extends CanvasBase { public onMeasure(widthMeasureSpec: number, heightMeasureSpec: number) { if (this._canvas) { - this.setMeasuredDimension(Utils.layout.getMeasureSpecSize(widthMeasureSpec), Utils.layout.getMeasureSpecSize(heightMeasureSpec)); + const width = Utils.layout.getMeasureSpecSize(widthMeasureSpec); + this.setMeasuredDimension(width, Utils.layout.getMeasureSpecSize(heightMeasureSpec)); + // Children never size the canvas. + const childWidth = Utils.layout.makeMeasureSpec(width, Utils.layout.EXACTLY); + const childHeight = Utils.layout.makeMeasureSpec(0, Utils.layout.UNSPECIFIED); + this.eachChildView((child) => { + View.measureChild(this, child, childWidth, childHeight); + return true; + }); } } diff --git a/packages/canvas/README.md b/packages/canvas/README.md index 4033ba7df..ec53b35ce 100644 --- a/packages/canvas/README.md +++ b/packages/canvas/README.md @@ -150,7 +150,7 @@ const canvas = OffscreenCanvas._fromHandle(data.canvas); ### Views in the canvas (HTML-in-Canvas) -The [HTML-in-Canvas](https://github.com/WICG/html-in-canvas) API, with NativeScript views as the elements (iOS and Android). With `content="drawable"` the canvas lays its children out (stacked, at its width) but never shows them; a context draws a snapshot of any descendant marked `drawable`, and touches reach the real view where it was drawn. +The [HTML-in-Canvas](https://github.com/WICG/html-in-canvas) API, with NativeScript views as the elements (iOS, Android and Linux). With `content="drawable"` the canvas lays its children out (stacked, at its width) but never shows them; a context draws a snapshot of any descendant marked `drawable`, and touches reach the real view where it was drawn. ```xml @@ -176,6 +176,7 @@ canvas.on('paint', () => { - WebGL: `gl.texElementSubImage2D(target, level, x, y, view)`. WebGPU: `device.queue.drawElementImageToTexture({ source: view }, { texture })`. Neither sets hit testing: call `canvas.updateElementGeometry(view, { canvasTransform })` yourself. - `canvas.captureElementImage(view)` returns an `ElementImage` that outlives the frame and is what an OffscreenCanvas's 2D context draws. - Snapshots are taken on the CPU (`layer.render(in:)` on iOS, `View.draw` on Android), so content that only exists on the GPU (video, other canvases, maps, blurs) is not in them. +- On Linux, GTK renders the snapshots from each view's render nodes, so GL content (other canvases) is in them. Hit testing follows the geometry of the canvas's own children only: a drawable view nested deeper gets input through its top-level ancestor's geometry. ## Linux diff --git a/tools/demo/canvas/spec/htmlincanvas.ts b/tools/demo/canvas/spec/htmlincanvas.ts index 31cb8e9b6..bc8ac84ab 100644 --- a/tools/demo/canvas/spec/htmlincanvas.ts +++ b/tools/demo/canvas/spec/htmlincanvas.ts @@ -5,6 +5,17 @@ import { Canvas } from '@nativescript/canvas'; import { suite, test, skip, ok, equal, closeTo, throws, getPageCanvas, pixelAt } from './harness'; declare const navigator: any; +declare const __LINUX__: boolean; +declare const gtk_widget_pick: any, gtk_widget_is_ancestor: any; + +const isLinux = typeof __LINUX__ !== 'undefined' && __LINUX__; + +/** Linux: whether the canvas picks `target` (or something in it) at (x, y) DIP. */ +function gtkHits(canvas: any, target: View, x: number, y: number) { + const picked = gtk_widget_pick(canvas.nativeView, x, y, 0); + const native: any = target.nativeView; + return !!picked && (picked.toNumber() === native.toNumber() || !!gtk_widget_is_ancestor(picked, native)); +} const RED: [number, number, number, number] = [255, 0, 0, 255]; const BLUE: [number, number, number, number] = [0, 0, 255, 255]; @@ -396,6 +407,29 @@ export function registerHtmlInCanvasSpec() { equal(touched, 0, 'the layout position does not reach the button'); send(150, 70); equal(touched, 2, 'the drawn position reaches the button (down + up)'); + } else if (isLinux) { + // Allocation catches up on GTK's next layout pass. + await wait(100); + ok(gtkHits(canvas, button, 150, 70), 'the drawn position picks the button'); + ok(!gtkHits(canvas, button, 20, 20), 'the layout position does not'); + } + }); + + mountedTest('a source rect limits hit testing to the drawn part', async () => { + const button = new Button(); + button.text = 'hit'; + button.width = 60; + button.height = 40; + button.horizontalAlignment = 'left'; + (button as any).drawable = true; + const { canvas, ctx } = await mount([button]); + await snapshotted(canvas, button); + // The left half, at (120, 50): the right half would have been at 150..180. + ctx.drawElementImage(button, 0, 0, 30, 40, 120, 50, 30, 40); + if (isLinux) { + await wait(100); + ok(gtkHits(canvas, button, 135, 70), 'the drawn half picks the button'); + ok(!gtkHits(canvas, button, 165, 70), 'the half left out does not'); } }); @@ -415,6 +449,9 @@ export function registerHtmlInCanvasSpec() { if (__APPLE__) { const hit = (canvas.nativeView as UIView).hitTestWithEvent(CGPointMake(20, 20), null); ok(hit !== button.nativeView && !hit?.isDescendantOfView?.(button.nativeView), 'no longer hits the button'); + } else if (isLinux) { + await wait(100); + ok(!gtkHits(canvas, button, 20, 20), 'no longer picks the button'); } }); }); diff --git a/tools/demo/canvas/spec/index.ts b/tools/demo/canvas/spec/index.ts index 82d836326..3e6bb4d19 100644 --- a/tools/demo/canvas/spec/index.ts +++ b/tools/demo/canvas/spec/index.ts @@ -6,7 +6,7 @@ * adb logcat -d | grep 'SPEC|' * * `--es suite ` narrows the run: a group (`2d`, `webgl`, `webgpu`, - * `imagebitmap`, `bitmaprenderer`, `lifecycle`, `offscreen` (with `offscreen.worker`), `htmlincanvas` (iOS, Android), `audio` and `media` on Windows and Linux) or a single suite (`2d.path2d`). + * `imagebitmap`, `bitmaprenderer`, `lifecycle`, `offscreen` (with `offscreen.worker`), `htmlincanvas` (iOS, Android, Linux), `audio` and `media` on Windows and Linux) or a single suite (`2d.path2d`). * `contextlost` (Windows) runs on its own: it removes the GPU device. */ @@ -43,7 +43,7 @@ export async function runSpecTests(group: SpecGroup = 'all') { registerLifecycleSpec(); registerOffscreenCanvasSpec(); registerOffscreenWorkerSpec(); - if (!(typeof __WINDOWS__ !== 'undefined' && __WINDOWS__) && !(typeof __LINUX__ !== 'undefined' && __LINUX__)) { + if (!(typeof __WINDOWS__ !== 'undefined' && __WINDOWS__)) { registerHtmlInCanvasSpec(); } if ((typeof __WINDOWS__ !== 'undefined' && __WINDOWS__) || (typeof __LINUX__ !== 'undefined' && __LINUX__)) {