From 57b9484c5a9918d64d10c8c8e0a01bf0c4494a36 Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Wed, 18 Nov 2026 07:30:02 -0400 Subject: [PATCH] perf(canvas): native hosts and GPU snapshots for drawable views The drawable children of a content="drawable" canvas are now held by a native host view, so the per-frame work, touch routing and capture no longer cross into JS: - Android: NSCDrawableHost (Kotlin) lays the children out with NativeScript's own CommonLayoutParams helpers, tracks invalidations from onDescendantInvalidated, and routes touches from a geometry table JS sets only when geometry changes. On API 29+ it records each changed view into a RenderNode and renders it with HardwareRenderer into an ImageReader, waiting on the image's fence on API 33+. 2D draws the HardwareBuffer through the existing EGLImage path video uses, and WebGPU imports it as a native texture. - iOS: NSCDrawableHost (Swift) does the hit testing and keeps the layer transforms in sync with layout. It renders into an IOSurface that 2D and WebGPU read as a Metal texture. A layer-tree signature taken every frame catches most changes without rendering; a quarter-size probe, checked less often while nothing changes, catches the rest. JS walks the NativeScript tree for drawable views and makes one call to the host per frame. drawElementImage no longer uploads the snapshot's bytes on every draw. WebGL still reads the snapshot back on the CPU. --- packages/canvas/Canvas/common.ts | 6 +- packages/canvas/Canvas/drawable/common.ts | 141 ++--- .../canvas/Canvas/drawable/index.android.ts | 325 ++++------ packages/canvas/Canvas/drawable/index.ios.ts | 294 +++------- .../CanvasRenderingContext2D/index.ts | 26 +- packages/canvas/WebGPU/GPUQueue.ts | 20 + .../nativescript/canvas/NSCDrawableHost.kt | 554 ++++++++++++++++++ .../CanvasNative.xcodeproj/project.pbxproj | 4 + .../CanvasNative/Source/NSCDrawableHost.swift | 410 +++++++++++++ tools/demo/canvas/spec/htmlincanvas.ts | 24 + 10 files changed, 1291 insertions(+), 513 deletions(-) create mode 100644 packages/canvas/src-native/canvas-android/canvas/src/main/java/org/nativescript/canvas/NSCDrawableHost.kt create mode 100644 packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCDrawableHost.swift diff --git a/packages/canvas/Canvas/common.ts b/packages/canvas/Canvas/common.ts index 340a9062..b526674f 100644 --- a/packages/canvas/Canvas/common.ts +++ b/packages/canvas/Canvas/common.ts @@ -1558,8 +1558,12 @@ export abstract class CanvasBase extends ContainerView implements ICanvasBase { if (!rec) { throw domError('InvalidStateError', `Failed to execute '${method}': The element has no snapshot yet; draw it from a paint event.`); } + const controller = this._drawables; return { - asset: rec.asset, + get asset() { + return controller.assetOf(view, rec); + }, + nativeSnapshot: () => controller.nativeSnapshot(view), width: rec.width, height: rec.height, defaultWidth: rec.defaultWidth, diff --git a/packages/canvas/Canvas/drawable/common.ts b/packages/canvas/Canvas/drawable/common.ts index 7956f95b..2df0bbaf 100644 --- a/packages/canvas/Canvas/drawable/common.ts +++ b/packages/canvas/Canvas/drawable/common.ts @@ -172,6 +172,10 @@ export function rectFrom(init: any): Rect | null { }; } +export function sameRect(a: Rect | null, b: Rect | null): boolean { + return a === b || (!!a && !!b && a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height); +} + export function rectContains(rect: Rect, x: number, y: number) { const left = Math.min(rect.x, rect.x + rect.width); const top = Math.min(rect.y, rect.y + rect.height); @@ -233,7 +237,10 @@ export class ElementImage { } export interface ElementSource { - asset: ImageAsset; + /** Built on first use where the snapshot lives on the GPU. */ + readonly asset: ImageAsset; + /** A retained native snapshot (close it), or null. */ + nativeSnapshot?: () => any; width: number; height: number; /** Grid units. */ @@ -264,9 +271,6 @@ export interface SnapshotRecord { [key: string]: any; } -export const HOT_FRAMES = 30; -export const COLD_INTERVAL = 10; - export abstract class DrawableController { protected readonly canvasRef: WeakRef; protected snapshots = new Map(); @@ -274,7 +278,11 @@ export abstract class DrawableController { /** Bottom to top. */ protected hitOrder: View[] = []; protected paintRequested = false; - protected hotUntil = 0; + private generation = 0; + /** What the native host last got for each view. */ + private sent = new Map(); + /** The view last raised above the others for hit testing. */ + private top: View | null = null; protected disposed = false; constructor(canvas: any) { @@ -289,15 +297,10 @@ export abstract class DrawableController { abstract detachChild(child: View): void; - /** False when the pixels did not change. */ - protected abstract capture(view: View, rec: SnapshotRecord, width: number, height: number, scaleX: number, scaleY: number, nested: View[]): boolean; - - protected abstract needsCapture(view: View, rec: SnapshotRecord, frame: number): boolean; - /** `source` is in DIP. */ abstract render(view: View, width: number, height: number, source?: Rect): ImageAsset | null; - protected abstract applyGeometry(view: View, geometry: ElementGeometry | null): void; + protected abstract applyGeometry(view: View, geometry: ElementGeometry | null, moveToTop?: boolean): void; protected releaseRecord(rec: SnapshotRecord) { rec.asset = null; @@ -311,6 +314,8 @@ export abstract class DrawableController { } this.snapshots.clear(); this.geometry.clear(); + this.sent.clear(); + this.top = null; this.hitOrder = []; } @@ -350,22 +355,6 @@ export abstract class DrawableController { return out; } - protected nestedDrawables(view: View): View[] { - const out: View[] = []; - const visit = (node: View) => { - node.eachChildView((child) => { - if (isDrawable(child)) { - out.push(child); - } else { - visit(child); - } - return true; - }); - }; - visit(view); - return out; - } - contains(view: View): boolean { const canvas = this.canvas; let node: any = view?.parent; @@ -391,13 +380,21 @@ export abstract class DrawableController { snapshot(view: View): SnapshotRecord | null { const rec = this.snapshots.get(view); - return rec?.asset ? rec : null; + return rec?.asset || rec?.native ? rec : null; } - markHot(frames = HOT_FRAMES) { - this.hotUntil = Math.max(this.hotUntil, DrawableController.frameNumber + frames); + /** The snapshot as an ImageAsset, for consumers with no GPU path. */ + assetOf(view: View, rec: SnapshotRecord): ImageAsset | null { + return rec.asset; } + /** A retained native snapshot the GPU paths can draw (close it when done), or null. */ + nativeSnapshot(view: View): any { + return null; + } + + markHot() {} + requestPaint() { this.paintRequested = true; DrawableController.activate(this); @@ -421,13 +418,14 @@ export abstract class DrawableController { geometry.clip = rectFrom(options.canvasClip); } const index = this.hitOrder.indexOf(view); - if (!(options?.preserveHitTestOrder && known && index > -1)) { + const moveToTop = !(options?.preserveHitTestOrder && known && index > -1); + if (moveToTop) { if (index > -1) { this.hitOrder.splice(index, 1); } this.hitOrder.push(view); } - this.applyGeometry(view, geometry); + this.applyGeometry(view, geometry, moveToTop); } clearGeometry(view: View) { @@ -450,6 +448,27 @@ export abstract class DrawableController { return mat4.multiply(mat4.scale(1 / scale.x, 1 / scale.y), geometry.transform); } + /** + * drawElementImage sets geometry on every draw, and most of it is unchanged: whether the host + * needs `m` (null: cleared) and `clip` for `view`, and whether it still needs raising. + */ + protected geometryUpdate(view: View, native: any, m: Mat4 | null, clip: Rect | null, moveToTop: boolean): { send: boolean; raise: boolean } { + const last = this.sent.get(view); + if (!m) { + this.sent.delete(view); + return { send: !!last, raise: false }; + } + const raise = moveToTop && this.top !== view; + if (!raise && last && last.native === native && mat4.equals(last.m, m) && sameRect(last.clip, clip)) { + return { send: false, raise: false }; + } + this.sent.set(view, { native, m, clip: clip ? { ...clip } : null }); + if (raise) { + this.top = view; + } + return { send: true, raise }; + } + protected syncGeometry() { for (const [view, geometry] of this.geometry) { this.applyGeometry(view, geometry); @@ -478,42 +497,7 @@ export abstract class DrawableController { this.clearGeometry(view); } } - const scale = this.gridScale(); - for (const view of list) { - const css = this.cssSize(view); - if (css.width <= 0 || css.height <= 0 || !view.nativeViewProtected) { - continue; - } - let rec = this.snapshots.get(view); - if (!rec) { - rec = { asset: null, width: 0, height: 0, cssWidth: 0, cssHeight: 0, defaultWidth: 0, defaultHeight: 0 }; - this.snapshots.set(view, rec); - } - const width = Math.max(1, Math.ceil(css.width * scale.x - 1e-3)); - const height = Math.max(1, Math.ceil(css.height * scale.y - 1e-3)); - const resized = !rec.asset || rec.width !== width || rec.height !== height || rec.scaleX !== scale.x || rec.scaleY !== scale.y; - if (!resized && frameNumber > this.hotUntil && !this.needsCapture(view, rec, frameNumber)) { - continue; - } - let didChange = false; - try { - didChange = this.capture(view, rec, width, height, scale.x, scale.y, this.nestedDrawables(view)); - } catch (e) { - console.error('Canvas: could not snapshot a drawable element', e); - } - if (didChange || resized) { - rec.cssWidth = css.width; - rec.cssHeight = css.height; - rec.defaultWidth = css.width * scale.x; - rec.defaultHeight = css.height * scale.y; - rec.scaleX = scale.x; - rec.scaleY = scale.y; - rec.hotUntil = frameNumber + HOT_FRAMES; - if (rec.asset) { - changed.push(view); - } - } - } + changed.push(...this.refresh(list)); this.syncGeometry(); } if (changed.length || this.paintRequested) { @@ -525,6 +509,27 @@ export abstract class DrawableController { } } + /** Brings snapshots up to date; returns the views that got a new one. */ + protected abstract refresh(list: View[]): View[]; + + /** Notes that `view` has a new native snapshot, taken at `scale`. */ + protected recordSnapshot(view: View, scale: { x: number; y: number }) { + const css = this.cssSize(view); + let rec = this.snapshots.get(view); + if (!rec) { + rec = { asset: null, width: 0, height: 0, cssWidth: 0, cssHeight: 0, defaultWidth: 0, defaultHeight: 0 }; + this.snapshots.set(view, rec); + } + rec.native = true; + rec.width = Math.max(1, Math.ceil(css.width * scale.x - 1e-3)); + rec.height = Math.max(1, Math.ceil(css.height * scale.y - 1e-3)); + rec.cssWidth = css.width; + rec.cssHeight = css.height; + rec.defaultWidth = css.width * scale.x; + rec.defaultHeight = css.height * scale.y; + rec.generation = ++this.generation; + } + private static active = new Set(); private static scheduled = false; static frameNumber = 0; diff --git a/packages/canvas/Canvas/drawable/index.android.ts b/packages/canvas/Canvas/drawable/index.android.ts index 09aba6ad..a3d450b6 100644 --- a/packages/canvas/Canvas/drawable/index.android.ts +++ b/packages/canvas/Canvas/drawable/index.android.ts @@ -1,82 +1,22 @@ import { View, Screen } from '@nativescript/core'; import { ImageAsset } from '../../ImageAsset'; -import { DrawableController, ElementGeometry, Mat4, mat4, Rect, rectContains, SnapshotRecord, COLD_INTERVAL } from './common'; +import { DrawableController, ElementGeometry, mat4, Rect, SnapshotRecord } from './common'; export * from './common'; -const SDK = android.os.Build.VERSION.SDK_INT; -// Below it drawables are polled. -const HAS_INVALIDATION_HOOK = SDK >= 26; - -@NativeClass() -class NSCDrawableHost extends org.nativescript.widgets.StackLayout { - owner: WeakRef; - - constructor(context: android.content.Context) { - super(context); - return global.__native(this); - } - - dispatchDraw(canvas: android.graphics.Canvas) {} - - // Reports only a size the canvas already has, so children never resize it. - onMeasure(widthMeasureSpec: number, heightMeasureSpec: number) { - const MeasureSpec = android.view.View.MeasureSpec; - const parent = this.getParent() as android.view.ViewGroup; - let siblingWidth = 0; - let siblingHeight = 0; - if (parent instanceof android.view.ViewGroup) { - for (let i = 0; i < parent.getChildCount(); i++) { - const child = parent.getChildAt(i); - if (child !== this && child.getVisibility() !== android.view.View.GONE) { - siblingWidth = Math.max(siblingWidth, child.getMeasuredWidth()); - siblingHeight = Math.max(siblingHeight, child.getMeasuredHeight()); - } - } - } - const width = MeasureSpec.getMode(widthMeasureSpec) === MeasureSpec.EXACTLY ? MeasureSpec.getSize(widthMeasureSpec) : siblingWidth; - const height = MeasureSpec.getMode(heightMeasureSpec) === MeasureSpec.EXACTLY ? MeasureSpec.getSize(heightMeasureSpec) : siblingHeight; - super.onMeasure(MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY), MeasureSpec.makeMeasureSpec(0, MeasureSpec.UNSPECIFIED)); - this.setMeasuredDimension(width, height); - } - - onDescendantInvalidated(child: android.view.View, target: android.view.View) { - super.onDescendantInvalidated(child, target); - this.owner?.deref()?.invalidated(target); - } - - dispatchTouchEvent(event: android.view.MotionEvent): boolean { - return this.owner?.deref()?.dispatchTouch(event) ?? false; - } -} - -function toAndroidMatrix(m: Mat4): android.graphics.Matrix { - const matrix = new android.graphics.Matrix(); - // Drops z: row-major 3x3 of x, y and w. - matrix.setValues([m[0], m[4], m[12], m[1], m[5], m[13], m[3], m[7], m[15]]); - return matrix; -} - -interface TouchTarget { - native: android.view.View; - inverse: android.graphics.Matrix; -} +declare const org: any; +/** Layout, invalidation, touch routing and capture live in `NSCDrawableHost` (Kotlin). */ class AndroidDrawableController extends DrawableController { - readonly host: NSCDrawableHost; - private capturing = false; - private touchTarget: TouchTarget | null = null; - private byNative = new Map(); + readonly host: any; + private list: View[] = []; + private scaleX = 0; + private scaleY = 0; constructor(canvas: any) { super(canvas); const native = canvas.nativeViewProtected as android.widget.FrameLayout; - this.host = new NSCDrawableHost(native.getContext()); - this.host.owner = new WeakRef(this); - this.host.setOrientation(org.nativescript.widgets.Orientation.vertical); - // Above the surfaces, for touch order. - this.host.setTranslationZ(1e4); - this.host.setClipChildren(false); + this.host = new org.nativescript.canvas.NSCDrawableHost(native.getContext()); native.addView(this.host, new android.widget.FrameLayout.LayoutParams(android.view.ViewGroup.LayoutParams.MATCH_PARENT, android.view.ViewGroup.LayoutParams.MATCH_PARENT)); } @@ -97,203 +37,144 @@ class AndroidDrawableController extends DrawableController { } } - drawables(): View[] { - const list = super.drawables(); - this.byNative.clear(); - for (const view of list) { - if (view.nativeViewProtected) { - this.byNative.set(view.nativeViewProtected, view); - } - } - return list; - } - - invalidated(target: android.view.View) { - if (this.capturing) { - return; - } - let node: any = target; - while (node && node !== this.host) { - const view = this.byNative.get(node); - if (view) { - const rec = this.snapshots.get(view); - if (rec) { - rec.dirty = true; - } - return; - } - node = node.getParent?.(); - } - // New since the last frame. - for (const rec of this.snapshots.values()) { - rec.dirty = true; - } - } - private density() { return Screen.mainScreen.scale || 1; } - private pixelTransform(geometry: ElementGeometry): Mat4 | null { - const view = this.viewTransform(geometry); - if (!view) { - return null; + cssSize(view: View): { width: number; height: number } { + const native = view.nativeViewProtected as android.view.View; + if (!native) { + return { width: 0, height: 0 }; } const d = this.density(); - return mat4.chain(mat4.scale(d, d), view, mat4.scale(1 / d, 1 / d)); + return { width: native.getWidth() / d, height: native.getHeight() / d }; } - private findTouchTarget(x: number, y: number): TouchTarget | null { + protected refresh(list: View[]): View[] { + const live = list.filter((view) => !!view.nativeViewProtected); + if (live.length !== this.list.length || live.some((view, i) => view !== this.list[i])) { + this.list = live; + const array = Array.create(android.view.View, live.length); + live.forEach((view, i) => (array[i] = view.nativeViewProtected)); + this.host.setDrawables(array); + } + const scale = this.gridScale(); const d = this.density(); - for (let i = this.hitOrder.length - 1; i >= 0; i--) { - const view = this.hitOrder[i]; - const geometry = this.geometry.get(view); - const native = view.nativeViewProtected as android.view.View; - if (!geometry?.transform || !native || native.getVisibility() !== android.view.View.VISIBLE) { + const indices = this.host.update(scale.x / d, scale.y / d); + if (!indices) { + return []; + } + const changed: View[] = []; + for (let i = 0; i < indices.length; i++) { + const view = this.list[indices[i]]; + if (!view) { continue; } - const forward = this.pixelTransform(geometry); - const inverse = forward ? mat4.invert(forward) : null; - if (!inverse) { - continue; - } - const local = mat4.apply(inverse, x, y); - if (local.x < 0 || local.y < 0 || local.x >= native.getWidth() || local.y >= native.getHeight()) { - continue; - } - if (geometry.clip && !rectContains(geometry.clip, local.x / d, local.y / d)) { - continue; - } - return { native, inverse: toAndroidMatrix(inverse) }; + this.recordSnapshot(view, scale); + changed.push(view); } - return null; + return changed; } - // The canvas still gets every event, as if it bubbled. - dispatchTouch(event: android.view.MotionEvent): boolean { - const MotionEvent = android.view.MotionEvent; - const action = event.getActionMasked(); - if (action === MotionEvent.ACTION_DOWN) { - this.touchTarget = this.findTouchTarget(event.getX(), event.getY()); - } - const target = this.touchTarget; - if (!target) { - return false; - } - const copy = MotionEvent.obtain(event); - copy.transform(target.inverse); - let handled = false; - try { - handled = target.native.dispatchTouchEvent(copy); - } finally { - copy.recycle(); - } - if (action === MotionEvent.ACTION_UP || action === MotionEvent.ACTION_CANCEL) { - this.touchTarget = null; - } - if (action === MotionEvent.ACTION_DOWN && !handled) { - this.touchTarget = null; - return false; - } - const canvas = this.canvas?.nativeViewProtected as android.view.View; - canvas?.onTouchEvent(event); - return true; + nativeSnapshot(view: View): any { + const native = view.nativeViewProtected; + return native ? this.host.snapshot(native) : null; } - // dispatchTouch reads the geometry; the views keep their layout position. - protected applyGeometry(view: View, geometry: ElementGeometry | null) {} - - protected syncGeometry() {} - - protected needsCapture(view: View, rec: SnapshotRecord, frame: number): boolean { - if (rec.dirty) { - return true; + assetOf(view: View, rec: SnapshotRecord): ImageAsset | null { + if (rec.asset && rec.assetGeneration === rec.generation) { + return rec.asset; } - if (HAS_INVALIDATION_HOOK) { - return false; + const snapshot = this.nativeSnapshot(view); + if (!snapshot) { + return rec.asset; } - rec.phase ??= Math.floor(Math.random() * COLD_INTERVAL); - return (frame + rec.phase) % COLD_INTERVAL === 0; - } - - private draw(native: android.view.View, canvas: android.graphics.Canvas, scaleX: number, scaleY: number, source: Rect | null, nested: View[]) { - const d = this.density(); - const hide = nested.map((v) => v.nativeViewProtected as android.view.View).filter((v) => v && v.getVisibility() === android.view.View.VISIBLE); - const save = canvas.save(); - this.capturing = true; try { - for (const v of hide) { - v.setVisibility(android.view.View.INVISIBLE); + const bitmap = snapshot.toBitmap(); + if (bitmap) { + rec.asset ??= new ImageAsset(); + rec.asset.loadFromNativeSync(bitmap); + rec.assetGeneration = rec.generation; } - canvas.scale(scaleX / d, scaleY / d); - canvas.translate(-native.getScrollX() - (source ? source.x * d : 0), -native.getScrollY() - (source ? source.y * d : 0)); - native.draw(canvas); } finally { - for (const v of hide) { - v.setVisibility(android.view.View.VISIBLE); - } - this.capturing = false; - canvas.restoreToCount(save); - } - } - - protected capture(view: View, rec: SnapshotRecord, width: number, height: number, scaleX: number, scaleY: number, nested: View[]): boolean { - const native = view.nativeViewProtected as android.view.View; - if (!native || native.getWidth() <= 0 || native.getHeight() <= 0) { - return false; - } - let bitmap = rec.bitmap as android.graphics.Bitmap; - if (!bitmap || bitmap.getWidth() !== width || bitmap.getHeight() !== height) { - bitmap?.recycle(); - bitmap = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888); - rec.bitmap = bitmap; - rec.canvas = new android.graphics.Canvas(bitmap); - } else { - bitmap.eraseColor(0); + snapshot.close(); } - this.draw(native, rec.canvas, scaleX, scaleY, null, nested); - if (!rec.asset) { - rec.asset = new ImageAsset(); - } - rec.asset.loadFromNativeSync(bitmap); - rec.width = width; - rec.height = height; - rec.dirty = false; - return true; + return rec.asset; } render(view: View, width: number, height: number, source?: Rect): ImageAsset | null { const native = view.nativeViewProtected as android.view.View; width = Math.max(1, Math.round(width)); height = Math.max(1, Math.round(height)); - if (!native || native.getWidth() <= 0 || native.getHeight() <= 0) { + if (!native) { return null; } + const d = this.density(); const css = this.cssSize(view); const rect = source ?? { x: 0, y: 0, width: css.width, height: css.height }; if (rect.width <= 0 || rect.height <= 0) { return null; } - const bitmap = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888); - this.draw(native, new android.graphics.Canvas(bitmap), width / rect.width, height / rect.height, source ?? null, this.nestedDrawables(view)); - const asset = new ImageAsset(); - asset.loadFromNativeSync(bitmap); - bitmap.recycle(); - return asset; + const snapshot = this.host.capture(native, width, height, width / (rect.width * d), height / (rect.height * d), rect.x * d, rect.y * d); + if (!snapshot) { + return null; + } + try { + const asset = new ImageAsset(); + asset.loadFromNativeSync(snapshot.toBitmap()); + return asset; + } finally { + snapshot.close(); + } + } + + protected applyGeometry(view: View, geometry: ElementGeometry | null, moveToTop = false) { + const native = view.nativeViewProtected; + if (!native) { + return; + } + const forward = geometry ? this.viewTransform(geometry) : null; + const clip = geometry?.clip ?? null; + const { send, raise } = this.geometryUpdate(view, native, forward, clip, moveToTop); + if (!send) { + return; + } + if (!forward) { + this.host.clearGeometry(native); + return; + } + const d = this.density(); + // Element pixels to host pixels. + const m = mat4.chain(mat4.scale(d, d), forward, mat4.scale(1 / d, 1 / d)); + const left = clip ? Math.min(clip.x, clip.x + clip.width) * d : 0; + const top = clip ? Math.min(clip.y, clip.y + clip.height) * d : 0; + const right = clip ? left + Math.abs(clip.width) * d : 0; + const bottom = clip ? top + Math.abs(clip.height) * d : 0; + this.host.setGeometry(native, m[0], m[4], m[12], m[1], m[5], m[13], m[3], m[7], m[15], !!clip, left, top, right, bottom, !raise); + } + + /** Geometry is in grid units: re-sent only when the grid's scale changes. */ + protected syncGeometry() { + const scale = this.gridScale(); + if (scale.x === this.scaleX && scale.y === this.scaleY) { + return; + } + this.scaleX = scale.x; + this.scaleY = scale.y; + for (const [view, geometry] of this.geometry) { + this.applyGeometry(view, geometry, false); + } } protected releaseRecord(rec: SnapshotRecord) { super.releaseRecord(rec); - (rec.bitmap as android.graphics.Bitmap)?.recycle(); - rec.bitmap = rec.canvas = null; + rec.native = null; } dispose() { super.dispose(); - this.touchTarget = null; - this.byNative.clear(); - const parent = this.host.getParent() as android.view.ViewGroup; - parent?.removeView(this.host); + this.host.dispose(); + (this.host.getParent() as android.view.ViewGroup)?.removeView(this.host); } } diff --git a/packages/canvas/Canvas/drawable/index.ios.ts b/packages/canvas/Canvas/drawable/index.ios.ts index 3701c123..e1397564 100644 --- a/packages/canvas/Canvas/drawable/index.ios.ts +++ b/packages/canvas/Canvas/drawable/index.ios.ts @@ -1,125 +1,21 @@ import { View } from '@nativescript/core'; import { ImageAsset } from '../../ImageAsset'; -import { DrawableController, ElementGeometry, Mat4, mat4, Rect, rectContains, SnapshotRecord, COLD_INTERVAL } from './common'; +import { DrawableController, ElementGeometry, Rect, SnapshotRecord } from './common'; export * from './common'; -// UIKit does not hit test views at alpha 0, hence the override. -@NativeClass() -class NSCDrawableHost extends UIView { - _owner: WeakRef; - - static initWithOwner(owner: IOSDrawableController): NSCDrawableHost { - const host = NSCDrawableHost.alloc().initWithFrame(CGRectZero) as NSCDrawableHost; - host._owner = new WeakRef(owner); - host.alpha = 0; - host.clipsToBounds = false; - host.autoresizingMask = UIViewAutoresizing.FlexibleWidth | UIViewAutoresizing.FlexibleHeight; - return host; - } - - hitTestWithEvent(point: CGPoint, event: _UIEvent): UIView { - return this._owner?.deref()?.hitTest(this, point, event) ?? null; - } - - pointInsideWithEvent(point: CGPoint, event: _UIEvent): boolean { - return !!this.hitTestWithEvent(point, event); - } -} - -const BITMAP_INFO = CGImageAlphaInfo.kCGImageAlphaPremultipliedLast | CGBitmapInfo.kCGBitmapByteOrder32Big; - -let colorSpace: any; -function deviceRGB() { - if (!colorSpace) { - colorSpace = CGColorSpaceCreateDeviceRGB(); - } - return colorSpace; -} - -interface Buffer { - data: NSMutableData; - context: any; - bytes: Uint8Array; -} - -function makeBuffer(width: number, height: number): Buffer { - const data = NSMutableData.dataWithLength(width * height * 4); - const context = CGBitmapContextCreate(data.mutableBytes, width, height, 8, width * 4, deviceRGB(), BITMAP_INFO); - return { data, context, bytes: new Uint8Array(interop.bufferFromData(data)) }; -} - -function toCATransform(m: Mat4): CATransform3D { - const t = new CATransform3D(CATransform3DIdentity); - t.m11 = m[0]; - t.m12 = m[1]; - t.m13 = m[2]; - t.m14 = m[3]; - t.m21 = m[4]; - t.m22 = m[5]; - t.m23 = m[6]; - t.m24 = m[7]; - t.m31 = m[8]; - t.m32 = m[9]; - t.m33 = m[10]; - t.m34 = m[11]; - t.m41 = m[12]; - t.m42 = m[13]; - t.m43 = m[14]; - t.m44 = m[15]; - return t; -} - -/** Assumes nothing in between is 3D. */ -function chainToHost(superview: UIView, host: UIView): Mat4 { - const o = superview.convertPointToView(CGPointMake(0, 0), host); - const x = superview.convertPointToView(CGPointMake(1, 0), host); - const y = superview.convertPointToView(CGPointMake(0, 1), host); - return mat4.from({ a: x.x - o.x, b: x.y - o.y, c: y.x - o.x, d: y.y - o.y, e: o.x, f: o.y }); -} - -function renderLayer(nativeView: UIView, context: any, width: number, height: number, scaleX: number, scaleY: number, source: Rect | null, hide: UIView[]) { - CGContextClearRect(context, CGRectMake(0, 0, width, height)); - CGContextSaveGState(context); - CGContextTranslateCTM(context, 0, height); - CGContextScaleCTM(context, scaleX, -scaleY); - const origin = nativeView.layer.bounds.origin; - CGContextTranslateCTM(context, -origin.x - (source?.x ?? 0), -origin.y - (source?.y ?? 0)); - let layer: CALayer = nativeView.layer; - if (hide.length) { - CATransaction.begin(); - CATransaction.setDisableActions(true); - for (const view of hide) { - view.layer.hidden = true; - } - } else { - // Catches in-flight animations. - layer = nativeView.layer.presentationLayer() ?? nativeView.layer; - } - try { - layer.renderInContext(context); - } finally { - if (hide.length) { - for (const view of hide) { - view.layer.hidden = false; - } - CATransaction.commit(); - } - CGContextRestoreGState(context); - } -} +declare const NSCDrawableHost: any; +/** Hit testing, layer transforms, change detection and capture live in `NSCDrawableHost` (Swift). */ class IOSDrawableController extends DrawableController { - readonly host: NSCDrawableHost; + readonly host: any; + private list: View[] = []; constructor(canvas: any) { super(canvas); - this.host = NSCDrawableHost.initWithOwner(this); const native = canvas.nativeViewProtected as UIView; - if (native) { - native.addSubview(this.host); - this.host.frame = native.bounds; - } + this.host = NSCDrawableHost.alloc().initWithFrame(native.bounds); + native.addSubview(this.host); } // Core's size reads the frame, which follows the hit-testing transform. @@ -155,119 +51,54 @@ class IOSDrawableController extends DrawableController { layoutHost() { const canvas = this.canvas?.nativeViewProtected as UIView; - if (!canvas) { - return; + if (canvas) { + this.host.frame = canvas.bounds; } - if (this.host.superview !== canvas) { - canvas.addSubview(this.host); - } else if (canvas.subviews.lastObject !== this.host) { - canvas.bringSubviewToFront(this.host); - } - this.host.frame = canvas.bounds; } - hitTest(host: UIView, point: CGPoint, event: _UIEvent): UIView | null { - this.markHot(); - for (let i = this.hitOrder.length - 1; i >= 0; i--) { - const view = this.hitOrder[i]; - const geometry = this.geometry.get(view); - const native = view.nativeViewProtected as UIView; - if (!geometry?.transform || !native || native.hidden || !native.userInteractionEnabled) { - continue; - } - const local = host.convertPointToView(point, native); - const origin = native.bounds.origin; - if (geometry.clip && !rectContains(geometry.clip, local.x - origin.x, local.y - origin.y)) { - continue; - } - const hit = native.hitTestWithEvent(local, event); - if (hit) { - return hit; - } - } - return null; + markHot() { + this.host.markHot(); } - protected applyGeometry(view: View, geometry: ElementGeometry | null) { - const native = view.nativeViewProtected as UIView; - if (!native) { - return; - } - const target = geometry ? this.viewTransform(geometry) : null; - if (!target) { - if ((view as any)._drawableTransform) { - (view as any)._drawableTransform = null; - native.layer.transform = CATransform3DIdentity; - (view as any)._isTransformed = false; - (view as any).updateNativeTransform?.(); - } - return; - } - const superview = native.superview; - if (!superview) { - return; + protected refresh(list: View[]): View[] { + const live = list.filter((view) => !!view.nativeViewProtected); + if (live.length !== this.list.length || live.some((view, i) => view !== this.list[i])) { + this.list = live; + this.host.setDrawables(live.map((view) => view.nativeViewProtected)); } - const toHost = superview === this.host ? null : chainToHost(superview, this.host); - const fromHost = toHost ? mat4.invert(toHost) : null; - if (toHost && !fromHost) { - return; + const scale = this.gridScale(); + const indices = this.host.update(scale.x, scale.y); + if (!indices) { + return []; } - // p -> position + M * (p - anchor) - const size = native.bounds.size; - const anchor = native.layer.anchorPoint; - const position = native.layer.position; - const parts = [mat4.translate(-position.x, -position.y)]; - if (fromHost) { - parts.push(fromHost); - } - parts.push(target, mat4.translate(anchor.x * size.width, anchor.y * size.height)); - const m = mat4.chain(...parts); - if (mat4.equals(m, (view as any)._drawableTransform)) { - return; + const changed: View[] = []; + for (let i = 0; i < indices.count; i++) { + const view = this.list[indices.objectAtIndex(i)]; + if (!view) { + continue; + } + this.recordSnapshot(view, scale); + changed.push(view); } - (view as any)._drawableTransform = m; - // Core then resets the transform around its frame updates. - (view as any)._isTransformed = true; - CATransaction.begin(); - CATransaction.setDisableActions(true); - native.layer.transform = toCATransform(m); - CATransaction.commit(); + return changed; } - protected needsCapture(view: View, rec: SnapshotRecord, frame: number): boolean { - if (frame <= (rec.hotUntil ?? 0)) { - return true; - } - // No redraw signal on iOS: idle drawables are polled. - rec.phase ??= Math.floor(Math.random() * COLD_INTERVAL); - return (frame + rec.phase) % COLD_INTERVAL === 0; + nativeSnapshot(view: View): any { + const native = view.nativeViewProtected; + return native ? this.host.snapshot(native) : null; } - protected capture(view: View, rec: SnapshotRecord, width: number, height: number, scaleX: number, scaleY: number, nested: View[]): boolean { - const native = view.nativeViewProtected as UIView; - if (!native) { - return false; + assetOf(view: View, rec: SnapshotRecord): ImageAsset | null { + if (rec.asset && rec.assetGeneration === rec.generation) { + return rec.asset; } - const resized = rec.width !== width || rec.height !== height || !rec.front; - if (resized) { - rec.front = makeBuffer(width, height); - rec.back = makeBuffer(width, height); + const image = this.nativeSnapshot(view)?.image(); + if (image) { + rec.asset ??= new ImageAsset(); + rec.asset.loadFromNativeSync(image); + rec.assetGeneration = rec.generation; } - const back: Buffer = rec.back; - const hide = nested.map((v) => v.nativeViewProtected as UIView).filter((v) => !!v); - renderLayer(native, back.context, width, height, scaleX, scaleY, null, hide); - if (!resized && rec.asset && back.data.isEqualToData(rec.front.data)) { - return false; - } - rec.back = rec.front; - rec.front = back; - if (!rec.asset) { - rec.asset = new ImageAsset(); - } - rec.asset.loadFromBytesSync(width, height, back.bytes, true); - rec.width = width; - rec.height = height; - return true; + return rec.asset; } render(view: View, width: number, height: number, source?: Rect): ImageAsset | null { @@ -282,27 +113,48 @@ class IOSDrawableController extends DrawableController { if (rect.width <= 0 || rect.height <= 0) { return null; } - const buffer = makeBuffer(width, height); - const hide = this.nestedDrawables(view) - .map((v) => v.nativeViewProtected as UIView) - .filter((v) => !!v); - renderLayer(native, buffer.context, width, height, width / rect.width, height / rect.height, source ?? null, hide); + const image = this.host.capture(native, width, height, width / rect.width, height / rect.height, rect.x, rect.y)?.image(); + if (!image) { + return null; + } const asset = new ImageAsset(); - asset.loadFromBytesSync(width, height, buffer.bytes, true); + asset.loadFromNativeSync(image); return asset; } + protected applyGeometry(view: View, geometry: ElementGeometry | null, moveToTop = false) { + const native = view.nativeViewProtected; + if (!native) { + return; + } + const m = geometry ? this.viewTransform(geometry) : null; + const clip = geometry?.clip ?? null; + const { send, raise } = this.geometryUpdate(view, native, m, clip, moveToTop); + if (!send) { + return; + } + if (!m) { + this.host.clearGeometry(native); + (view as any)._isTransformed = false; + (view as any).updateNativeTransform?.(); + return; + } + // Core then resets the transform around its frame updates. + (view as any)._isTransformed = true; + this.host.setGeometry(native, Array.from(m), !!clip, clip?.x ?? 0, clip?.y ?? 0, clip?.width ?? 0, clip?.height ?? 0, !raise); + } + + /** The host re-syncs transforms with layout every frame. */ + protected syncGeometry() {} + protected releaseRecord(rec: SnapshotRecord) { super.releaseRecord(rec); - rec.front = rec.back = null; + rec.native = null; } dispose() { - for (const view of Array.from(this.geometry.keys())) { - this.applyGeometry(view, null); - } super.dispose(); - this.host.removeFromSuperview(); + this.host.dispose(); } } diff --git a/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts b/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts index b93597fc..ca255167 100644 --- a/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts +++ b/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts @@ -1014,7 +1014,31 @@ export class CanvasRenderingContext2D implements CanvasRenderingContext { if (![sx, sy, sw, sh, dx, dy, dw, dh].every(Number.isFinite) || !sw || !sh || !dw || !dh) { return; } - this.context.drawImage(source.asset.native, sx, sy, sw, sh, dx, dy, dw, dh); + let drawn = false; + if (__ANDROID__) { + const snapshot = source.nativeSnapshot?.(); + if (snapshot) { + try { + drawn = (org).nativescript.canvas.NSCDrawableHost.draw2D(long(this.context.__getPointer()), snapshot, sx, sy, sw, sh, dx, dy, dw, dh); + } finally { + snapshot.close(); + } + if (drawn) { + this.context.__makeDirty(); + } + } + } else if (__APPLE__) { + const snapshot = source.nativeSnapshot?.(); + if (snapshot) { + drawn = snapshot.draw2D(this.context.__getPointer(), sx, sy, sw, sh, dx, dy, dw, dh); + if (drawn) { + this.context.__makeDirty(); + } + } + } + if (!drawn) { + this.context.drawImage(source.asset.native, sx, sy, sw, sh, dx, dy, dw, dh); + } if (options?.preserveElementGeometry) { return; diff --git a/packages/canvas/WebGPU/GPUQueue.ts b/packages/canvas/WebGPU/GPUQueue.ts index 0f529dc0..7235bf11 100644 --- a/packages/canvas/WebGPU/GPUQueue.ts +++ b/packages/canvas/WebGPU/GPUQueue.ts @@ -81,6 +81,20 @@ export class GPUQueue { width = size.width; height = size.height ?? 1; } + const whole = !crop && (width === undefined || width === resolved.width) && (height === undefined || height === resolved.height); + const renderable = ((destination?.texture as any)?.usage ?? 0) & GPUTextureUsage.RENDER_ATTACHMENT; + const snapshot = whole && renderable ? resolved.nativeSnapshot?.() : null; + if (snapshot) { + try { + const pointer = __ANDROID__ ? snapshot.getTexturePointer() : snapshot.texturePointer; + if (pointer && String(pointer) !== '0') { + this.copyExternalImageToTexture({ nativeTexture: pointer, width: resolved.width, height: resolved.height } as never, target, { width: resolved.width, height: resolved.height, depthOrArrayLayers: 1 }); + return; + } + } finally { + snapshot.close?.(); + } + } const image = elementAsset(resolved, 'drawElementImageToTexture', crop, width, height); this.copyExternalImageToTexture({ source: image.asset as never }, target, { width: image.width, height: image.height, depthOrArrayLayers: 1 }); } @@ -134,6 +148,12 @@ export class GPUQueue { } } + if ((source as any).nativeTexture) { + (src as any).nativeTexture = (source as any).nativeTexture; + (src as any).width = (source as any).width; + (src as any).height = (source as any).height; + } + if (typeof source.flipY === 'boolean') { src.flipY = source.flipY; } diff --git a/packages/canvas/src-native/canvas-android/canvas/src/main/java/org/nativescript/canvas/NSCDrawableHost.kt b/packages/canvas/src-native/canvas-android/canvas/src/main/java/org/nativescript/canvas/NSCDrawableHost.kt new file mode 100644 index 00000000..ac9f5198 --- /dev/null +++ b/packages/canvas/src-native/canvas-android/canvas/src/main/java/org/nativescript/canvas/NSCDrawableHost.kt @@ -0,0 +1,554 @@ +package org.nativescript.canvas + +import android.annotation.SuppressLint +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.HardwareRenderer +import android.graphics.Matrix +import android.graphics.PixelFormat +import android.graphics.RectF +import android.graphics.RenderNode +import android.hardware.HardwareBuffer +import android.media.Image +import android.media.ImageReader +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.view.MotionEvent +import android.view.View +import android.view.ViewGroup +import android.widget.FrameLayout +import androidx.annotation.RequiresApi +import java.lang.reflect.Method + +/** + * Holds a `content="drawable"` canvas's children: laid out in the window, never drawn. Snapshots + * them when they change and routes touches to wherever the page last drew them. + */ +@SuppressLint("ViewConstructor") +class NSCDrawableHost(context: Context) : ViewGroup(context) { + + /** A captured frame of one view. Drawn by reference: each user closes it once. */ + class Snapshot internal constructor( + val width: Int, + val height: Int, + private val image: Image?, + bitmap: Bitmap?, + private val onRelease: (() -> Unit)?, + ) : AutoCloseable { + val hardwareBuffer: HardwareBuffer? = + if (image != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) image.hardwareBuffer else null + private var bitmap: Bitmap? = bitmap + private var refs = 1 + + @Synchronized + fun retain(): Snapshot { + refs++ + return this + } + + @Synchronized + override fun close() { + if (refs <= 0) return + refs-- + if (refs == 0) { + hardwareBuffer?.close() + image?.close() + bitmap?.recycle() + bitmap = null + onRelease?.invoke() + } + } + + /** `AHardwareBuffer *`, or 0. Valid while the snapshot is. */ + val texturePointer: Long + get() = Utils.hardwareBufferPointer(hardwareBuffer) + + /** A software copy, for consumers that take no hardware buffer. */ + @Synchronized + fun toBitmap(): Bitmap? { + bitmap?.let { if (!it.isRecycled) return it } + val buffer = hardwareBuffer ?: return null + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) return null + val wrapped = Bitmap.wrapHardwareBuffer(buffer, null) ?: return null + val copy = wrapped.copy(Bitmap.Config.ARGB_8888, false) + wrapped.recycle() + bitmap = copy + return copy + } + } + + private class Entry(val view: View) { + @Volatile + var dirty = true + var forward: Matrix? = null + var inverse: Matrix? = null + var clip: RectF? = null + var snapshot: Snapshot? = null + var pending: Snapshot? = null + var gpu: Any? = null + var width = 0 + var height = 0 + /** Next poll (uptime ms) below API 26, staggered across entries. */ + var nextPoll = 0L + } + + private val entries = LinkedHashMap() + /** Bottom to top. */ + private val hitOrder = ArrayList() + private val handler = Handler(Looper.getMainLooper()) + private var capturing = false + private var touchTarget: Entry? = null + private var touchInverse: Matrix? = null + + init { + setWillNotDraw(true) + clipChildren = false + // Above the canvas's surfaces, which it adds and swaps later, for touch order. + translationZ = 10000f + } + + + override fun dispatchDraw(canvas: Canvas) {} + + override fun generateDefaultLayoutParams(): LayoutParams = newLayoutParams() + + override fun generateLayoutParams(p: LayoutParams?): LayoutParams = + if (p is FrameLayout.LayoutParams) p else newLayoutParams() + + override fun checkLayoutParams(p: LayoutParams?): Boolean = p is FrameLayout.LayoutParams + + override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) { + var siblingWidth = 0 + var siblingHeight = 0 + (parent as? ViewGroup)?.let { group -> + for (i in 0 until group.childCount) { + val child = group.getChildAt(i) + if (child !== this && child.visibility != GONE) { + siblingWidth = maxOf(siblingWidth, child.measuredWidth) + siblingHeight = maxOf(siblingHeight, child.measuredHeight) + } + } + } + val width = if (MeasureSpec.getMode(widthMeasureSpec) == MeasureSpec.EXACTLY) MeasureSpec.getSize(widthMeasureSpec) else siblingWidth + val height = if (MeasureSpec.getMode(heightMeasureSpec) == MeasureSpec.EXACTLY) MeasureSpec.getSize(heightMeasureSpec) else siblingHeight + val childWidth = MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY) + val childHeight = MeasureSpec.makeMeasureSpec(0, MeasureSpec.UNSPECIFIED) + for (i in 0 until childCount) { + val child = getChildAt(i) + if (child.visibility == GONE) continue + if (!NsLayout.measure(child, childWidth, childHeight)) { + measureChildWithMargins(child, childWidth, 0, childHeight, 0) + } + } + // Only a size the canvas already has, so the children never resize it. + setMeasuredDimension(width, height) + } + + override fun onLayout(changed: Boolean, l: Int, t: Int, r: Int, b: Int) { + val width = r - l + var y = 0 + for (i in 0 until childCount) { + val child = getChildAt(i) + if (child.visibility == GONE) continue + val height = NsLayout.desiredHeight(child) ?: run { + val lp = child.layoutParams as? MarginLayoutParams + child.measuredHeight + (lp?.topMargin ?: 0) + (lp?.bottomMargin ?: 0) + } + if (!NsLayout.layout(child, 0, y, width, y + height)) { + val lp = child.layoutParams as? MarginLayoutParams + val left = lp?.leftMargin ?: 0 + val top = y + (lp?.topMargin ?: 0) + child.layout(left, top, left + child.measuredWidth, top + child.measuredHeight) + } + y += height + } + } + + + /** The views to snapshot, in tree order. */ + fun setDrawables(views: Array) { + val keep = HashSet(views.size) + for (view in views) { + keep.add(view) + entries.getOrPut(view) { + Entry(view).also { it.nextPoll = SystemClock.uptimeMillis() + (entries.size * POLL_STAGGER_MS) % POLL_INTERVAL_MS } + } + } + val iterator = entries.entries.iterator() + while (iterator.hasNext()) { + val (view, entry) = iterator.next() + if (!keep.contains(view)) { + release(entry) + hitOrder.remove(entry) + iterator.remove() + } + } + if (entries.size != views.size || !entries.keys.toTypedArray().contentEquals(views)) { + val ordered = LinkedHashMap() + for (view in views) ordered[view] = entries[view]!! + entries.clear() + entries.putAll(ordered) + } + } + + /** Snapshots what changed; returns the indices (in [setDrawables] order) of new snapshots. */ + fun update(scaleX: Float, scaleY: Float): IntArray? { + val now = SystemClock.uptimeMillis() + var changed: IntArray? = null + var count = 0 + var index = -1 + for (entry in entries.values) { + index++ + val view = entry.view + if (view.width <= 0 || view.height <= 0) continue + entry.pending?.let { next -> + entry.pending = null + entry.snapshot?.close() + entry.snapshot = next + if (changed == null) changed = IntArray(entries.size) + changed!![count++] = index + } + val width = maxOf(1, Math.ceil((view.width * scaleX).toDouble() - 1e-3).toInt()) + val height = maxOf(1, Math.ceil((view.height * scaleY).toDouble() - 1e-3).toInt()) + val resized = width != entry.width || height != entry.height + val poll = Build.VERSION.SDK_INT < Build.VERSION_CODES.O && now >= entry.nextPoll + if (poll) entry.nextPoll = now + POLL_INTERVAL_MS + if (!entry.dirty && !resized && !poll) continue + entry.dirty = false + entry.width = width + entry.height = height + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && captureGpu(entry, width, height, scaleX, scaleY)) { + continue + } + val bitmap = captureBitmap(entry, width, height, scaleX, scaleY) ?: continue + entry.snapshot?.close() + entry.snapshot = Snapshot(width, height, null, bitmap, null) + if (changed == null) changed = IntArray(entries.size) + changed!![count++] = index + } + return changed?.copyOf(count) + } + + /** The latest snapshot of [view], retained: close it when done. */ + fun snapshot(view: View): Snapshot? = entries[view]?.snapshot?.retain() + + /** A snapshot of [view] that is not shared with the per-frame one. */ + fun capture(view: View, width: Int, height: Int, scaleX: Float, scaleY: Float, sourceX: Float, sourceY: Float): Snapshot? { + if (view.width <= 0 || view.height <= 0 || width <= 0 || height <= 0) return null + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + drawInto(Canvas(bitmap), view, scaleX, scaleY, sourceX, sourceY, nestedOf(view)) + return Snapshot(width, height, null, bitmap, null) + } + + fun markDirty(view: View) { + entries[view]?.dirty = true + } + + + /** The matrix (Android Matrix values) maps [view]'s pixels to this view's; the clip is in its pixels. */ + fun setGeometry( + view: View, + scaleX: Float, skewX: Float, transX: Float, + skewY: Float, scaleY: Float, transY: Float, + persp0: Float, persp1: Float, persp2: Float, + hasClip: Boolean, left: Float, top: Float, right: Float, bottom: Float, + preserveOrder: Boolean, + ) { + val entry = entries[view] ?: Entry(view).also { entries[view] = it } + val forward = Matrix().apply { + setValues(floatArrayOf(scaleX, skewX, transX, skewY, scaleY, transY, persp0, persp1, persp2)) + } + val inverse = Matrix() + if (!forward.invert(inverse)) return + entry.forward = forward + entry.inverse = inverse + entry.clip = if (hasClip) RectF(left, top, right, bottom) else null + val known = hitOrder.indexOf(entry) + if (!(preserveOrder && known >= 0)) { + if (known >= 0) hitOrder.removeAt(known) + hitOrder.add(entry) + } + } + + fun clearGeometry(view: View) { + val entry = entries[view] ?: return + entry.forward = null + entry.inverse = null + entry.clip = null + hitOrder.remove(entry) + if (touchTarget === entry) touchTarget = null + } + + + override fun dispatchTouchEvent(event: MotionEvent): Boolean { + val action = event.actionMasked + if (action == MotionEvent.ACTION_DOWN) { + touchTarget = null + touchInverse = null + val point = FloatArray(2) + for (i in hitOrder.indices.reversed()) { + val entry = hitOrder[i] + val inverse = entry.inverse ?: continue + val view = entry.view + if (view.visibility != VISIBLE) continue + point[0] = event.x + point[1] = event.y + inverse.mapPoints(point) + if (point[0] < 0 || point[1] < 0 || point[0] >= view.width || point[1] >= view.height) continue + if (entry.clip?.contains(point[0], point[1]) == false) continue + touchTarget = entry + touchInverse = Matrix(inverse) + break + } + } + val target = touchTarget ?: return false + val copy = MotionEvent.obtain(event) + copy.transform(touchInverse) + val handled = try { + target.view.dispatchTouchEvent(copy) + } finally { + copy.recycle() + } + if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) { + touchTarget = null + } + if (action == MotionEvent.ACTION_DOWN && !handled) { + touchTarget = null + return false + } + // The canvas still sees the event, as if it bubbled. + (parent as? View)?.onTouchEvent(event) + return true + } + + + override fun onDescendantInvalidated(child: View, target: View) { + super.onDescendantInvalidated(child, target) + if (capturing) return + var node: Any? = target + while (node is View && node !== this) { + val entry = entries[node] + if (entry != null) { + entry.dirty = true + return + } + node = node.parent + } + } + + + private fun nestedOf(view: View): List { + val out = ArrayList() + fun visit(node: View) { + if (node !is ViewGroup) return + for (i in 0 until node.childCount) { + val child = node.getChildAt(i) + if (entries.containsKey(child)) out.add(child) else visit(child) + } + } + visit(view) + return out + } + + private fun drawInto(canvas: Canvas, view: View, scaleX: Float, scaleY: Float, sourceX: Float, sourceY: Float, nested: List) { + val hidden = nested.filter { it.visibility == VISIBLE } + val save = canvas.save() + capturing = true + try { + for (v in hidden) v.visibility = INVISIBLE + canvas.scale(scaleX, scaleY) + canvas.translate(-view.scrollX - sourceX, -view.scrollY - sourceY) + view.draw(canvas) + } finally { + for (v in hidden) v.visibility = VISIBLE + capturing = false + canvas.restoreToCount(save) + } + } + + private fun captureBitmap(entry: Entry, width: Int, height: Int, scaleX: Float, scaleY: Float): Bitmap? { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + drawInto(Canvas(bitmap), entry.view, scaleX, scaleY, 0f, 0f, nestedOf(entry.view)) + return bitmap + } + + @RequiresApi(Build.VERSION_CODES.Q) + private inner class Gpu(val width: Int, val height: Int) { + val reader: ImageReader = ImageReader.newInstance( + width, height, PixelFormat.RGBA_8888, MAX_IMAGES, + HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE or HardwareBuffer.USAGE_GPU_COLOR_OUTPUT, + ) + val node = RenderNode("NSCDrawable") + val renderer = HardwareRenderer() + var outstanding = 0 + var retired = false + + init { + node.setPosition(0, 0, width, height) + renderer.setContentRoot(node) + renderer.setSurface(reader.surface) + renderer.isOpaque = false + val density = resources.displayMetrics.density + renderer.setLightSourceGeometry(width / 2f, 0f, 600f * density, 8f * density) + renderer.setLightSourceAlpha(0.04f, 0.19f) + } + + fun destroyIfDone() { + if (retired && outstanding == 0) { + renderer.destroy() + node.discardDisplayList() + reader.close() + } + } + } + + @RequiresApi(Build.VERSION_CODES.Q) + private fun captureGpu(entry: Entry, width: Int, height: Int, scaleX: Float, scaleY: Float): Boolean { + if (!entry.view.isHardwareAccelerated) return false + val current = entry.gpu as? Gpu + val gpu: Gpu = if (current != null && current.width == width && current.height == height) { + current + } else { + current?.let { retire(it) } + val created = try { + Gpu(width, height) + } catch (e: Throwable) { + return false + } + entry.gpu = created + created.reader.setOnImageAvailableListener({ reader -> + val image = try { + reader.acquireLatestImage() + } catch (e: IllegalStateException) { + null + } ?: return@setOnImageAvailableListener + // The buffer arrives before the GPU finishes it: readers would see a partial frame. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + try { + image.fence.await(java.time.Duration.ofMillis(FENCE_WAIT_MS)) + } catch (e: Throwable) { + } + } + created.outstanding++ + val snapshot = Snapshot(created.width, created.height, image, null) { + handler.post { + created.outstanding-- + created.destroyIfDone() + } + } + if (entry.gpu === created) { + entry.pending?.close() + entry.pending = snapshot + } else { + snapshot.close() + } + }, handler) + created + } + val canvas = gpu.node.beginRecording(width, height) + try { + drawInto(canvas, entry.view, scaleX, scaleY, 0f, 0f, nestedOf(entry.view)) + } finally { + gpu.node.endRecording() + } + gpu.renderer.createRenderRequest() + .setWaitForPresent(Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) + .syncAndDraw() + return true + } + + @RequiresApi(Build.VERSION_CODES.Q) + private fun retire(gpu: Gpu) { + gpu.retired = true + gpu.reader.setOnImageAvailableListener(null, null) + gpu.destroyIfDone() + } + + private fun release(entry: Entry) { + entry.snapshot?.close() + entry.snapshot = null + entry.pending?.close() + entry.pending = null + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + (entry.gpu as? Gpu)?.let { retire(it) } + } + entry.gpu = null + } + + /** Releases every snapshot and renderer. */ + fun dispose() { + for (entry in entries.values) release(entry) + entries.clear() + hitOrder.clear() + touchTarget = null + } + + companion object { + private const val MAX_IMAGES = 4 + private const val FENCE_WAIT_MS = 100L + /** Below API 26 there is no onDescendantInvalidated: idle views are polled (ms, not frames: refresh rates differ). */ + private const val POLL_INTERVAL_MS = 166L + private const val POLL_STAGGER_MS = 17L + + private fun newLayoutParams(): LayoutParams = + NsLayout.newParams() ?: FrameLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.WRAP_CONTENT) + + /** Draws [snapshot] into a 2D context on the GPU. False when it can't: draw its bitmap instead. */ + @JvmStatic + fun draw2D( + context: Long, snapshot: Snapshot, + sx: Float, sy: Float, sw: Float, sh: Float, + dx: Float, dy: Float, dw: Float, dh: Float, + ): Boolean { + val buffer = snapshot.hardwareBuffer ?: return false + snapshot.retain() + val drawn = Utils.drawVideoFrameHardwareBuffer( + context, buffer, snapshot, snapshot.width, snapshot.height, + sx, sy, sw, sh, dx, dy, dw, dh, + ) + if (!drawn) snapshot.close() + return drawn + } + } + + /** NativeScript's own measure/layout; widgets ships with the app, so it is looked up at runtime. */ + private object NsLayout { + private val params: Class<*>? = try { + Class.forName("org.nativescript.widgets.CommonLayoutParams") + } catch (e: Throwable) { + null + } + private val measureChild: Method? = find("measureChild", View::class.java, Int::class.java, Int::class.java) + private val layoutChild: Method? = find("layoutChild", View::class.java, Int::class.java, Int::class.java, Int::class.java, Int::class.java) + private val desiredHeight: Method? = find("getDesiredHeight", View::class.java) + + private fun find(name: String, vararg types: Class<*>): Method? = try { + params?.getDeclaredMethod(name, *types)?.apply { isAccessible = true } + } catch (e: Throwable) { + null + } + + fun newParams(): LayoutParams? = try { + params?.getConstructor()?.newInstance() as? LayoutParams + } catch (e: Throwable) { + null + } + + fun measure(child: View, widthSpec: Int, heightSpec: Int): Boolean { + val method = measureChild ?: return false + method.invoke(null, child, widthSpec, heightSpec) + return true + } + + fun layout(child: View, left: Int, top: Int, right: Int, bottom: Int): Boolean { + val method = layoutChild ?: return false + method.invoke(null, child, left, top, right, bottom) + return true + } + + fun desiredHeight(child: View): Int? = desiredHeight?.invoke(null, child) as? Int + } +} diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative.xcodeproj/project.pbxproj b/packages/canvas/src-native/canvas-ios/CanvasNative.xcodeproj/project.pbxproj index 1297200c..b283c6d8 100644 --- a/packages/canvas/src-native/canvas-ios/CanvasNative.xcodeproj/project.pbxproj +++ b/packages/canvas/src-native/canvas-ios/CanvasNative.xcodeproj/project.pbxproj @@ -18,6 +18,7 @@ F18165C72621CECC0051F06B /* NSCRender.swift in Sources */ = {isa = PBXBuildFile; fileRef = F18165C62621CECC0051F06B /* NSCRender.swift */; }; F181661D26228CBA0051F06B /* NSCImageBitmap.swift in Sources */ = {isa = PBXBuildFile; fileRef = F181661C26228CBA0051F06B /* NSCImageBitmap.swift */; }; F1886A102C7B12FE00B2892C /* NSCCanvasVideoFrame.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1886A0F2C7B12FE00B2892C /* NSCCanvasVideoFrame.swift */; }; + F1A0DA0D2EB8000100C0FFEE /* NSCDrawableHost.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1A0DA0C2EB8000100C0FFEE /* NSCDrawableHost.swift */; }; F1A87C75255D00BA00394387 /* NSCCanvas.xib in Resources */ = {isa = PBXBuildFile; fileRef = F1A87C74255D00BA00394387 /* NSCCanvas.xib */; }; F1CB77132BBFC05900481C3F /* NSCWebGLRenderingContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1CB77122BBFC05900481C3F /* NSCWebGLRenderingContext.swift */; }; F1D1AE292ADAAEAA005C140B /* canvas_ios.h in Headers */ = {isa = PBXBuildFile; fileRef = F1D1AE252ADAAEAA005C140B /* canvas_ios.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -83,6 +84,7 @@ F18165C62621CECC0051F06B /* NSCRender.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NSCRender.swift; sourceTree = ""; }; F181661C26228CBA0051F06B /* NSCImageBitmap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NSCImageBitmap.swift; sourceTree = ""; }; F1886A0F2C7B12FE00B2892C /* NSCCanvasVideoFrame.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = NSCCanvasVideoFrame.swift; path = CanvasNative/Source/NSCCanvasVideoFrame.swift; sourceTree = ""; }; + F1A0DA0C2EB8000100C0FFEE /* NSCDrawableHost.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = NSCDrawableHost.swift; path = CanvasNative/Source/NSCDrawableHost.swift; sourceTree = ""; }; F1A87C74255D00BA00394387 /* NSCCanvas.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = NSCCanvas.xib; sourceTree = ""; }; F1CB77122BBFC05900481C3F /* NSCWebGLRenderingContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NSCWebGLRenderingContext.swift; sourceTree = ""; }; F1D1AE252ADAAEAA005C140B /* canvas_ios.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = canvas_ios.h; path = CanvasNative/include/canvas_ios.h; sourceTree = ""; }; @@ -128,6 +130,7 @@ F1DC52652CB59F16000DD64F /* NSCMTLView.h */, F1DC52632CB59EFA000DD64F /* NSCMTLView.m */, F1886A0F2C7B12FE00B2892C /* NSCCanvasVideoFrame.swift */, + F1A0DA0C2EB8000100C0FFEE /* NSCDrawableHost.swift */, F1CB77122BBFC05900481C3F /* NSCWebGLRenderingContext.swift */, F14908962AEA0F8D007C6693 /* NSCTouchHandler.swift */, F1D1AE252ADAAEAA005C140B /* canvas_ios.h */, @@ -372,6 +375,7 @@ F1FF6A6C25248CFF001DAE0D /* NSCCanvasListener.swift in Sources */, F134FE5C29D005BE0051057B /* CanvasHelpers.swift in Sources */, F1886A102C7B12FE00B2892C /* NSCCanvasVideoFrame.swift in Sources */, + F1A0DA0D2EB8000100C0FFEE /* NSCDrawableHost.swift in Sources */, F1DC52642CB59EFA000DD64F /* NSCMTLView.m in Sources */, F1FF6A7825248CFF001DAE0D /* NSCCanvas.swift in Sources */, F1FF6A8225248CFF001DAE0D /* GLUtils.swift in Sources */, diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCDrawableHost.swift b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCDrawableHost.swift new file mode 100644 index 00000000..99240d42 --- /dev/null +++ b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCDrawableHost.swift @@ -0,0 +1,410 @@ +import Foundation +import UIKit +import Metal +import IOSurface +import QuartzCore + +private let metalDevice: MTLDevice? = MTLCreateSystemDefaultDevice() +private let deviceRGB = CGColorSpaceCreateDeviceRGB() +private let bgraInfo = CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue + +/// One captured frame of a drawable view: premultiplied BGRA in an IOSurface Metal reads directly. +@objcMembers +@objc(NSCDrawableSnapshot) +public class NSCDrawableSnapshot: NSObject { + public let width: Int + public let height: Int + let surface: IOSurface + private var _texture: MTLTexture? + + init?(width: Int, height: Int) { + let properties: [IOSurfacePropertyKey: Any] = [ + .width: width, + .height: height, + .bytesPerElement: 4, + .pixelFormat: kCVPixelFormatType_32BGRA, + ] + guard width > 0, height > 0, let surface = IOSurface(properties: properties) else { return nil } + self.width = width + self.height = height + self.surface = surface + } + + var texture: MTLTexture? { + if let texture = _texture { return texture } + guard let device = metalDevice else { return nil } + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false) + descriptor.usage = .shaderRead + descriptor.storageMode = .shared + _texture = device.makeTexture(descriptor: descriptor, iosurface: surface, plane: 0) + return _texture + } + + /// `id` for WebGPU, or 0. Valid while the snapshot is alive. + public var texturePointer: Int64 { + guard let texture = texture else { return 0 } + return Int64(Int(bitPattern: Unmanaged.passUnretained(texture).toOpaque())) + } + + /// Draws into a 2D context on the GPU. False when it can't: draw `image()` instead. + public func draw2D(_ context: Int64, _ sx: Float, _ sy: Float, _ sw: Float, _ sh: Float, _ dx: Float, _ dy: Float, _ dw: Float, _ dh: Float) -> Bool { + guard context != 0, let texture = texture else { return false } + let owner = Unmanaged.passRetained(self).toOpaque() + return canvas_native_ios_context_draw_external_metal_texture( + context, Unmanaged.passUnretained(texture).toOpaque(), owner, + Float(width), Float(height), + sx, sy, sw, sh, dx, dy, dw, dh + ) + } + + /// A CPU copy, for consumers with no Metal path. + public func image() -> UIImage? { + surface.lock(options: .readOnly, seed: nil) + defer { surface.unlock(options: .readOnly, seed: nil) } + guard let context = CGContext(data: surface.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: surface.bytesPerRow, space: deviceRGB, bitmapInfo: bgraInfo), + let image = context.makeImage() else { return nil } + return UIImage(cgImage: image) + } + + func render(_ view: UIView, scaleX: CGFloat, scaleY: CGFloat, sourceX: CGFloat, sourceY: CGFloat, hide: [UIView]) { + surface.lock(options: [], seed: nil) + defer { surface.unlock(options: [], seed: nil) } + guard let context = CGContext(data: surface.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: surface.bytesPerRow, space: deviceRGB, bitmapInfo: bgraInfo) else { return } + context.clear(CGRect(x: 0, y: 0, width: width, height: height)) + context.translateBy(x: 0, y: CGFloat(height)) + context.scaleBy(x: scaleX, y: -scaleY) + let origin = view.layer.bounds.origin + context.translateBy(x: -origin.x - sourceX, y: -origin.y - sourceY) + if hide.isEmpty { + // The presentation layer catches in-flight animations. + (view.layer.presentation() ?? view.layer).render(in: context) + return + } + CATransaction.begin() + CATransaction.setDisableActions(true) + for v in hide { v.layer.isHidden = true } + view.layer.render(in: context) + for v in hide { v.layer.isHidden = false } + CATransaction.commit() + } + + func samePixels(as other: NSCDrawableSnapshot) -> Bool { + guard other.width == width, other.height == height else { return false } + surface.lock(options: .readOnly, seed: nil) + other.surface.lock(options: .readOnly, seed: nil) + defer { + other.surface.unlock(options: .readOnly, seed: nil) + surface.unlock(options: .readOnly, seed: nil) + } + let rowBytes = width * 4 + let a = surface.baseAddress, b = other.surface.baseAddress + let strideA = surface.bytesPerRow, strideB = other.surface.bytesPerRow + if strideA == strideB { return memcmp(a, b, strideA * height) == 0 } + for y in 0.. [NSNumber]? { + let now = CACurrentMediaTime() + updates += 1 + keepOnTop() + var changed: [NSNumber]? + for (index, entry) in entries.enumerated() { + guard let view = entry.view else { continue } + applyTransform(entry) + let size = view.bounds.size + guard size.width > 0, size.height > 0 else { continue } + let width = max(1, Int(ceil(Double(size.width) * scaleX - 1e-3))) + let height = max(1, Int(ceil(Double(size.height) * scaleY - 1e-3))) + let resized = entry.snapshot.map { $0.width != width || $0.height != height } ?? true + let signature = layerSignature(view.layer) + let signatureChanged = signature != entry.signature + entry.signature = signature + let hide = nested(of: view) + if !resized && !signatureChanged { + let hot = now <= entry.hotUntil || now <= hotAll + if !hot && now < entry.nextCheck { continue } + if !probeChanged(entry, view, width, height, scaleX, scaleY, hide) { + if !hot { entry.interval = min(entry.interval * 2, NSCDrawableHost.maxInterval) } + entry.nextCheck = now + entry.interval + continue + } + } + + let target: NSCDrawableSnapshot + if let scratch = entry.scratch, scratch.width == width, scratch.height == height { + target = scratch + } else if let fresh = NSCDrawableSnapshot(width: width, height: height) { + target = fresh + } else { + continue + } + target.render(view, scaleX: CGFloat(scaleX), scaleY: CGFloat(scaleY), sourceX: 0, sourceY: 0, hide: hide) + if signatureChanged || resized { _ = probeChanged(entry, view, width, height, scaleX, scaleY, hide) } + if !resized, let current = entry.snapshot, target.samePixels(as: current) { + entry.scratch = target + entry.nextCheck = now + entry.interval + continue + } + // The old snapshot may still be read by a pending draw: never render into it again. + entry.scratch = nil + entry.snapshot = target + entry.interval = NSCDrawableHost.minInterval + entry.nextCheck = now + entry.interval + entry.hotUntil = now + NSCDrawableHost.hotDuration + if changed == nil { changed = [] } + changed?.append(NSNumber(value: index)) + } + return changed + } + + /// Renders the quarter-size probe; true when it differs from the last one. + private func probeChanged(_ entry: Entry, _ view: UIView, _ width: Int, _ height: Int, _ scaleX: Double, _ scaleY: Double, _ hide: [UIView]) -> Bool { + let s = NSCDrawableHost.probeScale + let w = max(1, Int(Double(width) * s)), h = max(1, Int(Double(height) * s)) + let target: NSCDrawableSnapshot + if let scratch = entry.probeScratch, scratch.width == w, scratch.height == h { + target = scratch + } else if let fresh = NSCDrawableSnapshot(width: w, height: h) { + target = fresh + } else { + return true + } + target.render(view, scaleX: CGFloat(scaleX * s), scaleY: CGFloat(scaleY * s), sourceX: 0, sourceY: 0, hide: hide) + let same = entry.probe.map { target.samePixels(as: $0) } ?? false + entry.probeScratch = entry.probe + entry.probe = target + return !same + } + + public func snapshot(_ view: UIView) -> NSCDrawableSnapshot? { + return byView[ObjectIdentifier(view)]?.snapshot + } + + /// A snapshot of `view` that is not shared with the per-frame one. + public func capture(_ view: UIView, _ width: Int, _ height: Int, _ scaleX: Double, _ scaleY: Double, _ sourceX: Double, _ sourceY: Double) -> NSCDrawableSnapshot? { + guard let snapshot = NSCDrawableSnapshot(width: width, height: height) else { return nil } + snapshot.render(view, scaleX: CGFloat(scaleX), scaleY: CGFloat(scaleY), sourceX: CGFloat(sourceX), sourceY: CGFloat(sourceY), hide: nested(of: view)) + return snapshot + } + + private func nested(of view: UIView) -> [UIView] { + var out: [UIView] = [] + func visit(_ node: UIView) { + for child in node.subviews { + if byView[ObjectIdentifier(child)] != nil { out.append(child) } else { visit(child) } + } + } + visit(view) + return out + } + + /// Cheap, render-free fingerprint of what a layer tree looks like. + private func layerSignature(_ root: CALayer) -> Int { + var hasher = Hasher() + // The root's own position and transform are where it is drawn, not what it looks like. + func visit(_ model: CALayer, root: Bool = false) { + let layer = model.presentation() ?? model + hasher.combine(layer.bounds.origin.x) + hasher.combine(layer.bounds.origin.y) + hasher.combine(layer.bounds.size.width) + hasher.combine(layer.bounds.size.height) + if !root { + hasher.combine(layer.position.x) + hasher.combine(layer.position.y) + } + hasher.combine(layer.opacity) + hasher.combine(layer.isHidden) + hasher.combine(layer.cornerRadius) + hasher.combine(layer.borderWidth) + if let color = layer.backgroundColor?.components { hasher.combine(color) } + if let color = layer.borderColor?.components { hasher.combine(color) } + if let contents = layer.contents { hasher.combine(ObjectIdentifier(contents as AnyObject)) } + if !root && !CATransform3DIsIdentity(layer.transform) { + let t = layer.transform + hasher.combine(t.m11); hasher.combine(t.m12); hasher.combine(t.m21); hasher.combine(t.m22) + hasher.combine(t.m41); hasher.combine(t.m42); hasher.combine(t.m34) + } + // Animating: a new signature every update. + if model.animationKeys()?.isEmpty == false { hasher.combine(updates) } + let sublayers = model.sublayers ?? [] + hasher.combine(sublayers.count) + for sublayer in sublayers { visit(sublayer) } + } + visit(root, root: true) + return hasher.finalize() + } + + // MARK: geometry + + /// `matrix` (DOMMatrix order) maps `view`'s border box to this view; the clip is in its border box. + public func setGeometry(_ view: UIView, _ matrix: [NSNumber], _ hasClip: Bool, _ clipX: Double, _ clipY: Double, _ clipWidth: Double, _ clipHeight: Double, _ preserveOrder: Bool) { + guard matrix.count == 16 else { return } + let key = ObjectIdentifier(view) + let entry = byView[key] ?? { + let created = Entry(view) + byView[key] = created + return created + }() + entry.transform = matrix.map { CGFloat($0.doubleValue) } + entry.clip = hasClip ? CGRect(x: clipX, y: clipY, width: clipWidth, height: clipHeight).standardized : nil + let known = hitOrder.firstIndex { $0 === entry } + if !(preserveOrder && known != nil) { + if let known = known { hitOrder.remove(at: known) } + hitOrder.append(entry) + } + applyTransform(entry) + } + + public func clearGeometry(_ view: UIView) { + guard let entry = byView[ObjectIdentifier(view)] else { return } + entry.transform = nil + entry.clip = nil + hitOrder.removeAll { $0 === entry } + CATransaction.begin() + CATransaction.setDisableActions(true) + view.layer.transform = CATransform3DIdentity + CATransaction.commit() + } + + /// The layer maps p to position + M * (p - anchor). + private func applyTransform(_ entry: Entry) { + guard let view = entry.view, let m = entry.transform, let superview = view.superview else { return } + var target = CATransform3D( + m11: m[0], m12: m[1], m13: m[2], m14: m[3], + m21: m[4], m22: m[5], m23: m[6], m24: m[7], + m31: m[8], m32: m[9], m33: m[10], m34: m[11], + m41: m[12], m42: m[13], m43: m[14], m44: m[15] + ) + if superview !== self { + // Assumes nothing between is 3D. + let o = superview.convert(CGPoint.zero, to: self) + let x = superview.convert(CGPoint(x: 1, y: 0), to: self) + let y = superview.convert(CGPoint(x: 0, y: 1), to: self) + let toHost = CGAffineTransform(a: x.x - o.x, b: x.y - o.y, c: y.x - o.x, d: y.y - o.y, tx: o.x, ty: o.y) + target = CATransform3DConcat(target, CATransform3DMakeAffineTransform(toHost.inverted())) + } + let size = view.bounds.size + let anchor = view.layer.anchorPoint + let position = view.layer.position + var transform = CATransform3DConcat(CATransform3DMakeTranslation(anchor.x * size.width, anchor.y * size.height, 0), target) + transform = CATransform3DConcat(transform, CATransform3DMakeTranslation(-position.x, -position.y, 0)) + if CATransform3DEqualToTransform(transform, view.layer.transform) { return } + CATransaction.begin() + CATransaction.setDisableActions(true) + view.layer.transform = transform + CATransaction.commit() + } + + private func keepOnTop() { + guard let superview = superview else { return } + if superview.subviews.last !== self { superview.bringSubviewToFront(self) } + if frame != superview.bounds { frame = superview.bounds } + } + + // MARK: hit testing + + // UIKit skips views at alpha 0, so hit testing is done here. + public override func hitTest(_ point: CGPoint, with event: UIEvent?) -> UIView? { + markHot() + for entry in hitOrder.reversed() { + guard entry.transform != nil, let view = entry.view, !view.isHidden, view.isUserInteractionEnabled else { continue } + let local = convert(point, to: view) + if let clip = entry.clip { + let origin = view.bounds.origin + if !clip.contains(CGPoint(x: local.x - origin.x, y: local.y - origin.y)) { continue } + } + if let hit = view.hitTest(local, with: event) { return hit } + } + return nil + } + + public override func point(inside point: CGPoint, with event: UIEvent?) -> Bool { + return hitTest(point, with: event) != nil + } + + public func dispose() { + for entry in entries + hitOrder { + entry.snapshot = nil + entry.scratch = nil + if entry.transform != nil, let view = entry.view { + view.layer.transform = CATransform3DIdentity + } + } + entries = [] + hitOrder = [] + byView = [:] + removeFromSuperview() + } +} diff --git a/tools/demo/canvas/spec/htmlincanvas.ts b/tools/demo/canvas/spec/htmlincanvas.ts index c6156b7c..31cb8e9b 100644 --- a/tools/demo/canvas/spec/htmlincanvas.ts +++ b/tools/demo/canvas/spec/htmlincanvas.ts @@ -330,6 +330,30 @@ export function registerHtmlInCanvasSpec() { }); }); + suite('htmlincanvas.gpu', () => { + mountedTest('snapshots live on the GPU and draw from there', async () => { + const child = block(50, 50, 'red'); + const { canvas, ctx } = await mount([child]); + await snapshotted(canvas, child); + const snapshot = canvas._drawables.nativeSnapshot(child); + if (__ANDROID__ && android.os.Build.VERSION.SDK_INT >= 29) { + ok(snapshot?.getHardwareBuffer?.(), 'expected a HardwareBuffer'); + const drawn = (org as any).nativescript.canvas.NSCDrawableHost.draw2D(long(ctx.context.__getPointer()), snapshot, 0, 0, snapshot.getWidth(), snapshot.getHeight(), 100, 0, 50, 50); + ok(drawn, 'expected the GPU draw to be taken'); + ctx.context.__makeDirty(); + const pixel = pixelAt(ctx, 125, 25); + ok(near(pixel, RED), `expected red from the GPU draw, got ${show(pixel)}`); + } else if (__APPLE__) { + ok(snapshot, 'expected a native snapshot'); + ok(snapshot.draw2D(ctx.context.__getPointer(), 0, 0, snapshot.width, snapshot.height, 100, 0, 50, 50), 'expected the Metal draw to be taken'); + ctx.context.__makeDirty?.(); + const pixel = pixelAt(ctx, 125, 25); + ok(near(pixel, RED), `expected red from the Metal draw, got ${show(pixel)}`); + } + snapshot?.close?.(); + }); + }); + suite('htmlincanvas.hittest', () => { mountedTest('touches reach the element where it is drawn, not where it is laid out', async () => { const button = new Button();