diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 0587166d..364d9264 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Realize Off-Screen Items in Virtualized Lists / Grids + +Reach a row that isn't scrolled into view yet — the "element not found in a long list" fix. Full reference: [`docs/source/Eng/doc/new_features/v195_features_doc.rst`](docs/source/Eng/doc/new_features/v195_features_doc.rst). + +- **`realize_item`** (`AC_realize_item`): long lists / data grids / trees only materialize visible rows, so an off-screen row has no accessibility element at all — `list_accessibility_elements` / `read_control_table` / `select_control_item` can't see it, and `scroll_control_into_view` can't help because the element doesn't exist yet. This locates the item by property (UIA `ItemContainerPattern.FindItemByProperty`) and realizes it (`VirtualizedItemPattern.Realize`) so it becomes a real, clickable element. Match `by` name (default) or `automation_id`; locate the container by name/role/app. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Per-Run Step Timeline (waterfall + bottleneck steps) Read why *this* run was slow — a step waterfall and its bottlenecks. Full reference: [`docs/source/Eng/doc/new_features/v194_features_doc.rst`](docs/source/Eng/doc/new_features/v194_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v195_features_doc.rst b/docs/source/Eng/doc/new_features/v195_features_doc.rst new file mode 100644 index 00000000..43a8d486 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v195_features_doc.rst @@ -0,0 +1,48 @@ +Realize Off-Screen Items in Virtualized Lists / Grids +===================================================== + +Long lists, data grids and trees (WPF / WinUI / File Explorer / virtual +treeviews) only materialize the rows that are scrolled into view — a row that is +off-screen has **no** accessibility element at all. So +``list_accessibility_elements`` / ``read_control_table`` / ``select_control_item`` +simply cannot see it, and ``scroll_control_into_view`` can't help because the +target element does not exist yet. This is the classic "element not found in a +long list" wall. + +``realize_item`` closes that gap: it locates the item inside its container by +property (UI Automation ``ItemContainerPattern.FindItemByProperty``) and realizes +it (``VirtualizedItemPattern.Realize``) so it materializes as a real element you +can then click or read. + +It is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam (the same seam the rest of the accessibility module uses) — headless-testable +on any platform by injecting a fake backend; the real UIA calls live in the +Windows backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import realize_item, click_accessibility_element + + # Bring a far-down row into existence, then act on it: + row = realize_item("Order 5000", container_name="Orders") + if row is not None: + click_accessibility_element(name=row.name) # now a real element + + realize_item("row-42", by="automation_id", container_name="DataGrid") + +``item_name`` is matched against the item's Name (``by="name"``, default) or its +AutomationId (``by="automation_id"``). The container is located by +``container_name`` / ``container_role`` / ``app_name`` / ``automation_id`` (the +same matchers as the other native-control actions). Returns the realized +``AccessibilityElement``, or ``None`` if the container or item isn't found. + +Executor commands +----------------- + +``AC_realize_item`` (``item_name`` / ``by`` / ``container_name`` / +``container_role`` / ``app_name`` / ``automation_id``) returns +``{found, element}``. It is exposed as the read-only ``ac_realize_item`` MCP tool +and as a Script Builder command under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v195_features_doc.rst b/docs/source/Zh/doc/new_features/v195_features_doc.rst new file mode 100644 index 00000000..ee1cbc8e --- /dev/null +++ b/docs/source/Zh/doc/new_features/v195_features_doc.rst @@ -0,0 +1,41 @@ +實體化虛擬化清單 / 格線中的離畫面項目 +====================================== + +長清單、資料格線與樹(WPF / WinUI / 檔案總管 / 虛擬化 treeview)只會實體化已捲入視野的列—— +離畫面的列**完全沒有**無障礙元素。因此 ``list_accessibility_elements`` / +``read_control_table`` / ``select_control_item`` 根本看不到它,而 ``scroll_control_into_view`` +也幫不上忙,因為目標元素根本還不存在。這就是經典的「長清單裡找不到元素」的牆。 + +``realize_item`` 補上這個缺口:它以屬性在容器內定位該項目(UI Automation +``ItemContainerPattern.FindItemByProperty``)並將其實體化(``VirtualizedItemPattern.Realize``), +使其成為一個真正、可點擊或可讀取的元素。 + +它是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派(與無障礙模組其餘部分相同的 +接縫)——可在任何平台透過注入 fake backend 進行無頭測試;真正的 UIA 呼叫位於 Windows 後端。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import realize_item, click_accessibility_element + + # 讓一個很下方的列「存在」,然後對它操作: + row = realize_item("Order 5000", container_name="Orders") + if row is not None: + click_accessibility_element(name=row.name) # 現在是真正的元素 + + realize_item("row-42", by="automation_id", container_name="DataGrid") + +``item_name`` 會比對項目的 Name(``by="name"``,預設)或其 AutomationId +(``by="automation_id"``)。容器以 ``container_name`` / ``container_role`` / ``app_name`` / +``automation_id`` 定位(與其他原生控制動作相同的比對方式)。回傳實體化後的 +``AccessibilityElement``,若找不到容器或項目則回傳 ``None``。 + +執行器指令 +---------- + +``AC_realize_item``(``item_name`` / ``by`` / ``container_name`` / ``container_role`` / +``app_name`` / ``automation_id``)回傳 ``{found, element}``。以唯讀 ``ac_realize_item`` MCP +工具及 Script Builder 指令(位於 **Native UI** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 81bc5b59..4113e55c 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -66,6 +66,8 @@ from je_auto_control.utils.focus_order import ( audit_focus_order, focus_control, is_interactive_role, tab_order, ) +# Realize off-screen items in virtualized lists / grids (UIA VirtualizedItem) +from je_auto_control.utils.virtualized import realize_item # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1666,6 +1668,7 @@ def start_autocontrol_gui(*args, **kwargs): "control_type_name", "humanize_role", "humanize_tree", "assign_node_paths", "find_by_path", "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", + "realize_item", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index d4d5bce4..c407d839 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1608,6 +1608,19 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: fields=fields, description="Scroll a control into view (ScrollItemPattern).", )) + specs.append(CommandSpec( + "AC_realize_item", "Native UI", "Realize Virtualized Item", + fields=( + FieldSpec("item_name", FieldType.STRING), + FieldSpec("by", FieldType.ENUM, optional=True, default="name", + choices=("name", "automation_id")), + FieldSpec("container_name", FieldType.STRING, optional=True), + FieldSpec("container_role", FieldType.STRING, optional=True), + FieldSpec("app_name", FieldType.STRING, optional=True), + FieldSpec("automation_id", FieldType.STRING, optional=True), + ), + description="Realize an off-screen item in a virtualized list/grid.", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 82d5798c..2f40c8af 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -131,6 +131,23 @@ def set_focus(self, name: Optional[str] = None, role: Optional[str] = None, """Set keyboard focus on the matched control (SetFocus); True on success.""" self._unsupported("set_focus") + # --- virtualized items (realize off-screen list / grid items) ----------- + + def find_virtual_item(self, item_name: Optional[str] = None, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[AccessibilityElement]: + """Find a (possibly virtualized) item inside a container and realize it. + + Long virtualized lists / grids only materialize visible rows; this locates + the item by property (``ItemContainerPattern``) and realizes it + (``VirtualizedItemPattern``) so it exists as a real element. Returns the + realized element, or None if the container or item isn't found. + """ + self._unsupported("find_virtual_item") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index cbfcba60..0d0594fc 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -30,6 +30,9 @@ _UIA_RANGEVALUE_PATTERN_ID = 10003 _UIA_SCROLLITEM_PATTERN_ID = 10017 _UIA_TEXT_PATTERN_ID = 10014 +_UIA_ITEMCONTAINER_PATTERN_ID = 10019 +_UIA_VIRTUALIZEDITEM_PATTERN_ID = 10020 +_UIA_AUTOMATIONID_PROPERTY = 30011 _EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} @@ -264,6 +267,37 @@ def get_range(self, name=None, role=None, app_name=None, except (OSError, AttributeError, ValueError, TypeError): return None + def _realize(self, raw) -> None: + """Realize a virtualized element so it materializes (VirtualizedItemPattern).""" + pattern = self._pattern(raw, _UIA_VIRTUALIZEDITEM_PATTERN_ID, + "IUIAutomationVirtualizedItemPattern") + if pattern is None: + return + try: + pattern.Realize() + except (OSError, AttributeError): + pass + + def find_virtual_item(self, item_name=None, by="name", container_name=None, + container_role=None, app_name=None, automation_id=None): + container = self._find_raw(container_name, container_role, app_name, + automation_id) + pattern = self._pattern(container, _UIA_ITEMCONTAINER_PATTERN_ID, + "IUIAutomationItemContainerPattern" + ) if container else None + if pattern is None: + return None + property_id = (_UIA_AUTOMATIONID_PROPERTY if by == "automation_id" + else _UIA_NAME_PROPERTY) + try: + found = pattern.FindItemByProperty(None, property_id, item_name) + except (OSError, AttributeError, ValueError): + return None + if not found: + return None + self._realize(found) + return _convert_uia(found) + def _text_pattern(self, name, role, app_name, automation_id): """Find a control and return its IUIAutomationTextPattern, or None.""" raw = self._find_raw(name, role, app_name, automation_id) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index edff6eda..be846d81 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2462,6 +2462,20 @@ def _scroll_control_into_view(name: Optional[str] = None, role: Optional[str] = automation_id=automation_id) +def _realize_item(item_name: str, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: find + realize a virtualized list/grid item (VirtualizedItem).""" + from je_auto_control.utils.virtualized import realize_item + element = realize_item(item_name, by=str(by), container_name=container_name, + container_role=container_role, app_name=app_name, + automation_id=automation_id) + return {"found": element is not None, + "element": element.to_dict() if element else None} + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6405,6 +6419,7 @@ def __init__(self): "AC_control_range": _control_range, "AC_set_control_range": _set_control_range, "AC_scroll_control_into_view": _scroll_control_into_view, + "AC_realize_item": _realize_item, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index d32d15ae..fb805544 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1157,6 +1157,27 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.scroll_control_into_view, annotations=DESTRUCTIVE, ), + MCPTool( + name="ac_realize_item", + description=("Find and REALIZE an off-screen item in a virtualized " + "list/grid (ItemContainer + VirtualizedItem patterns) so " + "it materializes as a real element — rows that aren't " + "scrolled into view have no element until realized. " + "'item_name' matched by 'name' (default) or " + "'automation_id'; the container by container_name/" + "container_role/app_name/automation_id. Returns " + "{found, element}."), + input_schema=schema({ + "item_name": {"type": "string"}, + "by": {"type": "string", "enum": ["name", "automation_id"]}, + "container_name": {"type": "string"}, + "container_role": {"type": "string"}, + "app_name": {"type": "string"}, + "automation_id": {"type": "string"}}, + required=["item_name"]), + handler=h.realize_item, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index ab6d91cc..e9caff10 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -795,6 +795,13 @@ def get_control_text(name=None, role=None, app_name=None, automation_id=None): return _get_control_text(name, role, app_name, automation_id) +def realize_item(item_name, by="name", container_name=None, container_role=None, + app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _realize_item + return _realize_item(item_name, by, container_name, container_role, + app_name, automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/je_auto_control/utils/virtualized/__init__.py b/je_auto_control/utils/virtualized/__init__.py new file mode 100644 index 00000000..8a010951 --- /dev/null +++ b/je_auto_control/utils/virtualized/__init__.py @@ -0,0 +1,4 @@ +"""Realize off-screen items in virtualized lists / grids (ItemContainer + VirtualizedItem).""" +from je_auto_control.utils.virtualized.virtualized import realize_item + +__all__ = ["realize_item"] diff --git a/je_auto_control/utils/virtualized/virtualized.py b/je_auto_control/utils/virtualized/virtualized.py new file mode 100644 index 00000000..7615ea34 --- /dev/null +++ b/je_auto_control/utils/virtualized/virtualized.py @@ -0,0 +1,39 @@ +"""Realize off-screen items in virtualized lists / grids / trees. + +Long lists, data grids and trees (WPF / WinUI / Explorer / virtual treeviews) +only materialize the rows that are scrolled into view — a row that is off-screen +has *no* accessibility element at all, so ``list_accessibility_elements`` / +``read_control_table`` / ``select_control_item`` simply cannot see it, and +``scroll_control_into_view`` can't help because the target doesn't exist yet. + +``realize_item`` closes that gap: it locates the item inside its container by +property (UI Automation ``ItemContainerPattern``) and realizes it +(``VirtualizedItemPattern``) so it materializes as a real element you can then +click / read. It is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam — headless-testable on any platform +by injecting a fake backend; the real UIA calls live in the Windows backend. +Imports no ``PySide6``. +""" +from typing import Optional + +from je_auto_control.utils.accessibility.element import AccessibilityElement + + +def realize_item(item_name: str, *, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[AccessibilityElement]: + """Find and realize a virtualized item, returning its element (or None). + + ``item_name`` is matched against the item's Name (``by="name"``) or its + AutomationId (``by="automation_id"``). The container is located by + ``container_name`` / ``container_role`` / ``app_name`` / ``automation_id`` + (the same matchers as the other native-control actions). + """ + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().find_virtual_item( + item_name, by=by, container_name=container_name, + container_role=container_role, app_name=app_name, + automation_id=automation_id) diff --git a/test/unit_test/headless/test_virtualized_batch.py b/test/unit_test/headless/test_virtualized_batch.py new file mode 100644 index 00000000..b01c6a65 --- /dev/null +++ b/test/unit_test/headless/test_virtualized_batch.py @@ -0,0 +1,90 @@ +"""Headless tests for realizing virtualized list/grid items (fake backend seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.accessibility.element import AccessibilityElement +from je_auto_control.utils.virtualized import realize_item + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, items=None): + # items: {name: AccessibilityElement} the container can realize + self.items = items or {} + self.calls = [] + + def find_virtual_item(self, item_name=None, by="name", container_name=None, + container_role=None, app_name=None, automation_id=None): + self.calls.append({"item_name": item_name, "by": by, + "container_name": container_name}) + return self.items.get(item_name) + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_realize_item_returns_realized_element(monkeypatch): + row = AccessibilityElement(name="Order 5000", role="ControlType_50007", + bounds=(10, 900, 200, 24), app_name="grid.exe") + fake = _FakeBackend({"Order 5000": row}) + _inject(monkeypatch, fake) + element = realize_item("Order 5000", container_name="Orders") + assert element is row + assert fake.calls[0]["item_name"] == "Order 5000" + assert fake.calls[0]["by"] == "name" + assert fake.calls[0]["container_name"] == "Orders" + + +def test_realize_item_not_found_returns_none(monkeypatch): + _inject(monkeypatch, _FakeBackend()) + assert realize_item("missing-row", container_name="Orders") is None + + +def test_realize_item_by_automation_id(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + realize_item("row-42", by="automation_id", container_name="Grid") + assert fake.calls[0]["by"] == "automation_id" + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) # all _unsupported + try: + realize_item("x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_wraps_element(monkeypatch): + row = AccessibilityElement(name="Row 9", role="ControlType_50007", + bounds=(1, 2, 3, 4), app_name="x") + _inject(monkeypatch, _FakeBackend({"Row 9": row})) + from je_auto_control.utils.executor.action_executor import _realize_item + out = _realize_item("Row 9", container_name="List") + assert out["found"] is True + assert out["element"]["name"] == "Row 9" + assert _realize_item("nope")["found"] is False + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_realize_item" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_realize_item" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_realize_item" in specs + + +def test_facade_export(): + assert hasattr(ac, "realize_item") and "realize_item" in ac.__all__