From 0c76ec5f8b2c0d507b50a47e85083849ded92ab5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 02:44:13 +0800 Subject: [PATCH] Add ax_props: read rich UIA element properties before acting The flat element list carries only name/role/bounds/app/id, but automation needs more before it acts: is the control enabled (don't click a disabled button), is it off-screen, its item_status, help_text (tooltip), accelerator_key. get_element_properties reads those UIA properties; is_element_enabled is the common pre-action guard. Extends the backend ABC + Windows UIA backend via the same fake-backend seam. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v196_features_doc.rst | 44 +++++++++ .../Zh/doc/new_features/v196_features_doc.rst | 40 ++++++++ je_auto_control/__init__.py | 5 + .../gui/script_builder/command_schema.py | 5 + .../utils/accessibility/backends/base.py | 14 +++ .../accessibility/backends/windows_backend.py | 34 +++++++ je_auto_control/utils/ax_props/__init__.py | 6 ++ je_auto_control/utils/ax_props/ax_props.py | 38 ++++++++ .../utils/executor/action_executor.py | 11 +++ .../utils/mcp_server/tools/_factories.py | 10 ++ .../utils/mcp_server/tools/_handlers.py | 7 ++ .../unit_test/headless/test_ax_props_batch.py | 93 +++++++++++++++++++ 13 files changed, 313 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v196_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v196_features_doc.rst create mode 100644 je_auto_control/utils/ax_props/__init__.py create mode 100644 je_auto_control/utils/ax_props/ax_props.py create mode 100644 test/unit_test/headless/test_ax_props_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 364d9264..c8bf18ea 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Rich UIA Element Properties + +Know if a control is enabled / off-screen / has a tooltip before you act. Full reference: [`docs/source/Eng/doc/new_features/v196_features_doc.rst`](docs/source/Eng/doc/new_features/v196_features_doc.rst). + +- **`get_element_properties` / `is_element_enabled`** (`AC_get_element_properties`): the flat element list carries only name/role/bounds/app/id, but automation needs more before it acts — **is the control enabled** (don't click a disabled button), **is it off-screen**, its **item_status** (field validation/error), **help_text** (tooltip), and **accelerator_key** (drive via hotkey). This reads those high-value UIA properties (`enabled`/`offscreen`/`help_text`/`item_status`/`accelerator_key`/`access_key`/`orientation`); `is_element_enabled` is the common pre-action guard. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA reads in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Realize Off-Screen Items in Virtualized Lists / Grids Reach a row that isn't scrolled into view yet — the "element not found in a long list" fix. Full reference: [`docs/source/Eng/doc/new_features/v195_features_doc.rst`](docs/source/Eng/doc/new_features/v195_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v196_features_doc.rst b/docs/source/Eng/doc/new_features/v196_features_doc.rst new file mode 100644 index 00000000..3a097a4d --- /dev/null +++ b/docs/source/Eng/doc/new_features/v196_features_doc.rst @@ -0,0 +1,44 @@ +Rich UIA Element Properties +=========================== + +``list_accessibility_elements`` / ``AccessibilityElement`` carry only name / role / +bounds / app / pid / automation_id. Automation routinely needs more *before it +acts*: **is the control enabled** (don't click a disabled button), **is it +off-screen** (is it really visible?), its **item_status** (validation / error text +on a field), **help_text** (tooltip), and **accelerator_key** (drive it via a +hotkey instead of a click). ``ax_props`` exposes those high-value UIA properties. + +* :func:`get_element_properties` — the full property dict, +* :func:`is_element_enabled` — the common pre-action guard. + +Each function is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam — headless-testable on any platform +by injecting a fake backend; the real UIA property reads live in the Windows +backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import get_element_properties, is_element_enabled + + get_element_properties(name="Save", role="button") + # {"enabled": False, "offscreen": False, "help_text": "Save the file", + # "item_status": "", "accelerator_key": "Ctrl+S", "access_key": "S", + # "orientation": 0} + + if is_element_enabled(name="Submit"): + click_text("Submit") # don't click a disabled button + +The control is located by ``name`` / ``role`` / ``app_name`` / ``automation_id`` +(same as the other native-control reads). ``get_element_properties`` returns the +property dict or ``None`` when the control isn't found; ``is_element_enabled`` +returns the ``enabled`` flag (or ``None`` if not found). + +Executor commands +----------------- + +``AC_get_element_properties`` returns ``{found, properties}``. It is exposed as the +read-only ``ac_get_element_properties`` MCP tool and as a Script Builder command +under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v196_features_doc.rst b/docs/source/Zh/doc/new_features/v196_features_doc.rst new file mode 100644 index 00000000..a13380c9 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v196_features_doc.rst @@ -0,0 +1,40 @@ +豐富的 UIA 元素屬性 +=================== + +``list_accessibility_elements`` / ``AccessibilityElement``只帶有 name / role / bounds / +app / pid / automation_id。自動化在*動作之前*常需要更多資訊:**控制項是否啟用**(別點停用的 +按鈕)、**是否在畫面外**(是否真的可見)、其 **item_status**(欄位的驗證 / 錯誤文字)、 +**help_text**(工具提示),以及 **accelerator_key**(以快捷鍵而非點擊來驅動它)。``ax_props`` +就提供這些高價值的 UIA 屬性。 + +* :func:`get_element_properties` ——完整的屬性字典, +* :func:`is_element_enabled` ——常見的動作前守衛。 + +每個函式都是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派——可在任何平台透過 +注入 fake backend 進行無頭測試;真正的 UIA 屬性讀取位於 Windows 後端。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import get_element_properties, is_element_enabled + + get_element_properties(name="Save", role="button") + # {"enabled": False, "offscreen": False, "help_text": "Save the file", + # "item_status": "", "accelerator_key": "Ctrl+S", "access_key": "S", + # "orientation": 0} + + if is_element_enabled(name="Submit"): + click_text("Submit") # 別點停用的按鈕 + +控制項以 ``name`` / ``role`` / ``app_name`` / ``automation_id`` 定位(與其他原生控制讀取相同)。 +``get_element_properties`` 回傳屬性字典,找不到控制項時回傳 ``None``;``is_element_enabled`` +回傳 ``enabled`` 旗標(找不到則為 ``None``)。 + +執行器指令 +---------- + +``AC_get_element_properties`` 回傳 ``{found, properties}``。以唯讀 +``ac_get_element_properties`` MCP 工具及 Script Builder 指令(位於 **Native UI** 分類下) +形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 4113e55c..6e0a0d26 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -68,6 +68,10 @@ ) # Realize off-screen items in virtualized lists / grids (UIA VirtualizedItem) from je_auto_control.utils.virtualized import realize_item +# Rich UIA element properties (enabled / offscreen / help / status / keys) +from je_auto_control.utils.ax_props import ( + get_element_properties, is_element_enabled, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1669,6 +1673,7 @@ def start_autocontrol_gui(*args, **kwargs): "assign_node_paths", "find_by_path", "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", "realize_item", + "get_element_properties", "is_element_enabled", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index c407d839..18746e54 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1621,6 +1621,11 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: ), description="Realize an off-screen item in a virtualized list/grid.", )) + specs.append(CommandSpec( + "AC_get_element_properties", "Native UI", "Get Element Properties", + fields=fields, + description="Read rich UIA props (enabled/offscreen/help/status/keys).", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 2f40c8af..9379c68b 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -148,6 +148,20 @@ def find_virtual_item(self, item_name: Optional[str] = None, by: str = "name", """ self._unsupported("find_virtual_item") + # --- rich element properties ------------------------------------------- + + def get_properties(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return rich UIA properties of the matched control, or None. + + Surfaces the high-value properties the flat element list omits — + ``enabled`` / ``offscreen`` / ``help_text`` / ``item_status`` / + ``accelerator_key`` / ``access_key`` / ``orientation``. + """ + self._unsupported("get_properties") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index 0d0594fc..a11394ae 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -298,6 +298,13 @@ def find_virtual_item(self, item_name=None, by="name", container_name=None, self._realize(found) return _convert_uia(found) + def get_properties(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + if not raw: + return None + return _read_properties(raw) + def _text_pattern(self, name, role, app_name, automation_id): """Find a control and return its IUIAutomationTextPattern, or None.""" raw = self._find_raw(name, role, app_name, automation_id) @@ -366,6 +373,33 @@ def _read_row(pattern, row: int, cols: int): return cells +def _as_text(value) -> str: + return str(value or "") + + +# (key, UIA element attribute, cast) for the rich properties the flat list omits. +_PROPERTY_READS = ( + ("enabled", "CurrentIsEnabled", bool), + ("offscreen", "CurrentIsOffscreen", bool), + ("help_text", "CurrentHelpText", _as_text), + ("item_status", "CurrentItemStatus", _as_text), + ("accelerator_key", "CurrentAcceleratorKey", _as_text), + ("access_key", "CurrentAccessKey", _as_text), + ("orientation", "CurrentOrientation", int), +) + + +def _read_properties(raw) -> Dict[str, Any]: + """Read the rich UIA properties of a raw element into a plain dict.""" + properties: Dict[str, Any] = {} + for key, attribute, cast in _PROPERTY_READS: + try: + properties[key] = cast(getattr(raw, attribute)) + except (OSError, AttributeError, ValueError, TypeError): + properties[key] = None + return properties + + def _convert_uia(raw) -> Optional[AccessibilityElement]: try: name = str(raw.CurrentName or "") diff --git a/je_auto_control/utils/ax_props/__init__.py b/je_auto_control/utils/ax_props/__init__.py new file mode 100644 index 00000000..9d3bcebb --- /dev/null +++ b/je_auto_control/utils/ax_props/__init__.py @@ -0,0 +1,6 @@ +"""Read rich UIA element properties (enabled / offscreen / help / status / keys).""" +from je_auto_control.utils.ax_props.ax_props import ( + get_element_properties, is_element_enabled, +) + +__all__ = ["get_element_properties", "is_element_enabled"] diff --git a/je_auto_control/utils/ax_props/ax_props.py b/je_auto_control/utils/ax_props/ax_props.py new file mode 100644 index 00000000..8e185325 --- /dev/null +++ b/je_auto_control/utils/ax_props/ax_props.py @@ -0,0 +1,38 @@ +"""Read rich UI Automation properties the flat element list omits. + +``list_accessibility_elements`` / ``AccessibilityElement`` carry only name / role / +bounds / app / pid / automation_id. Automation routinely needs more before it +acts: **is the control enabled** (don't click a disabled button), **is it +off-screen** (is it really visible?), its **item_status** (validation / error text +on a field), **help_text** (tooltip), and **accelerator_key** (drive it via a +hotkey instead of a click). ``ax_props`` exposes those high-value UIA properties. + +Each function is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam, so the headless core is +unit-testable on any platform by injecting a fake backend; the real UIA property +reads live in the Windows backend. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional + + +def get_element_properties(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return ``{enabled, offscreen, help_text, item_status, accelerator_key, + access_key, orientation}`` for the matched control, or None if not found.""" + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().get_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def is_element_enabled(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[bool]: + """Return whether the matched control is enabled (None if not found). + + The common guard before acting — don't click a disabled button. + """ + properties = get_element_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return properties.get("enabled") if properties else None diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index be846d81..487bf76d 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2476,6 +2476,16 @@ def _realize_item(item_name: str, by: str = "name", "element": element.to_dict() if element else None} +def _get_element_properties(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: read rich UIA properties (enabled/offscreen/help/status/keys).""" + from je_auto_control.utils.ax_props import get_element_properties + props = get_element_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return {"found": props is not None, "properties": props} + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6420,6 +6430,7 @@ def __init__(self): "AC_set_control_range": _set_control_range, "AC_scroll_control_into_view": _scroll_control_into_view, "AC_realize_item": _realize_item, + "AC_get_element_properties": _get_element_properties, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fb805544..6369d8d2 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1178,6 +1178,16 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.realize_item, annotations=READ_ONLY, ), + MCPTool( + name="ac_get_element_properties", + description=("Read rich UIA properties the flat list omits: " + "{found, properties:{enabled, offscreen, help_text, " + "item_status, accelerator_key, access_key, " + "orientation}}. Check 'enabled' before clicking."), + input_schema=schema(dict(_M)), + handler=h.get_element_properties, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index e9caff10..7942f130 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -802,6 +802,13 @@ def realize_item(item_name, by="name", container_name=None, container_role=None, app_name, automation_id) +def get_element_properties(name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import ( + _get_element_properties) + return _get_element_properties(name, role, app_name, automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/test/unit_test/headless/test_ax_props_batch.py b/test/unit_test/headless/test_ax_props_batch.py new file mode 100644 index 00000000..cf253988 --- /dev/null +++ b/test/unit_test/headless/test_ax_props_batch.py @@ -0,0 +1,93 @@ +"""Headless tests for rich UIA element property reads (fake backend seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.ax_props import ( + get_element_properties, is_element_enabled, +) + +_PROPS = {"enabled": False, "offscreen": True, "help_text": "Save the file", + "item_status": "invalid", "accelerator_key": "Ctrl+S", + "access_key": "S", "orientation": 0} + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, props=None): + self.props = props + self.calls = [] + + def get_properties(self, name=None, role=None, app_name=None, + automation_id=None): + self.calls.append({"name": name, "role": role}) + return self.props + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_get_element_properties_dispatch(monkeypatch): + fake = _FakeBackend(dict(_PROPS)) + _inject(monkeypatch, fake) + props = get_element_properties(name="Save", role="button") + assert props == _PROPS + assert fake.calls[0] == {"name": "Save", "role": "button"} + + +def test_get_element_properties_not_found(monkeypatch): + _inject(monkeypatch, _FakeBackend(None)) + assert get_element_properties(name="missing") is None + + +def test_is_element_enabled_reads_flag(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_PROPS))) + assert is_element_enabled(name="Save") is False + _inject(monkeypatch, _FakeBackend({"enabled": True})) + assert is_element_enabled(name="OK") is True + + +def test_is_element_enabled_none_when_not_found(monkeypatch): + _inject(monkeypatch, _FakeBackend(None)) + assert is_element_enabled(name="x") is None + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) # all _unsupported + try: + get_element_properties(name="x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_wraps_props(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_PROPS))) + from je_auto_control.utils.executor.action_executor import ( + _get_element_properties) + out = _get_element_properties(name="Save") + assert out["found"] is True + assert out["properties"]["accelerator_key"] == "Ctrl+S" + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_get_element_properties" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_get_element_properties" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_get_element_properties" in specs + + +def test_facade_exports(): + for name in ("get_element_properties", "is_element_enabled"): + assert hasattr(ac, name) and name in ac.__all__