diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 41e34e20..22a8444d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — MSAA Bridge for Legacy Controls (LegacyIAccessible) + +Automate the long tail of old Win32 controls that expose nothing via modern UIA. Full reference: [`docs/source/Eng/doc/new_features/v199_features_doc.rst`](docs/source/Eng/doc/new_features/v199_features_doc.rst). + +- **`legacy_info` / `legacy_default_action`** (`AC_legacy_info`, `AC_legacy_default_action`): many legacy Win32 / MFC / Delphi controls expose nothing useful via modern UIA patterns (`control_get_value` / `control_invoke` / `control_toggle` all return None), yet they're fully described through the MSAA `IAccessible` bridge — Name, Value, Description, Role, State and a **DefaultAction**. This reads that info and fires the default action via `LegacyIAccessiblePattern` — the last-resort fallback that makes old apps automatable. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Move / Resize Elements + Window State (UIA Transform + Window) Move a floating panel, resize a control, and know if a window is modal-blocked. Full reference: [`docs/source/Eng/doc/new_features/v198_features_doc.rst`](docs/source/Eng/doc/new_features/v198_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v199_features_doc.rst b/docs/source/Eng/doc/new_features/v199_features_doc.rst new file mode 100644 index 00000000..15f7de18 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v199_features_doc.rst @@ -0,0 +1,43 @@ +MSAA Bridge for Legacy Controls (LegacyIAccessible) +=================================================== + +Many legacy Win32 / MFC / Delphi controls expose **nothing useful** via the modern +UI Automation patterns — ``control_get_value`` / ``control_invoke`` / +``control_toggle`` all return None or do nothing — yet they are fully described +through the MSAA ``IAccessible`` bridge: a Name, Value, Description, Role, State +and a **DefaultAction**. ``legacy_accessible`` is the last-resort fallback that +still reads that info and fires the default action, making the long tail of old +apps automatable. + +* :func:`legacy_info` — the MSAA fields ``{name, value, description, + default_action, role, state}``, +* :func:`legacy_default_action` — fire the control's default action. + +Each is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam — headless-testable via a fake backend; the real ``LegacyIAccessiblePattern`` +calls live in the Windows backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (legacy_info, legacy_default_action, + control_invoke) + + # Modern patterns came up empty? Fall back to MSAA: + if not control_invoke(name="Apply"): + info = legacy_info(name="Apply") # {"default_action": "Press", ...} + legacy_default_action(name="Apply") # fires the MSAA default action + +The control is located by ``name`` / ``role`` / ``app_name`` / ``automation_id`` +(same as the other native-control actions). ``legacy_info`` returns the MSAA info +dict (``role`` / ``state`` are the raw MSAA numbers) or ``None`` if the control or +pattern isn't found; ``legacy_default_action`` returns ``bool``. + +Executor commands +----------------- + +``AC_legacy_info`` (``{found, info}``) and ``AC_legacy_default_action``. They are +exposed as the matching ``ac_*`` MCP tools (info read-only, the action +destructive) and as Script Builder commands under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v199_features_doc.rst b/docs/source/Zh/doc/new_features/v199_features_doc.rst new file mode 100644 index 00000000..ea5d746d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v199_features_doc.rst @@ -0,0 +1,38 @@ +舊式控制項的 MSAA 橋接(LegacyIAccessible) +========================================== + +許多舊式 Win32 / MFC / Delphi 控制項透過現代 UI Automation 模式**完全不提供有用資訊**—— +``control_get_value`` / ``control_invoke`` / ``control_toggle`` 都回 None 或毫無作用——但它們透過 +MSAA ``IAccessible`` 橋接卻有完整描述:Name、Value、Description、Role、State,以及一個 +**DefaultAction**。``legacy_accessible`` 就是那個最後手段的後備:仍能讀取這些資訊並觸發預設動作, +讓大量舊應用程式得以自動化。 + +* :func:`legacy_info` ——MSAA 欄位 ``{name, value, description, default_action, role, state}``, +* :func:`legacy_default_action` ——觸發控制項的預設動作。 + +每個都是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派——可透過注入 fake +backend 進行無頭測試;真正的 ``LegacyIAccessiblePattern`` 呼叫位於 Windows 後端。不匯入 +``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (legacy_info, legacy_default_action, + control_invoke) + + # 現代模式撲空?退回 MSAA: + if not control_invoke(name="Apply"): + info = legacy_info(name="Apply") # {"default_action": "Press", ...} + legacy_default_action(name="Apply") # 觸發 MSAA 預設動作 + +控制項以 ``name`` / ``role`` / ``app_name`` / ``automation_id`` 定位(與其他原生控制動作相同)。 +``legacy_info`` 回傳 MSAA 資訊字典(``role`` / ``state`` 為原始 MSAA 數字),找不到控制項或模式 +則回傳 ``None``;``legacy_default_action`` 回傳 ``bool``。 + +執行器指令 +---------- + +``AC_legacy_info``(``{found, info}``)與 ``AC_legacy_default_action``。皆以對應的 ``ac_*`` MCP +工具(info 為唯讀、動作為破壞性)及 Script Builder 指令(位於 **Native UI** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 88c0d772..19631fe4 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -80,6 +80,10 @@ from je_auto_control.utils.transform_window import ( move_element, resize_element, set_window_state, window_interaction_state, ) +# MSAA bridge for old controls UIA can't model (LegacyIAccessiblePattern) +from je_auto_control.utils.legacy_accessible import ( + legacy_default_action, legacy_info, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1685,6 +1689,7 @@ def start_autocontrol_gui(*args, **kwargs): "table_headers", "table_cell", "cell_by_header", "move_element", "resize_element", "set_window_state", "window_interaction_state", + "legacy_info", "legacy_default_action", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index b09e34ab..4b3ed68b 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1666,6 +1666,16 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: fields=fields, description="Read window readiness (ready/blocked_by_modal/...).", )) + specs.append(CommandSpec( + "AC_legacy_info", "Native UI", "Legacy (MSAA) Info", + fields=fields, + description="Read an old control's MSAA info (LegacyIAccessible).", + )) + specs.append(CommandSpec( + "AC_legacy_default_action", "Native UI", "Legacy (MSAA) Default Action", + fields=fields, + description="Fire an old control's MSAA default action (fallback).", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index dc3a6722..586e1ddc 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -213,6 +213,28 @@ def window_interaction_state(self, name: Optional[str] = None, ``not_responding`` / ``running`` / ``closing`` (WindowPattern), or None.""" self._unsupported("window_interaction_state") + # --- MSAA bridge (LegacyIAccessiblePattern) ---------------------------- + + def legacy_info(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return the MSAA ``IAccessible`` info of an old control, or None. + + ``{name, value, description, default_action, role, state}`` — the + last-resort read for legacy Win32 controls that expose nothing useful via + the modern UIA patterns. + """ + self._unsupported("legacy_info") + + def legacy_default_action(self, name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Fire an old control's MSAA default action (DoDefaultAction); True on + success — the fallback when Value / Invoke / Toggle all do nothing.""" + self._unsupported("legacy_default_action") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index 8df71c30..a606c155 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -36,6 +36,7 @@ _UIA_GRIDITEM_PATTERN_ID = 10007 _UIA_TRANSFORM_PATTERN_ID = 10016 _UIA_WINDOW_PATTERN_ID = 10009 +_UIA_LEGACYIACCESSIBLE_PATTERN_ID = 10018 _UIA_AUTOMATIONID_PROPERTY = 30011 _EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} _WINDOW_VISUAL_STATES = {"normal": 0, "maximized": 1, "minimized": 2} @@ -351,6 +352,24 @@ def window_interaction_state(self, name=None, role=None, app_name=None, except (OSError, AttributeError, ValueError, TypeError): return None + def legacy_info(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + pattern = self._pattern(raw, _UIA_LEGACYIACCESSIBLE_PATTERN_ID, + "IUIAutomationLegacyIAccessiblePattern" + ) if raw else None + if pattern is None: + return None + return _read_legacy(pattern) + + def legacy_default_action(self, name=None, role=None, app_name=None, + automation_id=None): + return self._invoke_pattern_method( + name, role, app_name, automation_id, + _UIA_LEGACYIACCESSIBLE_PATTERN_ID, + "IUIAutomationLegacyIAccessiblePattern", + lambda pattern: pattern.DoDefaultAction()) + def get_table_headers(self, name=None, role=None, app_name=None, automation_id=None) -> Optional[Dict[str, Any]]: raw = self._find_raw(name, role, app_name, automation_id) @@ -493,6 +512,28 @@ def _as_text(value) -> str: return str(value or "") +# (key, LegacyIAccessiblePattern attribute, cast) for the MSAA bridge read. +_LEGACY_READS = ( + ("name", "CurrentName", _as_text), + ("value", "CurrentValue", _as_text), + ("description", "CurrentDescription", _as_text), + ("default_action", "CurrentDefaultAction", _as_text), + ("role", "CurrentRole", int), + ("state", "CurrentState", int), +) + + +def _read_legacy(pattern) -> Dict[str, Any]: + """Read a LegacyIAccessiblePattern's MSAA fields into a plain dict.""" + info: Dict[str, Any] = {} + for key, attribute, cast in _LEGACY_READS: + try: + info[key] = cast(getattr(pattern, attribute)) + except (OSError, AttributeError, ValueError, TypeError): + info[key] = None + return info + + # (key, UIA element attribute, cast) for the rich properties the flat list omits. _PROPERTY_READS = ( ("enabled", "CurrentIsEnabled", bool), diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cfb7ecb3..35072adf 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2554,6 +2554,25 @@ def _window_interaction_state(name: Optional[str] = None, automation_id=automation_id)} +def _legacy_info(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: MSAA IAccessible info of an old control (LegacyIAccessible).""" + from je_auto_control.utils.legacy_accessible import legacy_info + info = legacy_info(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return {"found": info is not None, "info": info} + + +def _legacy_default_action(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Adapter: fire an old control's MSAA default action (Value/Invoke fallback).""" + from je_auto_control.utils.legacy_accessible import legacy_default_action + return legacy_default_action(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6506,6 +6525,8 @@ def __init__(self): "AC_resize_element": _resize_element, "AC_set_window_state": _set_window_state, "AC_window_interaction_state": _window_interaction_state, + "AC_legacy_info": _legacy_info, + "AC_legacy_default_action": _legacy_default_action, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/legacy_accessible/__init__.py b/je_auto_control/utils/legacy_accessible/__init__.py new file mode 100644 index 00000000..bb7b5a5a --- /dev/null +++ b/je_auto_control/utils/legacy_accessible/__init__.py @@ -0,0 +1,6 @@ +"""MSAA bridge for old controls UIA can't model (LegacyIAccessiblePattern).""" +from je_auto_control.utils.legacy_accessible.legacy_accessible import ( + legacy_default_action, legacy_info, +) + +__all__ = ["legacy_info", "legacy_default_action"] diff --git a/je_auto_control/utils/legacy_accessible/legacy_accessible.py b/je_auto_control/utils/legacy_accessible/legacy_accessible.py new file mode 100644 index 00000000..16c4ccfd --- /dev/null +++ b/je_auto_control/utils/legacy_accessible/legacy_accessible.py @@ -0,0 +1,39 @@ +"""MSAA bridge for old controls UIA can't model (LegacyIAccessiblePattern). + +Many legacy Win32 / MFC / Delphi controls expose **nothing useful** via the modern +UIA patterns — ``control_get_value`` / ``control_invoke`` / ``control_toggle`` all +return None / do nothing — yet they are fully described through the MSAA +``IAccessible`` bridge: a Name, Value, Description, Role, State and a +**DefaultAction**. ``legacy_accessible`` is the last-resort fallback that still +reads that info and fires the default action, making the long tail of old apps +automatable. + +* :func:`legacy_info` — the MSAA fields ``{name, value, description, + default_action, role, state}``, +* :func:`legacy_default_action` — fire the control's default action. + +Each is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam — headless-testable via a fake backend; the real ``LegacyIAccessiblePattern`` +calls live in the Windows backend. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional + + +def legacy_info(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return the MSAA ``IAccessible`` info of an old control, or None if absent.""" + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().legacy_info(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def legacy_default_action(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + """Fire an old control's MSAA default action (the Value/Invoke/Toggle fallback).""" + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().legacy_default_action(name=name, role=role, + app_name=app_name, + automation_id=automation_id) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 36af99dd..805fa98e 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1262,6 +1262,27 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.window_interaction_state, annotations=READ_ONLY, ), + MCPTool( + name="ac_legacy_info", + description=("Read an old control's MSAA IAccessible info via " + "LegacyIAccessiblePattern: {found, info:{name, value, " + "description, default_action, role, state}}. The " + "last-resort read for legacy Win32 controls UIA can't " + "model."), + input_schema=schema(dict(_M)), + handler=h.legacy_info, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_legacy_default_action", + description=("Fire an old control's MSAA default action " + "(LegacyIAccessible.DoDefaultAction) — the fallback when " + "Value/Invoke/Toggle all do nothing. Returns True on " + "success."), + input_schema=schema(dict(_M)), + handler=h.legacy_default_action, + annotations=DESTRUCTIVE, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index b3536bbc..6b6e2538 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -851,6 +851,18 @@ def window_interaction_state(name=None, role=None, app_name=None, return _window_interaction_state(name, role, app_name, automation_id) +def legacy_info(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _legacy_info + return _legacy_info(name, role, app_name, automation_id) + + +def legacy_default_action(name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import ( + _legacy_default_action) + return _legacy_default_action(name, role, app_name, automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/test/unit_test/headless/test_legacy_accessible_batch.py b/test/unit_test/headless/test_legacy_accessible_batch.py new file mode 100644 index 00000000..e0b59c9b --- /dev/null +++ b/test/unit_test/headless/test_legacy_accessible_batch.py @@ -0,0 +1,87 @@ +"""Headless tests for the MSAA bridge (LegacyIAccessiblePattern, fake backend).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.legacy_accessible import ( + legacy_default_action, legacy_info, +) + +_INFO = {"name": "OK", "value": "", "description": "Accept the dialog", + "default_action": "Press", "role": 43, "state": 1048576} + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, info=None): + self.info = info + self.actions = [] + + def legacy_info(self, name=None, role=None, app_name=None, automation_id=None): + return self.info + + def legacy_default_action(self, name=None, role=None, app_name=None, + automation_id=None): + self.actions.append(name) + return True + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_legacy_info_dispatch(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_INFO))) + assert legacy_info(name="OK", role="button") == _INFO + + +def test_legacy_info_not_found(monkeypatch): + _inject(monkeypatch, _FakeBackend(None)) + assert legacy_info(name="missing") is None + + +def test_legacy_default_action(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + assert legacy_default_action(name="OK") is True + assert fake.actions == ["OK"] + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) + try: + legacy_info(name="x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapters(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_INFO))) + from je_auto_control.utils.executor.action_executor import ( + _legacy_default_action, _legacy_info) + out = _legacy_info(name="OK") + assert out["found"] is True and out["info"]["default_action"] == "Press" + assert _legacy_default_action(name="OK") is True + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_legacy_info", "AC_legacy_default_action"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_legacy_info", "ac_legacy_default_action"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_legacy_info", "AC_legacy_default_action"} <= specs + + +def test_facade_exports(): + for name in ("legacy_info", "legacy_default_action"): + assert hasattr(ac, name) and name in ac.__all__