diff --git a/WHATS_NEW.md b/WHATS_NEW.md index d7bf15b2..ce781384 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Trial and Force Action Modes (Playwright-style) + +Dry-run "is this control ready?" without clicking, or force a click past the gate. Full reference: [`docs/source/Eng/doc/new_features/v222_features_doc.rst`](docs/source/Eng/doc/new_features/v222_features_doc.rst). + +- **`act_with_mode`** (`AC_act_with_mode`): `actionability.act_when_ready` only waits-then-acts. Real flows need two more modes Playwright codified: **trial** (run every actionability check but *don't* act — a side-effect-free "would this work?" dry run) and **force** (skip the checks and act now — the escape hatch when the gate misjudges a control as occluded/disabled). `act_with_mode` adds both alongside the default `auto`, over the same injectable seams as the gate, returning `{mode, acted, actionable, reason, point, result}`. Reuses `actionability.wait_actionable`; fully testable without a screen. Completes the ROUND-15 input-fidelity lane (7/7). No `PySide6`. + ### Act In View — Scroll to a Target, Then Act When Actionable Click the row three pages down: scroll it into view, then gate on actionability before clicking. Full reference: [`docs/source/Eng/doc/new_features/v221_features_doc.rst`](docs/source/Eng/doc/new_features/v221_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v222_features_doc.rst b/docs/source/Eng/doc/new_features/v222_features_doc.rst new file mode 100644 index 00000000..820a9a4b --- /dev/null +++ b/docs/source/Eng/doc/new_features/v222_features_doc.rst @@ -0,0 +1,49 @@ +Trial and Force Action Modes (Playwright-style) +=============================================== + +``actionability.act_when_ready`` has one behaviour: wait for the target to be +actionable, then act (or raise on timeout). Real flows need two more modes that +Playwright codified: + +* **trial** — run every actionability check but *don't* perform the action; just + report whether it *would* have acted. The dry run for "is this control ready?" + without side effects. +* **force** — skip the checks and act *now*, the deliberate escape hatch when the + gate is wrong (a control the heuristics misjudge as occluded / disabled). + +:func:`act_with_mode` adds both alongside the default gated (``auto``) behaviour, +over the same injectable seams as the gate, so each mode is testable without a +screen. Reuses :func:`actionability.wait_actionable`. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import act_with_mode + + bbox = lambda: (x, y, w, h) + click = lambda point: do_click(point[0], point[1]) + + act_with_mode(click, bbox, mode="auto") # gate, then click if ready + report = act_with_mode(click, bbox, mode="trial") # dry run, never clicks + if report["actionable"]: + ... + act_with_mode(click, bbox, mode="force") # click now, no checks + +Every mode returns ``{mode, acted, actionable, reason, point, result}``: +``acted`` says whether the action ran, ``actionable`` / ``reason`` come from the +gate (``trial`` reports these without acting), and ``result`` is the action's +return value. The actionability probes (``region_sampler`` / ``enabled_probe`` / +``hit_tester``) and ``config`` are forwarded to the gate as usual. An unknown +``mode`` raises ``ValueError``. + +Executor commands +----------------- + +``AC_act_with_mode`` (``x`` / ``y`` + ``mode`` / ``button`` → ``{mode, acted, +actionable, reason, point}``) clicks a point under the chosen mode — ``trial`` +is a dry-run probe that never clicks, ``force`` clicks unconditionally. It is the +matching ``ac_act_with_mode`` MCP tool and a Script Builder command under +**Flow**. :func:`act_with_mode` (which takes an arbitrary action) is the +Python-API surface. diff --git a/docs/source/Zh/doc/new_features/v222_features_doc.rst b/docs/source/Zh/doc/new_features/v222_features_doc.rst new file mode 100644 index 00000000..3a82d3d8 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v222_features_doc.rst @@ -0,0 +1,41 @@ +試行與強制動作模式(Playwright 風格) +===================================== + +``actionability.act_when_ready`` 只有一種行為:等待目標可操作,再動作(或逾時丟例外)。真實流程還需要 +Playwright 定義的另外兩種模式: + +* **trial(試行)**——執行每一項 actionability 檢查,但*不*真正動作;只回報它*是否會*動作。 + 「這個控制項準備好了嗎?」的無副作用乾跑。 +* **force(強制)**——跳過檢查,*立即*動作;當閘控判斷錯誤(把控制項誤判為被遮擋 / 停用)時的刻意逃生口。 + +:func:`act_with_mode` 在預設的閘控(``auto``)行為之外加上這兩種,使用與閘控相同的可注入接縫, +故每種模式都能在沒有螢幕的情況下測試。重用 :func:`actionability.wait_actionable`。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import act_with_mode + + bbox = lambda: (x, y, w, h) + click = lambda point: do_click(point[0], point[1]) + + act_with_mode(click, bbox, mode="auto") # 閘控後若就緒則點擊 + report = act_with_mode(click, bbox, mode="trial") # 乾跑,絕不點擊 + if report["actionable"]: + ... + act_with_mode(click, bbox, mode="force") # 立即點擊,不檢查 + +每種模式皆回傳 ``{mode, acted, actionable, reason, point, result}``:``acted`` 表示動作是否執行, +``actionable`` / ``reason`` 來自閘控(``trial`` 不動作即回報這些),``result`` 為 action 的回傳值。 +actionability 探針(``region_sampler`` / ``enabled_probe`` / ``hit_tester``)與 ``config`` 一如往常轉發給閘控。 +未知的 ``mode`` 會丟出 ``ValueError``。 + +執行器指令 +---------- + +``AC_act_with_mode``(``x`` / ``y`` 加上 ``mode`` / ``button`` → ``{mode, acted, +actionable, reason, point}``)以所選模式點擊一個點——``trial`` 是絕不點擊的乾跑探測,``force`` 無條件點擊。 +以對應的 ``ac_act_with_mode`` MCP 工具及 Script Builder 指令(位於 **Flow** 分類下)形式提供。 +:func:`act_with_mode`(接受任意 action)則是 Python API 介面。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 7817f77d..84c51200 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -153,6 +153,8 @@ from je_auto_control.utils.element_proposal import propose_elements, tag_kinds # Scroll a target into view, then act on it once it is actionable from je_auto_control.utils.act_in_view import ScrollPlan, act_in_view +# Trial / force action modes over the actionability gate +from je_auto_control.utils.act_modes import act_with_mode # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1784,7 +1786,7 @@ def start_autocontrol_gui(*args, **kwargs): "localize_changes", "rank_changes", "classify_widget", "box_features", "classify_icon", "propose_elements", "tag_kinds", - "act_in_view", "ScrollPlan", + "act_in_view", "ScrollPlan", "act_with_mode", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 29885136..be1377da 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4539,6 +4539,18 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Scroll a target into view, then click it when actionable.", )) + specs.append(CommandSpec( + "AC_act_with_mode", "Flow", "Click with Mode (auto/trial/force)", + fields=( + FieldSpec("x", FieldType.INT, placeholder="x"), + FieldSpec("y", FieldType.INT, placeholder="y"), + FieldSpec("mode", FieldType.STRING, optional=True, default="auto", + placeholder="auto / trial / force"), + FieldSpec("button", FieldType.STRING, optional=True, + default="left"), + ), + description="Click a point under an action mode (gate / dry-run / force).", + )) specs.append(CommandSpec( "AC_simulate_cvd", "Image", "Simulate Colour-Vision Deficiency", fields=( diff --git a/je_auto_control/utils/act_modes/__init__.py b/je_auto_control/utils/act_modes/__init__.py new file mode 100644 index 00000000..30067368 --- /dev/null +++ b/je_auto_control/utils/act_modes/__init__.py @@ -0,0 +1,4 @@ +"""Trial and force action modes over the actionability gate.""" +from je_auto_control.utils.act_modes.act_modes import ACT_MODES, act_with_mode + +__all__ = ["act_with_mode", "ACT_MODES"] diff --git a/je_auto_control/utils/act_modes/act_modes.py b/je_auto_control/utils/act_modes/act_modes.py new file mode 100644 index 00000000..ed370fb9 --- /dev/null +++ b/je_auto_control/utils/act_modes/act_modes.py @@ -0,0 +1,63 @@ +"""Trial and force action modes over the actionability gate (Playwright-style). + +``actionability.act_when_ready`` has one behaviour: wait for the target to be +actionable, then act (or raise on timeout). Real flows need two more modes that +Playwright codified: + +* **trial** — run every actionability check but *don't* perform the action; just + report whether it *would* have acted. The dry-run for "is this control ready?" + without side effects. +* **force** — skip the checks and act *now*, for the deliberate escape hatch when + the gate is wrong (a control the heuristics misjudge as occluded / disabled). + +``act_with_mode`` adds both alongside the default gated behaviour, over the same +injectable seams as the gate, so each mode is testable without a screen. Reuses +:func:`actionability.wait_actionable`. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional + +from je_auto_control.utils.actionability import wait_actionable +from je_auto_control.utils.actionability.actionability import _center + +# The supported action modes. +ACT_MODES = ("auto", "trial", "force") + + +def _force_act(action: Callable[[List[int]], Any], + bbox_provider: Callable[[], Any]) -> Dict[str, Any]: + """Act at the target centre with no actionability checks.""" + bbox = bbox_provider() + if not bbox: + return {"mode": "force", "acted": False, "actionable": True, + "reason": "no target", "point": None, "result": None} + point = _center(bbox) + return {"mode": "force", "acted": True, "actionable": True, + "reason": "forced", "point": point, "result": action(point)} + + +def act_with_mode(action: Callable[[List[int]], Any], + bbox_provider: Callable[[], Any], *, mode: str = "auto", + region_sampler: Optional[Callable[[Any], Any]] = None, + enabled_probe: Optional[Callable[[], Optional[bool]]] = None, + hit_tester: Optional[Callable[[List[int]], bool]] = None, + config: Optional[Any] = None) -> Dict[str, Any]: + """Perform ``action`` on a target under a ``mode`` (auto / trial / force). + + ``auto`` waits for the actionability gate and acts only if it passes; + ``trial`` runs the gate but never acts (a dry run); ``force`` acts at once + with no checks. Returns ``{mode, acted, actionable, reason, point, result}``. + Raises ``ValueError`` for an unknown ``mode``. + """ + if mode not in ACT_MODES: + raise ValueError(f"unknown act mode: {mode!r}") + if mode == "force": + return _force_act(action, bbox_provider) + report = wait_actionable(bbox_provider, region_sampler=region_sampler, + enabled_probe=enabled_probe, hit_tester=hit_tester, + config=config) + point = list(report.point) if report.point is not None else None + base = {"mode": mode, "actionable": report.actionable, + "reason": report.reason, "point": point} + if mode == "trial" or not report.actionable: + return {**base, "acted": False, "result": None} + return {**base, "acted": True, "result": action(report.point)} diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 4f566c17..639a3acf 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2979,6 +2979,18 @@ def _act_in_view(target: Any, kind: Any = "image", direction: Any = "down", "scrolls": out["scrolls"]} +def _act_with_mode(x: Any, y: Any, mode: Any = "auto", + button: Any = "left") -> Dict[str, Any]: + """Adapter: click a point under an action mode (auto / trial / force).""" + from je_auto_control.utils.act_modes import act_with_mode + out = act_with_mode( + lambda point: click_mouse(str(button), int(point[0]), int(point[1])), + lambda: (int(x), int(y), 1, 1), mode=str(mode)) + return {"mode": out["mode"], "acted": out["acted"], + "actionable": out["actionable"], "reason": out["reason"], + "point": out["point"]} + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -7025,6 +7037,7 @@ def __init__(self): "AC_propose_elements": _propose_elements, "AC_tag_kinds": _tag_kinds, "AC_act_in_view": _act_in_view, + "AC_act_with_mode": _act_with_mode, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index e2910c74..a5130fc9 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1913,6 +1913,20 @@ def smart_wait_tools() -> List[MCPTool]: handler=h.act_in_view, annotations=SIDE_EFFECT_ONLY, ), + MCPTool( + name="ac_act_with_mode", + description=("Click point (x, y) under an action 'mode': 'auto' " + "(gate then click), 'trial' (gate but DON'T click — " + "dry run), or 'force' (click with no checks). Returns " + "{mode, acted, actionable, reason, point}."), + input_schema=schema({"x": {"type": "integer"}, + "y": {"type": "integer"}, + "mode": {"type": "string"}, + "button": {"type": "string"}}, + required=["x", "y"]), + handler=h.act_with_mode, + annotations=SIDE_EFFECT_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 4eadd1d0..5fdc6f29 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -816,6 +816,11 @@ def act_in_view(target, kind="image", direction="down", max_scrolls=10, button) +def act_with_mode(x, y, mode="auto", button="left"): + from je_auto_control.utils.executor.action_executor import _act_with_mode + return _act_with_mode(x, y, mode, button) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/test/unit_test/headless/test_act_modes_batch.py b/test/unit_test/headless/test_act_modes_batch.py new file mode 100644 index 00000000..2cfc5c0b --- /dev/null +++ b/test/unit_test/headless/test_act_modes_batch.py @@ -0,0 +1,102 @@ +"""Headless tests for act_with_mode (trial / force / auto over the gate).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.act_modes import ACT_MODES, act_with_mode +from je_auto_control.utils.actionability import GateConfig + + +def _gate_config(): + """A GateConfig whose clock advances so the gate can time out / poll.""" + ticks = iter([float(t) for t in range(0, 40)]) + return GateConfig(timeout_s=3.0, stable_for_s=0.0, poll_interval_s=1.0, + clock=lambda: next(ticks), sleep=lambda _s: None) + + +# --- force ---------------------------------------------------------------- + +def test_force_acts_without_any_checks(): + clicked = [] + # enabled_probe says disabled, but force ignores the gate entirely + out = act_with_mode(clicked.append, lambda: (10, 20, 4, 4), mode="force", + enabled_probe=lambda: False) + assert out["mode"] == "force" + assert out["acted"] is True + assert out["point"] == [12, 22] # centre of (10,20,4,4) + assert clicked == [[12, 22]] + + +def test_force_no_target_does_not_act(): + clicked = [] + out = act_with_mode(clicked.append, lambda: None, mode="force") + assert out["acted"] is False + assert clicked == [] + + +# --- trial ---------------------------------------------------------------- + +def test_trial_reports_but_never_acts(): + clicked = [] + out = act_with_mode(clicked.append, lambda: (0, 0, 2, 2), mode="trial", + config=_gate_config()) + assert out["mode"] == "trial" + assert out["acted"] is False # dry run: gate ran, no action + assert out["actionable"] is True + assert clicked == [] # never clicked + + +def test_trial_reports_not_actionable_without_acting(): + clicked = [] + out = act_with_mode(clicked.append, lambda: None, mode="trial", + config=_gate_config()) # no bbox -> not visible + assert out["acted"] is False + assert out["actionable"] is False + assert out["reason"] == "not visible" + assert clicked == [] + + +# --- auto ----------------------------------------------------------------- + +def test_auto_acts_when_actionable(): + clicked = [] + out = act_with_mode(clicked.append, lambda: (5, 5, 2, 2), mode="auto", + config=_gate_config()) + assert out["acted"] is True + assert clicked == [[6, 6]] # centre of (5,5,2,2) + + +def test_auto_does_not_act_when_gate_times_out(): + clicked = [] + out = act_with_mode(clicked.append, lambda: None, mode="auto", + config=_gate_config()) # never visible -> timeout + assert out["acted"] is False + assert out["actionable"] is False + assert clicked == [] + + +def test_unknown_mode_raises(): + with pytest.raises(ValueError): + act_with_mode(lambda p: None, lambda: (0, 0, 1, 1), mode="bogus") + + +def test_act_modes_constant(): + assert set(ACT_MODES) == {"auto", "trial", "force"} + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_act_with_mode" in known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert "ac_act_with_mode" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_act_with_mode" in specs + + +def test_facade_exports(): + assert hasattr(ac, "act_with_mode") and "act_with_mode" in ac.__all__