From 82d034461a273199e02c9c61a29215134e0decd9 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 07:03:58 +0800 Subject: [PATCH 1/3] Add match_ensemble: multi-template consensus matching A button renders in several states (default/hover/pressed) but is one logical target; ab_locator picks one strategy and match_template(scales=) sweeps one template - neither fuses multiple references. Match each reference, cluster the hit centres, and accept only when >= min_votes agree within agree_px, returning the consensus point. Reuses visual_match.match_template + grounding_consensus; vote_centers is the pure voting core. --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v178_features_doc.rst | 43 ++++++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v178_features_doc.rst | 39 +++++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 24 ++++++++ .../utils/executor/action_executor.py | 28 +++++++++ .../utils/match_ensemble/__init__.py | 6 ++ .../utils/match_ensemble/match_ensemble.py | 57 +++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 32 +++++++++++ .../utils/mcp_server/tools/_handlers.py | 10 ++++ .../headless/test_match_ensemble_batch.py | 55 ++++++++++++++++++ 15 files changed, 320 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v178_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v178_features_doc.rst create mode 100644 je_auto_control/utils/match_ensemble/__init__.py create mode 100644 je_auto_control/utils/match_ensemble/match_ensemble.py create mode 100644 test/unit_test/headless/test_match_ensemble_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 04613662..6830012a 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 多模板共识匹配 + +把同一目标的多个参考裁切投票成单一可信位置。完整参考:[`docs/source/Zh/doc/new_features/v178_features_doc.rst`](../docs/source/Zh/doc/new_features/v178_features_doc.rst)。 + +- **`match_ensemble` / `vote_centers`**(`AC_match_ensemble`、`AC_vote_centers`):一个按钮以多种状态呈现(默认/悬停/按下)但是单一逻辑目标;`ab_locator` 只选一个策略、`match_template(scales=...)` 只扫一个模板——两者都不融合多参考。本功能匹配每个参考,聚类命中中心,只有在 ≥ `min_votes` 个于 `agree_px` 内一致时才接受,返回 `{point, votes, n_candidates, spread}`——减少换肤/动画 UI 的误判。重用 `visual_match.match_template` + `grounding_consensus`;`vote_centers` 为纯投票核心。不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 逐步评审特征 + 规则式步骤评分 把为代理步骤评分所需的证据打包,并内建规则式评分器。完整参考:[`docs/source/Zh/doc/new_features/v177_features_doc.rst`](../docs/source/Zh/doc/new_features/v177_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index a2cf33fd..ad80bb93 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 多模板共識比對 + +把同一目標的多個參考裁切投票成單一可信位置。完整參考:[`docs/source/Zh/doc/new_features/v178_features_doc.rst`](../docs/source/Zh/doc/new_features/v178_features_doc.rst)。 + +- **`match_ensemble` / `vote_centers`**(`AC_match_ensemble`、`AC_vote_centers`):一個按鈕以多種狀態呈現(預設/懸停/按下)但是單一邏輯目標;`ab_locator` 只選一個策略、`match_template(scales=...)` 只掃一個模板——兩者都不融合多參考。本功能比對每個參考,聚類命中中心,只有在 ≥ `min_votes` 個於 `agree_px` 內一致時才接受,回傳 `{point, votes, n_candidates, spread}`——減少換膚/動畫 UI 的誤判。重用 `visual_match.match_template` + `grounding_consensus`;`vote_centers` 為純投票核心。不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 逐步評審特徵 + 規則式步驟評分 把為代理步驟評分所需的證據打包,並內建規則式評分器。完整參考:[`docs/source/Zh/doc/new_features/v177_features_doc.rst`](../docs/source/Zh/doc/new_features/v177_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index ed2d61ce..0fcbf971 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Multi-Template Consensus Matching + +Vote several reference crops of one target into a single trustworthy location. Full reference: [`docs/source/Eng/doc/new_features/v178_features_doc.rst`](docs/source/Eng/doc/new_features/v178_features_doc.rst). + +- **`match_ensemble` / `vote_centers`** (`AC_match_ensemble`, `AC_vote_centers`): a button renders in several states (default/hover/pressed) but is one logical target; `ab_locator` picks one strategy and `match_template(scales=...)` sweeps one template — neither fuses multiple references. This matches each reference, clusters the hit centres, and accepts a target only when ≥ `min_votes` agree within `agree_px`, returning `{point, votes, n_candidates, spread}` — cutting false positives on themed/animated UI. Reuses `visual_match.match_template` + `grounding_consensus`; `vote_centers` is the pure voting core. No `PySide6`. + ## What's new (2026-06-24) — Per-Step Critic Features + Rule-Based Step Scorer Bundle the evidence to score an agent step, with a built-in rule-based scorer. Full reference: [`docs/source/Eng/doc/new_features/v177_features_doc.rst`](docs/source/Eng/doc/new_features/v177_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v178_features_doc.rst b/docs/source/Eng/doc/new_features/v178_features_doc.rst new file mode 100644 index 00000000..b3436a75 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v178_features_doc.rst @@ -0,0 +1,43 @@ +Multi-Template Consensus Matching +================================= + +A button often renders in several states — default / hover / pressed / disabled — yet it is a +single logical target. ``ab_locator`` A/B-tests *which single strategy wins* and +``match_template(scales=...)`` sweeps *one* template across scales — neither fuses *multiple +reference crops* into one vote. ``match_ensemble`` matches each reference, clusters the hit +centres and accepts a target only when at least ``min_votes`` references agree within +``agree_px`` — sharply cutting false positives on themed / animated UI. + +The voting core (``vote_centers``) is pure-stdlib and reuses ``grounding_consensus`` for the +clustering, so it is unit-testable with no image; only ``match_ensemble`` itself calls +``visual_match.match_template`` (testable on a synthetic injected ``haystack``). Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import match_ensemble, vote_centers + + # three reference crops of the same button (default / hover / pressed) + result = match_ensemble(["btn_default.png", "btn_hover.png", "btn_pressed.png"], + min_score=0.8, agree_px=10, min_votes=2) + if result: + click(*result["point"]) # {point, votes, n_candidates, spread} + + # or vote hit centres you already have + vote_centers([[100, 100], [102, 98], [400, 400]], agree_px=10, min_votes=2) + +``match_ensemble`` returns ``{point, votes, n_candidates, spread}`` for the consensus location, +or ``None`` if fewer than ``min_votes`` references agree. ``vote_centers`` is the pure voting +step over candidate centres you supply. + +Executor commands +----------------- + +``AC_match_ensemble`` (``templates`` / ``min_score`` / ``agree_px`` / ``min_votes`` / +``region`` → ``{found, result}``) and ``AC_vote_centers`` (``centers`` / ``agree_px`` / +``min_votes`` → ``{found, result}``). They are exposed as the MCP tools ``ac_match_ensemble`` / +``ac_vote_centers`` (read-only) and as the Script Builder commands **Match Ensemble (vote +references)** / **Vote Centers (consensus)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index a1cf10d5..b2b8eabf 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -200,6 +200,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v175_features_doc doc/new_features/v176_features_doc doc/new_features/v177_features_doc + doc/new_features/v178_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v178_features_doc.rst b/docs/source/Zh/doc/new_features/v178_features_doc.rst new file mode 100644 index 00000000..b4d801e7 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v178_features_doc.rst @@ -0,0 +1,39 @@ +多模板共識比對 +============== + +一個按鈕常以多種狀態呈現——預設 / 懸停 / 按下 / 停用——但它是單一邏輯目標。``ab_locator`` +A/B 測試*哪個單一策略勝出*,``match_template(scales=...)`` 跨縮放掃描*一個*模板——兩者都不把 +*多個參考裁切*融合成一次投票。``match_ensemble`` 比對每個參考,聚類命中中心,只有在至少 +``min_votes`` 個參考於 ``agree_px`` 內一致時才接受目標——大幅減少換膚 / 動畫 UI 上的誤判。 + +投票核心(``vote_centers``)為純標準函式庫,並重用 ``grounding_consensus`` 做聚類,因此可在 +無影像下單元測試;只有 ``match_ensemble`` 本身呼叫 ``visual_match.match_template``(可於注入的 +合成 ``haystack`` 上測試)。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_ensemble, vote_centers + + # 同一按鈕的三個參考裁切(預設 / 懸停 / 按下) + result = match_ensemble(["btn_default.png", "btn_hover.png", "btn_pressed.png"], + min_score=0.8, agree_px=10, min_votes=2) + if result: + click(*result["point"]) # {point, votes, n_candidates, spread} + + # 或對你已有的命中中心投票 + vote_centers([[100, 100], [102, 98], [400, 400]], agree_px=10, min_votes=2) + +``match_ensemble`` 回傳共識位置的 ``{point, votes, n_candidates, spread}``,或在少於 +``min_votes`` 個參考一致時回傳 ``None``。``vote_centers`` 是對你提供的候選中心的純投票步驟。 + +執行器指令 +---------- + +``AC_match_ensemble``(``templates`` / ``min_score`` / ``agree_px`` / ``min_votes`` / +``region`` → ``{found, result}``)與 ``AC_vote_centers``(``centers`` / ``agree_px`` / +``min_votes`` → ``{found, result}``)。兩者以 MCP 工具 ``ac_match_ensemble`` / +``ac_vote_centers``(唯讀)及 Script Builder 指令 **Match Ensemble (vote references)** / +**Vote Centers (consensus)**(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 65497aa5..302048f7 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -200,6 +200,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v175_features_doc doc/new_features/v176_features_doc doc/new_features/v177_features_doc + doc/new_features/v178_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 7f645d83..0a11743c 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -291,6 +291,10 @@ from je_auto_control.utils.edge_match import ( chamfer_distance, edge_match, edge_match_all, ) +# Multi-template consensus matching (vote references onto one location) +from je_auto_control.utils.match_ensemble import ( + match_ensemble, vote_centers, +) # Otsu auto-thresholding for template matching (no hand-tuned min_score) from je_auto_control.utils.match_autothresh import ( auto_threshold, match_auto, @@ -1274,6 +1278,8 @@ def start_autocontrol_gui(*args, **kwargs): "edge_match", "edge_match_all", "chamfer_distance", + "match_ensemble", + "vote_centers", "SubPixelMatch", "match_subpixel", "refine_peak", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 9f0ab51b..e57268aa 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -407,6 +407,30 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Match with a sub-pixel-refined centre (drag / slider precision).", )) + specs.append(CommandSpec( + "AC_match_ensemble", "Image", "Match Ensemble (vote references)", + fields=( + FieldSpec("templates", FieldType.STRING, + placeholder='["default.png", "hover.png", "pressed.png"]'), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("agree_px", FieldType.INT, optional=True, default=10), + FieldSpec("min_votes", FieldType.INT, optional=True, default=2), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Vote several reference crops onto one consensus location.", + )) + specs.append(CommandSpec( + "AC_vote_centers", "Image", "Vote Centers (consensus)", + fields=( + FieldSpec("centers", FieldType.STRING, + placeholder="[[100, 100], [102, 98]]"), + FieldSpec("agree_px", FieldType.INT, optional=True, default=10), + FieldSpec("min_votes", FieldType.INT, optional=True, default=2), + ), + description="Vote candidate hit centres into one consensus target.", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 8b66803b..72298ed6 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3400,6 +3400,32 @@ def _match_subpixel(template: str, min_score: Any = 0.0, region: Any = None, "match": match.to_dict() if match else None} +def _vote_centers(centers: Any, agree_px: Any = 10, + min_votes: Any = 2) -> Dict[str, Any]: + """Adapter: vote candidate hit centres into a consensus target.""" + import json + from je_auto_control.utils.match_ensemble import vote_centers + if isinstance(centers, str): + centers = json.loads(centers) + result = vote_centers(centers, agree_px=float(agree_px), + min_votes=int(min_votes)) + return {"found": result is not None, "result": result} + + +def _match_ensemble(templates: Any, min_score: Any = 0.8, agree_px: Any = 10, + min_votes: Any = 2, region: Any = None) -> Dict[str, Any]: + """Adapter: vote several template references onto one consensus location.""" + import json + from je_auto_control.utils.match_ensemble import match_ensemble + if isinstance(templates, str): + templates = json.loads(templates) + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + result = match_ensemble(templates, region=region, min_score=float(min_score), + agree_px=float(agree_px), min_votes=int(min_votes)) + return {"found": result is not None, "result": result} + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -6123,6 +6149,8 @@ def __init__(self): "AC_edge_match": _edge_match, "AC_edge_match_all": _edge_match_all, "AC_match_subpixel": _match_subpixel, + "AC_match_ensemble": _match_ensemble, + "AC_vote_centers": _vote_centers, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/match_ensemble/__init__.py b/je_auto_control/utils/match_ensemble/__init__.py new file mode 100644 index 00000000..b109eb5d --- /dev/null +++ b/je_auto_control/utils/match_ensemble/__init__.py @@ -0,0 +1,6 @@ +"""Multi-template consensus matching (vote several references onto one location).""" +from je_auto_control.utils.match_ensemble.match_ensemble import ( + match_ensemble, vote_centers, +) + +__all__ = ["match_ensemble", "vote_centers"] diff --git a/je_auto_control/utils/match_ensemble/match_ensemble.py b/je_auto_control/utils/match_ensemble/match_ensemble.py new file mode 100644 index 00000000..64c568d2 --- /dev/null +++ b/je_auto_control/utils/match_ensemble/match_ensemble.py @@ -0,0 +1,57 @@ +"""Multi-template consensus matching (vote several references onto one location). + +A button often renders in several states — default / hover / pressed / disabled — yet it is a +single logical target. ``ab_locator`` A/B-tests *which single strategy wins* and +``match_template(scales=...)`` sweeps *one* template across scales — neither fuses *multiple +reference crops* into one vote. ``match_ensemble`` matches each reference, clusters the hit +centres and accepts a target only when at least ``min_votes`` references agree within +``agree_px`` — sharply cutting false positives on themed / animated UI. + +The voting core (``vote_centers``) is pure-stdlib and reuses ``grounding_consensus`` for the +clustering, so it is unit-testable with no image; only ``match_ensemble`` itself calls +``visual_match.match_template`` (and is testable on a synthetic injected ``haystack``). Imports +no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +ImageSource = Any + + +def vote_centers(centers: Sequence[Sequence[int]], *, agree_px: float = 10, + min_votes: int = 2) -> Optional[Dict[str, Any]]: + """Cluster candidate hit centres and return the agreed target if enough agree. + + Returns ``{point, votes, n_candidates, spread}`` for the largest cluster, or ``None`` + when fewer than ``min_votes`` candidates fall within ``agree_px`` of each other. + """ + from je_auto_control.utils.grounding_consensus import consensus_point + result = consensus_point(centers, cluster_radius=float(agree_px)) + if result is None: + return None + votes = round(result.agreement * len(centers)) + if votes < int(min_votes): + return None + return {"point": result.point, "votes": votes, + "n_candidates": len(centers), "spread": result.spread} + + +def match_ensemble(templates: Sequence[ImageSource], *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + scales: Sequence[float] = (1.0,), min_score: float = 0.8, + agree_px: float = 10, min_votes: int = 2 + ) -> Optional[Dict[str, Any]]: + """Match each reference in ``templates`` and return their voted consensus target. + + Each template is located with ``visual_match.match_template``; the hit centres are then + fused by :func:`vote_centers`. Returns ``{point, votes, n_candidates, spread}`` or + ``None`` if too few references agree. + """ + from je_auto_control.utils.visual_match import match_template + centers: List[List[int]] = [] + for template in templates: + match = match_template(template, haystack=haystack, region=region, + scales=tuple(scales), min_score=float(min_score)) + if match is not None: + centers.append(match.center) + return vote_centers(centers, agree_px=agree_px, min_votes=min_votes) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index a2c6829c..e804eb1b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3847,6 +3847,38 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.match_subpixel, annotations=READ_ONLY, ), + MCPTool( + name="ac_match_ensemble", + description=("Match several reference crops of one target ('templates': " + "image paths — e.g. default / hover / pressed states) and " + "return their VOTED consensus location: {found, result:" + "{point, votes, n_candidates, spread}}. Accepts only when " + ">= 'min_votes' references agree within 'agree_px'. Cuts false " + "positives on themed / animated UI. 'min_score', 'region'."), + input_schema=schema({ + "templates": {"type": "array", "items": {"type": "string"}}, + "min_score": {"type": "number"}, + "agree_px": {"type": "number"}, + "min_votes": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["templates"]), + handler=h.match_ensemble, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_vote_centers", + description=("Vote candidate hit 'centers' ([[x,y],...]) into one consensus " + "target: {found, result:{point, votes, n_candidates, spread}}. " + "Accepts only when >= 'min_votes' agree within 'agree_px'."), + input_schema=schema({ + "centers": {"type": "array", + "items": {"type": "array", "items": {"type": "integer"}}}, + "agree_px": {"type": "number"}, + "min_votes": {"type": "integer"}}, + required=["centers"]), + handler=h.vote_centers, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 38ec81cd..8ec9d6d3 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2142,6 +2142,16 @@ def match_subpixel(template, min_score=0.0, region=None, method="ccoeff_normed") return _match_subpixel(template, min_score, region, method) +def vote_centers(centers, agree_px=10, min_votes=2): + from je_auto_control.utils.executor.action_executor import _vote_centers + return _vote_centers(centers, agree_px, min_votes) + + +def match_ensemble(templates, min_score=0.8, agree_px=10, min_votes=2, region=None): + from je_auto_control.utils.executor.action_executor import _match_ensemble + return _match_ensemble(templates, min_score, agree_px, min_votes, region) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/test/unit_test/headless/test_match_ensemble_batch.py b/test/unit_test/headless/test_match_ensemble_batch.py new file mode 100644 index 00000000..67e57799 --- /dev/null +++ b/test/unit_test/headless/test_match_ensemble_batch.py @@ -0,0 +1,55 @@ +"""Headless tests for multi-template consensus matching.""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.match_ensemble import match_ensemble, vote_centers + + +def test_vote_centers_majority_agrees(): + # three references agree near (100,100); one outlier + result = vote_centers([[100, 100], [102, 98], [97, 103], [400, 400]], + agree_px=10, min_votes=2) + assert result is not None + assert abs(result["point"][0] - 99) <= 4 + assert result["votes"] == 3 and result["n_candidates"] == 4 + + +def test_vote_centers_too_few_votes_is_none(): + assert vote_centers([[0, 0], [500, 500]], agree_px=10, min_votes=2) is None + + +def test_vote_centers_empty(): + assert vote_centers([], min_votes=1) is None + + +def test_match_ensemble_with_injected_haystack(): + np = pytest.importorskip("numpy") + pytest.importorskip("cv2") + tmpl = np.zeros((24, 24), dtype=np.uint8) + tmpl[:, :12] = 200 + hay = np.zeros((120, 160), dtype=np.uint8) + hay[40:64, 50:74] = tmpl + # three reference crops (here identical) all land on the same spot + result = match_ensemble([tmpl, tmpl, tmpl], haystack=hay, min_score=0.8, + agree_px=8, min_votes=2) + assert result is not None + assert result["votes"] == 3 + assert abs(result["point"][0] - 62) <= 2 # centre of the 24px patch at x=50 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_match_ensemble", "AC_vote_centers"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_match_ensemble", "ac_vote_centers"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_match_ensemble", "AC_vote_centers"} <= specs + + +def test_facade_exports(): + for name in ("match_ensemble", "vote_centers"): + assert hasattr(ac, name) and name in ac.__all__ From 612c4ebadb7f70001ff2b8c41f54b72ce431a96c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 07:20:13 +0800 Subject: [PATCH 2/3] Add color_match: colour-aware (HSV) template matching Every visual_match matcher grayscales first, so red vs green of identical shape is indistinguishable; color_region finds known-colour blobs but can't template-match a multi-colour glyph. Match on HSV hue/saturation with a colour-distance metric (TM_SQDIFF_NORMED, not correlation which normalises away the absolute hue). Reuses color_region's RGB loaders and visual_match's resize/NMS/Match. --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v179_features_doc.rst | 47 +++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v179_features_doc.rst | 42 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 27 ++++++ je_auto_control/utils/color_match/__init__.py | 6 ++ .../utils/color_match/color_match.py | 95 +++++++++++++++++++ .../utils/executor/action_executor.py | 35 +++++++ .../utils/mcp_server/tools/_factories.py | 34 +++++++ .../utils/mcp_server/tools/_handlers.py | 12 +++ .../headless/test_color_match_batch.py | 68 +++++++++++++ 15 files changed, 392 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v179_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v179_features_doc.rst create mode 100644 je_auto_control/utils/color_match/__init__.py create mode 100644 je_auto_control/utils/color_match/color_match.py create mode 100644 test/unit_test/headless/test_color_match_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 6830012a..7fcf0b0b 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 色彩感知模板匹配(HSV) + +区分形状相同的红色与绿色状态点。完整参考:[`docs/source/Zh/doc/new_features/v179_features_doc.rst`](../docs/source/Zh/doc/new_features/v179_features_doc.rst)。 + +- **`match_color` / `match_color_all`**(`AC_match_color`、`AC_match_color_all`):`visual_match` 每个匹配器都先转灰阶,故形状相同的红 vs 绿无法区分;`color_region` 找已知颜色的 blob 却无法对多色字形做模板匹配。本功能在 HSV 色相/饱和度上以色彩*距离*度量(`TM_SQDIFF_NORMED`——相关会把绝对色相正规化掉,使红→绿边与黑→蓝边同分)。重用 `color_region` 的 RGB 加载器 + `visual_match` 的 resize/NMS/`Match`。`channels` 默认 `("h","s")`(平坦饱和度目标用 `("h",)`);纯色 blob 请用 `find_color_region`。不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 多模板共识匹配 把同一目标的多个参考裁切投票成单一可信位置。完整参考:[`docs/source/Zh/doc/new_features/v178_features_doc.rst`](../docs/source/Zh/doc/new_features/v178_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index ad80bb93..86513731 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 色彩感知樣板比對(HSV) + +區分形狀相同的紅色與綠色狀態點。完整參考:[`docs/source/Zh/doc/new_features/v179_features_doc.rst`](../docs/source/Zh/doc/new_features/v179_features_doc.rst)。 + +- **`match_color` / `match_color_all`**(`AC_match_color`、`AC_match_color_all`):`visual_match` 每個比對器都先轉灰階,故形狀相同的紅 vs 綠無法區分;`color_region` 找已知顏色的 blob 卻無法對多色字形做樣板比對。本功能在 HSV 色相/飽和度上以色彩*距離*度量(`TM_SQDIFF_NORMED`——相關會把絕對色相正規化掉,使紅→綠邊與黑→藍邊同分)。重用 `color_region` 的 RGB 載入器 + `visual_match` 的 resize/NMS/`Match`。`channels` 預設 `("h","s")`(平坦飽和度目標用 `("h",)`);純色 blob 請用 `find_color_region`。不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 多模板共識比對 把同一目標的多個參考裁切投票成單一可信位置。完整參考:[`docs/source/Zh/doc/new_features/v178_features_doc.rst`](../docs/source/Zh/doc/new_features/v178_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 0fcbf971..47d1db4c 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Colour-Aware Template Matching (HSV) + +Tell a red status dot from a green one of identical shape. Full reference: [`docs/source/Eng/doc/new_features/v179_features_doc.rst`](docs/source/Eng/doc/new_features/v179_features_doc.rst). + +- **`match_color` / `match_color_all`** (`AC_match_color`, `AC_match_color_all`): every `visual_match` matcher grayscales first, so red vs green of identical shape is indistinguishable; `color_region` finds known-colour blobs but can't template-match a multi-colour glyph. This matches on HSV hue/saturation with a colour-*distance* metric (`TM_SQDIFF_NORMED` — correlation would normalise away the absolute hue, scoring a red→green edge same as black→blue). Reuses `color_region`'s RGB loaders + `visual_match`'s resize/NMS/`Match`. `channels` default `("h","s")` (use `("h",)` for flat-saturation targets); for solid blobs use `find_color_region`. No `PySide6`. + ## What's new (2026-06-24) — Multi-Template Consensus Matching Vote several reference crops of one target into a single trustworthy location. Full reference: [`docs/source/Eng/doc/new_features/v178_features_doc.rst`](docs/source/Eng/doc/new_features/v178_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v179_features_doc.rst b/docs/source/Eng/doc/new_features/v179_features_doc.rst new file mode 100644 index 00000000..8aeb2de1 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v179_features_doc.rst @@ -0,0 +1,47 @@ +Colour-Aware Template Matching (HSV) +==================================== + +Every matcher in ``visual_match`` converts to grayscale first, so a red versus green status +indicator of identical shape is *indistinguishable* to ``match_template`` — the discriminating +signal is thrown away. ``color_region`` finds blobs of a *known* colour but cannot +template-match a multi-colour glyph by appearance. ``color_match`` matches on the HSV +hue / saturation channels using a colour-*distance* metric (``TM_SQDIFF_NORMED``, not a +correlation — correlation normalises away the absolute hue, so a red→green edge and a +black→blue edge would score the same), locating colour-discriminated targets that grayscale +matching collapses. + +It reuses ``color_region``'s RGB loaders and ``visual_match``'s resize / NMS / ``Match``. The +``haystack`` is injectable; the search is unit-testable on synthetic arrays. Imports no +``PySide6``. + +Note: like any window metric, a *flat* single-colour patch has no per-channel variance — for +solid colour blobs use ``find_color_region``; ``color_match`` is for targets with colour +*structure*. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import match_color, match_color_all + + # locate a red status chip, not the green one of the same shape + hit = match_color("status_red.png", channels=("h", "s"), min_score=0.7) + if hit: + click(*hit.center) + + for m in match_color_all("tag_green.png", channels=("h",)): + print(m.center, m.score) + +``match_color`` returns the best ``Match`` over the chosen ``channels`` (default ``("h", "s")``; +use ``("h",)`` for flat-saturation targets), or ``None``. ``match_color_all`` returns every +match at or above ``min_score`` with overlaps removed by NMS. + +Executor commands +----------------- + +``AC_match_color`` (``template`` / ``channels`` / ``min_score`` / ``scales`` / ``region`` → +``{found, match}``) and ``AC_match_color_all`` (adds ``max_results`` / ``nms_iou`` → +``{count, matches}``). They are exposed as the MCP tools ``ac_match_color`` / +``ac_match_color_all`` (read-only) and as the Script Builder commands **Match Template +(colour/HSV)** / **Match Template All (colour/HSV)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index b2b8eabf..15c8b6ed 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -201,6 +201,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v176_features_doc doc/new_features/v177_features_doc doc/new_features/v178_features_doc + doc/new_features/v179_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v179_features_doc.rst b/docs/source/Zh/doc/new_features/v179_features_doc.rst new file mode 100644 index 00000000..e422a720 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v179_features_doc.rst @@ -0,0 +1,42 @@ +色彩感知樣板比對(HSV) +======================== + +``visual_match`` 的每個比對器都先轉灰階,因此形狀相同的紅色與綠色狀態指示燈對 +``match_template`` 而言*無法區分*——具辨別力的訊號被丟棄了。``color_region`` 找*已知*顏色的 +blob,卻無法以外觀對多色字形做樣板比對。``color_match`` 在 HSV 色相 / 飽和度通道上以色彩 +*距離*度量(``TM_SQDIFF_NORMED``,而非相關——相關會把絕對色相正規化掉,使紅→綠邊與黑→藍邊 +得分相同)進行比對,定位灰階比對會塌掉的色彩辨識目標。 + +本功能重用 ``color_region`` 的 RGB 載入器與 ``visual_match`` 的 resize / NMS / ``Match``。 +``haystack`` 可注入;搜尋可在合成陣列上單元測試。不匯入 ``PySide6``。 + +註:如同任何視窗度量,*平坦*的單色 patch 在各通道無變異——純色 blob 請用 +``find_color_region``;``color_match`` 適用於有色彩*結構*的目標。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_color, match_color_all + + # 定位紅色狀態 chip,而非同形狀的綠色 + hit = match_color("status_red.png", channels=("h", "s"), min_score=0.7) + if hit: + click(*hit.center) + + for m in match_color_all("tag_green.png", channels=("h",)): + print(m.center, m.score) + +``match_color`` 在選定 ``channels``(預設 ``("h", "s")``;平坦飽和度目標用 ``("h",)``)中回傳 +最佳 ``Match``,或 ``None``。``match_color_all`` 回傳所有達到 ``min_score`` 的匹配,重疊以 NMS +移除。 + +執行器指令 +---------- + +``AC_match_color``(``template`` / ``channels`` / ``min_score`` / ``scales`` / ``region`` → +``{found, match}``)與 ``AC_match_color_all``(另加 ``max_results`` / ``nms_iou`` → +``{count, matches}``)。兩者以 MCP 工具 ``ac_match_color`` / ``ac_match_color_all``(唯讀)及 +Script Builder 指令 **Match Template (colour/HSV)** / **Match Template All (colour/HSV)** +(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 302048f7..ac3b1012 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -201,6 +201,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v176_features_doc doc/new_features/v177_features_doc doc/new_features/v178_features_doc + doc/new_features/v179_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 0a11743c..cd7097fb 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -295,6 +295,10 @@ from je_auto_control.utils.match_ensemble import ( match_ensemble, vote_centers, ) +# Colour-aware template matching on HSV channels +from je_auto_control.utils.color_match import ( + match_color, match_color_all, +) # Otsu auto-thresholding for template matching (no hand-tuned min_score) from je_auto_control.utils.match_autothresh import ( auto_threshold, match_auto, @@ -1280,6 +1284,8 @@ def start_autocontrol_gui(*args, **kwargs): "chamfer_distance", "match_ensemble", "vote_centers", + "match_color", + "match_color_all", "SubPixelMatch", "match_subpixel", "refine_peak", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index e57268aa..d7039b96 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -431,6 +431,33 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Vote candidate hit centres into one consensus target.", )) + specs.append(CommandSpec( + "AC_match_color", "Image", "Match Template (colour/HSV)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("channels", FieldType.STRING, optional=True, + placeholder='["h", "s"]'), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.7, + min_value=0.0, max_value=1.0), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Match by colour on HSV channels (red vs green, not grayscale).", + )) + specs.append(CommandSpec( + "AC_match_color_all", "Image", "Match Template All (colour/HSV)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("channels", FieldType.STRING, optional=True, + placeholder='["h", "s"]'), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.7, + min_value=0.0, max_value=1.0), + FieldSpec("max_results", FieldType.INT, optional=True, default=20), + FieldSpec("nms_iou", FieldType.FLOAT, optional=True, default=0.3, + min_value=0.0, max_value=1.0), + ), + description="Find every colour (HSV) match of a template (NMS-deduped).", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/color_match/__init__.py b/je_auto_control/utils/color_match/__init__.py new file mode 100644 index 00000000..b79a3b44 --- /dev/null +++ b/je_auto_control/utils/color_match/__init__.py @@ -0,0 +1,6 @@ +"""Colour-aware template matching on HSV channels.""" +from je_auto_control.utils.color_match.color_match import ( + match_color, match_color_all, +) + +__all__ = ["match_color", "match_color_all"] diff --git a/je_auto_control/utils/color_match/color_match.py b/je_auto_control/utils/color_match/color_match.py new file mode 100644 index 00000000..a0858fc3 --- /dev/null +++ b/je_auto_control/utils/color_match/color_match.py @@ -0,0 +1,95 @@ +"""Colour-aware template matching on HSV channels. + +Every matcher in ``visual_match`` converts to grayscale first, so a red versus green status +indicator of identical shape is *indistinguishable* to ``match_template`` — the discriminating +signal is thrown away. ``color_region`` finds blobs of a *known* colour but cannot template-match +a multi-colour glyph by appearance. ``color_match`` correlates on the HSV hue / saturation +channels (hue is illumination-invariant), so it locates colour-discriminated targets — a +coloured chip, a syntax-highlighted token, a status light with structure — that grayscale +matching collapses. + +It reuses ``color_region``'s RGB loaders and ``visual_match``'s resize / NMS / ``Match`` (no new +image or matching code). The ``haystack`` is injectable; the search is unit-testable on synthetic +arrays. OpenCV + NumPy are imported lazily. Imports no ``PySide6``. + +Note: like any NCC, a *flat* single-colour patch has no per-channel variance to correlate; for +solid colour blobs use ``color_region``. ``color_match`` is for targets with colour *structure*. +""" +from typing import Any, List, Optional, Sequence + +from je_auto_control.utils.color_region.color_region import _grab_rgb, _to_rgb +from je_auto_control.utils.visual_match.visual_match import Match, _nms, _resize + +ImageSource = Any +_CHANNEL_INDEX = {"h": 0, "s": 1, "v": 2} + + +def _hsv(source, region, is_haystack: bool): + import cv2 + rgb = (_to_rgb(source) if source is not None + else _grab_rgb(region)) if is_haystack else _to_rgb(source) + return cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) + + +def _score_map(template_hsv, haystack_hsv, channels: Sequence[str]): + """Per-channel colour-distance score (higher = better) over the chosen channels. + + Uses ``TM_SQDIFF_NORMED`` (not a correlation): correlation methods normalise away the + *absolute* hue, so a red→green edge and a black→blue edge correlate identically. Squared + distance keeps the absolute colour, so red is told from green. Score is ``1 - mean(sqdiff)``; + a flat channel (no variance, sqdiff undefined) contributes the worst score. + """ + import cv2 + import numpy as np + accumulator = None + for channel in channels: + index = _CHANNEL_INDEX[channel] + result = cv2.matchTemplate(haystack_hsv[:, :, index], + template_hsv[:, :, index], cv2.TM_SQDIFF_NORMED) + result = np.nan_to_num(result, nan=1.0, posinf=1.0) + accumulator = result if accumulator is None else accumulator + result + return 1.0 - accumulator / len(channels) + + +def match_color(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + channels: Sequence[str] = ("h", "s"), + scales: Sequence[float] = (1.0,), + min_score: float = 0.7) -> Optional[Match]: + """Return the best colour (HSV-channel) match at or above ``min_score``, or ``None``.""" + import cv2 + template_hsv = _hsv(template, None, is_haystack=False) + haystack_hsv = _hsv(haystack, region, is_haystack=True) + best: Optional[Match] = None + for scale in scales: + scaled = _resize(template_hsv, float(scale)) + if scaled.shape[0] > haystack_hsv.shape[0] \ + or scaled.shape[1] > haystack_hsv.shape[1]: + continue + _, max_val, _, max_loc = cv2.minMaxLoc( + _score_map(scaled, haystack_hsv, channels)) + if max_val >= min_score and (best is None or max_val > best.score): + best = Match(int(max_loc[0]), int(max_loc[1]), scaled.shape[1], + scaled.shape[0], round(float(max_val), 4), float(scale)) + return best + + +def match_color_all(template: ImageSource, *, + haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + channels: Sequence[str] = ("h", "s"), min_score: float = 0.7, + max_results: int = 20, nms_iou: float = 0.3) -> List[Match]: + """Return every colour match >= ``min_score`` (scale 1.0), overlaps removed (NMS).""" + import numpy as np + template_hsv = _hsv(template, None, is_haystack=False) + haystack_hsv = _hsv(haystack, region, is_haystack=True) + if template_hsv.shape[0] > haystack_hsv.shape[0] \ + or template_hsv.shape[1] > haystack_hsv.shape[1]: + return [] + score_map = _score_map(template_hsv, haystack_hsv, channels) + height, width = template_hsv.shape[:2] + ys, xs = np.nonzero(score_map >= float(min_score)) + candidates = [Match(int(x), int(y), width, height, + round(float(score_map[y, x]), 4), 1.0) + for y, x in zip(ys, xs)] + return _nms(candidates, float(nms_iou))[:int(max_results)] diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 72298ed6..df506c60 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3426,6 +3426,39 @@ def _match_ensemble(templates: Any, min_score: Any = 0.8, agree_px: Any = 10, return {"found": result is not None, "result": result} +def _match_color(template: str, channels: Any = None, min_score: Any = 0.7, + scales: Any = None, region: Any = None) -> Dict[str, Any]: + """Adapter: best colour (HSV-channel) template match on the screen.""" + import json + from je_auto_control.utils.color_match import match_color + if isinstance(channels, str): + channels = json.loads(channels) if channels.strip() else None + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + match = match_color(template, region=region, + channels=tuple(channels) if channels else ("h", "s"), + scales=_seq_arg(scales, (1.0,)), min_score=float(min_score)) + return {"found": match is not None, + "match": match.to_dict() if match else None} + + +def _match_color_all(template: str, channels: Any = None, min_score: Any = 0.7, + max_results: Any = 20, nms_iou: Any = 0.3, + region: Any = None) -> Dict[str, Any]: + """Adapter: every colour (HSV-channel) match on the screen (NMS).""" + import json + from je_auto_control.utils.color_match import match_color_all + if isinstance(channels, str): + channels = json.loads(channels) if channels.strip() else None + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = match_color_all(template, region=region, + channels=tuple(channels) if channels else ("h", "s"), + min_score=float(min_score), + max_results=int(max_results), nms_iou=float(nms_iou)) + return {"count": len(matches), "matches": [m.to_dict() for m in matches]} + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -6151,6 +6184,8 @@ def __init__(self): "AC_match_subpixel": _match_subpixel, "AC_match_ensemble": _match_ensemble, "AC_vote_centers": _vote_centers, + "AC_match_color": _match_color, + "AC_match_color_all": _match_color_all, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index e804eb1b..dcce606a 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3879,6 +3879,40 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.vote_centers, annotations=READ_ONLY, ), + MCPTool( + name="ac_match_color", + description=("Find 'template' by COLOUR on the HSV hue/saturation channels " + "(not grayscale): tells a red status dot from a green one of " + "identical shape. Returns {found, match}. 'channels' " + "(default [\"h\",\"s\"]; use [\"h\"] for flat-saturation " + "targets), 'min_score', 'scales', 'region'. For solid colour " + "blobs use find_color_region instead."), + input_schema=schema({ + "template": {"type": "string"}, + "channels": {"type": "array", "items": {"type": "string"}}, + "min_score": {"type": "number"}, + "scales": {"type": "array", "items": {"type": "number"}}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.match_color, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_color_all", + description=("Find EVERY colour (HSV-channel) match of 'template' >= " + "'min_score', overlaps removed by NMS. Returns " + "{count, matches}."), + input_schema=schema({ + "template": {"type": "string"}, + "channels": {"type": "array", "items": {"type": "string"}}, + "min_score": {"type": "number"}, + "max_results": {"type": "integer"}, + "nms_iou": {"type": "number"}, + "region": {"type": "array", "items": {"type": "integer"}}}, + required=["template"]), + handler=h.match_color_all, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 8ec9d6d3..7e382e88 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2152,6 +2152,18 @@ def match_ensemble(templates, min_score=0.8, agree_px=10, min_votes=2, region=No return _match_ensemble(templates, min_score, agree_px, min_votes, region) +def match_color(template, channels=None, min_score=0.7, scales=None, region=None): + from je_auto_control.utils.executor.action_executor import _match_color + return _match_color(template, channels, min_score, scales, region) + + +def match_color_all(template, channels=None, min_score=0.7, max_results=20, + nms_iou=0.3, region=None): + from je_auto_control.utils.executor.action_executor import _match_color_all + return _match_color_all(template, channels, min_score, max_results, nms_iou, + region) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/test/unit_test/headless/test_color_match_batch.py b/test/unit_test/headless/test_color_match_batch.py new file mode 100644 index 00000000..803d3c79 --- /dev/null +++ b/test/unit_test/headless/test_color_match_batch.py @@ -0,0 +1,68 @@ +"""Headless tests for colour-aware (HSV) template matching.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.color_match import ( # noqa: E402 + match_color, match_color_all, +) + +RED, GREEN, BLUE, YELLOW = [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 0] + + +def _patch(left, right): + patch = np.zeros((24, 24, 3), dtype=np.uint8) + patch[:, :12] = left + patch[:, 12:] = right + return patch + + +def _scene(): + hay = np.zeros((160, 200, 3), dtype=np.uint8) + hay[50:74, 40:64] = _patch(RED, GREEN) # the target + hay[50:74, 140:164] = _patch(BLUE, YELLOW) # same shape, different colours + return hay + + +def test_finds_the_colour_target(): + match = match_color(_patch(RED, GREEN), haystack=_scene(), channels=("h",), + min_score=0.7) + assert match is not None + assert abs(match.x - 40) <= 1 and abs(match.y - 50) <= 1 + assert match.score >= 0.99 + + +def test_discriminates_colour_not_just_shape(): + # at a high threshold only the matching-colour patch survives — the blue/yellow + # decoy of identical shape is rejected (grayscale matching would accept it) + hits = match_color_all(_patch(RED, GREEN), haystack=_scene(), channels=("h",), + min_score=0.95) + assert len(hits) == 1 + assert abs(hits[0].x - 40) <= 1 + + +def test_absent_returns_none(): + blank = np.zeros((160, 200, 3), dtype=np.uint8) + assert match_color(_patch(BLUE, YELLOW), haystack=blank, channels=("h",), + min_score=0.7) is None + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_match_color", "AC_match_color_all"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_match_color", "ac_match_color_all"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_match_color", "AC_match_color_all"} <= specs + + +def test_facade_exports(): + for name in ("match_color", "match_color_all"): + assert hasattr(ac, name) and name in ac.__all__ From 6e9f116aff3ac8d7449d0b8abe96087b30402654 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 07:34:10 +0800 Subject: [PATCH 3/3] Add match_stability: pre-match settle gating + match persistence Matching mid-animation is a top flakiness source; wait_until_screen_stable gates a live loop with a boolean and can't score an injectable frame sequence or check a match held steady. region_stability scores consecutive-frame SSIM; match_persistence confirms a template is found in every frame with the centres agreeing within agree_px. Reuses ssim + visual_match + grounding_consensus. --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v180_features_doc.rst | 43 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v180_features_doc.rst | 41 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 22 +++++ .../utils/executor/action_executor.py | 22 +++++ .../utils/match_stability/__init__.py | 6 ++ .../utils/match_stability/match_stability.py | 62 ++++++++++++++ .../utils/mcp_server/tools/_factories.py | 30 +++++++ .../utils/mcp_server/tools/_handlers.py | 10 +++ .../headless/test_match_stability_batch.py | 81 +++++++++++++++++++ 15 files changed, 343 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v180_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v180_features_doc.rst create mode 100644 je_auto_control/utils/match_stability/__init__.py create mode 100644 je_auto_control/utils/match_stability/match_stability.py create mode 100644 test/unit_test/headless/test_match_stability_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 7fcf0b0b..d4c92bce 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 匹配前安定门 + 命中稳定性 + +避免在动画进行中匹配,并确认命中跨帧维持稳定。完整参考:[`docs/source/Zh/doc/new_features/v180_features_doc.rst`](../docs/source/Zh/doc/new_features/v180_features_doc.rst)。 + +- **`region_stability` / `match_persistence`**(`AC_region_stability`、`AC_match_persistence`):`smart_waits.wait_until_screen_stable` 以布尔门控实时循环——无法对可注入帧序列评分稳定度,也无法检查某*命中*是否维持。`region_stability` 以相邻帧 SSIM 评分(`{stable, mean_ssim, min_ssim}`);`match_persistence` 确认 template 在*每一*帧都找到且中心于 `agree_px` 内一致(`{persisted, n_hits, jitter}`)。重用 `ssim` + `visual_match` + `grounding_consensus`;帧可注入;不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 色彩感知模板匹配(HSV) 区分形状相同的红色与绿色状态点。完整参考:[`docs/source/Zh/doc/new_features/v179_features_doc.rst`](../docs/source/Zh/doc/new_features/v179_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 86513731..bb67dab1 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 比對前安定閘 + 命中穩定性 + +避免在動畫進行中比對,並確認命中跨幀維持穩定。完整參考:[`docs/source/Zh/doc/new_features/v180_features_doc.rst`](../docs/source/Zh/doc/new_features/v180_features_doc.rst)。 + +- **`region_stability` / `match_persistence`**(`AC_region_stability`、`AC_match_persistence`):`smart_waits.wait_until_screen_stable` 以布林閘控即時迴圈——無法對可注入幀序列評分穩定度,也無法檢查某*命中*是否維持。`region_stability` 以相鄰幀 SSIM 評分(`{stable, mean_ssim, min_ssim}`);`match_persistence` 確認 template 在*每一*幀都找到且中心於 `agree_px` 內一致(`{persisted, n_hits, jitter}`)。重用 `ssim` + `visual_match` + `grounding_consensus`;幀可注入;不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 色彩感知樣板比對(HSV) 區分形狀相同的紅色與綠色狀態點。完整參考:[`docs/source/Zh/doc/new_features/v179_features_doc.rst`](../docs/source/Zh/doc/new_features/v179_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 47d1db4c..1104cf7a 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Pre-Match Settle Gating + Match Persistence + +Avoid matching mid-animation, and confirm a hit holds steady across frames. Full reference: [`docs/source/Eng/doc/new_features/v180_features_doc.rst`](docs/source/Eng/doc/new_features/v180_features_doc.rst). + +- **`region_stability` / `match_persistence`** (`AC_region_stability`, `AC_match_persistence`): `smart_waits.wait_until_screen_stable` gates a live loop with a boolean — it can't score stability on an injectable frame sequence or check whether a *match* held steady. `region_stability` scores consecutive-frame SSIM (`{stable, mean_ssim, min_ssim}`); `match_persistence` confirms a template is found in *every* frame with the centres agreeing within `agree_px` (`{persisted, n_hits, jitter}`). Reuses `ssim` + `visual_match` + `grounding_consensus`; injectable frames; no `PySide6`. + ## What's new (2026-06-24) — Colour-Aware Template Matching (HSV) Tell a red status dot from a green one of identical shape. Full reference: [`docs/source/Eng/doc/new_features/v179_features_doc.rst`](docs/source/Eng/doc/new_features/v179_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v180_features_doc.rst b/docs/source/Eng/doc/new_features/v180_features_doc.rst new file mode 100644 index 00000000..16f78134 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v180_features_doc.rst @@ -0,0 +1,43 @@ +Pre-Match Settle Gating + Match Persistence +=========================================== + +Matching mid-animation is a top flakiness source: the spinner is still spinning, a transition +is half-done, and the matcher fires on a transient. ``smart_waits.wait_until_screen_stable`` +gates a *live polling loop* and returns a boolean — it cannot score stability on an *injectable* +sequence of frames, nor tell you whether a *match* held steady across them. ``match_stability`` +adds both, on injected frames so it is unit-testable: ``region_stability`` scores how settled a +frame sequence is (consecutive-frame SSIM), and ``match_persistence`` checks that a template's +hit holds at the same place across the frames (low jitter), not just in one lucky frame. + +It reuses ``ssim.ssim_compare``, ``visual_match.match_template`` and +``grounding_consensus.consensus_point`` — no new matching code. The frames are injectable +(ndarray / path / PIL). Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import region_stability, match_persistence + + frames = [grab(), grab(), grab()] # a few snapshots of the same region + if region_stability(frames, settle_threshold=0.99)["stable"]: + do_the_match() + + result = match_persistence("button.png", frames, min_score=0.8, agree_px=8) + if result["persisted"]: + print("steady hit, jitter", result["jitter"]) + +``region_stability`` returns ``{stable, mean_ssim, min_ssim}`` — ``stable`` when the lowest +consecutive-frame SSIM clears ``settle_threshold``. ``match_persistence`` returns +``{persisted, n_hits, jitter}`` — ``persisted`` when the template is found in *every* frame and +the hit centres agree within ``agree_px`` (``jitter`` is their spread; 0 = rock steady). + +Executor commands +----------------- + +``AC_region_stability`` (``frames`` / ``settle_threshold`` → ``{stable, mean_ssim, min_ssim}``) +and ``AC_match_persistence`` (``template`` / ``frames`` / ``min_score`` / ``agree_px`` → +``{persisted, n_hits, jitter}``). They are exposed as the MCP tools ``ac_region_stability`` / +``ac_match_persistence`` (read-only) and as the Script Builder commands **Region Stability +(frames)** / **Match Persistence (frames)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 15c8b6ed..e2c5eed5 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -202,6 +202,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v177_features_doc doc/new_features/v178_features_doc doc/new_features/v179_features_doc + doc/new_features/v180_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v180_features_doc.rst b/docs/source/Zh/doc/new_features/v180_features_doc.rst new file mode 100644 index 00000000..f420a11a --- /dev/null +++ b/docs/source/Zh/doc/new_features/v180_features_doc.rst @@ -0,0 +1,41 @@ +比對前安定閘 + 命中穩定性 +========================== + +在動畫進行中比對是主要的不穩定來源:轉圈圖示還在轉、轉場做到一半,比對器命中暫態。 +``smart_waits.wait_until_screen_stable`` 閘控*即時輪詢迴圈*並回傳布林——它無法對*可注入*的 +幀序列評分穩定度,也無法告訴你某個*命中*是否跨幀維持。``match_stability`` 補上兩者,且作用於 +注入幀因此可單元測試:``region_stability`` 以相鄰幀 SSIM 評分幀序列的安定度,``match_persistence`` +檢查某 template 的命中是否跨幀維持在同一處(jitter 小),而非只在某一幸運幀。 + +本功能重用 ``ssim.ssim_compare``、``visual_match.match_template`` 與 +``grounding_consensus.consensus_point``——不新增比對程式。幀可注入(ndarray / 路徑 / PIL)。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import region_stability, match_persistence + + frames = [grab(), grab(), grab()] # 同一區域的幾張快照 + if region_stability(frames, settle_threshold=0.99)["stable"]: + do_the_match() + + result = match_persistence("button.png", frames, min_score=0.8, agree_px=8) + if result["persisted"]: + print("穩定命中, jitter", result["jitter"]) + +``region_stability`` 回傳 ``{stable, mean_ssim, min_ssim}``——當最低相鄰幀 SSIM 超過 +``settle_threshold`` 時 ``stable``。``match_persistence`` 回傳 ``{persisted, n_hits, jitter}`` +——當 template 在*每一*幀都找到且命中中心於 ``agree_px`` 內一致時 ``persisted``(``jitter`` +為其離散度;0 = 穩如磐石)。 + +執行器指令 +---------- + +``AC_region_stability``(``frames`` / ``settle_threshold`` → ``{stable, mean_ssim, min_ssim}``) +與 ``AC_match_persistence``(``template`` / ``frames`` / ``min_score`` / ``agree_px`` → +``{persisted, n_hits, jitter}``)。兩者以 MCP 工具 ``ac_region_stability`` / +``ac_match_persistence``(唯讀)及 Script Builder 指令 **Region Stability (frames)** / +**Match Persistence (frames)**(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index ac3b1012..261eedf0 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -202,6 +202,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v177_features_doc doc/new_features/v178_features_doc doc/new_features/v179_features_doc + doc/new_features/v180_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index cd7097fb..5a741775 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -299,6 +299,10 @@ from je_auto_control.utils.color_match import ( match_color, match_color_all, ) +# Pre-match settle gating + match persistence over a frame sequence +from je_auto_control.utils.match_stability import ( + match_persistence, region_stability, +) # Otsu auto-thresholding for template matching (no hand-tuned min_score) from je_auto_control.utils.match_autothresh import ( auto_threshold, match_auto, @@ -1286,6 +1290,8 @@ def start_autocontrol_gui(*args, **kwargs): "vote_centers", "match_color", "match_color_all", + "region_stability", + "match_persistence", "SubPixelMatch", "match_subpixel", "refine_peak", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index d7039b96..c46132e9 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -458,6 +458,28 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Find every colour (HSV) match of a template (NMS-deduped).", )) + specs.append(CommandSpec( + "AC_region_stability", "Image", "Region Stability (frames)", + fields=( + FieldSpec("frames", FieldType.STRING, + placeholder='["f1.png", "f2.png", "f3.png"]'), + FieldSpec("settle_threshold", FieldType.FLOAT, optional=True, + default=0.99, min_value=0.0, max_value=1.0), + ), + description="Is a frame sequence settled? (consecutive-frame SSIM).", + )) + specs.append(CommandSpec( + "AC_match_persistence", "Image", "Match Persistence (frames)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("frames", FieldType.STRING, + placeholder='["f1.png", "f2.png", "f3.png"]'), + FieldSpec("min_score", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("agree_px", FieldType.INT, optional=True, default=8), + ), + description="Does a template match hold steady across frames? (jitter).", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index df506c60..3534e300 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3459,6 +3459,26 @@ def _match_color_all(template: str, channels: Any = None, min_score: Any = 0.7, return {"count": len(matches), "matches": [m.to_dict() for m in matches]} +def _region_stability(frames: Any, settle_threshold: Any = 0.99) -> Dict[str, Any]: + """Adapter: how settled an injected frame sequence is (consecutive SSIM).""" + import json + from je_auto_control.utils.match_stability import region_stability + if isinstance(frames, str): + frames = json.loads(frames) + return region_stability(frames, settle_threshold=float(settle_threshold)) + + +def _match_persistence(template: str, frames: Any, min_score: Any = 0.8, + agree_px: Any = 8) -> Dict[str, Any]: + """Adapter: whether a template match holds steady across frames.""" + import json + from je_auto_control.utils.match_stability import match_persistence + if isinstance(frames, str): + frames = json.loads(frames) + return match_persistence(template, frames, min_score=float(min_score), + agree_px=float(agree_px)) + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -6186,6 +6206,8 @@ def __init__(self): "AC_vote_centers": _vote_centers, "AC_match_color": _match_color, "AC_match_color_all": _match_color_all, + "AC_region_stability": _region_stability, + "AC_match_persistence": _match_persistence, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/match_stability/__init__.py b/je_auto_control/utils/match_stability/__init__.py new file mode 100644 index 00000000..1d0bcc84 --- /dev/null +++ b/je_auto_control/utils/match_stability/__init__.py @@ -0,0 +1,6 @@ +"""Pre-match settle gating and match persistence over a sequence of frames.""" +from je_auto_control.utils.match_stability.match_stability import ( + match_persistence, region_stability, +) + +__all__ = ["region_stability", "match_persistence"] diff --git a/je_auto_control/utils/match_stability/match_stability.py b/je_auto_control/utils/match_stability/match_stability.py new file mode 100644 index 00000000..c7efe05d --- /dev/null +++ b/je_auto_control/utils/match_stability/match_stability.py @@ -0,0 +1,62 @@ +"""Pre-match settle gating and match persistence over a sequence of frames. + +Matching mid-animation is a top flakiness source: the spinner is still spinning, a transition +is half-done, and the matcher fires on a transient. ``smart_waits.wait_until_screen_stable`` +gates a *live polling loop* and returns a boolean — it cannot score stability on an *injectable* +sequence of frames, nor tell you whether a *match* held steady across them. ``match_stability`` +adds both, on injected frames so it is unit-testable: ``region_stability`` scores how settled a +frame sequence is (consecutive-frame SSIM), and ``match_persistence`` checks that a template's +hit holds at the same place across the frames (low jitter), not just in one lucky frame. + +Reuses ``ssim.ssim_compare``, ``visual_match.match_template`` and +``grounding_consensus.consensus_point``; no new matching code. The frames are injectable +(ndarray / path / PIL); OpenCV + NumPy arrive lazily via those modules. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +ImageSource = Any + + +def region_stability(frames: Sequence[ImageSource], *, + settle_threshold: float = 0.99) -> Dict[str, Any]: + """Score how settled a frame sequence is via consecutive-frame SSIM. + + Returns ``{stable, mean_ssim, min_ssim}`` — ``stable`` is true when the lowest + consecutive-frame SSIM is at least ``settle_threshold`` (nothing moved between frames). + A sequence shorter than two frames is trivially stable. + """ + from je_auto_control.utils.ssim import ssim_compare + frame_list = list(frames) + if len(frame_list) < 2: + return {"stable": True, "mean_ssim": 1.0, "min_ssim": 1.0} + scores = [ssim_compare(frame_list[i], frame_list[i + 1]) + for i in range(len(frame_list) - 1)] + lowest = min(scores) + return {"stable": lowest >= float(settle_threshold), + "mean_ssim": round(sum(scores) / len(scores), 4), + "min_ssim": round(lowest, 4)} + + +def match_persistence(template: ImageSource, frames: Sequence[ImageSource], *, + min_score: float = 0.8, agree_px: float = 8) -> Dict[str, Any]: + """Check that ``template`` matches the same place across ``frames``. + + Returns ``{persisted, n_hits, jitter}`` — ``persisted`` is true when the template is + found in *every* frame and the hit centres agree (cluster within ``agree_px``); + ``jitter`` is the spread of the agreeing centres (0 = rock steady). + """ + from je_auto_control.utils.grounding_consensus import consensus_point + from je_auto_control.utils.visual_match import match_template + frame_list = list(frames) + centers: List[List[int]] = [] + for frame in frame_list: + match = match_template(template, haystack=frame, min_score=float(min_score)) + if match is not None: + centers.append(match.center) + if not centers: + return {"persisted": False, "n_hits": 0, "jitter": None} + result = consensus_point(centers, cluster_radius=float(agree_px)) + jitter: Optional[float] = result.spread if result else None + persisted = (len(centers) == len(frame_list) and result is not None + and result.agreement >= 0.9) + return {"persisted": persisted, "n_hits": len(centers), "jitter": jitter} diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index dcce606a..0db7d0a3 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3913,6 +3913,36 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.match_color_all, annotations=READ_ONLY, ), + MCPTool( + name="ac_region_stability", + description=("Score how settled a sequence of 'frames' (image paths) is by " + "consecutive-frame SSIM: {stable, mean_ssim, min_ssim}. " + "stable=true when nothing moved between frames " + "(min_ssim >= 'settle_threshold') - match only when stable to " + "avoid mid-animation hits."), + input_schema=schema({ + "frames": {"type": "array", "items": {"type": "string"}}, + "settle_threshold": {"type": "number"}}, + required=["frames"]), + handler=h.region_stability, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_persistence", + description=("Check a 'template' matches the same place across 'frames' " + "(image paths): {persisted, n_hits, jitter}. persisted=true " + "when found in every frame and the centres agree within " + "'agree_px' - a steady match, not one lucky frame. " + "'min_score'."), + input_schema=schema({ + "template": {"type": "string"}, + "frames": {"type": "array", "items": {"type": "string"}}, + "min_score": {"type": "number"}, + "agree_px": {"type": "number"}}, + required=["template", "frames"]), + handler=h.match_persistence, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 7e382e88..5d7a8051 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2164,6 +2164,16 @@ def match_color_all(template, channels=None, min_score=0.7, max_results=20, region) +def region_stability(frames, settle_threshold=0.99): + from je_auto_control.utils.executor.action_executor import _region_stability + return _region_stability(frames, settle_threshold) + + +def match_persistence(template, frames, min_score=0.8, agree_px=8): + from je_auto_control.utils.executor.action_executor import _match_persistence + return _match_persistence(template, frames, min_score, agree_px) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/test/unit_test/headless/test_match_stability_batch.py b/test/unit_test/headless/test_match_stability_batch.py new file mode 100644 index 00000000..92fe9f0b --- /dev/null +++ b/test/unit_test/headless/test_match_stability_batch.py @@ -0,0 +1,81 @@ +"""Headless tests for pre-match settle gating + match persistence.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.match_stability import ( # noqa: E402 + match_persistence, region_stability, +) + + +def _frame(extra=None): + img = np.zeros((120, 160), dtype=np.uint8) + cv2.rectangle(img, (30, 30), (90, 80), 255, 2) # stable content + if extra is not None: + cv2.circle(img, extra, 8, 200, -1) # moving element + return img + + +def _template(): + tmpl = np.zeros((24, 24), dtype=np.uint8) + tmpl[:, :12] = 200 + return tmpl + + +def _haystack(left): + hay = np.zeros((120, 160), dtype=np.uint8) + hay[40:64, left:left + 24] = _template() + return hay + + +def test_identical_frames_are_stable(): + frames = [_frame(), _frame(), _frame()] + result = region_stability(frames, settle_threshold=0.99) + assert result["stable"] is True + assert result["min_ssim"] >= 0.99 + + +def test_moving_element_is_unstable(): + frames = [_frame((10, 10)), _frame((60, 60)), _frame((110, 100))] + result = region_stability(frames, settle_threshold=0.99) + assert result["stable"] is False + assert result["min_ssim"] < 0.99 + + +def test_single_frame_is_trivially_stable(): + assert region_stability([_frame()])["stable"] is True + + +def test_match_persists_across_frames(): + frames = [_haystack(50), _haystack(51), _haystack(50)] # ~steady + result = match_persistence(_template(), frames, min_score=0.8, agree_px=8) + assert result["persisted"] is True + assert result["n_hits"] == 3 + + +def test_match_not_persistent_when_missing(): + frames = [_haystack(50), np.zeros((120, 160), np.uint8), _haystack(50)] + result = match_persistence(_template(), frames, min_score=0.8) + assert result["persisted"] is False + assert result["n_hits"] == 2 + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_region_stability", "AC_match_persistence"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_region_stability", "ac_match_persistence"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_region_stability", "AC_match_persistence"} <= specs + + +def test_facade_exports(): + for name in ("region_stability", "match_persistence"): + assert hasattr(ac, name) and name in ac.__all__