From 620bd562048d6c4bd97de4cf536be1d7a3b16c66 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 03:14:29 +0800 Subject: [PATCH 1/2] Add auto-thresholded template matching (Otsu on score map) Every match_template_all call forces a hand-tuned min_score: too low floods NMS, too high drops re-themed targets, and the right value differs per asset. Run Otsu on the correlation score histogram to find the valley between background correlation and real matches, returning that cut-off plus a separability score so a unimodal (no-match) surface is flagged. match_auto returns one peak per above-threshold region via connected_boxes. --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v164_features_doc.rst | 47 +++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v164_features_doc.rst | 43 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 6 ++ .../gui/script_builder/command_schema.py | 21 ++++ .../utils/executor/action_executor.py | 25 +++++ .../utils/match_autothresh/__init__.py | 6 ++ .../match_autothresh/match_autothresh.py | 99 +++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 31 ++++++ .../utils/mcp_server/tools/_handlers.py | 11 +++ .../headless/test_match_autothresh_batch.py | 70 +++++++++++++ 15 files changed, 379 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v164_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v164_features_doc.rst create mode 100644 je_auto_control/utils/match_autothresh/__init__.py create mode 100644 je_auto_control/utils/match_autothresh/match_autothresh.py create mode 100644 test/unit_test/headless/test_match_autothresh_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index ea5d810f..af6eb0f1 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 自动门槛模板匹配(对分数图做 Otsu) + +不再手调 `min_score`——由分数图推导匹配门槛。完整参考:[`docs/source/Zh/doc/new_features/v164_features_doc.rst`](../docs/source/Zh/doc/new_features/v164_features_doc.rst)。 + +- **`match_auto` / `auto_threshold`**(`AC_match_auto`、`AC_auto_threshold`):每次 `match_template_all` 都迫使你猜 `min_score`(太低充满 NMS 噪声、太高漏掉换肤目标,且因素材而异)。本功能对*相关性分数直方图*套用 Otsu,找出背景相关与真正匹配之间的谷,返回该门槛加上 *separability* 分离度(接近 0 = 单峰、无明确匹配 → 不要信任)。`match_auto` 每个过门槛区域只返回单一峰(通过 `connected_boxes`,避免宽峰上的重复命中),并以 `floor` 夹住。重用新增的 `visual_match._score_map`;`haystack` 可注入;不导入 `PySide6`。 + ## 本次更新 (2026-06-24) — 令牌预算化的观测增量(变更了什么) 告诉代理*变更了什么*,而非再次整个屏幕。完整参考:[`docs/source/Zh/doc/new_features/v163_features_doc.rst`](../docs/source/Zh/doc/new_features/v163_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 5f73a0a3..ffae59c7 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-24) — 自動門檻樣板比對(對分數圖做 Otsu) + +不再手調 `min_score`——由分數圖推導比對門檻。完整參考:[`docs/source/Zh/doc/new_features/v164_features_doc.rst`](../docs/source/Zh/doc/new_features/v164_features_doc.rst)。 + +- **`match_auto` / `auto_threshold`**(`AC_match_auto`、`AC_auto_threshold`):每次 `match_template_all` 都迫使你猜 `min_score`(太低充滿 NMS 雜訊、太高漏掉換膚目標,且因素材而異)。本功能對*相關性分數直方圖*套用 Otsu,找出背景相關與真正匹配之間的谷,回傳該門檻加上 *separability* 分離度(接近 0 = 單峰、無明確匹配 → 不要信任)。`match_auto` 每個過門檻區域只回傳單一峰(透過 `connected_boxes`,避免寬峰上的重複命中),並以 `floor` 夾住。重用新增的 `visual_match._score_map`;`haystack` 可注入;不匯入 `PySide6`。 + ## 本次更新 (2026-06-24) — 權杖預算化的觀測差異(變更了什麼) 告訴代理*變更了什麼*,而非再次整個畫面。完整參考:[`docs/source/Zh/doc/new_features/v163_features_doc.rst`](../docs/source/Zh/doc/new_features/v163_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 798c4238..d24bf68d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-24) — Auto-Thresholded Template Matching (Otsu on the Score Map) + +No more hand-tuned `min_score` — derive the match threshold from the score map. Full reference: [`docs/source/Eng/doc/new_features/v164_features_doc.rst`](docs/source/Eng/doc/new_features/v164_features_doc.rst). + +- **`match_auto` / `auto_threshold`** (`AC_match_auto`, `AC_auto_threshold`): every `match_template_all` call forces you to guess `min_score` (too low floods NMS, too high drops re-themed targets, and it differs per asset). This runs Otsu on the *correlation score histogram* to find the valley between background correlation and real matches, returns that cut-off plus a *separability* score (near 0 = unimodal, no clear match → don't trust it). `match_auto` returns one peak per above-threshold region (via `connected_boxes`, avoiding duplicate hits on a wide peak), clamped by a `floor`. Reuses the new `visual_match._score_map`; injectable `haystack`; no `PySide6`. + ## What's new (2026-06-24) — Token-Budgeted Observation Delta (What Changed) Tell an agent *what changed* since the last step, not the whole screen again. Full reference: [`docs/source/Eng/doc/new_features/v163_features_doc.rst`](docs/source/Eng/doc/new_features/v163_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v164_features_doc.rst b/docs/source/Eng/doc/new_features/v164_features_doc.rst new file mode 100644 index 00000000..9138c109 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v164_features_doc.rst @@ -0,0 +1,47 @@ +Auto-Thresholded Template Matching (Otsu on the Score Map) +========================================================== + +Every call to ``match_template_all`` forces the caller to guess ``min_score``: too low +floods NMS with background noise, too high drops scaled / re-themed targets, and the right +value differs per asset and per screen. ``match_autothresh`` removes the magic number — it +runs Otsu's method on the *correlation score histogram* (not pixel intensities, the way +``preprocess.binarize`` does) to find the valley between the "background correlation" mass +and the "real match" mass, and returns that cut-off plus a *separability* number so the +caller knows when the histogram is unimodal (no clear match → don't trust the threshold). + +It reuses ``visual_match._score_map`` (the full ``matchTemplate`` surface the public matchers +discard) and ``cv2_utils.blobs.connected_boxes`` — each above-threshold region yields a single +peak, avoiding the duplicate hits a raw pixel scan + NMS leaves on a wide correlation peak. The +``haystack`` is injectable; the analysis is unit-testable on synthetic arrays. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import match_auto, auto_threshold + + # no min_score to tune — the threshold is derived from the score map + for hit in match_auto("save_button.png", floor=0.5): + print(hit.center, hit.score) + + info = auto_threshold("save_button.png") + # {"threshold": 0.83, "separability": 0.61, "n_above": 2} + if info["separability"] < 0.3: + print("no clear match — threshold not trustworthy") + +``match_auto`` returns one ``Match`` per above-threshold region, ordered by score and capped at +``max_results``; the cut-off is ``max(floor, otsu_threshold)`` so a unimodal / noisy surface +cannot drag it below a sane floor. ``auto_threshold`` returns ``{threshold, separability, +n_above}`` — ``separability`` near 0 means the score histogram is unimodal and the threshold +should be treated as unreliable. + +Executor commands +----------------- + +``AC_match_auto`` (``template`` / ``floor`` / ``max_results`` / ``region`` / ``method`` → +``{count, matches}``) and ``AC_auto_threshold`` (``template`` / ``region`` / ``method`` → +``{found, info}``). They are exposed as the MCP tools ``ac_match_auto`` / ``ac_auto_threshold`` +(read-only) and as the Script Builder commands **Match Template (auto-threshold)** / **Auto +Threshold (Otsu on scores)** under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 8e7dc0c1..cc670fc5 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -186,6 +186,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v161_features_doc doc/new_features/v162_features_doc doc/new_features/v163_features_doc + doc/new_features/v164_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v164_features_doc.rst b/docs/source/Zh/doc/new_features/v164_features_doc.rst new file mode 100644 index 00000000..ca033afe --- /dev/null +++ b/docs/source/Zh/doc/new_features/v164_features_doc.rst @@ -0,0 +1,43 @@ +自動門檻樣板比對(對分數圖做 Otsu) +==================================== + +每次呼叫 ``match_template_all`` 都迫使呼叫者猜 ``min_score``:太低會讓 NMS 充滿背景雜訊, +太高會漏掉縮放 / 換膚的目標,而正確值因素材與畫面而異。``match_autothresh`` 移除這個魔術 +數字——它對*相關性分數直方圖*(而非像素強度,後者是 ``preprocess.binarize`` 的做法)套用 +Otsu 法,找出「背景相關」團與「真正匹配」團之間的谷,並回傳該門檻加上一個 *separability* +(分離度)值,讓呼叫端知道何時直方圖為單峰(無明確匹配 → 不要信任該門檻)。 + +本功能重用 ``visual_match._score_map``(公開比對器丟棄的完整 ``matchTemplate`` 曲面)與 +``cv2_utils.blobs.connected_boxes``——每個過門檻區域只回傳單一峰,避免原始像素掃描 + NMS +在寬相關峰上留下的重複命中。``haystack`` 可注入;分析可在合成陣列上單元測試。不匯入 +``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_auto, auto_threshold + + # 無需手調 min_score——門檻由分數圖推導 + for hit in match_auto("save_button.png", floor=0.5): + print(hit.center, hit.score) + + info = auto_threshold("save_button.png") + # {"threshold": 0.83, "separability": 0.61, "n_above": 2} + if info["separability"] < 0.3: + print("無明確匹配——門檻不可信") + +``match_auto`` 每個過門檻區域回傳一個 ``Match``,依分數排序並以 ``max_results`` 為上限; +門檻為 ``max(floor, otsu_threshold)``,使單峰 / 雜訊曲面無法把它拉到合理下限之下。 +``auto_threshold`` 回傳 ``{threshold, separability, n_above}``——``separability`` 接近 0 +表示分數直方圖為單峰,該門檻應視為不可靠。 + +執行器指令 +---------- + +``AC_match_auto``(``template`` / ``floor`` / ``max_results`` / ``region`` / ``method`` → +``{count, matches}``)與 ``AC_auto_threshold``(``template`` / ``region`` / ``method`` → +``{found, info}``)。兩者以 MCP 工具 ``ac_match_auto`` / ``ac_auto_threshold``(唯讀)及 +Script Builder 指令 **Match Template (auto-threshold)** / **Auto Threshold (Otsu on scores)** +(位於 **Image** 分類下)形式提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index a3eec4cc..4f0b963d 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -186,6 +186,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v161_features_doc doc/new_features/v162_features_doc doc/new_features/v163_features_doc + doc/new_features/v164_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 43013a6d..26d604d0 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -287,6 +287,10 @@ from je_auto_control.utils.match_trust import ( TrustedMatch, match_with_trust, score_peaks, ) +# Otsu auto-thresholding for template matching (no hand-tuned min_score) +from je_auto_control.utils.match_autothresh import ( + auto_threshold, match_auto, +) # Coarse labelled cell grid for VLM grounding (point <-> cell mapping) from je_auto_control.utils.screen_grid import ( GridCell, cell_for_point, grid_cells, point_for_cell, @@ -1213,6 +1217,8 @@ def start_autocontrol_gui(*args, **kwargs): "TrustedMatch", "match_with_trust", "score_peaks", + "auto_threshold", + "match_auto", "GridCell", "grid_cells", "cell_for_point", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 31e2de7d..95419552 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -350,6 +350,27 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Match a template and flag if it is ambiguous (duplicate peak).", )) + specs.append(CommandSpec( + "AC_match_auto", "Image", "Match Template (auto-threshold)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("floor", FieldType.FLOAT, optional=True, default=0.5, + min_value=0.0, max_value=1.0), + FieldSpec("max_results", FieldType.INT, optional=True, default=20), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Find a template with an Otsu auto-threshold (no min_score).", + )) + specs.append(CommandSpec( + "AC_auto_threshold", "Image", "Auto Threshold (Otsu on scores)", + fields=( + FieldSpec("template", FieldType.FILE_PATH), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder=_REGION_PLACEHOLDER), + ), + description="Derive a match threshold + separability from the score map.", + )) specs.append(CommandSpec( "AC_grid_cells", "Image", "Grid Cells (coarse grounding)", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index b37519b9..cea97608 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3339,6 +3339,29 @@ def _match_with_trust(template: str, min_score: Any = 0.0, scales: Any = None, "match": match.to_dict() if match else None} +def _auto_threshold(template: str, region: Any = None, + method: str = "ccoeff_normed") -> Dict[str, Any]: + """Adapter: Otsu-derived accept threshold for a template (+ separability).""" + import json + from je_auto_control.utils.match_autothresh import auto_threshold + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + info = auto_threshold(template, region=region, method=method) + return {"found": info is not None, "info": info} + + +def _match_auto(template: str, floor: Any = 0.5, max_results: Any = 20, + region: Any = None, method: str = "ccoeff_normed") -> Dict[str, Any]: + """Adapter: matches above the auto-derived (Otsu) threshold, one per region.""" + import json + from je_auto_control.utils.match_autothresh import match_auto + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = match_auto(template, region=region, floor=float(floor), + max_results=int(max_results), method=method) + return {"count": len(matches), "matches": [m.to_dict() for m in matches]} + + def _region_arg(value: Any) -> Optional[List[int]]: """Coerce a JSON-string / list region arg into a list of ints, or None.""" import json @@ -5829,6 +5852,8 @@ def __init__(self): "AC_match_rotated": _match_rotated, "AC_match_rotated_all": _match_rotated_all, "AC_match_with_trust": _match_with_trust, + "AC_auto_threshold": _auto_threshold, + "AC_match_auto": _match_auto, "AC_grid_cells": _grid_cells, "AC_cell_for_point": _cell_for_point, "AC_point_for_cell": _point_for_cell, diff --git a/je_auto_control/utils/match_autothresh/__init__.py b/je_auto_control/utils/match_autothresh/__init__.py new file mode 100644 index 00000000..312f21d1 --- /dev/null +++ b/je_auto_control/utils/match_autothresh/__init__.py @@ -0,0 +1,6 @@ +"""Otsu auto-thresholding for template matching (no hand-tuned min_score).""" +from je_auto_control.utils.match_autothresh.match_autothresh import ( + auto_threshold, match_auto, +) + +__all__ = ["auto_threshold", "match_auto"] diff --git a/je_auto_control/utils/match_autothresh/match_autothresh.py b/je_auto_control/utils/match_autothresh/match_autothresh.py new file mode 100644 index 00000000..60b2a952 --- /dev/null +++ b/je_auto_control/utils/match_autothresh/match_autothresh.py @@ -0,0 +1,99 @@ +"""Auto-derive a template-match threshold from the score map (Otsu). + +Every call to ``match_template_all`` forces the caller to guess ``min_score``: too low +floods NMS with background noise, too high drops scaled / re-themed targets, and the right +value differs per asset and per screen. ``match_autothresh`` removes the magic number — it +runs Otsu's method on the *correlation score histogram* (not pixel intensities, the way +``preprocess.binarize`` does) to find the valley between the "background correlation" mass +and the "real match" mass, and returns that cut-off plus a *separability* number so the +caller knows when the histogram is unimodal (no clear match → don't trust the threshold). + +It reuses ``visual_match._score_map`` — the full ``matchTemplate`` surface the public +matchers discard — plus the shared ``Match`` / ``_nms``. The ``haystack`` is injectable +(ndarray / path / PIL); the analysis is unit-testable on synthetic arrays. OpenCV + NumPy +are imported lazily. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +from je_auto_control.utils.visual_match.visual_match import Match, _score_map + +ImageSource = Any + + +def _separability(scaled, threshold: float) -> float: + """Otsu separability eta = between-class variance / total variance (0..1).""" + total_var = float(scaled.var()) + if total_var < 1e-9: + return 0.0 + below, above = scaled[scaled <= threshold], scaled[scaled > threshold] + if below.size == 0 or above.size == 0: + return 0.0 + weight0, weight1 = below.size / scaled.size, above.size / scaled.size + between = weight0 * weight1 * (float(below.mean()) - float(above.mean())) ** 2 + return max(0.0, min(1.0, between / total_var)) + + +def _otsu_on_scores(score_map): + """Return ``(threshold_in_score_units, separability)`` for a correlation surface.""" + import cv2 + import numpy as np + low, high = float(score_map.min()), float(score_map.max()) + if high - low < 1e-9: + return low, 0.0 + scaled = ((score_map - low) / (high - low) * 255.0).astype(np.uint8) + level, _ = cv2.threshold(scaled, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + threshold = low + (float(level) / 255.0) * (high - low) + return threshold, _separability(scaled, float(level)) + + +def auto_threshold(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, + method: str = "ccoeff_normed") -> Optional[Dict[str, Any]]: + """Return an Otsu-derived accept cut-off for ``template``, or ``None``. + + ``{threshold, separability, n_above}`` — ``separability`` near 0 means the score + histogram is unimodal (no clear match) and the threshold should not be trusted. + """ + import numpy as np + score_map, _ = _score_map(template, haystack, region=region, method=method) + if score_map is None: + return None + threshold, separability = _otsu_on_scores(score_map) + return {"threshold": round(threshold, 4), + "separability": round(separability, 4), + "n_above": int(np.count_nonzero(score_map >= threshold))} + + +def _peak_in_box(score_map, box: Dict[str, Any]): + """Return ``(x, y, score)`` of the highest score inside one connected blob box.""" + import numpy as np + x, y, width, height = box["x"], box["y"], box["width"], box["height"] + sub = score_map[y:y + height, x:x + width] + iy, ix = np.unravel_index(int(np.argmax(sub)), sub.shape) + return x + int(ix), y + int(iy), float(sub[iy, ix]) + + +def match_auto(template: ImageSource, *, haystack: Optional[ImageSource] = None, + region: Optional[Sequence[int]] = None, floor: float = 0.5, + method: str = "ccoeff_normed", max_results: int = 20) -> List[Match]: + """Return one match per above-threshold region (auto cut-off clamped by ``floor``). + + The cut-off is ``max(floor, otsu_threshold)`` so a unimodal / noisy surface cannot + drag the threshold below a sane floor. Each connected above-threshold region yields + a single peak (its highest score), avoiding the duplicate hits a raw pixel scan + + NMS leaves on a wide correlation peak. Ordered by score, capped at ``max_results``. + """ + import numpy as np + from je_auto_control.utils.cv2_utils.blobs import connected_boxes + score_map, tmpl = _score_map(template, haystack, region=region, method=method) + if score_map is None: + return [] + threshold, _ = _otsu_on_scores(score_map) + cutoff = max(float(floor), threshold) + mask = (score_map >= cutoff).astype(np.uint8) + height, width = tmpl.shape[:2] + matches = [Match(px, py, width, height, round(score, 4), 1.0) + for px, py, score in + (_peak_in_box(score_map, box) for box in connected_boxes(mask))] + matches.sort(key=lambda m: m.score, reverse=True) + return matches[:int(max_results)] diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index a2adadf3..8a6ad74f 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3638,6 +3638,37 @@ def rotated_match_tools() -> List[MCPTool]: handler=h.match_with_trust, annotations=READ_ONLY, ), + MCPTool( + name="ac_auto_threshold", + description=("Derive an accept threshold for 'template' by Otsu on the " + "match score map (no hand-tuned min_score). Returns " + "{found, info:{threshold, separability, n_above}}. " + "separability near 0 = unimodal (no clear match) - do NOT " + "trust the threshold. 'region', 'method'."), + input_schema=schema({ + "template": {"type": "string"}, + "region": {"type": "array", "items": {"type": "integer"}}, + "method": {"type": "string"}}, + required=["template"]), + handler=h.auto_threshold, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_match_auto", + description=("Find every occurrence of 'template' above an AUTO-derived " + "(Otsu) threshold - one peak per region, no min_score to " + "tune. 'floor' (default 0.5) clamps the threshold so a noisy " + "surface can't match junk. Returns {count, matches}."), + input_schema=schema({ + "template": {"type": "string"}, + "floor": {"type": "number"}, + "max_results": {"type": "integer"}, + "region": {"type": "array", "items": {"type": "integer"}}, + "method": {"type": "string"}}, + required=["template"]), + handler=h.match_auto, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 38fc6bf2..7c06c8b4 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2115,6 +2115,17 @@ def match_with_trust(template, min_score=0.0, scales=None, ambiguous_ratio=0.9, region, method) +def auto_threshold(template, region=None, method="ccoeff_normed"): + from je_auto_control.utils.executor.action_executor import _auto_threshold + return _auto_threshold(template, region, method) + + +def match_auto(template, floor=0.5, max_results=20, region=None, + method="ccoeff_normed"): + from je_auto_control.utils.executor.action_executor import _match_auto + return _match_auto(template, floor, max_results, region, method) + + def grid_cells(rows, cols, region=None): from je_auto_control.utils.executor.action_executor import _grid_cells return _grid_cells(rows, cols, region) diff --git a/test/unit_test/headless/test_match_autothresh_batch.py b/test/unit_test/headless/test_match_autothresh_batch.py new file mode 100644 index 00000000..ebd546ee --- /dev/null +++ b/test/unit_test/headless/test_match_autothresh_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for Otsu auto-thresholded template matching.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +pytest.importorskip("cv2") + +from je_auto_control.utils.match_autothresh import ( # noqa: E402 + auto_threshold, match_auto, +) + + +def _template(): + tmpl = np.zeros((24, 24), dtype=np.uint8) + tmpl[:, :12] = 200 + return tmpl + + +def _haystack(*tops_lefts): + hay = np.zeros((160, 220), dtype=np.uint8) + tmpl = _template() + for top, left in tops_lefts: + hay[top:top + 24, left:left + 24] = tmpl + return hay + + +def test_auto_threshold_reports_metrics(): + info = auto_threshold(_template(), haystack=_haystack((20, 30), (20, 150))) + assert set(info) == {"threshold", "separability", "n_above"} + assert 0.0 < info["threshold"] < 1.0 + assert info["separability"] > 0.3 # clearly bimodal: matches vs background + + +def test_match_auto_finds_both_occurrences(): + matches = match_auto(_template(), haystack=_haystack((20, 30), (20, 150)), + floor=0.5) + assert len(matches) == 2 + xs = sorted(m.x for m in matches) + assert abs(xs[0] - 30) <= 1 and abs(xs[1] - 150) <= 1 + + +def test_floor_prevents_noise_on_blank(): + # a blank haystack has no bimodal split; floor keeps it from matching noise + matches = match_auto(_template(), haystack=np.zeros((160, 220), np.uint8), + floor=0.6) + assert matches == [] + + +def test_blank_separability_is_low(): + info = auto_threshold(_template(), haystack=np.zeros((160, 220), np.uint8)) + assert info["separability"] < 0.3 # unimodal → do not trust the threshold + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_match_auto", "AC_auto_threshold"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_match_auto", "ac_auto_threshold"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_match_auto", "AC_auto_threshold"} <= specs + + +def test_facade_exports(): + for name in ("match_auto", "auto_threshold"): + assert hasattr(ac, name) and name in ac.__all__ From 29d440dadf532754995478d90c47269c9ddc9a1f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 24 Jun 2026 03:21:32 +0800 Subject: [PATCH 2/2] Make auto_threshold test robust across OpenCV builds ccoeff_normed scores span [-1, 1], so the Otsu cut-off can legitimately be negative on some OpenCV builds (match_auto clamps it with floor anyway). Assert the threshold is below a perfect match and use a relative separability check (bimodal > flat) instead of absolute bounds. --- test/unit_test/headless/test_match_autothresh_batch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/unit_test/headless/test_match_autothresh_batch.py b/test/unit_test/headless/test_match_autothresh_batch.py index ebd546ee..ffd852fd 100644 --- a/test/unit_test/headless/test_match_autothresh_batch.py +++ b/test/unit_test/headless/test_match_autothresh_batch.py @@ -28,8 +28,14 @@ def _haystack(*tops_lefts): def test_auto_threshold_reports_metrics(): info = auto_threshold(_template(), haystack=_haystack((20, 30), (20, 150))) assert set(info) == {"threshold", "separability", "n_above"} - assert 0.0 < info["threshold"] < 1.0 - assert info["separability"] > 0.3 # clearly bimodal: matches vs background + # ccoeff_normed spans [-1, 1]; the cut-off just has to sit below a perfect + # match (its exact value depends on the OpenCV build's score distribution). + assert info["threshold"] < 1.0 + assert info["n_above"] >= 2 + # a clearly bimodal surface (matches vs background) is more separable than a + # flat one — a relative check that is stable across OpenCV builds + blank = auto_threshold(_template(), haystack=np.zeros((160, 220), np.uint8)) + assert info["separability"] > blank["separability"] def test_match_auto_finds_both_occurrences():