diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index a9818323..b925557e 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 以边缘 / 轮廓定位 UI 元素(免模板) + +在从未见过的画面上找出可点击的方框。完整参考:[`docs/source/Zh/doc/new_features/v132_features_doc.rst`](../docs/source/Zh/doc/new_features/v132_features_doc.rst)。 + +- **`find_shapes` / `find_rectangles`**(`AC_find_shapes`、`AC_find_rectangles`):其他定位器都需要一个寻找对象——模板、颜色或文字。这两个什么都不需要:Canny 边缘检测 + 轮廓提取返回各个形状的边界框(`{x,y,width,height,area,center,aspect}`,由大到小),让脚本能结构性地列举卡片 / 按钮 / 输入框并点击第 N 个。`find_rectangles` 只保留凸四边形,并加上 `aspect_range=(min,max)` 宽高比过滤(`(1.5,8)` 取宽按钮)。可注入 haystack → 无头可测。 + ## 本次更新 (2026-06-23) — ORB 特征匹配(对旋转 / 缩放 / 主题稳健) 即使目标旋转、缩放或换主题也能找到。完整参考:[`docs/source/Zh/doc/new_features/v131_features_doc.rst`](../docs/source/Zh/doc/new_features/v131_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 119b87ab..18f2c014 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 以邊緣 / 輪廓定位 UI 元素(免模板) + +在從未見過的畫面上找出可點擊的方框。完整參考:[`docs/source/Zh/doc/new_features/v132_features_doc.rst`](../docs/source/Zh/doc/new_features/v132_features_doc.rst)。 + +- **`find_shapes` / `find_rectangles`**(`AC_find_shapes`、`AC_find_rectangles`):其他定位器都需要一個尋找對象——模板、顏色或文字。這兩個什麼都不需要:Canny 邊緣偵測 + 輪廓擷取回傳各個形狀的邊界框(`{x,y,width,height,area,center,aspect}`,由大到小),讓腳本能結構性地列舉卡片 / 按鈕 / 輸入框並點擊第 N 個。`find_rectangles` 只保留凸四邊形,並加上 `aspect_range=(min,max)` 寬高比過濾(`(1.5,8)` 取寬按鈕)。可注入 haystack → 無頭可測。 + ## 本次更新 (2026-06-23) — ORB 特徵比對(對旋轉 / 縮放 / 主題穩健) 即使目標旋轉、縮放或換主題也能找到。完整參考:[`docs/source/Zh/doc/new_features/v131_features_doc.rst`](../docs/source/Zh/doc/new_features/v131_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 56530e83..c2b54731 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Locate UI Elements by Edge / Contour (No Template) + +Find the clickable boxes on a screen you have never seen. Full reference: [`docs/source/Eng/doc/new_features/v132_features_doc.rst`](docs/source/Eng/doc/new_features/v132_features_doc.rst). + +- **`find_shapes` / `find_rectangles`** (`AC_find_shapes`, `AC_find_rectangles`): every other locator needs something to look *for* — a template, a colour, some text. These need nothing: Canny edge detection + contour extraction returns the bounding boxes (`{x,y,width,height,area,center,aspect}`, largest first) of the distinct shapes, so a script can enumerate cards / buttons / input fields structurally and click the Nth one. `find_rectangles` keeps only convex quads and adds an `aspect_range=(min,max)` w/h filter (`(1.5,8)` wide buttons). Injectable haystack → headless-testable. + ## What's new (2026-06-23) — ORB Feature Matching (Rotation / Scale / Theme Robust) Find a target even when it is rotated, rescaled or re-themed. Full reference: [`docs/source/Eng/doc/new_features/v131_features_doc.rst`](docs/source/Eng/doc/new_features/v131_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v132_features_doc.rst b/docs/source/Eng/doc/new_features/v132_features_doc.rst new file mode 100644 index 00000000..612d4229 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v132_features_doc.rst @@ -0,0 +1,47 @@ +Locate UI Elements by Edge / Contour (No Template) +================================================== + +Every locator so far needs something to look *for*: ``match_template`` and +``feature_match`` need a reference image, ``find_color_region`` needs a colour, +``locate_text`` needs the text. None of them answers the structural question +"where are the clickable boxes on this screen?". ``find_shapes`` and +``find_rectangles`` run Canny edge detection plus contour extraction and return +the bounding boxes of the distinct shapes — so a script can enumerate the cards, +buttons or input fields on a screen it has never seen and act on the Nth one, +without ever supplying a sample. + +Both run on an injectable ``haystack`` image (ndarray / path / PIL), so they are +unit-testable on synthetic arrays without a real screen. OpenCV + NumPy come in +via ``je_open_cv``. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import find_shapes, find_rectangles + + # Every distinct shape, largest first. + for shape in find_shapes(min_area=500): + print(shape["x"], shape["y"], shape["width"], shape["height"]) + + # Just the wide, button-shaped rectangles, then click the first. + buttons = find_rectangles(min_area=800, aspect_range=(1.5, 8.0)) + if buttons: + click(*buttons[0]["center"]) + +``find_shapes`` returns ``{x, y, width, height, area, center, aspect}`` for every +contour (``area`` is the bounding-box area), largest first; ``min_area`` / +``max_area`` drop specks and the full-frame border. ``find_rectangles`` keeps only +contours that approximate to a convex quadrilateral (``epsilon`` is the +``approxPolyDP`` tolerance as a fraction of the perimeter) and adds an optional +``aspect_range`` ``(min, max)`` width/height filter — ``(1.5, 8)`` for wide +buttons, ``(0.8, 1.2)`` for square icons. + +Executor commands +----------------- + +``AC_find_shapes`` (``region`` / ``min_area`` / ``max_area`` → ``{count, shapes}``) +and ``AC_find_rectangles`` (also ``aspect_range`` / ``epsilon`` → +``{count, rectangles}``). They are exposed as the MCP tools ``ac_find_shapes`` / +``ac_find_rectangles`` and as Script Builder commands under **Image**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 7bb264bf..7206eab2 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -154,6 +154,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v129_features_doc doc/new_features/v130_features_doc doc/new_features/v131_features_doc + doc/new_features/v132_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v132_features_doc.rst b/docs/source/Zh/doc/new_features/v132_features_doc.rst new file mode 100644 index 00000000..343de33c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v132_features_doc.rst @@ -0,0 +1,38 @@ +以邊緣 / 輪廓定位 UI 元素(免模板) +==================================== + +目前所有定位器都需要一個*尋找對象*:``match_template`` 與 ``feature_match`` 需要參考影像、``find_color_region`` +需要顏色、``locate_text`` 需要文字。它們都無法回答結構性問題「這個畫面上可點擊的方框在哪裡?」。``find_shapes`` +與 ``find_rectangles`` 執行 Canny 邊緣偵測加輪廓擷取,回傳各個形狀的邊界框——讓腳本能在從未見過的畫面上列舉 +卡片、按鈕或輸入框,並對第 N 個操作,完全不需提供樣本。 + +兩者都在可注入的 ``haystack`` 影像(ndarray / 路徑 / PIL)上執行,因此可在無真實螢幕下對合成陣列做單元測試。 +OpenCV + NumPy 透過 ``je_open_cv`` 引入。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import find_shapes, find_rectangles + + # 每個獨立形狀,由大到小。 + for shape in find_shapes(min_area=500): + print(shape["x"], shape["y"], shape["width"], shape["height"]) + + # 只取寬的按鈕形矩形,點擊第一個。 + buttons = find_rectangles(min_area=800, aspect_range=(1.5, 8.0)) + if buttons: + click(*buttons[0]["center"]) + +``find_shapes`` 為每個輪廓回傳 ``{x, y, width, height, area, center, aspect}``(``area`` 為邊界框面積),由大到小; +``min_area`` / ``max_area`` 去除雜點與整框邊界。``find_rectangles`` 只保留近似凸四邊形的輪廓(``epsilon`` 是 +``approxPolyDP`` 以周長比例表示的容差),並加上選用的 ``aspect_range``(min, max)寬高比過濾——``(1.5, 8)`` 取寬 +按鈕、``(0.8, 1.2)`` 取方形圖示。 + +執行器命令 +---------- + +``AC_find_shapes``(``region`` / ``min_area`` / ``max_area`` → ``{count, shapes}``)與 ``AC_find_rectangles`` +(另有 ``aspect_range`` / ``epsilon`` → ``{count, rectangles}``)。它們以 MCP 工具 ``ac_find_shapes`` / +``ac_find_rectangles`` 以及 Script Builder 中 **Image** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 47ae36dc..ab847e27 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -154,6 +154,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v129_features_doc doc/new_features/v130_features_doc doc/new_features/v131_features_doc + doc/new_features/v132_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 571c87ea..cabefeaa 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -290,6 +290,10 @@ # ORB feature matching (rotation / scale / theme-robust template location) from je_auto_control.utils.feature_match import feature_match from je_auto_control.utils.feature_match import FeatureMatch +# Locate UI elements by edge/contour detection (rectangles / shapes, no template) +from je_auto_control.utils.shape_locator import ( + find_rectangles, find_shapes, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1091,6 +1095,8 @@ def start_autocontrol_gui(*args, **kwargs): "ssim_changed_regions", "feature_match", "FeatureMatch", + "find_shapes", + "find_rectangles", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index f868ef18..b2472eeb 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -354,6 +354,30 @@ def _add_image_specs(specs: List[CommandSpec]) -> None: ), description="Locate a template under rotation / scale / theme change.", )) + specs.append(CommandSpec( + "AC_find_shapes", "Image", "Find Shapes", + fields=( + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[left, top, right, bottom]"), + FieldSpec("min_area", FieldType.INT, optional=True, default=400), + FieldSpec("max_area", FieldType.INT, optional=True), + ), + description="Locate distinct shapes by edge/contour detection (no template).", + )) + specs.append(CommandSpec( + "AC_find_rectangles", "Image", "Find Rectangles", + fields=( + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[left, top, right, bottom]"), + FieldSpec("min_area", FieldType.INT, optional=True, default=400), + FieldSpec("max_area", FieldType.INT, optional=True), + FieldSpec("aspect_range", FieldType.STRING, optional=True, + placeholder="[1.5, 8.0]"), + FieldSpec("epsilon", FieldType.FLOAT, optional=True, default=0.04, + min_value=0.0, max_value=1.0), + ), + description="Locate rectangular regions (buttons / cards / input fields).", + )) def _add_ocr_specs(specs: List[CommandSpec]) -> None: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index c75fe5aa..e541111d 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3315,6 +3315,36 @@ def _feature_match(template: str, region: Any = None, max_features: Any = 500, "match": match.to_dict() if match else None} +def _find_shapes(region: Any = None, min_area: Any = 400, + max_area: Any = None) -> Dict[str, Any]: + """Adapter: bounding boxes of all distinct on-screen shapes, largest first.""" + import json + from je_auto_control.utils.shape_locator import find_shapes + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + shapes = find_shapes(region=region, min_area=int(min_area), + max_area=int(max_area) if max_area is not None else None) + return {"count": len(shapes), "shapes": shapes} + + +def _find_rectangles(region: Any = None, min_area: Any = 400, max_area: Any = None, + aspect_range: Any = None, epsilon: Any = 0.04 + ) -> Dict[str, Any]: + """Adapter: boxes of the ~rectangular shapes (buttons / cards), largest first.""" + import json + from je_auto_control.utils.shape_locator import find_rectangles + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + if isinstance(aspect_range, str): + aspect_range = json.loads(aspect_range) if aspect_range.strip() else None + rects = find_rectangles( + region=region, min_area=int(min_area), + max_area=int(max_area) if max_area is not None else None, + aspect_range=tuple(aspect_range) if aspect_range else None, + epsilon=float(epsilon)) + return {"count": len(rects), "rectangles": rects} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -5039,6 +5069,8 @@ def __init__(self): "AC_ssim_compare": _ssim_compare, "AC_ssim_changed_regions": _ssim_changed_regions, "AC_feature_match": _feature_match, + "AC_find_shapes": _find_shapes, + "AC_find_rectangles": _find_rectangles, "AC_find_color_region": _find_color_region, "AC_detect_drift": _detect_drift, "AC_categorical_drift": _categorical_drift, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b5e00280..d935e2ad 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2671,6 +2671,44 @@ def feature_match_tools() -> List[MCPTool]: ] +def shape_locator_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_find_shapes", + description=("Locate distinct on-screen shapes by edge/contour " + "detection — NO template/colour/text needed. Returns " + "{count, shapes:[{x,y,width,height,area,center,aspect}]} " + "(largest first). 'min_area'/'max_area' filter by " + "bounding-box area."), + input_schema=schema({ + "region": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "max_area": {"type": "integer"}}, + required=[]), + handler=h.find_shapes, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_find_rectangles", + description=("Locate ~rectangular regions (buttons / cards / input " + "fields) by contour approximation. Returns {count, " + "rectangles:[{x,y,width,height,area,center,aspect}]} " + "(largest first). 'aspect_range' [min,max] filters w/h " + "(e.g. [1.5,8] wide buttons); 'epsilon' the approx " + "tolerance."), + input_schema=schema({ + "region": {"type": "array", "items": {"type": "integer"}}, + "min_area": {"type": "integer"}, + "max_area": {"type": "integer"}, + "aspect_range": {"type": "array", "items": {"type": "number"}}, + "epsilon": {"type": "number"}}, + required=[]), + handler=h.find_rectangles, + annotations=READ_ONLY, + ), + ] + + def ssim_tools() -> List[MCPTool]: return [ MCPTool( @@ -6172,8 +6210,8 @@ def media_assert_tools() -> List[MCPTool]: process_doc_tools, tween_drag_tools, mouse_path_tools, field_entry_tools, key_hold_tools, mouse_relative_tools, text_unicode_tools, modifier_state_tools, grid_locator_tools, visual_match_tools, - color_region_tools, ssim_tools, feature_match_tools, plugin_sdk_tools, - governance_tools, + color_region_tools, ssim_tools, feature_match_tools, shape_locator_tools, + plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 216ddace..875b3d5b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2119,6 +2119,17 @@ def feature_match(template, region=None, max_features=500, ratio=0.75, return _feature_match(template, region, max_features, ratio, min_inliers) +def find_shapes(region=None, min_area=400, max_area=None): + from je_auto_control.utils.executor.action_executor import _find_shapes + return _find_shapes(region, min_area, max_area) + + +def find_rectangles(region=None, min_area=400, max_area=None, aspect_range=None, + epsilon=0.04): + from je_auto_control.utils.executor.action_executor import _find_rectangles + return _find_rectangles(region, min_area, max_area, aspect_range, epsilon) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/shape_locator/__init__.py b/je_auto_control/utils/shape_locator/__init__.py new file mode 100644 index 00000000..971fffad --- /dev/null +++ b/je_auto_control/utils/shape_locator/__init__.py @@ -0,0 +1,6 @@ +"""Locate UI elements by edge/contour detection (rectangles / shapes, no template).""" +from je_auto_control.utils.shape_locator.shape_locator import ( + find_rectangles, find_shapes, +) + +__all__ = ["find_rectangles", "find_shapes"] diff --git a/je_auto_control/utils/shape_locator/shape_locator.py b/je_auto_control/utils/shape_locator/shape_locator.py new file mode 100644 index 00000000..f1a01f6d --- /dev/null +++ b/je_auto_control/utils/shape_locator/shape_locator.py @@ -0,0 +1,99 @@ +"""Locate UI elements by edge/contour detection — buttons, cards, fields, no template. + +Template matching needs a reference image, colour-region needs a known colour, OCR +needs text — none of them answers "where are the rectangular, clickable regions on +this screen?". This runs Canny edge detection plus contour extraction and returns +the bounding boxes of the distinct shapes, optionally filtered to rectangles, so a +script can enumerate cards / buttons / inputs structurally and act on the Nth one +without ever supplying a sample image. + +It runs on an injectable ``haystack`` image (ndarray / path / PIL), so it is +unit-testable on synthetic arrays without a real screen. OpenCV + NumPy come in via +the project's ``je_open_cv`` dependency and are imported lazily. Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.visual_match.visual_match import _haystack_gray + +ImageSource = Any +AspectRange = Optional[Tuple[float, float]] +_CANNY_LOW = 50 +_CANNY_HIGH = 150 + + +def _contours(gray): + """Canny edges (dilated to close gaps) -> external contours, version-agnostic.""" + import cv2 + edges = cv2.Canny(gray, _CANNY_LOW, _CANNY_HIGH) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) + edges = cv2.dilate(edges, kernel, iterations=1) + found = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + return found[0] if len(found) == 2 else found[1] + + +def _box(contour) -> Dict[str, Any]: + """Bounding box of one contour as ``{x,y,width,height,area,center,aspect}``.""" + import cv2 + x, y, width, height = cv2.boundingRect(contour) + return {"x": int(x), "y": int(y), "width": int(width), "height": int(height), + "area": int(width * height), + "center": [int(x + width // 2), int(y + height // 2)], + "aspect": round(width / height, 3) if height else 0.0} + + +def _passes(box: Dict[str, Any], min_area: int, max_area: Optional[int], + aspect_range: AspectRange) -> bool: + """Apply the size / aspect-ratio filters to one box.""" + if box["area"] < int(min_area): + return False + if max_area is not None and box["area"] > int(max_area): + return False + if aspect_range is not None and not ( + aspect_range[0] <= box["aspect"] <= aspect_range[1]): + return False + return True + + +def _is_rectangle(contour, epsilon: float) -> bool: + """True when the contour approximates to a convex 4-gon (a rectangle).""" + import cv2 + perimeter = cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon * perimeter, True) + return len(approx) == 4 and cv2.isContourConvex(approx) + + +def find_shapes(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_area: int = 400, + max_area: Optional[int] = None) -> List[Dict[str, Any]]: + """Return bounding boxes of all distinct shapes on screen, largest first. + + Each result is ``{x, y, width, height, area, center, aspect}`` where ``area`` + is the bounding-box area. ``haystack`` is an ndarray / path / PIL image + (default: grab the screen / ``region``); ``min_area`` / ``max_area`` drop + specks and full-frame borders. + """ + boxes = [_box(contour) for contour in _contours(_haystack_gray(haystack, region))] + boxes = [box for box in boxes if _passes(box, min_area, max_area, None)] + boxes.sort(key=lambda box: box["area"], reverse=True) + return boxes + + +def find_rectangles(haystack: Optional[ImageSource] = None, *, + region: Optional[Sequence[int]] = None, min_area: int = 400, + max_area: Optional[int] = None, + aspect_range: AspectRange = None, + epsilon: float = 0.04) -> List[Dict[str, Any]]: + """Return boxes of the ~rectangular shapes (buttons / cards / fields), largest first. + + Keeps only contours that approximate to a convex quadrilateral (``epsilon`` is + the ``approxPolyDP`` tolerance as a fraction of the perimeter). ``aspect_range`` + is an optional ``(min, max)`` width/height filter — e.g. ``(1.5, 8)`` for wide + buttons, ``(0.8, 1.2)`` for square icons. + """ + gray = _haystack_gray(haystack, region) + boxes = [_box(contour) for contour in _contours(gray) + if _is_rectangle(contour, float(epsilon))] + boxes = [box for box in boxes if _passes(box, min_area, max_area, aspect_range)] + boxes.sort(key=lambda box: box["area"], reverse=True) + return boxes diff --git a/test/unit_test/headless/test_shape_locator_batch.py b/test/unit_test/headless/test_shape_locator_batch.py new file mode 100644 index 00000000..2ff2f66e --- /dev/null +++ b/test/unit_test/headless/test_shape_locator_batch.py @@ -0,0 +1,75 @@ +"""Headless tests for edge/contour shape location. No Qt.""" +import pytest + +import je_auto_control as ac + +np = pytest.importorskip("numpy") +cv2 = pytest.importorskip("cv2") + +from je_auto_control.utils.shape_locator import ( # noqa: E402 + find_rectangles, find_shapes, +) + + +def _scene(): + """300x200 scene: a wide button, a square outline, and a circle.""" + img = np.zeros((200, 300, 3), dtype=np.uint8) + cv2.rectangle(img, (20, 20), (120, 70), (255, 255, 255), -1) # 100x50 button + cv2.rectangle(img, (160, 30), (200, 70), (200, 200, 200), 2) # 40x40 outline + cv2.circle(img, (80, 150), 30, (180, 180, 180), -1) # not a rectangle + return img + + +def _near(value, target, tol=4): + return abs(value - target) <= tol + + +def test_find_shapes_returns_all_three(): + shapes = find_shapes(_scene(), min_area=300) + assert len(shapes) == 3 # button, outline, circle bbox + biggest = shapes[0] # largest first + assert _near(biggest["width"], 100) and _near(biggest["height"], 50) + + +def test_find_rectangles_excludes_the_circle(): + rects = find_rectangles(_scene(), min_area=300) + assert len(rects) == 2 # circle dropped + assert all(0.7 < r["aspect"] for r in rects) + + +def test_aspect_range_keeps_only_wide_button(): + wide = find_rectangles(_scene(), min_area=300, aspect_range=(1.5, 8.0)) + assert len(wide) == 1 + box = wide[0] + assert _near(box["x"], 20) and _near(box["y"], 20) + assert _near(box["center"][0], 70) and _near(box["center"][1], 45) + + +def test_max_area_filters_large_shapes(): + small = find_shapes(_scene(), min_area=300, max_area=3000) + assert all(s["area"] <= 3000 for s in small) + assert small # the ~40x40 outline survives + + +def test_min_area_filters_specks(): + img = np.zeros((100, 100, 3), dtype=np.uint8) + cv2.rectangle(img, (10, 10), (15, 15), (255, 255, 255), -1) # tiny speck + assert find_shapes(img, min_area=500) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_find_shapes", "AC_find_rectangles"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_find_shapes", "ac_find_rectangles"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_find_shapes", "AC_find_rectangles"} <= specs + + +def test_facade_exports(): + for attr in ("find_shapes", "find_rectangles"): + assert hasattr(ac, attr) and attr in ac.__all__