From b1a3a9fb7dad6da981bbbaf6ed69c7ce96b87192 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 26 Jun 2026 07:32:16 +0800 Subject: [PATCH 1/4] Add cvd_simulate: simulate colour-vision deficiency and flag collisions Status UIs lean on colour (green ok / red error), but for the ~8% of men with a colour-vision deficiency those can be indistinguishable and nothing could check it. simulate_cvd maps RGB through a dichromat matrix (protan/deuter/tritan) at a severity; colors_collide reports whether two colours become confusable under it. Pure stdlib over RGB tuples, no deps. --- WHATS_NEW.md | 6 + .../doc/new_features/v214_features_doc.rst | 48 +++++++ .../Zh/doc/new_features/v214_features_doc.rst | 42 +++++++ je_auto_control/__init__.py | 5 + .../gui/script_builder/command_schema.py | 25 ++++ .../utils/cvd_simulate/__init__.py | 6 + .../utils/cvd_simulate/cvd_simulate.py | 119 ++++++++++++++++++ .../utils/executor/action_executor.py | 26 ++++ .../utils/mcp_server/tools/_factories.py | 30 +++++ .../utils/mcp_server/tools/_handlers.py | 11 ++ .../headless/test_cvd_simulate_batch.py | 119 ++++++++++++++++++ 11 files changed, 437 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v214_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v214_features_doc.rst create mode 100644 je_auto_control/utils/cvd_simulate/__init__.py create mode 100644 je_auto_control/utils/cvd_simulate/cvd_simulate.py create mode 100644 test/unit_test/headless/test_cvd_simulate_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 2732b174..7e97b9ed 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Colour-Vision-Deficiency Simulation + Collision Check + +Check whether your red/green status colours are distinguishable to colour-blind users. Full reference: [`docs/source/Eng/doc/new_features/v214_features_doc.rst`](docs/source/Eng/doc/new_features/v214_features_doc.rst). + +- **`simulate_cvd` / `colors_collide` / `color_distance`** (`AC_simulate_cvd`, `AC_colors_collide`): status UIs lean on colour (green "ok" vs red "error"), but for the ~8% of men with a colour-vision deficiency those can be indistinguishable — and nothing in the framework could check it. `simulate_cvd` maps an RGB colour through a dichromat simulation matrix (protanopia/deuteranopia/tritanopia) at a given `severity`; `colors_collide` simulates two colours and reports whether they become confusable (a perceptual `redmean` distance below `threshold`); `color_distance` is the underlying metric. Pure standard library — no numpy/OpenCV, operating on plain RGB tuples, fully testable. First feature of the ROUND-15 perception lane. No `PySide6`. + ### Wait Until the App Is Idle Hold off the next click until the busy/wait cursor settles — don't act mid-churn. Full reference: [`docs/source/Eng/doc/new_features/v213_features_doc.rst`](docs/source/Eng/doc/new_features/v213_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v214_features_doc.rst b/docs/source/Eng/doc/new_features/v214_features_doc.rst new file mode 100644 index 00000000..2bae89c4 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v214_features_doc.rst @@ -0,0 +1,48 @@ +Colour-Vision-Deficiency Simulation + Collision Check +===================================================== + +Status UIs lean on colour — a green "ok" vs a red "error" dot, a colour-coded +chart legend. For the ~8% of men with a colour-vision deficiency (CVD) those can +be indistinguishable, and nothing in the framework could check it. +``cvd_simulate`` adds the two primitives an accessibility / design check needs. + +* :func:`simulate_cvd` — map an ``(r, g, b)`` colour through a dichromat + simulation matrix (``protanopia`` / ``deuteranopia`` / ``tritanopia``) at a + given ``severity`` (0 = unaffected, 1 = full dichromacy). +* :func:`colors_collide` — simulate two colours under a CVD type and report + whether they become too similar to tell apart (a perceptual ``redmean`` + distance below ``threshold``). +* :func:`color_distance` — the underlying ``redmean`` colour-difference metric. + +Pure standard library — no numpy / OpenCV — operating on plain RGB tuples, so it +is fully testable. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import simulate_cvd, colors_collide + + # How does the "error red" look to a deuteranope? + simulate_cvd((220, 40, 40), "deuteranopia") # -> (r, g, b) + + # Are my ok-green and error-red distinguishable for them? + report = colors_collide((60, 200, 60), (220, 60, 60), kind="deuteranopia") + report["collide"] # True if the two are confusable + report["distance"] # the perceptual distance after simulation + +``simulate_cvd`` accepts friendly aliases (``protan`` / ``deutan`` / ``tritan``, +or ``red`` / ``green`` / ``blue``). ``severity`` interpolates between the +original colour and the full dichromat simulation, for the milder anomalous +trichromacies. ``colors_collide`` returns ``{collide, distance, kind, severity, +simulated_left, simulated_right}``. + +Executor commands +----------------- + +``AC_simulate_cvd`` (``rgb`` ``[r, g, b]`` + ``kind`` / ``severity`` → +``{rgb}``) and ``AC_colors_collide`` (``left`` / ``right`` ``[r, g, b]`` + +``kind`` / ``severity`` / ``threshold`` → the report). RGB inputs accept a JSON +list. They are the matching read-only ``ac_*`` MCP tools and Script Builder +commands under **Image**. diff --git a/docs/source/Zh/doc/new_features/v214_features_doc.rst b/docs/source/Zh/doc/new_features/v214_features_doc.rst new file mode 100644 index 00000000..74cfd3e6 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v214_features_doc.rst @@ -0,0 +1,42 @@ +色覺辨認障礙模擬 + 碰撞檢查 +========================== + +狀態 UI 仰賴顏色——綠色「正常」對紅色「錯誤」的圓點、以顏色編碼的圖表圖例。對約 8% 有色覺辨認障礙 +(CVD)的男性而言,這些可能難以分辨,而框架原本無從檢查。``cvd_simulate`` 補上無障礙 / 設計檢查 +所需的兩個原語。 + +* :func:`simulate_cvd` ——把 ``(r, g, b)`` 顏色透過二色覺模擬矩陣(``protanopia`` / + ``deuteranopia`` / ``tritanopia``)在給定 ``severity``(0 = 不受影響,1 = 完全二色覺)下映射。 +* :func:`colors_collide` ——在某 CVD 類型下模擬兩個顏色,並回報它們是否變得太相似而難以區分 + (模擬後的感知 ``redmean`` 距離低於 ``threshold``)。 +* :func:`color_distance` ——底層的 ``redmean`` 色差度量。 + +純標準函式庫——不需 numpy / OpenCV——以單純的 RGB tuple 運作,故能完整測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import simulate_cvd, colors_collide + + # 「錯誤紅」在綠色弱者眼中看起來如何? + simulate_cvd((220, 40, 40), "deuteranopia") # -> (r, g, b) + + # 我的正常綠與錯誤紅對他們是否可區分? + report = colors_collide((60, 200, 60), (220, 60, 60), kind="deuteranopia") + report["collide"] # 若兩者易混淆則為 True + report["distance"] # 模擬後的感知距離 + +``simulate_cvd`` 接受友善別名(``protan`` / ``deutan`` / ``tritan``,或 +``red`` / ``green`` / ``blue``)。``severity`` 在原色與完全二色覺模擬之間插值, +用於較輕微的異常三色覺。``colors_collide`` 回傳 ``{collide, distance, kind, severity, +simulated_left, simulated_right}``。 + +執行器指令 +---------- + +``AC_simulate_cvd``(``rgb`` ``[r, g, b]`` 加上 ``kind`` / ``severity`` → +``{rgb}``)與 ``AC_colors_collide``(``left`` / ``right`` ``[r, g, b]`` 加上 +``kind`` / ``severity`` / ``threshold`` → 報告)。RGB 輸入接受 JSON 清單。皆以對應的唯讀 +``ac_*`` MCP 工具及 Script Builder 指令(位於 **Image** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 21dd9129..9930a9e4 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -131,6 +131,10 @@ from je_auto_control.utils.ensure_state import ensure_state, ensure_toggle # Wait until an application stops being busy before the next step from je_auto_control.utils.app_idle import idle_point, wait_until_app_idle +# Colour-vision-deficiency simulation + colour-collision check +from je_auto_control.utils.cvd_simulate import ( + color_distance, colors_collide, simulate_cvd, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1755,6 +1759,7 @@ def start_autocontrol_gui(*args, **kwargs): "recommend_timeout", "timeout_stats", "ensure_state", "ensure_toggle", "wait_until_app_idle", "idle_point", + "simulate_cvd", "colors_collide", "color_distance", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 96092eb9..996c0594 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4522,6 +4522,31 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Index where a busy/idle series first settles idle.", )) + specs.append(CommandSpec( + "AC_simulate_cvd", "Image", "Simulate Colour-Vision Deficiency", + fields=( + FieldSpec("rgb", FieldType.STRING, placeholder="[r, g, b]"), + FieldSpec("kind", FieldType.STRING, optional=True, + default="deuteranopia", + placeholder="protanopia / deuteranopia / tritanopia"), + FieldSpec("severity", FieldType.FLOAT, optional=True, default=1.0), + ), + description="Map an RGB colour through a CVD simulation matrix.", + )) + specs.append(CommandSpec( + "AC_colors_collide", "Image", "Colours Collide Under CVD", + fields=( + FieldSpec("left", FieldType.STRING, placeholder="[r, g, b]"), + FieldSpec("right", FieldType.STRING, placeholder="[r, g, b]"), + FieldSpec("kind", FieldType.STRING, optional=True, + default="deuteranopia", + placeholder="protanopia / deuteranopia / tritanopia"), + FieldSpec("severity", FieldType.FLOAT, optional=True, default=1.0), + FieldSpec("threshold", FieldType.FLOAT, optional=True, + default=40.0), + ), + description="Whether two colours become confusable under a CVD type.", + )) specs.append(CommandSpec( "AC_normalize_ext", "Shell", "Normalize Extension", fields=( diff --git a/je_auto_control/utils/cvd_simulate/__init__.py b/je_auto_control/utils/cvd_simulate/__init__.py new file mode 100644 index 00000000..98c6b8df --- /dev/null +++ b/je_auto_control/utils/cvd_simulate/__init__.py @@ -0,0 +1,6 @@ +"""Simulate colour-vision deficiency and flag colours that collide under it.""" +from je_auto_control.utils.cvd_simulate.cvd_simulate import ( + CVD_KINDS, color_distance, colors_collide, simulate_cvd, +) + +__all__ = ["simulate_cvd", "colors_collide", "color_distance", "CVD_KINDS"] diff --git a/je_auto_control/utils/cvd_simulate/cvd_simulate.py b/je_auto_control/utils/cvd_simulate/cvd_simulate.py new file mode 100644 index 00000000..9835947b --- /dev/null +++ b/je_auto_control/utils/cvd_simulate/cvd_simulate.py @@ -0,0 +1,119 @@ +"""Simulate colour-vision deficiency and flag colours that collide under it. + +Status UIs lean on colour — a green "ok" vs a red "error" dot, a colour-coded +chart legend. For the ~8% of men with a colour-vision deficiency (CVD) those can +be indistinguishable, and nothing in the framework could check it. ``cvd_simulate`` +adds the two primitives an accessibility / design check needs: + +* :func:`simulate_cvd` — map an ``(r, g, b)`` colour through a dichromat + simulation matrix (protanopia / deuteranopia / tritanopia) at a given + ``severity`` (0 = unaffected, 1 = full dichromacy). +* :func:`colors_collide` — simulate two colours under a CVD type and report + whether they become too similar to tell apart (a perceptual ``redmean`` + distance below ``threshold``). + +Pure standard library (no numpy / OpenCV) — operates on plain RGB tuples, so it +is fully testable. Imports no ``PySide6``. +""" +import math +from typing import Any, Dict, List, Sequence, Tuple + +RGB = Tuple[int, int, int] + +# Dichromat simulation matrices (sRGB-space approximation, Brettel/Viénot +# lineage as used by daltonize tooling). Applied at severity 1.0; lower +# severities interpolate toward the identity. +_MATRICES: Dict[str, List[List[float]]] = { + "protanopia": [[0.567, 0.433, 0.000], + [0.558, 0.442, 0.000], + [0.000, 0.242, 0.758]], + "deuteranopia": [[0.625, 0.375, 0.000], + [0.700, 0.300, 0.000], + [0.000, 0.300, 0.700]], + "tritanopia": [[0.950, 0.050, 0.000], + [0.000, 0.433, 0.567], + [0.000, 0.475, 0.525]], +} + +# Friendly aliases for the canonical CVD kinds. +_ALIASES = { + "protan": "protanopia", "protanope": "protanopia", "red": "protanopia", + "deuter": "deuteranopia", "deutan": "deuteranopia", + "deuteranope": "deuteranopia", "green": "deuteranopia", + "tritan": "tritanopia", "tritanope": "tritanopia", "blue": "tritanopia", +} + +CVD_KINDS = tuple(_MATRICES) + + +def _canonical_kind(kind: str) -> str: + """Resolve a CVD kind name / alias to its canonical key.""" + key = str(kind).strip().lower() + canonical = _ALIASES.get(key, key) + if canonical not in _MATRICES: + raise ValueError(f"unknown CVD kind: {kind!r}") + return canonical + + +def _clamp_byte(value: float) -> int: + """Clamp a channel value to an integer in ``[0, 255]``.""" + return max(0, min(255, int(round(value)))) + + +def simulate_cvd(rgb: Sequence[float], kind: str = "deuteranopia", + severity: float = 1.0) -> RGB: + """Return ``rgb`` as seen under ``kind`` colour-vision deficiency. + + ``severity`` interpolates between the original colour (0) and the full + dichromat simulation (1). ``rgb`` channels are ``0..255``. + """ + matrix = _MATRICES[_canonical_kind(kind)] + strength = max(0.0, min(1.0, float(severity))) + channels = [float(rgb[0]), float(rgb[1]), float(rgb[2])] + result = [] + for index in range(3): + row = matrix[index] + simulated = row[0] * channels[0] + row[1] * channels[1] \ + + row[2] * channels[2] + blended = channels[index] * (1.0 - strength) + simulated * strength + result.append(_clamp_byte(blended)) + return result[0], result[1], result[2] + + +def color_distance(left: Sequence[float], right: Sequence[float]) -> float: + """Perceptual ``redmean`` distance between two RGB colours (pure). + + A low-cost approximation of perceived colour difference that weights the + channels by the average red level. + """ + red_mean = (float(left[0]) + float(right[0])) / 2.0 + delta_r = float(left[0]) - float(right[0]) + delta_g = float(left[1]) - float(right[1]) + delta_b = float(left[2]) - float(right[2]) + return math.sqrt((2 + red_mean / 256) * delta_r * delta_r + + 4 * delta_g * delta_g + + (2 + (255 - red_mean) / 256) * delta_b * delta_b) + + +def colors_collide(left: Sequence[float], right: Sequence[float], *, + kind: str = "deuteranopia", severity: float = 1.0, + threshold: float = 40.0) -> Dict[str, Any]: + """Report whether two colours become confusable under ``kind`` CVD. + + Simulates both colours and compares them with :func:`color_distance`; + ``collide`` is ``True`` when that distance is below ``threshold``. Returns + ``{collide, distance, kind, severity, simulated_left, simulated_right}``. + """ + canonical = _canonical_kind(kind) + strength = max(0.0, min(1.0, float(severity))) + sim_left = simulate_cvd(left, canonical, strength) + sim_right = simulate_cvd(right, canonical, strength) + distance = color_distance(sim_left, sim_right) + return { + "collide": distance < float(threshold), + "distance": round(distance, 3), + "kind": canonical, + "severity": strength, + "simulated_left": list(sim_left), + "simulated_right": list(sim_right), + } diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 56f4305b..42a80840 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2839,6 +2839,30 @@ def _idle_point(busy_samples: Any, quiet_samples: Any = 3) -> Dict[str, Any]: return {"index": idle_point(samples, quiet_samples=int(quiet_samples))} +def _coerce_rgb(value: Any) -> tuple: + """Normalise an RGB argument (JSON '[r,g,b]' string / list) to (r, g, b).""" + seq = _coerce_list(value) if isinstance(value, str) else list(value) + return (int(seq[0]), int(seq[1]), int(seq[2])) + + +def _simulate_cvd(rgb: Any, kind: Any = "deuteranopia", + severity: Any = 1.0) -> Dict[str, Any]: + """Adapter: map an RGB colour through a CVD simulation matrix (pure).""" + from je_auto_control.utils.cvd_simulate import simulate_cvd + result = simulate_cvd(_coerce_rgb(rgb), str(kind), float(severity)) + return {"rgb": list(result)} + + +def _colors_collide(left: Any, right: Any, kind: Any = "deuteranopia", + severity: Any = 1.0, threshold: Any = 40.0 + ) -> Dict[str, Any]: + """Adapter: whether two colours become confusable under a CVD type (pure).""" + from je_auto_control.utils.cvd_simulate import colors_collide + return colors_collide(_coerce_rgb(left), _coerce_rgb(right), + kind=str(kind), severity=float(severity), + threshold=float(threshold)) + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -6870,6 +6894,8 @@ def __init__(self): "AC_ensure_field_value": _ensure_field_value, "AC_wait_until_app_idle": _wait_until_app_idle, "AC_idle_point": _idle_point, + "AC_simulate_cvd": _simulate_cvd, + "AC_colors_collide": _colors_collide, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 91e56b8b..c57aba83 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -4000,6 +4000,36 @@ def img_histogram_tools() -> List[MCPTool]: handler=h.most_salient, annotations=READ_ONLY, ), + MCPTool( + name="ac_simulate_cvd", + description=("Map an 'rgb' colour [r,g,b] through a colour-vision-" + "deficiency simulation (kind=protanopia/deuteranopia/" + "tritanopia, severity 0..1). Returns {rgb}."), + input_schema=schema({"rgb": {"type": "array", + "items": {"type": "integer"}}, + "kind": {"type": "string"}, + "severity": {"type": "number"}}, + required=["rgb"]), + handler=h.simulate_cvd, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_colors_collide", + description=("Whether two colours 'left'/'right' [r,g,b] become " + "confusable under a CVD 'kind' (distance below " + "'threshold'). Returns {collide, distance, kind, " + "severity, simulated_left, simulated_right}."), + input_schema=schema({"left": {"type": "array", + "items": {"type": "integer"}}, + "right": {"type": "array", + "items": {"type": "integer"}}, + "kind": {"type": "string"}, + "severity": {"type": "number"}, + "threshold": {"type": "number"}}, + required=["left", "right"]), + handler=h.colors_collide, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index ef5debeb..2d400a6d 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -733,6 +733,17 @@ def idle_point(busy_samples, quiet_samples=3): return _idle_point(busy_samples, quiet_samples) +def simulate_cvd(rgb, kind="deuteranopia", severity=1.0): + from je_auto_control.utils.executor.action_executor import _simulate_cvd + return _simulate_cvd(rgb, kind, severity) + + +def colors_collide(left, right, kind="deuteranopia", severity=1.0, + threshold=40.0): + from je_auto_control.utils.executor.action_executor import _colors_collide + return _colors_collide(left, right, kind, severity, threshold) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/test/unit_test/headless/test_cvd_simulate_batch.py b/test/unit_test/headless/test_cvd_simulate_batch.py new file mode 100644 index 00000000..0501abc7 --- /dev/null +++ b/test/unit_test/headless/test_cvd_simulate_batch.py @@ -0,0 +1,119 @@ +"""Headless tests for cvd_simulate (pure colour-vision-deficiency math).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.cvd_simulate import ( + CVD_KINDS, color_distance, colors_collide, simulate_cvd, +) + + +# --- simulate_cvd --------------------------------------------------------- + +def test_simulate_severity_zero_is_identity(): + assert simulate_cvd((120, 200, 30), "deuteranopia", severity=0.0) == \ + (120, 200, 30) + + +def test_simulate_grey_is_unchanged(): + # rows of every matrix sum to 1, so a neutral grey maps to itself + for kind in CVD_KINDS: + assert simulate_cvd((128, 128, 128), kind) == (128, 128, 128) + + +def test_simulate_shifts_red_under_protanopia(): + simulated = simulate_cvd((255, 0, 0), "protanopia") + # pure red is strongly altered for a protanope (red cones missing) + assert simulated != (255, 0, 0) + assert all(0 <= channel <= 255 for channel in simulated) + + +def test_simulate_accepts_aliases(): + assert simulate_cvd((10, 20, 30), "green") == \ + simulate_cvd((10, 20, 30), "deuteranopia") + + +def test_simulate_unknown_kind_raises(): + with pytest.raises(ValueError): + simulate_cvd((1, 2, 3), "tetrachromacy") + + +def test_simulate_severity_clamped(): + # severity above 1 behaves like 1 (full simulation) + assert simulate_cvd((200, 50, 10), "protanopia", severity=5.0) == \ + simulate_cvd((200, 50, 10), "protanopia", severity=1.0) + + +# --- color_distance ------------------------------------------------------- + +def test_color_distance_zero_for_identical(): + assert color_distance((10, 20, 30), (10, 20, 30)) == pytest.approx(0.0) + + +def test_color_distance_positive_and_symmetric(): + forward = color_distance((255, 0, 0), (0, 255, 0)) + backward = color_distance((0, 255, 0), (255, 0, 0)) + assert forward > 0 + assert forward == pytest.approx(backward) + + +# --- colors_collide ------------------------------------------------------- + +def test_cvd_reduces_red_green_distance(): + # deuteranopia pulls red and green closer together than normal vision + red, green = (220, 60, 60), (60, 200, 60) + normal = color_distance(red, green) + cvd = color_distance(simulate_cvd(red, "deuteranopia"), + simulate_cvd(green, "deuteranopia")) + assert cvd < normal + + +def test_colors_collide_close_pair(): + # two muddy shades that map to nearly the same colour for a deuteranope + result = colors_collide((150, 120, 110), (135, 130, 110), + kind="deuteranopia") + assert result["collide"] is True + assert result["kind"] == "deuteranopia" + assert len(result["simulated_left"]) == 3 + + +def test_colors_collide_black_white_never_collide(): + result = colors_collide((0, 0, 0), (255, 255, 255)) + assert result["collide"] is False + assert result["distance"] > 40.0 + + +def test_colors_collide_threshold_respected(): + red, green = (220, 60, 60), (60, 200, 60) + assert colors_collide(red, green, threshold=40.0)["collide"] is False + assert colors_collide(red, green, threshold=200.0)["collide"] is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _colors_collide, _simulate_cvd, + ) + out = _simulate_cvd("[128, 128, 128]", "protanopia", 1.0) + assert out["rgb"] == [128, 128, 128] + collide = _colors_collide([150, 120, 110], [135, 130, 110], + "deuteranopia", 1.0, 40.0) + assert collide["collide"] is True + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_simulate_cvd", "AC_colors_collide"} <= known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_simulate_cvd", "ac_colors_collide"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_simulate_cvd", "AC_colors_collide"} <= specs + + +def test_facade_exports(): + for name in ("simulate_cvd", "colors_collide", "color_distance"): + assert hasattr(ac, name) and name in ac.__all__ From d798e9d0bbb83d82572a5e8e8e9014bd38312180 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 26 Jun 2026 07:48:39 +0800 Subject: [PATCH 2/4] Add marks_layout: non-overlapping Set-of-Marks labels with readable colour set_of_marks draws each numbered label at a fixed offset, so on dense UIs the numbers pile up and a dark label on a dark element vanishes. place_labels does greedy non-overlap placement over a candidate ring, staying in bounds; label_color picks black/white by better WCAG contrast (reusing a11y_audit.contrast_ratio). Pure geometry, fully testable. --- WHATS_NEW.md | 6 + .../doc/new_features/v215_features_doc.rst | 43 +++++++ .../Zh/doc/new_features/v215_features_doc.rst | 37 ++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 20 +++ .../utils/executor/action_executor.py | 19 +++ .../utils/marks_layout/__init__.py | 6 + .../utils/marks_layout/marks_layout.py | 118 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 27 ++++ .../utils/mcp_server/tools/_handlers.py | 10 ++ .../headless/test_marks_layout_batch.py | 95 ++++++++++++++ 11 files changed, 384 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v215_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v215_features_doc.rst create mode 100644 je_auto_control/utils/marks_layout/__init__.py create mode 100644 je_auto_control/utils/marks_layout/marks_layout.py create mode 100644 test/unit_test/headless/test_marks_layout_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 7e97b9ed..0f537fc4 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Set-of-Marks Label Layout (No Overlap, Readable Colour) + +Number every element without the labels piling up or vanishing into the background. Full reference: [`docs/source/Eng/doc/new_features/v215_features_doc.rst`](docs/source/Eng/doc/new_features/v215_features_doc.rst). + +- **`place_labels` / `label_color`** (`AC_place_labels`, `AC_label_color`): Set-of-Marks draws each numbered label at a fixed offset, so on dense UIs the numbers pile on top of each other and a dark label on a dark element vanishes. `place_labels` is greedy non-overlap placement — for each mark it tries a ring of candidate positions around its box (above/below/inside, left/right aligned) and takes the first that stays in bounds and clears every already-placed label; `label_color` picks black or white by whichever has the better WCAG contrast against the element background (reusing `a11y_audit.contrast_ratio`). Pure standard library, deterministic, fully testable without rendering. Second feature of the ROUND-15 perception lane. No `PySide6`. + ### Colour-Vision-Deficiency Simulation + Collision Check Check whether your red/green status colours are distinguishable to colour-blind users. Full reference: [`docs/source/Eng/doc/new_features/v214_features_doc.rst`](docs/source/Eng/doc/new_features/v214_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v215_features_doc.rst b/docs/source/Eng/doc/new_features/v215_features_doc.rst new file mode 100644 index 00000000..4529494d --- /dev/null +++ b/docs/source/Eng/doc/new_features/v215_features_doc.rst @@ -0,0 +1,43 @@ +Set-of-Marks Label Layout (No Overlap, Readable Colour) +======================================================= + +Set-of-Marks overlays a numbered label on every element so a vision model can +say "click 7". ``set_of_marks`` draws each label at a fixed offset, so on dense +UIs the numbers pile on top of each other (unreadable) and a dark label on a +dark element vanishes. ``marks_layout`` fixes both with pure geometry. + +* :func:`place_labels` — greedy non-overlap placement: for each mark, try a ring + of candidate positions around its box (above, below, inside; left/right + aligned) and take the first that stays in bounds and clears every + already-placed label. +* :func:`label_color` — pick the label text colour (black or white) with the + better WCAG contrast against the element's background. + +Pure standard library; reuses :func:`a11y_audit.contrast_ratio`. Fully testable +without rendering. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import mark_elements, place_labels, label_color + + marks = mark_elements(elements) # [{id, bbox, ...}] + layout = place_labels(marks, bounds=(1920, 1080)) + # [{'id': 1, 'label': [x, y, 22, 16], 'anchor': [bx, by]}, ...] + + label_color((30, 30, 30)) # {'rgb': [255, 255, 255], 'contrast': ...} + +Feed the ``label`` boxes from :func:`place_labels` to your renderer instead of a +naive fixed offset, and pick each number's colour with :func:`label_color` so it +stays legible on its background. ``place_labels`` is deterministic and ordered by +the input marks, so the same screen always numbers the same way. + +Executor commands +----------------- + +``AC_place_labels`` (``marks`` JSON list + ``label_width`` / ``label_height`` / +``bounds`` ``[w, h]`` → ``{labels}``) and ``AC_label_color`` (``background`` +``[r, g, b]`` → ``{rgb, contrast}``). They are the matching read-only ``ac_*`` +MCP tools and Script Builder commands under **Image**. diff --git a/docs/source/Zh/doc/new_features/v215_features_doc.rst b/docs/source/Zh/doc/new_features/v215_features_doc.rst new file mode 100644 index 00000000..d5daad5d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v215_features_doc.rst @@ -0,0 +1,37 @@ +Set-of-Marks 標籤佈局(不重疊、可讀顏色) +========================================= + +Set-of-Marks 在每個元素上疊一個編號標籤,讓視覺模型能說「點 7」。``set_of_marks`` 以固定偏移繪製 +每個標籤,故在密集 UI 上數字會互相疊壓(難以辨讀),而深色標籤在深色元素上會消失。``marks_layout`` +以純幾何修正兩者。 + +* :func:`place_labels` ——貪婪式不重疊放置:對每個 mark,在其方框周圍嘗試一圈候選位置 + (上、下、內;左/右對齊),取第一個仍在邊界內且不與任何已放置標籤重疊者。 +* :func:`label_color` ——挑選標籤文字顏色(黑或白),取對元素背景 WCAG 對比較佳者。 + +純標準函式庫;重用 :func:`a11y_audit.contrast_ratio`。無需繪製即可完整測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import mark_elements, place_labels, label_color + + marks = mark_elements(elements) # [{id, bbox, ...}] + layout = place_labels(marks, bounds=(1920, 1080)) + # [{'id': 1, 'label': [x, y, 22, 16], 'anchor': [bx, by]}, ...] + + label_color((30, 30, 30)) # {'rgb': [255, 255, 255], 'contrast': ...} + +把 :func:`place_labels` 產生的 ``label`` 方框餵給你的繪製器(取代固定偏移),並用 :func:`label_color` +挑選每個編號的顏色,使其在背景上維持可讀。``place_labels`` 是確定性的且依輸入 marks 排序, +故同一畫面總是以相同方式編號。 + +執行器指令 +---------- + +``AC_place_labels``(``marks`` JSON 清單加上 ``label_width`` / ``label_height`` / +``bounds`` ``[w, h]`` → ``{labels}``)與 ``AC_label_color``(``background`` +``[r, g, b]`` → ``{rgb, contrast}``)。皆以對應的唯讀 ``ac_*`` MCP 工具及 Script Builder 指令 +(位於 **Image** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 9930a9e4..0b171abe 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -135,6 +135,8 @@ from je_auto_control.utils.cvd_simulate import ( color_distance, colors_collide, simulate_cvd, ) +# Lay out Set-of-Marks labels without overlap + readable colour +from je_auto_control.utils.marks_layout import label_color, place_labels # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1760,6 +1762,7 @@ def start_autocontrol_gui(*args, **kwargs): "ensure_state", "ensure_toggle", "wait_until_app_idle", "idle_point", "simulate_cvd", "colors_collide", "color_distance", + "place_labels", "label_color", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 996c0594..90af29b2 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4547,6 +4547,26 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Whether two colours become confusable under a CVD type.", )) + specs.append(CommandSpec( + "AC_place_labels", "Image", "Place Mark Labels", + fields=( + FieldSpec("marks", FieldType.STRING, + placeholder="JSON list of {id, bbox}"), + FieldSpec("label_width", FieldType.INT, optional=True, default=22), + FieldSpec("label_height", FieldType.INT, optional=True, + default=16), + FieldSpec("bounds", FieldType.STRING, optional=True, + placeholder="[width, height]"), + ), + description="Lay out non-overlapping Set-of-Marks label boxes.", + )) + specs.append(CommandSpec( + "AC_label_color", "Image", "Label Colour for Background", + fields=( + FieldSpec("background", FieldType.STRING, placeholder="[r, g, b]"), + ), + description="Higher-contrast label colour (black/white) for a background.", + )) specs.append(CommandSpec( "AC_normalize_ext", "Shell", "Normalize Extension", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 42a80840..01afd1ba 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2863,6 +2863,23 @@ def _colors_collide(left: Any, right: Any, kind: Any = "deuteranopia", threshold=float(threshold)) +def _place_labels(marks: Any, label_width: Any = 22, label_height: Any = 16, + bounds: Any = None) -> Dict[str, Any]: + """Adapter: lay out non-overlapping Set-of-Marks label boxes (pure).""" + from je_auto_control.utils.marks_layout import place_labels + items = _coerce_list(marks) if marks else [] + limit = _coerce_list(bounds) if bounds else None + labels = place_labels(items, label_width=int(label_width), + label_height=int(label_height), bounds=limit) + return {"labels": labels} + + +def _label_color(background: Any) -> Dict[str, Any]: + """Adapter: the higher-contrast label colour for a background (pure).""" + from je_auto_control.utils.marks_layout import label_color + return label_color(_coerce_rgb(background)) + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -6896,6 +6913,8 @@ def __init__(self): "AC_idle_point": _idle_point, "AC_simulate_cvd": _simulate_cvd, "AC_colors_collide": _colors_collide, + "AC_place_labels": _place_labels, + "AC_label_color": _label_color, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/marks_layout/__init__.py b/je_auto_control/utils/marks_layout/__init__.py new file mode 100644 index 00000000..5a742b5d --- /dev/null +++ b/je_auto_control/utils/marks_layout/__init__.py @@ -0,0 +1,6 @@ +"""Place Set-of-Marks labels without overlap, with readable label colours.""" +from je_auto_control.utils.marks_layout.marks_layout import ( + label_color, place_labels, +) + +__all__ = ["place_labels", "label_color"] diff --git a/je_auto_control/utils/marks_layout/marks_layout.py b/je_auto_control/utils/marks_layout/marks_layout.py new file mode 100644 index 00000000..bfcfb48d --- /dev/null +++ b/je_auto_control/utils/marks_layout/marks_layout.py @@ -0,0 +1,118 @@ +"""Place Set-of-Marks labels so they don't overlap, with readable label colours. + +Set-of-Marks overlays a numbered label on every element so a vision model can +say "click 7". ``set_of_marks`` draws each label at a fixed offset, so on dense +UIs the numbers pile on top of each other (unreadable) and a dark label on a +dark element vanishes. ``marks_layout`` fixes both with pure geometry: + +* :func:`place_labels` — greedy non-overlap placement: for each mark, try a ring + of candidate positions around its box and take the first that stays in bounds + and clears every already-placed label. +* :func:`label_color` — pick the label text colour (black or white) with the + better WCAG contrast against the element's background. + +Pure standard library; reuses :func:`a11y_audit.contrast_ratio`. Fully testable +without rendering. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +Rect = Tuple[int, int, int, int] + +_BLACK = (0, 0, 0) +_WHITE = (255, 255, 255) + + +def _overlap(first: Rect, second: Rect) -> bool: + """Whether two ``(x, y, w, h)`` rectangles overlap (pure).""" + ax, ay, aw, ah = first + bx, by, bw, bh = second + return not (ax + aw <= bx or bx + bw <= ax + or ay + ah <= by or by + bh <= ay) + + +def _in_bounds(rect: Rect, bounds: Tuple[int, int]) -> bool: + """Whether ``rect`` fits inside ``(width, height)`` (pure).""" + x, y, w, h = rect + return x >= 0 and y >= 0 and x + w <= int(bounds[0]) \ + and y + h <= int(bounds[1]) + + +def _candidates(bbox: Sequence[int], label_w: int, + label_h: int) -> List[Tuple[int, int]]: + """Candidate label top-left positions around an anchor box (pure).""" + bx, by, bw, bh = (int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])) + right = bx + bw - label_w + below = by + bh + return [ + (bx, by - label_h), # above, left-aligned (default SoM spot) + (right, by - label_h), # above, right-aligned + (bx, below), # below, left-aligned + (right, below), # below, right-aligned + (bx, by), # inside, top-left + (right, by), # inside, top-right + ] + + +def _clamp_to_bounds(rect: Rect, bounds: Tuple[int, int]) -> Rect: + """Shift ``rect`` to fit inside ``(width, height)`` (pure fallback).""" + x, y, w, h = rect + x = max(0, min(int(bounds[0]) - w, x)) + y = max(0, min(int(bounds[1]) - h, y)) + return (x, y, w, h) + + +def _pick_position(bbox: Sequence[int], label_w: int, label_h: int, + bounds: Optional[Tuple[int, int]], + placed: List[Rect]) -> Rect: + """Pick the first candidate that is in bounds and clears placed labels.""" + fallback: Optional[Rect] = None + for cx, cy in _candidates(bbox, label_w, label_h): + rect = (cx, cy, label_w, label_h) + if fallback is None: + fallback = rect + if bounds is not None and not _in_bounds(rect, bounds): + continue + if any(_overlap(rect, other) for other in placed): + continue + return rect + if bounds is not None and fallback is not None: + return _clamp_to_bounds(fallback, bounds) + return fallback if fallback is not None else (0, 0, label_w, label_h) + + +def place_labels(marks: Sequence[Dict[str, Any]], *, label_width: int = 22, + label_height: int = 16, + bounds: Optional[Sequence[int]] = None + ) -> List[Dict[str, Any]]: + """Lay out non-overlapping label boxes for ``marks`` (pure). + + ``marks`` is the :func:`set_of_marks.mark_elements` output (each has an + ``id`` and ``bbox`` ``[x, y, w, h]``). ``bounds`` is the ``(width, height)`` + the labels must stay within. Returns ``[{id, label, anchor}]`` where + ``label`` is the placed ``[x, y, w, h]`` box. + """ + size = (int(label_width), int(label_height)) + limit = (int(bounds[0]), int(bounds[1])) if bounds else None + placed: List[Rect] = [] + results: List[Dict[str, Any]] = [] + for mark in marks: + bbox = [int(value) for value in mark["bbox"][:4]] + rect = _pick_position(bbox, size[0], size[1], limit, placed) + placed.append(rect) + results.append({"id": mark.get("id"), "label": list(rect), + "anchor": [bbox[0], bbox[1]]}) + return results + + +def label_color(background: Sequence[float]) -> Dict[str, Any]: + """Pick the higher-contrast label text colour for ``background`` (pure). + + Returns ``{rgb, contrast}`` — black or white, whichever has the better WCAG + contrast ratio against the element background colour. + """ + from je_auto_control.utils.a11y_audit import contrast_ratio + black_contrast = contrast_ratio(background, _BLACK) + white_contrast = contrast_ratio(background, _WHITE) + if white_contrast >= black_contrast: + return {"rgb": list(_WHITE), "contrast": round(white_contrast, 3)} + return {"rgb": list(_BLACK), "contrast": round(black_contrast, 3)} diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index c57aba83..5c0431be 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -4030,6 +4030,33 @@ def img_histogram_tools() -> List[MCPTool]: handler=h.colors_collide, annotations=READ_ONLY, ), + MCPTool( + name="ac_place_labels", + description=("Lay out non-overlapping Set-of-Marks label boxes for " + "'marks' (each {id, bbox:[x,y,w,h]}). 'bounds' is " + "[width, height] to stay within. Pure. Returns " + "{labels:[{id, label:[x,y,w,h], anchor}]}."), + input_schema=schema({"marks": {"type": "array", + "items": {"type": "object"}}, + "label_width": {"type": "integer"}, + "label_height": {"type": "integer"}, + "bounds": {"type": "array", + "items": {"type": "integer"}}}, + required=["marks"]), + handler=h.place_labels, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_label_color", + description=("The higher-contrast label text colour (black or " + "white) for a 'background' [r,g,b], by WCAG contrast. " + "Returns {rgb, contrast}."), + input_schema=schema({"background": {"type": "array", + "items": {"type": "integer"}}}, + required=["background"]), + handler=h.label_color, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 2d400a6d..980b3197 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -744,6 +744,16 @@ def colors_collide(left, right, kind="deuteranopia", severity=1.0, return _colors_collide(left, right, kind, severity, threshold) +def place_labels(marks, label_width=22, label_height=16, bounds=None): + from je_auto_control.utils.executor.action_executor import _place_labels + return _place_labels(marks, label_width, label_height, bounds) + + +def label_color(background): + from je_auto_control.utils.executor.action_executor import _label_color + return _label_color(background) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/test/unit_test/headless/test_marks_layout_batch.py b/test/unit_test/headless/test_marks_layout_batch.py new file mode 100644 index 00000000..c38c4907 --- /dev/null +++ b/test/unit_test/headless/test_marks_layout_batch.py @@ -0,0 +1,95 @@ +"""Headless tests for marks_layout (pure label placement + colour).""" +import je_auto_control as ac +from je_auto_control.utils.marks_layout import label_color, place_labels + + +def _rects_overlap(a, b): + ax, ay, aw, ah = a + bx, by, bw, bh = b + return not (ax + aw <= bx or bx + bw <= ax + or ay + ah <= by or by + bh <= ay) + + +# --- place_labels --------------------------------------------------------- + +def test_place_labels_returns_one_per_mark(): + marks = [{"id": 1, "bbox": [100, 100, 40, 20]}, + {"id": 2, "bbox": [300, 300, 40, 20]}] + labels = place_labels(marks) + assert [item["id"] for item in labels] == [1, 2] + assert all(len(item["label"]) == 4 for item in labels) + + +def test_place_labels_no_overlap_on_stacked_marks(): + # three marks at the exact same spot would collide if placed naively; + # the candidate ring de-collides them + marks = [{"id": i, "bbox": [200, 200, 30, 18]} for i in range(1, 4)] + labels = place_labels(marks, bounds=[1920, 1080]) + boxes = [tuple(item["label"]) for item in labels] + for i in range(len(boxes)): + for j in range(i + 1, len(boxes)): + assert not _rects_overlap(boxes[i], boxes[j]) + + +def test_place_labels_stays_in_bounds(): + # a mark at the top-left corner can't put its label above the screen + marks = [{"id": 1, "bbox": [0, 0, 40, 20]}] + labels = place_labels(marks, label_width=22, label_height=16, + bounds=[800, 600]) + x, y, w, h = labels[0]["label"] + assert x >= 0 and y >= 0 + assert x + w <= 800 and y + h <= 600 + + +def test_place_labels_default_above_when_room(): + marks = [{"id": 1, "bbox": [100, 100, 40, 20]}] + labels = place_labels(marks, label_height=16, bounds=[800, 600]) + # default candidate is directly above the box (y = 100 - 16 = 84) + assert labels[0]["label"][1] == 84 + + +def test_place_labels_empty(): + assert place_labels([]) == [] + + +# --- label_color ---------------------------------------------------------- + +def test_label_color_white_on_dark(): + result = label_color((20, 20, 20)) + assert result["rgb"] == [255, 255, 255] + assert result["contrast"] > 1.0 + + +def test_label_color_black_on_light(): + result = label_color((240, 240, 240)) + assert result["rgb"] == [0, 0, 0] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _label_color, _place_labels, + ) + out = _place_labels('[{"id": 1, "bbox": [10, 10, 30, 20]}]', 22, 16, + "[800, 600]") + assert out["labels"][0]["id"] == 1 + assert _label_color([10, 10, 10])["rgb"] == [255, 255, 255] + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_place_labels", "AC_label_color"} <= known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_place_labels", "ac_label_color"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_place_labels", "AC_label_color"} <= specs + + +def test_facade_exports(): + for name in ("place_labels", "label_color"): + assert hasattr(ac, name) and name in ac.__all__ From e0035cfe0b9e9dc39c13a3b9a68c6b93c6401af3 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 26 Jun 2026 09:02:13 +0800 Subject: [PATCH 3/4] Add contrast_map: grade on-screen text contrast by sampling its colours a11y_audit.contrast_ratio grades a known foreground/background pair, but a button or label on screen is a patch of pixels, not two known colours. dominant_pair splits sampled pixels at the mean luminance into the dominant text (minority) and background (majority); grade_contrast grades a pair against WCAG AA/AAA; region_contrast samples a region (injectable sampler) and grades it. Grade + split are pure, reuse contrast_ratio. --- WHATS_NEW.md | 6 + .../doc/new_features/v216_features_doc.rst | 50 ++++++++ .../Zh/doc/new_features/v216_features_doc.rst | 42 +++++++ je_auto_control/__init__.py | 5 + .../gui/script_builder/command_schema.py | 24 ++++ .../utils/contrast_map/__init__.py | 6 + .../utils/contrast_map/contrast_map.py | 114 ++++++++++++++++++ .../utils/executor/action_executor.py | 23 ++++ .../utils/mcp_server/tools/_factories.py | 34 ++++++ .../utils/mcp_server/tools/_handlers.py | 15 +++ .../headless/test_contrast_map_batch.py | 101 ++++++++++++++++ 11 files changed, 420 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v216_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v216_features_doc.rst create mode 100644 je_auto_control/utils/contrast_map/__init__.py create mode 100644 je_auto_control/utils/contrast_map/contrast_map.py create mode 100644 test/unit_test/headless/test_contrast_map_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 0f537fc4..d82b808b 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Sample a Region's Text Contrast (WCAG) + +Grade the legibility of on-screen text when you only have a region, not the two colours. Full reference: [`docs/source/Eng/doc/new_features/v216_features_doc.rst`](docs/source/Eng/doc/new_features/v216_features_doc.rst). + +- **`grade_contrast` / `dominant_pair` / `region_contrast`** (`AC_grade_contrast`, `AC_dominant_pair`, `AC_region_contrast`): `a11y_audit.contrast_ratio` grades a foreground/background pair you already know — but a button or label on screen is a *patch of pixels*, not two known colours. `dominant_pair` splits sampled pixels at the mean luminance into the dominant foreground (minority, the text) and background (majority); `grade_contrast` grades a pair against the WCAG 2.x AA/AAA thresholds (normal + large text); `region_contrast` samples a screen region (through an injectable `sampler`) and grades it. The grading and split are pure and reuse `a11y_audit.contrast_ratio`, fully testable without a screen. Third feature of the ROUND-15 perception lane. No `PySide6`. + ### Set-of-Marks Label Layout (No Overlap, Readable Colour) Number every element without the labels piling up or vanishing into the background. Full reference: [`docs/source/Eng/doc/new_features/v215_features_doc.rst`](docs/source/Eng/doc/new_features/v215_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v216_features_doc.rst b/docs/source/Eng/doc/new_features/v216_features_doc.rst new file mode 100644 index 00000000..e4d6170b --- /dev/null +++ b/docs/source/Eng/doc/new_features/v216_features_doc.rst @@ -0,0 +1,50 @@ +Sample a Region's Text Contrast (WCAG) +====================================== + +:func:`a11y_audit.contrast_ratio` grades a foreground / background pair you +already know. But when you only have a *region* of the screen — a button, a +label — you don't know those two colours; you have a patch of pixels. +``contrast_map`` closes that gap: split a sampled region into its dominant +foreground (the minority — usually the text) and background (the majority) +colours, then grade their WCAG contrast. + +* :func:`grade_contrast` — pure: a foreground / background pair to + ``{ratio, aa, aaa, aa_large, aaa_large}`` against the WCAG 2.x thresholds. +* :func:`dominant_pair` — pure: split a list of sampled RGB pixels into the + dominant ``{foreground, background}`` colours by luminance. +* :func:`region_contrast` — sample a screen region and grade it, through an + injectable ``sampler`` (the real screen grab by default). + +The grading and split are pure and reuse :func:`a11y_audit.contrast_ratio`, so +they are fully testable without a screen. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import grade_contrast, dominant_pair, region_contrast + + # If you already know the colours: + grade_contrast((90, 90, 90), (255, 255, 255)) + # {'ratio': 3.9, 'aa': False, 'aaa': False, 'aa_large': True, ...} + + # If you only have a region of the screen, sample and grade it: + report = region_contrast(region=[x, y, w, h]) + if not report["aa"]: + print("low-contrast text", report["foreground"], report["background"]) + +``dominant_pair`` partitions the sampled pixels at the mean luminance and treats +the larger group as the background and the smaller as the text — a uniform patch +yields the same colour for both (no contrast). ``region_contrast`` accepts an +injectable ``sampler`` (``region -> list of RGB pixels``) so the logic is tested +without a real screen. + +Executor commands +----------------- + +``AC_grade_contrast`` (``foreground`` / ``background`` ``[r, g, b]`` → the +grade), ``AC_dominant_pair`` (``pixels`` JSON list of ``[r, g, b]`` → +``{foreground, background}``) and ``AC_region_contrast`` (``region`` +``[x, y, w, h]`` → the grade + colours + ``samples``). They are the matching +read-only ``ac_*`` MCP tools and Script Builder commands under **Image**. diff --git a/docs/source/Zh/doc/new_features/v216_features_doc.rst b/docs/source/Zh/doc/new_features/v216_features_doc.rst new file mode 100644 index 00000000..6b0c77bf --- /dev/null +++ b/docs/source/Zh/doc/new_features/v216_features_doc.rst @@ -0,0 +1,42 @@ +取樣區域的文字對比(WCAG) +========================== + +:func:`a11y_audit.contrast_ratio` 對你已知的前景 / 背景配對評分。但當你只有螢幕上的一個*區域*—— +一個按鈕、一個標籤——你並不知道那兩個顏色;你有的是一片像素。``contrast_map`` 補上這道缺口: +把取樣區域拆成其主要前景(少數——通常是文字)與背景(多數)顏色,再評其 WCAG 對比。 + +* :func:`grade_contrast` ——純函式:把前景 / 背景配對對 WCAG 2.x 門檻轉為 + ``{ratio, aa, aaa, aa_large, aaa_large}``。 +* :func:`dominant_pair` ——純函式:依亮度把一串取樣 RGB 像素拆成主要的 ``{foreground, background}``。 +* :func:`region_contrast` ——取樣螢幕區域並評分,透過可注入的 ``sampler``(預設為真實螢幕擷取)。 + +評分與拆分皆為純函式並重用 :func:`a11y_audit.contrast_ratio`,故能在沒有螢幕的情況下完整測試。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import grade_contrast, dominant_pair, region_contrast + + # 若你已知顏色: + grade_contrast((90, 90, 90), (255, 255, 255)) + # {'ratio': 3.9, 'aa': False, 'aaa': False, 'aa_large': True, ...} + + # 若你只有螢幕的一個區域,取樣並評分: + report = region_contrast(region=[x, y, w, h]) + if not report["aa"]: + print("低對比文字", report["foreground"], report["background"]) + +``dominant_pair`` 以平均亮度切分取樣像素,把較大的一群視為背景、較小的視為文字—— +均勻一片會讓兩者得到相同顏色(無對比)。``region_contrast`` 接受可注入的 ``sampler`` +(``region -> RGB 像素清單``),故邏輯能在沒有真實螢幕的情況下測試。 + +執行器指令 +---------- + +``AC_grade_contrast``(``foreground`` / ``background`` ``[r, g, b]`` → 評分)、 +``AC_dominant_pair``(``pixels`` JSON 清單 ``[r, g, b]`` → ``{foreground, background}``)與 +``AC_region_contrast``(``region`` ``[x, y, w, h]`` → 評分 + 顏色 + ``samples``)。皆以對應的 +唯讀 ``ac_*`` MCP 工具及 Script Builder 指令(位於 **Image** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 0b171abe..4c0193c0 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -137,6 +137,10 @@ ) # Lay out Set-of-Marks labels without overlap + readable colour from je_auto_control.utils.marks_layout import label_color, place_labels +# Grade on-screen text legibility by sampling its actual colours (WCAG) +from je_auto_control.utils.contrast_map import ( + dominant_pair, grade_contrast, region_contrast, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1763,6 +1767,7 @@ def start_autocontrol_gui(*args, **kwargs): "wait_until_app_idle", "idle_point", "simulate_cvd", "colors_collide", "color_distance", "place_labels", "label_color", + "grade_contrast", "dominant_pair", "region_contrast", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 90af29b2..42fcfc56 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4567,6 +4567,30 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Higher-contrast label colour (black/white) for a background.", )) + specs.append(CommandSpec( + "AC_grade_contrast", "Image", "Grade Contrast (WCAG)", + fields=( + FieldSpec("foreground", FieldType.STRING, placeholder="[r, g, b]"), + FieldSpec("background", FieldType.STRING, placeholder="[r, g, b]"), + ), + description="Grade a foreground/background colour pair against WCAG.", + )) + specs.append(CommandSpec( + "AC_dominant_pair", "Image", "Dominant FG/BG Colours", + fields=( + FieldSpec("pixels", FieldType.STRING, + placeholder="JSON list of [r, g, b] pixels"), + ), + description="Split sampled pixels into dominant foreground/background.", + )) + specs.append(CommandSpec( + "AC_region_contrast", "Image", "Region Text Contrast", + fields=( + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[x, y, w, h]"), + ), + description="Sample a screen region and grade its text contrast.", + )) specs.append(CommandSpec( "AC_normalize_ext", "Shell", "Normalize Extension", fields=( diff --git a/je_auto_control/utils/contrast_map/__init__.py b/je_auto_control/utils/contrast_map/__init__.py new file mode 100644 index 00000000..da004edf --- /dev/null +++ b/je_auto_control/utils/contrast_map/__init__.py @@ -0,0 +1,6 @@ +"""Grade the legibility of on-screen text by sampling its actual colours.""" +from je_auto_control.utils.contrast_map.contrast_map import ( + dominant_pair, grade_contrast, region_contrast, +) + +__all__ = ["grade_contrast", "dominant_pair", "region_contrast"] diff --git a/je_auto_control/utils/contrast_map/contrast_map.py b/je_auto_control/utils/contrast_map/contrast_map.py new file mode 100644 index 00000000..b92d161c --- /dev/null +++ b/je_auto_control/utils/contrast_map/contrast_map.py @@ -0,0 +1,114 @@ +"""Grade the legibility of on-screen text by sampling its actual colours. + +:func:`a11y_audit.contrast_ratio` grades a foreground / background pair you +already know. But when you only have a *region* of the screen — a button, a +label — you don't know those two colours; you have a patch of pixels. +``contrast_map`` closes that gap: split a sampled region into its dominant +foreground (the minority — usually the text) and background (the majority) +colours, then grade their WCAG contrast. + +* :func:`grade_contrast` — pure: a foreground / background pair to + ``{ratio, aa, aaa, aa_large, aaa_large}`` against the WCAG 2.x thresholds. +* :func:`dominant_pair` — pure: split a list of sampled RGB pixels into the + dominant ``{foreground, background}`` colours by luminance. +* :func:`region_contrast` — sample a screen region and grade it, through an + injectable ``sampler`` (the real screen grab by default). + +The grading and split are pure and reuse :func:`a11y_audit.contrast_ratio`, so +they are fully testable without a screen. Imports no ``PySide6``. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence + +from je_auto_control.utils.a11y_audit import contrast_ratio +from je_auto_control.utils.a11y_audit.audit import relative_luminance + +# WCAG 2.x contrast thresholds. +_AA_NORMAL = 4.5 +_AAA_NORMAL = 7.0 +_AA_LARGE = 3.0 +_AAA_LARGE = 4.5 + +RGB = List[int] +PixelSampler = Callable[[Optional[Sequence[int]]], List[Sequence[int]]] + + +def grade_contrast(foreground: Sequence[float], + background: Sequence[float]) -> Dict[str, Any]: + """Grade a foreground / background colour pair against WCAG (pure). + + Returns ``{ratio, aa, aaa, aa_large, aaa_large}`` — the contrast ratio and + whether it meets AA / AAA for normal and large text. + """ + ratio = contrast_ratio(foreground, background) + return { + "ratio": round(ratio, 3), + "aa": ratio >= _AA_NORMAL, + "aaa": ratio >= _AAA_NORMAL, + "aa_large": ratio >= _AA_LARGE, + "aaa_large": ratio >= _AAA_LARGE, + } + + +def _mean_color(pixels: Sequence[Sequence[int]]) -> RGB: + """Average a non-empty list of RGB pixels to one colour (pure).""" + count = len(pixels) + totals = [0, 0, 0] + for pixel in pixels: + totals[0] += int(pixel[0]) + totals[1] += int(pixel[1]) + totals[2] += int(pixel[2]) + return [totals[0] // count, totals[1] // count, totals[2] // count] + + +def _partition_by_luminance(rgbs: Sequence[Sequence[int]]) -> tuple: + """Split RGB pixels into (darker, lighter) groups at the mean luminance.""" + lums = [relative_luminance(rgb) for rgb in rgbs] + average = sum(lums) / len(lums) + low = [rgb for rgb, lum in zip(rgbs, lums) if lum < average] + high = [rgb for rgb, lum in zip(rgbs, lums) if lum >= average] + return low, high + + +def dominant_pair(pixels: Sequence[Sequence[int]]) -> Dict[str, RGB]: + """Split sampled pixels into dominant foreground / background colours (pure). + + Pixels are partitioned at the mean luminance; the larger group is the + background and the smaller the foreground (text). A uniform region yields + the same colour for both (no contrast). Returns ``{foreground, background}``. + """ + rgbs = [(int(p[0]), int(p[1]), int(p[2])) for p in pixels] + if not rgbs: + return {"foreground": [0, 0, 0], "background": [0, 0, 0]} + low, high = _partition_by_luminance(rgbs) + if not low or not high: + mean = _mean_color(rgbs) + return {"foreground": mean, "background": mean} + background, foreground = (high, low) if len(high) >= len(low) else (low, + high) + return {"foreground": _mean_color(foreground), + "background": _mean_color(background)} + + +def _default_sampler(region: Optional[Sequence[int]]) -> List[RGB]: + """Grab a screen region and return a capped list of its RGB pixels.""" + from je_auto_control.utils.color_region.color_region import _grab_rgb + array = _grab_rgb(region) + flat = array.reshape(-1, array.shape[-1]) + step = max(1, len(flat) // 4096) + return [[int(px[0]), int(px[1]), int(px[2])] for px in flat[::step]] + + +def region_contrast(*, sampler: Optional[PixelSampler] = None, + region: Optional[Sequence[int]] = None) -> Dict[str, Any]: + """Sample a screen ``region`` and grade its text contrast. + + Pass ``sampler`` (``region -> list of RGB pixels``) to supply pixels in + tests; the default grabs the real screen. Returns the + :func:`grade_contrast` result plus ``{foreground, background, samples}``. + """ + sample = sampler if sampler is not None else _default_sampler + pixels = list(sample(region)) + pair = dominant_pair(pixels) + grade = grade_contrast(pair["foreground"], pair["background"]) + return {**grade, "foreground": pair["foreground"], + "background": pair["background"], "samples": len(pixels)} diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 01afd1ba..4456d9ae 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2880,6 +2880,26 @@ def _label_color(background: Any) -> Dict[str, Any]: return label_color(_coerce_rgb(background)) +def _grade_contrast(foreground: Any, background: Any) -> Dict[str, Any]: + """Adapter: grade a foreground/background colour pair vs WCAG (pure).""" + from je_auto_control.utils.contrast_map import grade_contrast + return grade_contrast(_coerce_rgb(foreground), _coerce_rgb(background)) + + +def _dominant_pair(pixels: Any) -> Dict[str, Any]: + """Adapter: split sampled RGB pixels into fg/bg dominant colours (pure).""" + from je_auto_control.utils.contrast_map import dominant_pair + items = [_coerce_rgb(pixel) for pixel in _coerce_list(pixels)] \ + if pixels else [] + return dominant_pair(items) + + +def _region_contrast(region: Any = None) -> Dict[str, Any]: + """Adapter: sample a screen region and grade its text contrast (device).""" + from je_auto_control.utils.contrast_map import region_contrast + return region_contrast(region=_coerce_region(region)) + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -6915,6 +6935,9 @@ def __init__(self): "AC_colors_collide": _colors_collide, "AC_place_labels": _place_labels, "AC_label_color": _label_color, + "AC_grade_contrast": _grade_contrast, + "AC_dominant_pair": _dominant_pair, + "AC_region_contrast": _region_contrast, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 5c0431be..ab53fde9 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -4057,6 +4057,40 @@ def img_histogram_tools() -> List[MCPTool]: handler=h.label_color, annotations=READ_ONLY, ), + MCPTool( + name="ac_grade_contrast", + description=("Grade a 'foreground'/'background' [r,g,b] colour pair " + "against WCAG. Returns {ratio, aa, aaa, aa_large, " + "aaa_large}. Pure."), + input_schema=schema({"foreground": {"type": "array", + "items": {"type": "integer"}}, + "background": {"type": "array", + "items": {"type": "integer"}}}, + required=["foreground", "background"]), + handler=h.grade_contrast, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_dominant_pair", + description=("Split sampled RGB 'pixels' (list of [r,g,b]) into the " + "dominant {foreground, background} colours by " + "luminance. Pure."), + input_schema=schema({"pixels": {"type": "array", + "items": {"type": "array"}}}, + required=["pixels"]), + handler=h.dominant_pair, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_region_contrast", + description=("Sample a screen 'region' [x,y,w,h] and grade its text " + "contrast: {ratio, aa, aaa, aa_large, aaa_large, " + "foreground, background, samples}."), + input_schema=schema({"region": {"type": "array", + "items": {"type": "integer"}}}), + handler=h.region_contrast, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 980b3197..ed3bd986 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -754,6 +754,21 @@ def label_color(background): return _label_color(background) +def grade_contrast(foreground, background): + from je_auto_control.utils.executor.action_executor import _grade_contrast + return _grade_contrast(foreground, background) + + +def dominant_pair(pixels): + from je_auto_control.utils.executor.action_executor import _dominant_pair + return _dominant_pair(pixels) + + +def region_contrast(region=None): + from je_auto_control.utils.executor.action_executor import _region_contrast + return _region_contrast(region) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/test/unit_test/headless/test_contrast_map_batch.py b/test/unit_test/headless/test_contrast_map_batch.py new file mode 100644 index 00000000..c4366cc6 --- /dev/null +++ b/test/unit_test/headless/test_contrast_map_batch.py @@ -0,0 +1,101 @@ +"""Headless tests for contrast_map (pure WCAG grade + fg/bg split).""" +import je_auto_control as ac +from je_auto_control.utils.contrast_map import ( + dominant_pair, grade_contrast, region_contrast, +) + + +# --- grade_contrast ------------------------------------------------------- + +def test_grade_black_on_white_passes_all(): + grade = grade_contrast((0, 0, 0), (255, 255, 255)) + assert grade["ratio"] > 20.0 # 21:1 black/white + assert grade["aa"] is True and grade["aaa"] is True + assert grade["aa_large"] is True and grade["aaa_large"] is True + + +def test_grade_low_contrast_fails_normal(): + # mid-grey on light-grey: legible as large text only, fails normal AAA + grade = grade_contrast((120, 120, 120), (180, 180, 180)) + assert grade["aaa"] is False + + +def test_grade_symmetric(): + forward = grade_contrast((10, 20, 30), (200, 210, 220))["ratio"] + backward = grade_contrast((200, 210, 220), (10, 20, 30))["ratio"] + assert abs(forward - backward) < 1e-6 + + +# --- dominant_pair -------------------------------------------------------- + +def test_dominant_pair_text_on_background(): + # mostly white background with a few black "text" pixels + pixels = [(255, 255, 255)] * 8 + [(0, 0, 0)] * 2 + pair = dominant_pair(pixels) + assert pair["background"] == [255, 255, 255] + assert pair["foreground"] == [0, 0, 0] + + +def test_dominant_pair_uniform_region_no_contrast(): + pair = dominant_pair([(128, 128, 128)] * 5) + assert pair["foreground"] == pair["background"] == [128, 128, 128] + + +def test_dominant_pair_empty(): + pair = dominant_pair([]) + assert pair["foreground"] == [0, 0, 0] + assert pair["background"] == [0, 0, 0] + + +# --- region_contrast (injected sampler) ----------------------------------- + +def test_region_contrast_with_injected_sampler(): + pixels = [[245, 245, 245]] * 9 + [[20, 20, 20]] + result = region_contrast(sampler=lambda region: pixels) + assert result["background"] == [245, 245, 245] + assert result["foreground"] == [20, 20, 20] + assert result["samples"] == 10 + assert result["aa"] is True + + +def test_region_contrast_passes_region_to_sampler(): + seen = {} + + def sampler(region): + seen["region"] = region + return [(0, 0, 0), (255, 255, 255)] + + region_contrast(sampler=sampler, region=[10, 10, 50, 30]) + assert seen["region"] == [10, 10, 50, 30] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_pure_paths(): + from je_auto_control.utils.executor.action_executor import ( + _dominant_pair, _grade_contrast, + ) + assert _grade_contrast([0, 0, 0], [255, 255, 255])["aa"] is True + pair = _dominant_pair("[[255,255,255],[255,255,255],[0,0,0]]") + assert pair["background"] == [255, 255, 255] + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_grade_contrast", "AC_dominant_pair", + "AC_region_contrast"} <= known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_grade_contrast", "ac_dominant_pair", + "ac_region_contrast"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_grade_contrast", "AC_dominant_pair", + "AC_region_contrast"} <= specs + + +def test_facade_exports(): + for name in ("grade_contrast", "dominant_pair", "region_contrast"): + assert hasattr(ac, name) and name in ac.__all__ From b5191e40da165ef8251b3f402da1f030ee7e0b18 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 26 Jun 2026 09:19:50 +0800 Subject: [PATCH 4/4] Add theme_normalize: match a light-mode template in dark mode match_template correlates raw pixel intensities, so a light-mode template scores terribly against the same control in dark mode (polarity inverted). normalize_theme maps an image to a polarity-invariant single channel (sobel/laplacian gradient magnitude, identical for an image and its inverse) and match_theme normalizes both sides before locating via visual_match.match_template. cv2/numpy imported lazily. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v217_features_doc.rst | 49 +++++++++++ .../Zh/doc/new_features/v217_features_doc.rst | 41 +++++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 15 ++++ .../utils/executor/action_executor.py | 13 +++ .../utils/mcp_server/tools/_factories.py | 16 ++++ .../utils/mcp_server/tools/_handlers.py | 5 ++ .../utils/theme_normalize/__init__.py | 6 ++ .../utils/theme_normalize/theme_normalize.py | 86 +++++++++++++++++++ .../headless/test_theme_normalize_batch.py | 72 ++++++++++++++++ 11 files changed, 312 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v217_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v217_features_doc.rst create mode 100644 je_auto_control/utils/theme_normalize/__init__.py create mode 100644 je_auto_control/utils/theme_normalize/theme_normalize.py create mode 100644 test/unit_test/headless/test_theme_normalize_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index d82b808b..25fe4b24 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Theme-Invariant Matching (Light Template, Dark Mode) + +Find a button captured in light mode even after the app switches to dark mode. Full reference: [`docs/source/Eng/doc/new_features/v217_features_doc.rst`](docs/source/Eng/doc/new_features/v217_features_doc.rst). + +- **`normalize_theme` / `match_theme`** (`AC_match_theme`): `match_template` correlates raw pixel intensities, so a light-mode template scores terribly against the same control in dark mode — the polarity is inverted. The fix is to compare *structure*. `normalize_theme` maps an image to a polarity-invariant single channel (`sobel`/`laplacian` gradient magnitude — identical for an image and its colour inverse — or `zscore`); `match_theme` normalizes both the template and the screen, then locates the template via `visual_match.match_template`, finding it across a light/dark flip that defeats raw matching. cv2/numpy are imported lazily so the module stays importable everywhere. Fourth feature of the ROUND-15 perception lane. No `PySide6`. + ### Sample a Region's Text Contrast (WCAG) Grade the legibility of on-screen text when you only have a region, not the two colours. Full reference: [`docs/source/Eng/doc/new_features/v216_features_doc.rst`](docs/source/Eng/doc/new_features/v216_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v217_features_doc.rst b/docs/source/Eng/doc/new_features/v217_features_doc.rst new file mode 100644 index 00000000..dccaecca --- /dev/null +++ b/docs/source/Eng/doc/new_features/v217_features_doc.rst @@ -0,0 +1,49 @@ +Theme-Invariant Matching (Light Template, Dark Mode) +==================================================== + +``match_template`` correlates raw pixel intensities, so a template captured in +light mode scores terribly against the same control in dark mode — the polarity +is inverted. The fix is to compare *structure* (edges, gradients), which is the +same regardless of which way the colours run. ``theme_normalize`` turns an image +into a polarity-invariant representation before matching. + +* :func:`normalize_theme` — map an image to a normalised single-channel image. + ``sobel`` (default) and ``laplacian`` use gradient magnitude, which is + identical for an image and its colour-inverse; ``zscore`` standardises + intensity. +* :func:`match_theme` — :func:`normalize_theme` both the template and the + haystack (the screen by default), then locate the template — finding it across + a light/dark theme flip that defeats raw matching. + +``cv2`` / ``numpy`` are imported lazily, so importing the module never requires +them, and the locating logic reuses :func:`visual_match.match_template`. Imports +no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import match_theme, normalize_theme + + # A button template grabbed in light mode, found in the dark-mode app: + hit = match_theme("save_button_light.png", method="sobel", min_score=0.4) + if hit and hit["score"] >= 0.5: + click(hit["x"] + hit["width"] // 2, hit["y"] + hit["height"] // 2) + + # The transform itself (e.g. to feed your own matcher): + edges = normalize_theme("template.png", method="sobel") + +Because gradient magnitude is identical for an image and its inverse, +``normalize_theme(img, "sobel")`` equals ``normalize_theme(255 - img, "sobel")`` +— that invariance is exactly what lets one template match both themes. Use +``min_score`` lower than for raw matching (structure correlation runs cooler). + +Executor commands +----------------- + +``AC_match_theme`` (``template`` + ``region`` ``[x, y, w, h]`` / ``method`` / +``min_score`` → ``{found, x, y, width, height, score}``) locates a template +across a theme flip. It is the matching read-only ``ac_match_theme`` MCP tool and +a Script Builder command under **Image**. :func:`normalize_theme` (which returns +an image array) is the Python-API surface. diff --git a/docs/source/Zh/doc/new_features/v217_features_doc.rst b/docs/source/Zh/doc/new_features/v217_features_doc.rst new file mode 100644 index 00000000..83c82f8c --- /dev/null +++ b/docs/source/Zh/doc/new_features/v217_features_doc.rst @@ -0,0 +1,41 @@ +主題不變比對(淺色模板、深色模式) +================================== + +``match_template`` 以原始像素強度相關比對,故在淺色模式擷取的模板,對深色模式下同一控制項評分極差—— +極性反轉了。修法是比較*結構*(邊緣、梯度),不論顏色走向如何皆相同。``theme_normalize`` 在比對前 +把影像轉成極性不變的表示。 + +* :func:`normalize_theme` ——把影像映射為正規化的單通道影像。``sobel``(預設)與 ``laplacian`` + 使用梯度幅值,對影像與其顏色反相版本相同;``zscore`` 將強度標準化。 +* :func:`match_theme` ——對模板與 haystack(預設為螢幕)都做 :func:`normalize_theme`,再定位模板—— + 即使在會擊敗原始比對的淺/深主題切換下也能找到。 + +``cv2`` / ``numpy`` 採延遲匯入,故匯入本模組永遠不需要它們,定位邏輯則重用 +:func:`visual_match.match_template`。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import match_theme, normalize_theme + + # 淺色模式擷取的按鈕模板,在深色模式的 app 中找到: + hit = match_theme("save_button_light.png", method="sobel", min_score=0.4) + if hit and hit["score"] >= 0.5: + click(hit["x"] + hit["width"] // 2, hit["y"] + hit["height"] // 2) + + # 轉換本身(例如餵給你自己的比對器): + edges = normalize_theme("template.png", method="sobel") + +由於梯度幅值對影像與其反相版本相同,``normalize_theme(img, "sobel")`` 等於 +``normalize_theme(255 - img, "sobel")``——正是這個不變性讓單一模板能比對兩種主題。 +``min_score`` 請設得比原始比對低(結構相關分數較低)。 + +執行器指令 +---------- + +``AC_match_theme``(``template`` 加上 ``region`` ``[x, y, w, h]`` / ``method`` / +``min_score`` → ``{found, x, y, width, height, score}``)跨主題切換定位模板。以對應的唯讀 +``ac_match_theme`` MCP 工具及 Script Builder 指令(位於 **Image** 分類下)形式提供。 +:func:`normalize_theme`(回傳影像陣列)則是 Python API 介面。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 4c0193c0..bfb1a716 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -141,6 +141,8 @@ from je_auto_control.utils.contrast_map import ( dominant_pair, grade_contrast, region_contrast, ) +# Theme-invariant matching so a light template matches dark mode +from je_auto_control.utils.theme_normalize import match_theme, normalize_theme # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1768,6 +1770,7 @@ def start_autocontrol_gui(*args, **kwargs): "simulate_cvd", "colors_collide", "color_distance", "place_labels", "label_color", "grade_contrast", "dominant_pair", "region_contrast", + "normalize_theme", "match_theme", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 42fcfc56..1d1f6f0f 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4591,6 +4591,21 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Sample a screen region and grade its text contrast.", )) + specs.append(CommandSpec( + "AC_match_theme", "Image", "Match (Theme-Invariant)", + fields=( + FieldSpec("template", FieldType.STRING, + placeholder="template image path"), + FieldSpec("region", FieldType.STRING, optional=True, + placeholder="[x, y, w, h]"), + FieldSpec("method", FieldType.STRING, optional=True, + default="sobel", + placeholder="sobel / laplacian / zscore"), + FieldSpec("min_score", FieldType.FLOAT, optional=True, + default=0.5), + ), + description="Locate a template across a light/dark theme flip.", + )) specs.append(CommandSpec( "AC_normalize_ext", "Shell", "Normalize Extension", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 4456d9ae..6ba236f4 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2900,6 +2900,18 @@ def _region_contrast(region: Any = None) -> Dict[str, Any]: return region_contrast(region=_coerce_region(region)) +def _match_theme(template: Any, region: Any = None, method: Any = "sobel", + min_score: Any = 0.5) -> Dict[str, Any]: + """Adapter: locate a template across a light/dark theme flip (device).""" + from je_auto_control.utils.theme_normalize import match_theme + match = match_theme(str(template), method=str(method), + min_score=float(min_score), + region=_coerce_region(region)) + if match is None: + return {"found": False} + return {"found": True, **match} + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -6938,6 +6950,7 @@ def __init__(self): "AC_grade_contrast": _grade_contrast, "AC_dominant_pair": _dominant_pair, "AC_region_contrast": _region_contrast, + "AC_match_theme": _match_theme, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index ab53fde9..4369522b 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -4091,6 +4091,22 @@ def img_histogram_tools() -> List[MCPTool]: handler=h.region_contrast, annotations=READ_ONLY, ), + MCPTool( + name="ac_match_theme", + description=("Locate a 'template' image on screen across a " + "light/dark theme flip, by matching gradient " + "structure ('method' sobel/laplacian/zscore). " + "'region' [x,y,w,h] clips the search. Returns {found, " + "x, y, width, height, score}."), + input_schema=schema({"template": {"type": "string"}, + "region": {"type": "array", + "items": {"type": "integer"}}, + "method": {"type": "string"}, + "min_score": {"type": "number"}}, + required=["template"]), + handler=h.match_theme, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index ed3bd986..5006b182 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -769,6 +769,11 @@ def region_contrast(region=None): return _region_contrast(region) +def match_theme(template, region=None, method="sobel", min_score=0.5): + from je_auto_control.utils.executor.action_executor import _match_theme + return _match_theme(template, region, method, min_score) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/je_auto_control/utils/theme_normalize/__init__.py b/je_auto_control/utils/theme_normalize/__init__.py new file mode 100644 index 00000000..1c75d90e --- /dev/null +++ b/je_auto_control/utils/theme_normalize/__init__.py @@ -0,0 +1,6 @@ +"""Theme-invariant image normalisation so light templates match dark mode.""" +from je_auto_control.utils.theme_normalize.theme_normalize import ( + THEME_METHODS, match_theme, normalize_theme, +) + +__all__ = ["normalize_theme", "match_theme", "THEME_METHODS"] diff --git a/je_auto_control/utils/theme_normalize/theme_normalize.py b/je_auto_control/utils/theme_normalize/theme_normalize.py new file mode 100644 index 00000000..742d9d67 --- /dev/null +++ b/je_auto_control/utils/theme_normalize/theme_normalize.py @@ -0,0 +1,86 @@ +"""Theme-invariant image normalisation so light templates match dark mode. + +``match_template`` correlates raw pixel intensities, so a template captured in +light mode scores terribly against the same control in dark mode — the polarity +is inverted. The fix is to compare *structure* (edges, gradients), which is the +same regardless of which way the colours run. ``theme_normalize`` turns an image +into a polarity-invariant representation before matching: + +* :func:`normalize_theme` — map an image to a normalised single-channel image. + ``sobel`` (default) and ``laplacian`` use gradient magnitude, which is + identical for an image and its inverse; ``zscore`` standardises intensity. +* :func:`match_theme` — :func:`normalize_theme` both the template and the + haystack (the screen by default), then locate the template — finding it across + a light/dark theme flip that defeats raw matching. + +cv2 / numpy are imported lazily, so importing this module never requires them +(the package stays importable everywhere) and the locating logic reuses +:func:`visual_match.match_template`. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional, Sequence + +# A normalisation method name. +THEME_METHODS = ("sobel", "laplacian", "zscore") + + +def _to_uint8(array: Any) -> Any: + """Rescale a float array to a 0..255 uint8 image.""" + import cv2 + return cv2.normalize(array, None, 0, 255, cv2.NORM_MINMAX).astype("uint8") + + +def _zscore(gray: Any) -> Any: + """Standardise intensity to zero mean / unit variance (not inversion-safe).""" + import numpy as np + std = float(gray.std()) + if std < 1e-9: + return np.zeros_like(gray) + return (gray - gray.mean()) / std + + +def normalize_theme(source: Any, *, method: str = "sobel") -> Any: + """Return ``source`` as a theme-normalised single-channel ``uint8`` image. + + ``sobel`` / ``laplacian`` return gradient magnitude — identical for an image + and its colour-inverted (dark-mode) twin — and ``zscore`` standardises + intensity. Raises ``ValueError`` for an unknown ``method``. + """ + import cv2 + import numpy as np + from je_auto_control.utils.visual_match.visual_match import _to_gray + gray = _to_gray(source).astype("float64") + if method == "sobel": + gx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3) + gy = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3) + result = np.sqrt(gx * gx + gy * gy) + elif method == "laplacian": + result = np.abs(cv2.Laplacian(gray, cv2.CV_64F, ksize=3)) + elif method == "zscore": + result = _zscore(gray) + else: + raise ValueError(f"unknown theme-normalize method: {method!r}") + return _to_uint8(result) + + +def match_theme(template: Any, *, haystack: Optional[Any] = None, + method: str = "sobel", min_score: float = 0.5, + region: Optional[Sequence[int]] = None + ) -> Optional[Dict[str, Any]]: + """Locate ``template`` in ``haystack`` after theme-normalising both. + + ``haystack`` defaults to a fresh screen grab (optionally clipped to + ``region``). Returns ``{x, y, width, height, score}`` for the best match at + or above ``min_score``, or ``None``. Robust to a light/dark theme flip that + defeats raw :func:`visual_match.match_template`. + """ + from je_auto_control.utils.visual_match import match_template + from je_auto_control.utils.visual_match.visual_match import _grab_gray + raw_haystack = haystack if haystack is not None else _grab_gray(region) + norm_template = normalize_theme(template, method=method) + norm_haystack = normalize_theme(raw_haystack, method=method) + match = match_template(norm_template, haystack=norm_haystack, + min_score=float(min_score)) + if match is None: + return None + return {"x": match.x, "y": match.y, "width": match.width, + "height": match.height, "score": round(float(match.score), 4)} diff --git a/test/unit_test/headless/test_theme_normalize_batch.py b/test/unit_test/headless/test_theme_normalize_batch.py new file mode 100644 index 00000000..10174c38 --- /dev/null +++ b/test/unit_test/headless/test_theme_normalize_batch.py @@ -0,0 +1,72 @@ +"""Headless tests for theme_normalize (cv2 behaviour + cv2-free wiring).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.theme_normalize import match_theme, normalize_theme + + +# --- cv2 behaviour (gated per-function so wiring still runs without cv2) --- + +def test_normalize_theme_polarity_invariant(): + np = pytest.importorskip("numpy") + pytest.importorskip("cv2") + rng = np.random.default_rng(0) + image = rng.integers(0, 256, (60, 80)).astype("uint8") + light = normalize_theme(image, method="sobel") + dark = normalize_theme(255 - image, method="sobel") + assert light.shape == image.shape + # gradient magnitude is identical for an image and its colour inverse + assert np.array_equal(light, dark) + + +def test_normalize_theme_zscore_shape(): + np = pytest.importorskip("numpy") + pytest.importorskip("cv2") + rng = np.random.default_rng(1) + image = rng.integers(0, 256, (40, 40)).astype("uint8") + out = normalize_theme(image, method="zscore") + assert out.shape == image.shape + assert out.dtype == np.uint8 + + +def test_normalize_theme_unknown_method_raises(): + np = pytest.importorskip("numpy") + pytest.importorskip("cv2") + image = np.zeros((10, 10), dtype="uint8") + with pytest.raises(ValueError): + normalize_theme(image, method="bogus") + + +def test_match_theme_finds_template_across_inversion(): + np = pytest.importorskip("numpy") + pytest.importorskip("cv2") + haystack = np.full((100, 120), 128, dtype="uint8") + template = np.full((20, 20), 220, dtype="uint8") + template[5:15, 5:15] = 40 # internal edge structure + haystack[30:50, 40:60] = template # place at x=40, y=30 + dark_haystack = 255 - haystack # dark-mode: colours inverted + result = match_theme(template, haystack=dark_haystack, method="sobel", + min_score=0.3) + assert result is not None + assert abs(result["x"] - 40) <= 5 + assert abs(result["y"] - 30) <= 5 + + +# --- wiring (cv2-free: the module imports cv2 lazily) ---------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_match_theme" in known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert "ac_match_theme" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_match_theme" in specs + + +def test_facade_exports(): + for name in ("normalize_theme", "match_theme"): + assert hasattr(ac, name) and name in ac.__all__