diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 4c39e190..0587166d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,29 @@ # What's New — AutoControl +## What's new (2026-06-25) — Per-Run Step Timeline (waterfall + bottleneck steps) + +Read why *this* run was slow — a step waterfall and its bottlenecks. Full reference: [`docs/source/Eng/doc/new_features/v194_features_doc.rst`](docs/source/Eng/doc/new_features/v194_features_doc.rst). + +- **`build_timeline` / `critical_steps`** (`AC_build_timeline`, `AC_critical_steps`): the action profiler aggregates timings by step *name* across runs — useless for "why was *this* run slow". This turns one run's ordered steps into a waterfall (each step's offset, duration, and `pct` share of the total) with the `bottleneck` step and a `parallelism` ratio (`> 1` when steps overlap via explicit `start` times); `critical_steps` ranks the dominant steps to optimise. A step is any `{name, duration, start?}` dict. Pure stdlib. No `PySide6`. + +## What's new (2026-06-25) — Flaky-Test Co-Failure Clustering + +Find the tests that flake *together* — and the shared root cause behind them. Full reference: [`docs/source/Eng/doc/new_features/v193_features_doc.rst`](docs/source/Eng/doc/new_features/v193_features_doc.rst). + +- **`cofailure_pairs` / `failure_clusters`** (`AC_cofailure_pairs`, `AC_failure_clusters`): flaky tests are rarely independent — a wobbly fixture or noisy dependency makes a *group* fail in the same runs (~75% of flaky tests cluster). Ranking tests one-by-one by flip rate misses that. This measures how often each pair of tests fails in the *same* runs (Jaccard over their failing-run sets) and groups tests above a threshold into connected clusters with a cohesion score — so you chase one root cause instead of N symptoms. Input is a list of runs, each the test names that failed in it. Pure stdlib. No `PySide6`. + +## What's new (2026-06-25) — Run-Trace Diff (what changed between two executions) + +See exactly what changed between a passing run and a failing one. Full reference: [`docs/source/Eng/doc/new_features/v192_features_doc.rst`](docs/source/Eng/doc/new_features/v192_features_doc.rst). + +- **`diff_runs` / `summarize_run_diff`** (`AC_diff_runs`): a run history says a run *failed* but not *what changed* from the run that passed. This aligns two step sequences with a longest-common-subsequence walk (so an inserted/removed step shifts the rest into place instead of mis-pairing everything) and classifies the differences: **added**/**removed** steps, **status_flips** (an aligned step that changed status — with the new failure's `failure_signature` when it carries an error), and **timing_regressions** (a step that got `regress_factor`× slower). `summarize_run_diff` renders a one-line summary. Pure stdlib over lists of `{name,status,duration,error}` step dicts. No `PySide6`. + +## What's new (2026-06-25) — Stable Failure Signatures + +Match the *same kind* of failure across runs, despite differing paths and ids. Full reference: [`docs/source/Eng/doc/new_features/v191_features_doc.rst`](docs/source/Eng/doc/new_features/v191_features_doc.rst). + +- **`normalize_error` / `failure_signature` / `group_failures`** (`AC_failure_signature`, `AC_group_failures`): two runs that failed the same way rarely have byte-identical error text — paths, line numbers, addresses, ids and timestamps differ every time — which defeats "is this the same failure?" and "which tests fail together?". This strips the variable parts of an error to a canonical form and hashes it (SHA-256), so the same kind of failure gets the same short signature across runs — the join key the rest of the test-robustness tools (run diffing, flake clustering) group on. `group_failures` buckets a list of errors by signature, most frequent first. Pure stdlib (`re` + `hashlib`). No `PySide6`. + ## What's new (2026-06-24) — Visual Saliency (where to look — spectral-residual) Find the region that stands out, with no template / colour / text. Full reference: [`docs/source/Eng/doc/new_features/v190_features_doc.rst`](docs/source/Eng/doc/new_features/v190_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v191_features_doc.rst b/docs/source/Eng/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..994cf054 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v191_features_doc.rst @@ -0,0 +1,48 @@ +Stable Failure Signatures +========================= + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +* :func:`normalize_error` — collapse paths / hex addresses / UUIDs / timestamps / + line numbers / bare integers to placeholders, +* :func:`failure_signature` — a short stable SHA-256 of the normalised message, +* :func:`group_failures` — group a list of errors by signature, most frequent + first. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True — same failure + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows and POSIX paths, ``0x`` addresses, UUIDs, ISO timestamps, ``line N`` and +any leftover integers become placeholders; whitespace is squeezed. +``group_failures`` keeps up to three distinct raw examples per group and skips +empty / ``None`` messages. + +Executor commands +----------------- + +``AC_failure_signature`` (``error`` / ``length``) returns ``{signature, +normalized}``; ``AC_group_failures`` (``errors``) returns the grouped list. They +are exposed as read-only ``ac_*`` MCP tools and as Script Builder commands under +**Testing**. diff --git a/docs/source/Eng/doc/new_features/v192_features_doc.rst b/docs/source/Eng/doc/new_features/v192_features_doc.rst new file mode 100644 index 00000000..d717e2e1 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v192_features_doc.rst @@ -0,0 +1,48 @@ +Run-Trace Diff (what changed between two executions) +==================================================== + +A run history tells you a run *failed*, but not *what changed* from the run that +passed: which step was added or dropped, which step flipped pass→fail, which step +got slower. ``run_diff`` aligns the two step sequences with a longest-common- +subsequence walk — so an inserted or removed step shifts the rest into place +instead of mis-pairing everything — and classifies the differences: + +* **added** / **removed** — steps present in only one run, +* **status_flips** — an aligned step whose status changed, with the new failure's + :func:`failure_signature` when it carries an ``error``, +* **timing_regressions** — an aligned step that got ``regress_factor`` x slower. + +A step is any dict with a name key (default ``"name"``) and optional ``status`` / +``duration`` / ``error``. Pure standard library; no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import diff_runs, summarize_run_diff + + before = [{"name": "login", "status": "ok", "duration": 1.0}, + {"name": "submit", "status": "ok", "duration": 1.0}] + after = [{"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok"}, # inserted + {"name": "submit", "status": "error", "error": "Timeout ..."}] + + diff = diff_runs(before, after) + # {"added": [accept_cookies], "removed": [], + # "status_flips": [{"name": "submit", "from": "ok", "to": "error", + # "signature": "..."}], + # "timing_regressions": [], "aligned": 2, "identical": False} + + summarize_run_diff(diff) # "+1 added, 1 status flip(s)" + +``regress_factor`` (default ``1.5``) is the slowdown ratio that counts as a +regression; ``key`` selects the field steps are aligned on. ``summarize_run_diff`` +renders a one-line summary (``"no change"`` when identical). + +Executor commands +----------------- + +``AC_diff_runs`` (``before`` / ``after`` / ``key`` / ``regress_factor``) returns +the diff plus a ``summary`` field. It is exposed as the read-only ``ac_diff_runs`` +MCP tool and as a Script Builder command under **Testing**. diff --git a/docs/source/Eng/doc/new_features/v193_features_doc.rst b/docs/source/Eng/doc/new_features/v193_features_doc.rst new file mode 100644 index 00000000..3abad25e --- /dev/null +++ b/docs/source/Eng/doc/new_features/v193_features_doc.rst @@ -0,0 +1,46 @@ +Flaky-Test Co-Failure Clustering +================================ + +Flaky tests are rarely independent: a wobbly shared fixture, a slow dependency or +a noisy environment makes a *group* of tests fail in the same runs (research finds +~75% of flaky tests fall into co-failure clusters). Ranking tests one-by-one by +flip rate misses that shared root cause. ``flake_cluster`` measures how often each +pair of tests fails in the *same* runs — Jaccard similarity over the set of runs +each failed in — and groups tests whose co-failure exceeds a threshold, so you can +chase one root cause instead of N symptoms. + +* :func:`cofailure_pairs` — test pairs that fail together above a threshold, +* :func:`failure_clusters` — connected clusters of co-failing tests with a + cohesion score (mean pairwise Jaccard). + +Input is a list of runs, each a collection of the test names that failed in that +run. Pure standard library; no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import failure_clusters, cofailure_pairs + + runs = [["test_a", "test_b"], # both failed in this run + ["test_a", "test_b"], + ["test_c"], + ["test_a", "test_b", "test_c"]] + + failure_clusters(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "size": 2, "cohesion": 1.0}] + + cofailure_pairs(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "jaccard": 1.0, "co_failures": 3}] + +``threshold`` is the minimum co-failure Jaccard to link two tests; ``min_size`` +(default ``2``) drops singletons so only genuine clusters surface. Clusters come +back largest / most cohesive first. + +Executor commands +----------------- + +``AC_failure_clusters`` (``runs`` / ``threshold`` / ``min_size``) and +``AC_cofailure_pairs`` (``runs`` / ``threshold``). They are exposed as read-only +``ac_*`` MCP tools and as Script Builder commands under **Testing**. diff --git a/docs/source/Eng/doc/new_features/v194_features_doc.rst b/docs/source/Eng/doc/new_features/v194_features_doc.rst new file mode 100644 index 00000000..f8568eb6 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v194_features_doc.rst @@ -0,0 +1,51 @@ +Per-Run Step Timeline (waterfall + bottleneck steps) +==================================================== + +The action profiler aggregates timings by step *name* across many runs — great +for "which action is slow on average", useless for "why was *this* run slow". A +single run is an ordered timeline: step A ran, then B, then C, and one of them +dominated. ``step_timeline`` turns one run's steps into a waterfall (each step's +offset from the start, its duration and its share of the total) and ranks the +bottleneck steps, so you can read a single slow run instead of an average. + +* :func:`build_timeline` — the waterfall + total / busy / bottleneck / + parallelism, +* :func:`critical_steps` — the steps that dominate the run, longest first. + +A step is any dict with a name (default ``"name"``) and a ``duration``; an +optional ``start`` places it on an absolute timeline (overlapping / parallel +steps), else steps are laid out back-to-back. Pure standard library; no device, +no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import build_timeline, critical_steps + + steps = [{"name": "login", "duration": 1.0}, + {"name": "load_dashboard", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + build_timeline(steps) + # {"steps": [{"name": "login", "offset": 0.0, "duration": 1.0, "pct": 16.7}, + # {"name": "load_dashboard", "offset": 1.0, ..., "pct": 66.7}, ...], + # "total": 6.0, "busy": 6.0, + # "bottleneck": {"name": "load_dashboard", "duration": 4.0}, + # "parallelism": 1.0} + + critical_steps(steps, top=2) + # [{"name": "load_dashboard", "duration": 4.0, "pct": 66.7}, + # {"name": "login", "duration": 1.0, "pct": 16.7}] + +``total`` is the wall-clock span, ``busy`` the summed step time; ``parallelism`` = +busy / total is ``1.0`` for a purely sequential run and ``> 1`` when steps overlap +(supply ``start`` times). ``pct`` is each step's share of the total time. + +Executor commands +----------------- + +``AC_build_timeline`` (``steps``) and ``AC_critical_steps`` (``steps`` / ``top``). +They are exposed as read-only ``ac_*`` MCP tools and as Script Builder commands +under **Testing**. diff --git a/docs/source/Zh/doc/new_features/v191_features_doc.rst b/docs/source/Zh/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..f1d1f367 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v191_features_doc.rst @@ -0,0 +1,41 @@ +穩定的失敗簽章 +============== + +兩次以*相同方式*失敗的執行,幾乎不會有逐位元組相同的錯誤文字——路徑、行號、記憶體位址、id 與 +時間戳每次都不同。這使得「這和昨天是同一個失敗嗎?」或「哪些測試會*一起*失敗?」無從問起。 +``failure_signature`` 把錯誤的變動部分剝離成標準形式並雜湊(SHA-256),於是*相同類型*的失敗在 +不同執行間會得到相同的短簽章——即其餘 test-robustness 工具(執行比較、flaky 分群)所依據的 +join key。 + +* :func:`normalize_error` ——把路徑 / 十六進位位址 / UUID / 時間戳 / 行號 / 裸整數收斂成佔位符, +* :func:`failure_signature` ——正規化訊息的短而穩定的 SHA-256, +* :func:`group_failures` ——把一組錯誤依簽章分組,最常見者在前。 + +純標準庫(``re`` + ``hashlib``);不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True——同一個失敗 + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows 與 POSIX 路徑、``0x`` 位址、UUID、ISO 時間戳、``line N`` 與任何殘留整數都會變成佔位符; +空白會被壓縮。``group_failures`` 每組最多保留三個不同的原始範例,並略過空 / ``None`` 訊息。 + +執行器指令 +---------- + +``AC_failure_signature``(``error`` / ``length``)回傳 ``{signature, normalized}``; +``AC_group_failures``(``errors``)回傳分組清單。皆以唯讀 ``ac_*`` MCP 工具及 Script Builder +指令(位於 **Testing** 分類下)形式提供。 diff --git a/docs/source/Zh/doc/new_features/v192_features_doc.rst b/docs/source/Zh/doc/new_features/v192_features_doc.rst new file mode 100644 index 00000000..73116cf0 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v192_features_doc.rst @@ -0,0 +1,45 @@ +執行軌跡比較(兩次執行之間改變了什麼) +====================================== + +執行歷史告訴你某次執行*失敗*了,卻不告訴你相較於通過的那次*改變了什麼*:哪個步驟被加入或移除、 +哪個步驟由通過翻轉成失敗、哪個步驟變慢了。``run_diff`` 以最長共同子序列(LCS)走訪對齊兩個步驟 +序列——這樣插入或移除一個步驟會把其餘步驟順移到位,而非整個錯位配對——並將差異分類: + +* **added** / **removed** ——只存在於其中一次執行的步驟, +* **status_flips** ——某個已對齊步驟的狀態改變,若帶有 ``error`` 則附上新失敗的 + :func:`failure_signature`, +* **timing_regressions** ——某個已對齊步驟變慢了 ``regress_factor`` 倍。 + +步驟可為任何帶有名稱鍵(預設 ``"name"``)與選填 ``status`` / ``duration`` / ``error`` 的字典。 +純標準庫;不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import diff_runs, summarize_run_diff + + before = [{"name": "login", "status": "ok", "duration": 1.0}, + {"name": "submit", "status": "ok", "duration": 1.0}] + after = [{"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok"}, # 插入 + {"name": "submit", "status": "error", "error": "Timeout ..."}] + + diff = diff_runs(before, after) + # {"added": [accept_cookies], "removed": [], + # "status_flips": [{"name": "submit", "from": "ok", "to": "error", + # "signature": "..."}], + # "timing_regressions": [], "aligned": 2, "identical": False} + + summarize_run_diff(diff) # "+1 added, 1 status flip(s)" + +``regress_factor``(預設 ``1.5``)是算作退化的變慢比率;``key`` 選擇步驟對齊所依據的欄位。 +``summarize_run_diff`` 產生一行摘要(相同時為 ``"no change"``)。 + +執行器指令 +---------- + +``AC_diff_runs``(``before`` / ``after`` / ``key`` / ``regress_factor``)回傳該差異並附帶 +``summary`` 欄位。以唯讀 ``ac_diff_runs`` MCP 工具及 Script Builder 指令(位於 **Testing** +分類下)形式提供。 diff --git a/docs/source/Zh/doc/new_features/v193_features_doc.rst b/docs/source/Zh/doc/new_features/v193_features_doc.rst new file mode 100644 index 00000000..222fc3a7 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v193_features_doc.rst @@ -0,0 +1,41 @@ +不穩定測試的共同失敗分群 +======================== + +不穩定(flaky)測試很少是獨立的:搖晃的共用 fixture、緩慢的相依、或吵雜的環境,會讓*一群*測試在 +相同的執行中一起失敗(研究發現約 75% 的 flaky 測試落在共同失敗的群集裡)。逐一以翻轉率排名測試 +會錯過這個共同根因。``flake_cluster`` 量測每對測試多常在*相同*執行中失敗——即各自失敗的執行集合 +之間的 Jaccard 相似度——並把共同失敗超過門檻的測試分群,讓你能追一個根因,而非 N 個症狀。 + +* :func:`cofailure_pairs` ——共同失敗超過門檻的測試對, +* :func:`failure_clusters` ——共同失敗測試的連通群集,附凝聚度分數(群內平均成對 Jaccard)。 + +輸入是一份執行清單,每個元素為該次執行中失敗的測試名稱集合。純標準庫;不涉及裝置,不匯入 +``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import failure_clusters, cofailure_pairs + + runs = [["test_a", "test_b"], # 這次執行兩者皆失敗 + ["test_a", "test_b"], + ["test_c"], + ["test_a", "test_b", "test_c"]] + + failure_clusters(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "size": 2, "cohesion": 1.0}] + + cofailure_pairs(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "jaccard": 1.0, "co_failures": 3}] + +``threshold`` 是連結兩測試所需的最小共同失敗 Jaccard;``min_size``(預設 ``2``)會丟棄單例, +讓只有真正的群集浮現。群集以最大 / 最凝聚者在前回傳。 + +執行器指令 +---------- + +``AC_failure_clusters``(``runs`` / ``threshold`` / ``min_size``)與 +``AC_cofailure_pairs``(``runs`` / ``threshold``)。皆以唯讀 ``ac_*`` MCP 工具及 Script Builder +指令(位於 **Testing** 分類下)形式提供。 diff --git a/docs/source/Zh/doc/new_features/v194_features_doc.rst b/docs/source/Zh/doc/new_features/v194_features_doc.rst new file mode 100644 index 00000000..63a9e8db --- /dev/null +++ b/docs/source/Zh/doc/new_features/v194_features_doc.rst @@ -0,0 +1,44 @@ +單次執行的步驟時間軸(瀑布圖 + 瓶頸步驟) +========================================== + +動作 profiler 把計時按步驟*名稱*跨多次執行聚合——很適合「哪個動作平均較慢」,卻無助於「為什麼 +*這一次*執行很慢」。單次執行是一條有序時間軸:步驟 A 跑完、接著 B、再 C,其中某一步主導了時間。 +``step_timeline`` 把一次執行的步驟轉成瀑布圖(每步距起點的偏移、其時長、其占總時間的比例),並 +排名瓶頸步驟,讓你能讀懂單一慢執行,而非平均值。 + +* :func:`build_timeline` ——瀑布圖加上 total / busy / bottleneck / parallelism, +* :func:`critical_steps` ——主導該次執行的步驟,最長者在前。 + +步驟可為任何帶名稱(預設 ``"name"``)與 ``duration`` 的字典;選填 ``start`` 會把它放到絕對 +時間軸上(重疊 / 平行步驟),否則步驟會背靠背排列。純標準庫;不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import build_timeline, critical_steps + + steps = [{"name": "login", "duration": 1.0}, + {"name": "load_dashboard", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + build_timeline(steps) + # {"steps": [{"name": "login", "offset": 0.0, "duration": 1.0, "pct": 16.7}, + # {"name": "load_dashboard", "offset": 1.0, ..., "pct": 66.7}, ...], + # "total": 6.0, "busy": 6.0, + # "bottleneck": {"name": "load_dashboard", "duration": 4.0}, + # "parallelism": 1.0} + + critical_steps(steps, top=2) + # [{"name": "load_dashboard", "duration": 4.0, "pct": 66.7}, + # {"name": "login", "duration": 1.0, "pct": 16.7}] + +``total`` 是牆鐘時間跨度,``busy`` 是各步驟時長總和;``parallelism`` = busy / total,純序列執行 +為 ``1.0``,步驟重疊時 ``> 1``(需提供 ``start`` 時間)。``pct`` 是每步占總時間的比例。 + +執行器指令 +---------- + +``AC_build_timeline``(``steps``)與 ``AC_critical_steps``(``steps`` / ``top``)。皆以唯讀 +``ac_*`` MCP 工具及 Script Builder 指令(位於 **Testing** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 0cd404e9..81bc5b59 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -88,6 +88,16 @@ from je_auto_control.utils.saliency import ( most_salient, salient_regions, saliency_map, ) +# Stable failure signatures (normalise + hash error text; group failures) +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) +# Run-trace diff (LCS-aligned: added/removed steps, status flips, regressions) +from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff +# Flaky-test co-failure clustering (Jaccard over shared failing runs) +from je_auto_control.utils.flake_cluster import cofailure_pairs, failure_clusters +# Per-run step waterfall + bottleneck (critical) steps +from je_auto_control.utils.step_timeline import build_timeline, critical_steps # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1665,6 +1675,10 @@ def start_autocontrol_gui(*args, **kwargs): "image_quality", "is_blurry", "quality_gate", "detect_scale", "scale_sweep", "saliency_map", "salient_regions", "most_salient", + "normalize_error", "failure_signature", "group_failures", + "diff_runs", "summarize_run_diff", + "cofailure_pairs", "failure_clusters", + "build_timeline", "critical_steps", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 8019fb23..d4d5bce4 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2708,6 +2708,68 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: description="Aggregate the self-heal log (heal rate, brittle " "locators).", )) + specs.append(CommandSpec( + "AC_failure_signature", "Testing", "Failure Signature", + fields=( + FieldSpec("error", FieldType.STRING, + placeholder="Timeout at C:\\app.py line 42 (0x7ff..)"), + FieldSpec("length", FieldType.INT, optional=True, default=12), + ), + description="Normalise + hash an error to a stable failure signature.", + )) + specs.append(CommandSpec( + "AC_group_failures", "Testing", "Group Failures by Signature", + fields=(FieldSpec("errors", FieldType.STRING, + placeholder='["err one", "err two"]'),), + description="Group error messages by failure signature (most frequent).", + )) + specs.append(CommandSpec( + "AC_diff_runs", "Testing", "Diff Two Run Traces", + fields=( + FieldSpec("before", FieldType.STRING, + placeholder='[{"name": "login", "status": "ok"}]'), + FieldSpec("after", FieldType.STRING, + placeholder='[{"name": "login", "status": "error"}]'), + FieldSpec("key", FieldType.STRING, optional=True, default="name"), + FieldSpec("regress_factor", FieldType.FLOAT, optional=True, + default=1.5), + ), + description="LCS-align two run step-traces: added/removed/flips/regress.", + )) + specs.append(CommandSpec( + "AC_failure_clusters", "Testing", "Cluster Co-Failing Tests", + fields=( + FieldSpec("runs", FieldType.STRING, + placeholder='[["test_a", "test_b"], ["test_a", "test_b"]]'), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.5), + FieldSpec("min_size", FieldType.INT, optional=True, default=2), + ), + description="Cluster tests that flake together (co-failure Jaccard).", + )) + specs.append(CommandSpec( + "AC_cofailure_pairs", "Testing", "Co-Failing Test Pairs", + fields=( + FieldSpec("runs", FieldType.STRING, + placeholder='[["test_a", "test_b"]]'), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.5), + ), + description="Test pairs that fail together above a Jaccard threshold.", + )) + specs.append(CommandSpec( + "AC_build_timeline", "Testing", "Step Timeline (waterfall)", + fields=(FieldSpec("steps", FieldType.STRING, + placeholder='[{"name": "login", "duration": 1.2}]'),), + description="Per-run step waterfall: offsets, durations, bottleneck.", + )) + specs.append(CommandSpec( + "AC_critical_steps", "Testing", "Critical (Bottleneck) Steps", + fields=( + FieldSpec("steps", FieldType.STRING, + placeholder='[{"name": "login", "duration": 1.2}]'), + FieldSpec("top", FieldType.INT, optional=True, default=3), + ), + description="The steps that dominate a run's time, longest first.", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 39a99780..edff6eda 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4348,6 +4348,78 @@ def _most_salient(source: Any = None, region: Any = None, size: Any = 64, return {"found": result is not None, "region": result} +def _failure_signature(error: str, length: Any = 12) -> Dict[str, Any]: + """Adapter: normalise + hash an error message to a stable signature.""" + from je_auto_control.utils.failure_signature import ( + failure_signature, normalize_error) + return {"signature": failure_signature(str(error), length=int(length)), + "normalized": normalize_error(str(error))} + + +def _group_failures(errors: Any) -> Dict[str, Any]: + """Adapter: group error messages by failure signature.""" + import json + from je_auto_control.utils.failure_signature import group_failures + if isinstance(errors, str): + errors = json.loads(errors) + groups = group_failures(errors) + return {"groups": groups, "count": len(groups)} + + +def _diff_runs(before: Any, after: Any, key: str = "name", + regress_factor: Any = 1.5) -> Dict[str, Any]: + """Adapter: diff two run step-traces (added/removed/flips/regressions).""" + import json + from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff + if isinstance(before, str): + before = json.loads(before) + if isinstance(after, str): + after = json.loads(after) + diff = diff_runs(before, after, key=str(key), + regress_factor=float(regress_factor)) + return {**diff, "summary": summarize_run_diff(diff)} + + +def _failure_clusters(runs: Any, threshold: Any = 0.5, + min_size: Any = 2) -> Dict[str, Any]: + """Adapter: cluster tests that fail together (co-failure Jaccard).""" + import json + from je_auto_control.utils.flake_cluster import failure_clusters + if isinstance(runs, str): + runs = json.loads(runs) + clusters = failure_clusters(runs, threshold=float(threshold), + min_size=int(min_size)) + return {"clusters": clusters, "count": len(clusters)} + + +def _cofailure_pairs(runs: Any, threshold: Any = 0.5) -> Dict[str, Any]: + """Adapter: test pairs that fail together above a Jaccard threshold.""" + import json + from je_auto_control.utils.flake_cluster import cofailure_pairs + if isinstance(runs, str): + runs = json.loads(runs) + pairs = cofailure_pairs(runs, threshold=float(threshold)) + return {"pairs": pairs, "count": len(pairs)} + + +def _build_timeline(steps: Any) -> Dict[str, Any]: + """Adapter: a per-run step waterfall (offsets / durations / bottleneck).""" + import json + from je_auto_control.utils.step_timeline import build_timeline + if isinstance(steps, str): + steps = json.loads(steps) + return build_timeline(steps) + + +def _critical_steps(steps: Any, top: Any = 3) -> Dict[str, Any]: + """Adapter: the steps that dominate a run's time (bottlenecks).""" + import json + from je_auto_control.utils.step_timeline import critical_steps + if isinstance(steps, str): + steps = json.loads(steps) + return {"steps": critical_steps(steps, top=int(top))} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6576,6 +6648,13 @@ def __init__(self): "AC_scale_sweep": _scale_sweep, "AC_salient_regions": _salient_regions, "AC_most_salient": _most_salient, + "AC_failure_signature": _failure_signature, + "AC_group_failures": _group_failures, + "AC_diff_runs": _diff_runs, + "AC_failure_clusters": _failure_clusters, + "AC_cofailure_pairs": _cofailure_pairs, + "AC_build_timeline": _build_timeline, + "AC_critical_steps": _critical_steps, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/failure_signature/__init__.py b/je_auto_control/utils/failure_signature/__init__.py new file mode 100644 index 00000000..a9017441 --- /dev/null +++ b/je_auto_control/utils/failure_signature/__init__.py @@ -0,0 +1,6 @@ +"""Normalise error messages into stable SHA-256 failure signatures + grouping.""" +from je_auto_control.utils.failure_signature.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +__all__ = ["normalize_error", "failure_signature", "group_failures"] diff --git a/je_auto_control/utils/failure_signature/failure_signature.py b/je_auto_control/utils/failure_signature/failure_signature.py new file mode 100644 index 00000000..5743eec4 --- /dev/null +++ b/je_auto_control/utils/failure_signature/failure_signature.py @@ -0,0 +1,68 @@ +"""Normalise an error message into a stable failure signature. + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. +""" +import hashlib +import re +from typing import Any, Dict, Iterable, List + +# Ordered (pattern, replacement): the volatile parts of an error, most specific +# first so e.g. a path's trailing line number isn't half-collapsed by the digit rule. +_NORMALIZERS = [ + (re.compile(r"[A-Za-z]:\\[^\s:*?\"<>|]+"), ""), # Windows path + (re.compile(r"(?:/[\w.\-]+)+/[\w.\-]+"), ""), # POSIX path + (re.compile(r"0x[0-9A-Fa-f]+"), "0x"), # memory address + (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}" + r"-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), ""), + (re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?"), ""), + (re.compile(r"\bline\s+\d+\b", re.IGNORECASE), "line "), + (re.compile(r"\b\d+\b"), ""), # any leftover int +] +_WHITESPACE = re.compile(r"\s+") + + +def normalize_error(message: str) -> str: + """Collapse the volatile parts of an error message to a canonical form. + + Paths, hex addresses, UUIDs, timestamps, line numbers and bare integers + become placeholders, and whitespace is squeezed — so messages that differ + only in those details normalise to the same string. + """ + text = str(message) + for pattern, replacement in _NORMALIZERS: + text = pattern.sub(replacement, text) + return _WHITESPACE.sub(" ", text).strip() + + +def failure_signature(message: str, *, length: int = 12) -> str: + """Return a short stable SHA-256 signature of a normalised error message.""" + digest = hashlib.sha256(normalize_error(message).encode("utf-8")).hexdigest() + return digest[:max(1, int(length))] + + +def group_failures(messages: Iterable[str]) -> List[Dict[str, Any]]: + """Group error messages by signature, most frequent first. + + Returns ``[{signature, normalized, count, examples}]`` (up to three distinct + raw examples per group). ``None`` / empty messages are skipped. + """ + groups: Dict[str, Dict[str, Any]] = {} + for message in messages: + if not message: + continue + signature = failure_signature(message) + group = groups.setdefault(signature, { + "signature": signature, "normalized": normalize_error(message), + "count": 0, "examples": []}) + group["count"] += 1 + if len(group["examples"]) < 3 and str(message) not in group["examples"]: + group["examples"].append(str(message)) + return sorted(groups.values(), key=lambda group: group["count"], reverse=True) diff --git a/je_auto_control/utils/flake_cluster/__init__.py b/je_auto_control/utils/flake_cluster/__init__.py new file mode 100644 index 00000000..376dbb05 --- /dev/null +++ b/je_auto_control/utils/flake_cluster/__init__.py @@ -0,0 +1,6 @@ +"""Cluster tests that flake together by co-failure Jaccard similarity.""" +from je_auto_control.utils.flake_cluster.flake_cluster import ( + cofailure_pairs, failure_clusters, +) + +__all__ = ["cofailure_pairs", "failure_clusters"] diff --git a/je_auto_control/utils/flake_cluster/flake_cluster.py b/je_auto_control/utils/flake_cluster/flake_cluster.py new file mode 100644 index 00000000..d35d261c --- /dev/null +++ b/je_auto_control/utils/flake_cluster/flake_cluster.py @@ -0,0 +1,97 @@ +"""Cluster tests that flake *together* by co-failure similarity. + +Flaky tests are rarely independent: a wobbly shared fixture, a slow dependency or +a noisy environment makes a *group* of tests fail in the same runs (research finds +~75% of flaky tests fall into co-failure clusters). Ranking tests one-by-one by +flip rate misses that shared root cause. ``flake_cluster`` measures how often each +pair of tests fails in the *same* runs (Jaccard similarity over the set of runs +each failed in) and groups tests whose co-failure exceeds a threshold into +clusters — so you can chase one root cause instead of N symptoms. + +Input is a list of runs, each a collection of the test names that failed in that +run. Pure standard library; no device, no ``PySide6``. +""" +from itertools import combinations +from typing import Any, Dict, List, Sequence, Set + + +def _set_jaccard(left: Set[int], right: Set[int]) -> float: + union = left | right + return len(left & right) / len(union) if union else 0.0 + + +def _fail_runs(runs: Sequence[Sequence[str]]) -> Dict[str, Set[int]]: + """Map each test name to the set of run indices in which it failed.""" + fails: Dict[str, Set[int]] = {} + for index, run in enumerate(runs): + for test in set(run): + fails.setdefault(str(test), set()).add(index) + return fails + + +def cofailure_pairs(runs: Sequence[Sequence[str]], *, + threshold: float = 0.5) -> List[Dict[str, Any]]: + """Return test pairs whose co-failure Jaccard meets ``threshold``. + + Each entry is ``{tests:[a,b], jaccard, co_failures}``, most similar first. + """ + fails = _fail_runs(runs) + pairs = [] + for left, right in combinations(sorted(fails), 2): + score = _set_jaccard(fails[left], fails[right]) + if score >= float(threshold): + pairs.append({"tests": [left, right], "jaccard": round(score, 3), + "co_failures": len(fails[left] & fails[right])}) + pairs.sort(key=lambda pair: pair["jaccard"], reverse=True) + return pairs + + +def _connected_components(nodes: Sequence[str], + adjacency: Dict[str, Set[str]]) -> List[List[str]]: + seen: Set[str] = set() + components = [] + for node in nodes: + if node in seen: + continue + stack, component = [node], [] + while stack: + current = stack.pop() + if current in seen: + continue + seen.add(current) + component.append(current) + stack.extend(adjacency[current] - seen) + components.append(component) + return components + + +def _cohesion(component: Sequence[str], fails: Dict[str, Set[int]]) -> float: + scores = [_set_jaccard(fails[a], fails[b]) + for a, b in combinations(component, 2)] + return round(sum(scores) / len(scores), 3) if scores else 1.0 + + +def failure_clusters(runs: Sequence[Sequence[str]], *, threshold: float = 0.5, + min_size: int = 2) -> List[Dict[str, Any]]: + """Group tests that fail together into co-failure clusters. + + Builds a graph linking test pairs whose co-failure Jaccard meets + ``threshold``, then returns its connected components of at least ``min_size`` + tests as ``[{tests, size, cohesion}]`` (largest / most cohesive first). + ``cohesion`` is the mean pairwise Jaccard within the cluster. + """ + fails = _fail_runs(runs) + tests = sorted(fails) + adjacency: Dict[str, Set[str]] = {test: set() for test in tests} + for left, right in combinations(tests, 2): + if _set_jaccard(fails[left], fails[right]) >= float(threshold): + adjacency[left].add(right) + adjacency[right].add(left) + clusters = [] + for component in _connected_components(tests, adjacency): + if len(component) >= int(min_size): + clusters.append({"tests": sorted(component), "size": len(component), + "cohesion": _cohesion(component, fails)}) + clusters.sort(key=lambda cluster: (cluster["size"], cluster["cohesion"]), + reverse=True) + return clusters diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b84715a1..d32d15ae 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7652,6 +7652,98 @@ def flakiness_tools() -> List[MCPTool]: handler=h.flaky_report, annotations=READ_ONLY, ), + MCPTool( + name="ac_failure_signature", + description=("Normalise an error message (strip paths / addresses / " + "line numbers / timestamps / ids) and hash it to a stable " + "SHA-256 signature, so the same kind of failure matches " + "across runs. Returns {signature, normalized}."), + input_schema=schema({"error": {"type": "string"}, + "length": {"type": "integer"}}, + required=["error"]), + handler=h.failure_signature, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_group_failures", + description=("Group a list of error messages by failure signature, " + "most frequent first: [{signature, normalized, count, " + "examples}]."), + input_schema=schema({ + "errors": {"type": "array", "items": {"type": "string"}}}, + required=["errors"]), + handler=h.group_failures, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_diff_runs", + description=("Diff two run step-traces (lists of {name,status," + "duration,error}) — LCS-aligned so inserts shift rather " + "than mis-pair. Returns {added, removed, status_flips " + "(with failure signature), timing_regressions, aligned, " + "identical, summary}. 'regress_factor' = slowdown ratio."), + input_schema=schema({ + "before": {"type": "array", "items": {"type": "object"}}, + "after": {"type": "array", "items": {"type": "object"}}, + "key": {"type": "string"}, + "regress_factor": {"type": "number"}}, + required=["before", "after"]), + handler=h.diff_runs, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_failure_clusters", + description=("Cluster tests that flake TOGETHER: 'runs' is a list of " + "runs, each the list of test names that failed in that " + "run. Groups tests whose co-failure Jaccard >= " + "'threshold'. Returns {clusters:[{tests,size,cohesion}], " + "count} — chase one root cause, not N symptoms."), + input_schema=schema({ + "runs": {"type": "array", + "items": {"type": "array", "items": {"type": "string"}}}, + "threshold": {"type": "number"}, + "min_size": {"type": "integer"}}, + required=["runs"]), + handler=h.failure_clusters, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_cofailure_pairs", + description=("Test pairs that fail together above a Jaccard " + "'threshold' over shared failing runs: " + "{pairs:[{tests,jaccard,co_failures}], count}."), + input_schema=schema({ + "runs": {"type": "array", + "items": {"type": "array", "items": {"type": "string"}}}, + "threshold": {"type": "number"}}, + required=["runs"]), + handler=h.cofailure_pairs, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_build_timeline", + description=("Per-run step waterfall from 'steps' (list of {name," + "duration,start?}): {steps:[{name,offset,duration,pct}], " + "total, busy, bottleneck, parallelism}. Reads ONE slow " + "run, not a per-name average."), + input_schema=schema({ + "steps": {"type": "array", "items": {"type": "object"}}}, + required=["steps"]), + handler=h.build_timeline, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_critical_steps", + description=("The 'top' steps that dominate a run's time (bottlenecks " + "to optimise): {steps:[{name,duration,pct}]}, longest " + "first."), + input_schema=schema({ + "steps": {"type": "array", "items": {"type": "object"}}, + "top": {"type": "integer"}}, + required=["steps"]), + handler=h.critical_steps, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index c00840fe..ab6d91cc 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2543,6 +2543,41 @@ def most_salient(source=None, region=None, size=64, threshold=None, min_area=4): return _most_salient(source, region, size, threshold, min_area) +def failure_signature(error, length=12): + from je_auto_control.utils.executor.action_executor import _failure_signature + return _failure_signature(error, length) + + +def group_failures(errors): + from je_auto_control.utils.executor.action_executor import _group_failures + return _group_failures(errors) + + +def diff_runs(before, after, key="name", regress_factor=1.5): + from je_auto_control.utils.executor.action_executor import _diff_runs + return _diff_runs(before, after, key, regress_factor) + + +def failure_clusters(runs, threshold=0.5, min_size=2): + from je_auto_control.utils.executor.action_executor import _failure_clusters + return _failure_clusters(runs, threshold, min_size) + + +def cofailure_pairs(runs, threshold=0.5): + from je_auto_control.utils.executor.action_executor import _cofailure_pairs + return _cofailure_pairs(runs, threshold) + + +def build_timeline(steps): + from je_auto_control.utils.executor.action_executor import _build_timeline + return _build_timeline(steps) + + +def critical_steps(steps, top=3): + from je_auto_control.utils.executor.action_executor import _critical_steps + return _critical_steps(steps, top) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/je_auto_control/utils/run_diff/__init__.py b/je_auto_control/utils/run_diff/__init__.py new file mode 100644 index 00000000..05cee29c --- /dev/null +++ b/je_auto_control/utils/run_diff/__init__.py @@ -0,0 +1,4 @@ +"""Diff two run traces (LCS-aligned step diff: added/removed/flips/regressions).""" +from je_auto_control.utils.run_diff.run_diff import diff_runs, summarize_run_diff + +__all__ = ["diff_runs", "summarize_run_diff"] diff --git a/je_auto_control/utils/run_diff/run_diff.py b/je_auto_control/utils/run_diff/run_diff.py new file mode 100644 index 00000000..26b2f494 --- /dev/null +++ b/je_auto_control/utils/run_diff/run_diff.py @@ -0,0 +1,119 @@ +"""Diff two run traces — what changed between two executions of a flow. + +A run history tells you a run *failed*, but not *what changed* from the run that +passed: which step was added or dropped, which step flipped pass→fail, which step +got slower. ``run_diff`` aligns the two step sequences with a longest-common- +subsequence walk (so an inserted / removed step shifts the rest into place instead +of mis-pairing everything) and classifies the differences: + +* **added** / **removed** steps (present in only one run), +* **status flips** (an aligned step whose status changed — with the new failure's + :func:`failure_signature` when it carries an ``error``), +* **timing regressions** (an aligned step that got ``regress_factor`` x slower). + +A step is any dict with a name key (default ``"name"``) and optional ``status`` / +``duration`` / ``error``. Pure standard library; no device, no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence + +Step = Dict[str, Any] + + +def _lcs_pairs(left: Sequence[str], right: Sequence[str]) -> List[tuple]: + """Return aligned ``(i, j)`` index pairs of the longest common subsequence.""" + n, m = len(left), len(right) + table = [[0] * (m + 1) for _ in range(n + 1)] + for i in range(n - 1, -1, -1): + for j in range(m - 1, -1, -1): + if left[i] == right[j]: + table[i][j] = table[i + 1][j + 1] + 1 + else: + table[i][j] = max(table[i + 1][j], table[i][j + 1]) + pairs, i, j = [], 0, 0 + while i < n and j < m: + if left[i] == right[j]: + pairs.append((i, j)) + i, j = i + 1, j + 1 + elif table[i + 1][j] >= table[i][j + 1]: + i += 1 + else: + j += 1 + return pairs + + +def _status_flip(before: Step, after: Step, name: str) -> Dict[str, Any]: + """Build a status-flip record, attaching a signature for a new error.""" + flip = {"name": name, "from": before.get("status"), + "to": after.get("status")} + error = after.get("error") + if error: + from je_auto_control.utils.failure_signature import failure_signature + flip["signature"] = failure_signature(str(error)) + return flip + + +def _regression(before: Step, after: Step, name: str, + factor: float) -> Dict[str, Any]: + """Return a timing-regression record, or ``{}`` if not a regression.""" + prev, curr = before.get("duration"), after.get("duration") + if not isinstance(prev, (int, float)) or not isinstance(curr, (int, float)): + return {} + if prev > 0 and curr >= prev * float(factor): + return {"name": name, "before": float(prev), "after": float(curr), + "ratio": round(curr / prev, 3)} + return {} + + +def _aligned_changes(before: Sequence[Step], after: Sequence[Step], + names: Sequence[str], pairs: List[tuple], + factor: float) -> tuple: + """Classify the LCS-aligned pairs into (status flips, timing regressions).""" + flips, regressions = [], [] + for i, j in pairs: + if before[i].get("status") != after[j].get("status"): + flips.append(_status_flip(before[i], after[j], names[i])) + regression = _regression(before[i], after[j], names[i], factor) + if regression: + regressions.append(regression) + return flips, regressions + + +def _keys(steps: Sequence[Step], key: str) -> List[str]: + return [str(step.get(key, "")) for step in steps] + + +def _unmatched(steps: Sequence[Step], matched: set) -> List[Step]: + return [steps[k] for k in range(len(steps)) if k not in matched] + + +def diff_runs(before: Sequence[Step], after: Sequence[Step], *, + key: str = "name", regress_factor: float = 1.5) -> Dict[str, Any]: + """Diff two step sequences into ``{added, removed, status_flips, + timing_regressions, aligned, identical}``. + + Steps are aligned by their ``key`` value via LCS; ``regress_factor`` is the + slowdown ratio that counts as a timing regression. + """ + left, right = _keys(before, key), _keys(after, key) + pairs = _lcs_pairs(left, right) + flips, regressions = _aligned_changes(before, after, left, pairs, + regress_factor) + added = _unmatched(after, {j for _, j in pairs}) + removed = _unmatched(before, {i for i, _ in pairs}) + return {"added": added, "removed": removed, "status_flips": flips, + "timing_regressions": regressions, "aligned": len(pairs), + "identical": not any((added, removed, flips, regressions))} + + +def summarize_run_diff(diff: Dict[str, Any]) -> str: + """Render a one-line human summary of a :func:`diff_runs` result.""" + if diff.get("identical"): + return "no change" + parts = [] + for label, field in (("+{} added", "added"), ("-{} removed", "removed"), + ("{} status flip(s)", "status_flips"), + ("{} regression(s)", "timing_regressions")): + count = len(diff.get(field, [])) + if count: + parts.append(label.format(count)) + return ", ".join(parts) diff --git a/je_auto_control/utils/step_timeline/__init__.py b/je_auto_control/utils/step_timeline/__init__.py new file mode 100644 index 00000000..36cae102 --- /dev/null +++ b/je_auto_control/utils/step_timeline/__init__.py @@ -0,0 +1,6 @@ +"""Per-run step waterfall timeline + bottleneck (critical) step ranking.""" +from je_auto_control.utils.step_timeline.step_timeline import ( + build_timeline, critical_steps, +) + +__all__ = ["build_timeline", "critical_steps"] diff --git a/je_auto_control/utils/step_timeline/step_timeline.py b/je_auto_control/utils/step_timeline/step_timeline.py new file mode 100644 index 00000000..4f677bcd --- /dev/null +++ b/je_auto_control/utils/step_timeline/step_timeline.py @@ -0,0 +1,75 @@ +"""Build a per-run step waterfall and find the run's bottleneck steps. + +The action profiler aggregates timings by step *name* across many runs — great for +"which action is slow on average", useless for "why was *this* run slow". A single +run is an ordered timeline: step A ran, then B, then C, and one of them dominated. +``step_timeline`` turns one run's steps into a waterfall (each step's offset from +the start, duration and share of the total) and ranks the bottleneck steps, so you +can read a single slow run instead of an average. + +A step is any dict with a name (default ``"name"``) and a ``duration``; an optional +``start`` places it on an absolute timeline (overlapping / parallel steps), else +steps are laid out back-to-back. Pure standard library; no device, no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence + +Step = Dict[str, Any] + + +def _normalize(steps: Sequence[Step], name_key: str, start_key: str, + duration_key: str) -> List[Dict[str, Any]]: + """Resolve each step to ``{name, start, end, duration}`` (sequential if no start).""" + resolved, cursor = [], 0.0 + for step in steps: + duration = float(step.get(duration_key, 0.0) or 0.0) + raw_start = step.get(start_key) + start = float(raw_start) if raw_start is not None else cursor + end = start + duration + cursor = max(cursor, end) + resolved.append({"name": str(step.get(name_key, "")), "start": start, + "end": end, "duration": duration}) + return resolved + + +def build_timeline(steps: Sequence[Step], *, name_key: str = "name", + start_key: str = "start", + duration_key: str = "duration") -> Dict[str, Any]: + """Return a waterfall timeline for one run. + + ``{steps:[{name, offset, duration, pct}], total, busy, bottleneck, + parallelism}`` — ``total`` is the wall-clock span, ``busy`` the summed step + time, ``parallelism`` = busy / total (1.0 for a purely sequential run), + ``bottleneck`` the longest single step. + """ + resolved = _normalize(steps, name_key, start_key, duration_key) + if not resolved: + return {"steps": [], "total": 0.0, "busy": 0.0, "bottleneck": None, + "parallelism": 0.0} + base = min(step["start"] for step in resolved) + span = max(step["end"] for step in resolved) - base + busy = sum(step["duration"] for step in resolved) + rows = [{"name": step["name"], "offset": round(step["start"] - base, 6), + "duration": step["duration"], + "pct": round(step["duration"] / span * 100, 1) if span > 0 else 0.0} + for step in resolved] + bottleneck = max(resolved, key=lambda step: step["duration"]) + return {"steps": rows, "total": round(span, 6), "busy": round(busy, 6), + "bottleneck": {"name": bottleneck["name"], + "duration": bottleneck["duration"]}, + "parallelism": round(busy / span, 3) if span > 0 else 1.0} + + +def critical_steps(steps: Sequence[Step], *, name_key: str = "name", + start_key: str = "start", duration_key: str = "duration", + top: int = 3) -> List[Dict[str, Any]]: + """Return the ``top`` steps that dominate the run, longest first. + + Each entry is ``{name, duration, pct}`` where ``pct`` is the step's share of + the total step time — the bottlenecks worth optimising. + """ + resolved = _normalize(steps, name_key, start_key, duration_key) + busy = sum(step["duration"] for step in resolved) or 1.0 + ranked = sorted(resolved, key=lambda step: step["duration"], reverse=True) + return [{"name": step["name"], "duration": step["duration"], + "pct": round(step["duration"] / busy * 100, 1)} + for step in ranked[:max(1, int(top))]] diff --git a/test/unit_test/headless/test_failure_signature_batch.py b/test/unit_test/headless/test_failure_signature_batch.py new file mode 100644 index 00000000..c1f93933 --- /dev/null +++ b/test/unit_test/headless/test_failure_signature_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for error normalisation + stable failure signatures.""" +import je_auto_control as ac +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +_RUN_A = r"Timeout locating element at C:\Users\me\app.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" +_RUN_B = r"Timeout locating element at C:\Users\you\app.py line 99 (0x1234abcd) at 2026-06-25 09:15:00" +_OTHER = "Connection refused to /var/run/db.sock" + + +def test_normalize_collapses_volatile_parts(): + assert normalize_error(_RUN_A) == ( + "Timeout locating element at line (0x) at ") + + +def test_same_failure_same_signature_across_runs(): + assert failure_signature(_RUN_A) == failure_signature(_RUN_B) + + +def test_different_failure_differs(): + assert failure_signature(_RUN_A) != failure_signature(_OTHER) + + +def test_signature_length_param(): + assert len(failure_signature(_RUN_A, length=8)) == 8 + assert len(failure_signature(_RUN_A)) == 12 + + +def test_uuid_and_posix_path_normalised(): + msg = "row 00000000-1111-2222-3333-444444444444 at /tmp/x/data.json missing" + assert normalize_error(msg) == "row at missing" + + +def test_group_failures_counts_and_skips_empty(): + groups = group_failures([_RUN_A, _RUN_B, _OTHER, + "Connection refused to /tmp/other.sock", None, ""]) + assert len(groups) == 2 + assert groups[0]["count"] == 2 # most frequent first (tie → 2 each) + timeout = next(g for g in groups if "Timeout" in g["normalized"]) + assert timeout["count"] == 2 + assert len(timeout["examples"]) == 2 # both raw variants kept (max 3) + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _failure_signature, _group_failures) + sig = _failure_signature(_RUN_A) + assert sig["signature"] == failure_signature(_RUN_A) + assert sig["normalized"].endswith("at ") + grouped = _group_failures(f'["{_OTHER}", "{_OTHER}"]') + assert grouped["count"] == 1 and grouped["groups"][0]["count"] == 2 + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_failure_signature", "AC_group_failures"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_failure_signature", "ac_group_failures"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_failure_signature", "AC_group_failures"} <= specs + + +def test_facade_exports(): + for name in ("normalize_error", "failure_signature", "group_failures"): + assert hasattr(ac, name) and name in ac.__all__ diff --git a/test/unit_test/headless/test_flake_cluster_batch.py b/test/unit_test/headless/test_flake_cluster_batch.py new file mode 100644 index 00000000..e415668e --- /dev/null +++ b/test/unit_test/headless/test_flake_cluster_batch.py @@ -0,0 +1,83 @@ +"""Headless tests for co-failure flake clustering (Jaccard over failing runs).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.flake_cluster import cofailure_pairs, failure_clusters + + +def _runs(): + # a&b always fail together; c&d always together; e fails alone + return [ + ["a", "b"], + ["a", "b"], + ["c", "d"], + ["a", "b", "c", "d"], + ["e"], + ["c", "d"], + ] + + +def test_clusters_group_cofailing_tests(): + clusters = failure_clusters(_runs(), threshold=0.6) + grouped = sorted(tuple(c["tests"]) for c in clusters) + assert grouped == [("a", "b"), ("c", "d")] + assert all(c["cohesion"] == pytest.approx(1.0) for c in clusters) + assert all(c["size"] == 2 for c in clusters) + + +def test_singleton_excluded_by_min_size(): + # 'e' never co-fails, so it is not in any cluster of size >= 2 + tests_in_clusters = {t for c in failure_clusters(_runs()) for t in c["tests"]} + assert "e" not in tests_in_clusters + + +def test_min_size_includes_singletons_when_one(): + clusters = failure_clusters([["e"]], threshold=0.5, min_size=1) + assert clusters == [{"tests": ["e"], "size": 1, "cohesion": 1.0}] + + +def test_high_threshold_keeps_only_perfect_cofailure(): + # a&b co-fail perfectly (jaccard 1.0); a&c only sometimes + clusters = failure_clusters(_runs(), threshold=0.95) + assert sorted(tuple(c["tests"]) for c in clusters) == [("a", "b"), ("c", "d")] + + +def test_cofailure_pairs_scores_and_sorted(): + pairs = cofailure_pairs(_runs(), threshold=0.6) + assert {tuple(p["tests"]) for p in pairs} == {("a", "b"), ("c", "d")} + assert pairs[0]["jaccard"] == pytest.approx(1.0) + assert pairs[0]["co_failures"] == 3 + + +def test_empty_and_no_cofailure(): + assert failure_clusters([]) == [] + assert failure_clusters([["x"], ["y"], ["z"]]) == [] # nobody co-fails + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + import json + from je_auto_control.utils.executor.action_executor import ( + _cofailure_pairs, _failure_clusters) + runs_json = json.dumps(_runs()) + clusters = _failure_clusters(runs_json, threshold=0.6) + assert clusters["count"] == 2 + pairs = _cofailure_pairs(runs_json, threshold=0.6) + assert pairs["count"] == 2 + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_failure_clusters", "AC_cofailure_pairs"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_failure_clusters", "ac_cofailure_pairs"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_failure_clusters", "AC_cofailure_pairs"} <= specs + + +def test_facade_exports(): + for name in ("cofailure_pairs", "failure_clusters"): + assert hasattr(ac, name) and name in ac.__all__ diff --git a/test/unit_test/headless/test_run_diff_batch.py b/test/unit_test/headless/test_run_diff_batch.py new file mode 100644 index 00000000..a705afdc --- /dev/null +++ b/test/unit_test/headless/test_run_diff_batch.py @@ -0,0 +1,93 @@ +"""Headless tests for run-trace diffing (LCS-aligned step diff).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff + + +def _before(): + return [ + {"name": "login", "status": "ok", "duration": 1.0}, + {"name": "open_form", "status": "ok", "duration": 2.0}, + {"name": "submit", "status": "ok", "duration": 1.0}, + ] + + +def _after(): + return [ + {"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok", "duration": 0.5}, # inserted + {"name": "open_form", "status": "ok", "duration": 5.0}, # 2.5x slower + {"name": "submit", "status": "error", "duration": 1.0, + "error": r"Timeout at C:\app.py line 42 (0x7ff)"}, # flip + ] + + +def test_lcs_alignment_isolates_the_insert(): + diff = diff_runs(_before(), _after()) + # the inserted step is the only add; login/open_form/submit stay aligned + assert [s["name"] for s in diff["added"]] == ["accept_cookies"] + assert diff["removed"] == [] + assert diff["aligned"] == 3 + assert diff["identical"] is False + + +def test_status_flip_carries_failure_signature(): + flips = diff_runs(_before(), _after())["status_flips"] + assert len(flips) == 1 + assert flips[0]["name"] == "submit" + assert flips[0]["from"] == "ok" and flips[0]["to"] == "error" + assert len(flips[0]["signature"]) == 12 # failure_signature attached + + +def test_timing_regression_detected_with_ratio(): + regs = diff_runs(_before(), _after())["timing_regressions"] + assert len(regs) == 1 and regs[0]["name"] == "open_form" + assert regs[0]["ratio"] == pytest.approx(2.5) + # a small slowdown under the factor is not a regression + slow = diff_runs([{"name": "a", "duration": 1.0}], + [{"name": "a", "duration": 1.2}]) + assert slow["timing_regressions"] == [] + + +def test_removed_step_detected(): + diff = diff_runs(_before(), [_before()[0], _before()[2]]) # drop open_form + assert [s["name"] for s in diff["removed"]] == ["open_form"] + assert diff["added"] == [] + + +def test_identical_runs(): + diff = diff_runs(_before(), _before()) + assert diff["identical"] is True + assert summarize_run_diff(diff) == "no change" + + +def test_summary_lists_changes(): + summary = summarize_run_diff(diff_runs(_before(), _after())) + assert "added" in summary and "flip" in summary and "regression" in summary + + +# --- wiring --------------------------------------------------------------- + +def test_executor_path_includes_summary(): + import json + from je_auto_control.utils.executor.action_executor import _diff_runs + out = _diff_runs(json.dumps(_before()), json.dumps(_after())) + assert out["aligned"] == 3 + assert out["summary"] == summarize_run_diff(diff_runs(_before(), _after())) + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_diff_runs" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_diff_runs" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_diff_runs" in specs + + +def test_facade_exports(): + for name in ("diff_runs", "summarize_run_diff"): + assert hasattr(ac, name) and name in ac.__all__ diff --git a/test/unit_test/headless/test_step_timeline_batch.py b/test/unit_test/headless/test_step_timeline_batch.py new file mode 100644 index 00000000..5f8cb179 --- /dev/null +++ b/test/unit_test/headless/test_step_timeline_batch.py @@ -0,0 +1,75 @@ +"""Headless tests for per-run step timeline (waterfall + bottleneck steps).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.step_timeline import build_timeline, critical_steps + + +def _sequential(): + return [{"name": "login", "duration": 1.0}, + {"name": "load", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + +def test_sequential_waterfall_offsets_and_bottleneck(): + tl = build_timeline(_sequential()) + offsets = {s["name"]: s["offset"] for s in tl["steps"]} + assert offsets == {"login": 0.0, "load": 1.0, "submit": 5.0} + assert tl["total"] == pytest.approx(6.0) + assert tl["busy"] == pytest.approx(6.0) + assert tl["parallelism"] == pytest.approx(1.0) # purely sequential + assert tl["bottleneck"] == {"name": "load", "duration": 4.0} + + +def test_pct_share_of_total(): + pct = {s["name"]: s["pct"] for s in build_timeline(_sequential())["steps"]} + assert pct["load"] == pytest.approx(66.7, abs=0.1) + + +def test_overlapping_run_reports_parallelism(): + par = [{"name": "a", "start": 0.0, "duration": 3.0}, + {"name": "b", "start": 1.0, "duration": 3.0}] + tl = build_timeline(par) + assert tl["total"] == pytest.approx(4.0) # span 0..4 + assert tl["busy"] == pytest.approx(6.0) # 3 + 3 + assert tl["parallelism"] == pytest.approx(1.5) # overlap detected + + +def test_critical_steps_ranked_with_pct(): + crit = critical_steps(_sequential(), top=2) + assert [s["name"] for s in crit] == ["load", "login"] # longest first + assert crit[0]["pct"] == pytest.approx(66.7, abs=0.1) + assert len(critical_steps(_sequential(), top=1)) == 1 + + +def test_empty_run(): + assert build_timeline([]) == {"steps": [], "total": 0.0, "busy": 0.0, + "bottleneck": None, "parallelism": 0.0} + assert critical_steps([]) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + import json + from je_auto_control.utils.executor.action_executor import ( + _build_timeline, _critical_steps) + steps_json = json.dumps(_sequential()) + assert _build_timeline(steps_json)["bottleneck"]["name"] == "load" + assert _critical_steps(steps_json, top=1)["steps"][0]["name"] == "load" + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_build_timeline", "AC_critical_steps"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_build_timeline", "ac_critical_steps"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_build_timeline", "AC_critical_steps"} <= specs + + +def test_facade_exports(): + for name in ("build_timeline", "critical_steps"): + assert hasattr(ac, name) and name in ac.__all__