From 1d084cca66aebc0cd35e71f8c879cff0173c8df6 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 01:28:11 +0800 Subject: [PATCH 1/4] Add failure_signature: normalise + hash errors to stable signatures Two runs that failed the same way rarely have byte-identical error text (paths, line numbers, addresses, ids, timestamps differ), which defeats 'is this the same failure?' and 'which tests fail together?'. Strip the variable parts of an error to a canonical form and hash it (SHA-256) so the same kind of failure gets the same short signature across runs - the join key run diffing and flake clustering group on. group_failures buckets a list of errors by signature. Pure stdlib. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v191_features_doc.rst | 48 +++++++++++++ .../Zh/doc/new_features/v191_features_doc.rst | 41 +++++++++++ je_auto_control/__init__.py | 5 ++ .../gui/script_builder/command_schema.py | 15 ++++ .../utils/executor/action_executor.py | 20 ++++++ .../utils/failure_signature/__init__.py | 6 ++ .../failure_signature/failure_signature.py | 68 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 23 ++++++ .../utils/mcp_server/tools/_handlers.py | 10 +++ .../headless/test_failure_signature_batch.py | 70 +++++++++++++++++++ 11 files changed, 312 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v191_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v191_features_doc.rst create mode 100644 je_auto_control/utils/failure_signature/__init__.py create mode 100644 je_auto_control/utils/failure_signature/failure_signature.py create mode 100644 test/unit_test/headless/test_failure_signature_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 4c39e190..74f81365 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Stable Failure Signatures + +Match the *same kind* of failure across runs, despite differing paths and ids. Full reference: [`docs/source/Eng/doc/new_features/v191_features_doc.rst`](docs/source/Eng/doc/new_features/v191_features_doc.rst). + +- **`normalize_error` / `failure_signature` / `group_failures`** (`AC_failure_signature`, `AC_group_failures`): two runs that failed the same way rarely have byte-identical error text — paths, line numbers, addresses, ids and timestamps differ every time — which defeats "is this the same failure?" and "which tests fail together?". This strips the variable parts of an error to a canonical form and hashes it (SHA-256), so the same kind of failure gets the same short signature across runs — the join key the rest of the test-robustness tools (run diffing, flake clustering) group on. `group_failures` buckets a list of errors by signature, most frequent first. Pure stdlib (`re` + `hashlib`). No `PySide6`. + ## What's new (2026-06-24) — Visual Saliency (where to look — spectral-residual) Find the region that stands out, with no template / colour / text. Full reference: [`docs/source/Eng/doc/new_features/v190_features_doc.rst`](docs/source/Eng/doc/new_features/v190_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v191_features_doc.rst b/docs/source/Eng/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..994cf054 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v191_features_doc.rst @@ -0,0 +1,48 @@ +Stable Failure Signatures +========================= + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +* :func:`normalize_error` — collapse paths / hex addresses / UUIDs / timestamps / + line numbers / bare integers to placeholders, +* :func:`failure_signature` — a short stable SHA-256 of the normalised message, +* :func:`group_failures` — group a list of errors by signature, most frequent + first. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True — same failure + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows and POSIX paths, ``0x`` addresses, UUIDs, ISO timestamps, ``line N`` and +any leftover integers become placeholders; whitespace is squeezed. +``group_failures`` keeps up to three distinct raw examples per group and skips +empty / ``None`` messages. + +Executor commands +----------------- + +``AC_failure_signature`` (``error`` / ``length``) returns ``{signature, +normalized}``; ``AC_group_failures`` (``errors``) returns the grouped list. They +are exposed as read-only ``ac_*`` MCP tools and as Script Builder commands under +**Testing**. diff --git a/docs/source/Zh/doc/new_features/v191_features_doc.rst b/docs/source/Zh/doc/new_features/v191_features_doc.rst new file mode 100644 index 00000000..f1d1f367 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v191_features_doc.rst @@ -0,0 +1,41 @@ +穩定的失敗簽章 +============== + +兩次以*相同方式*失敗的執行,幾乎不會有逐位元組相同的錯誤文字——路徑、行號、記憶體位址、id 與 +時間戳每次都不同。這使得「這和昨天是同一個失敗嗎?」或「哪些測試會*一起*失敗?」無從問起。 +``failure_signature`` 把錯誤的變動部分剝離成標準形式並雜湊(SHA-256),於是*相同類型*的失敗在 +不同執行間會得到相同的短簽章——即其餘 test-robustness 工具(執行比較、flaky 分群)所依據的 +join key。 + +* :func:`normalize_error` ——把路徑 / 十六進位位址 / UUID / 時間戳 / 行號 / 裸整數收斂成佔位符, +* :func:`failure_signature` ——正規化訊息的短而穩定的 SHA-256, +* :func:`group_failures` ——把一組錯誤依簽章分組,最常見者在前。 + +純標準庫(``re`` + ``hashlib``);不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (normalize_error, failure_signature, + group_failures) + + a = r"Timeout at C:\app\run.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" + b = r"Timeout at C:\app\run.py line 88 (0x1234abcd) at 2026-06-25 09:15:00" + normalize_error(a) # "Timeout at line (0x) at " + failure_signature(a) == failure_signature(b) # True——同一個失敗 + + group_failures([a, b, "Connection refused to /tmp/x.sock"]) + # [{"signature": "...", "normalized": "...", "count": 2, "examples": [...]}, + # {"signature": "...", "count": 1, ...}] + +Windows 與 POSIX 路徑、``0x`` 位址、UUID、ISO 時間戳、``line N`` 與任何殘留整數都會變成佔位符; +空白會被壓縮。``group_failures`` 每組最多保留三個不同的原始範例,並略過空 / ``None`` 訊息。 + +執行器指令 +---------- + +``AC_failure_signature``(``error`` / ``length``)回傳 ``{signature, normalized}``; +``AC_group_failures``(``errors``)回傳分組清單。皆以唯讀 ``ac_*`` MCP 工具及 Script Builder +指令(位於 **Testing** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 0cd404e9..446a003b 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -88,6 +88,10 @@ from je_auto_control.utils.saliency import ( most_salient, salient_regions, saliency_map, ) +# Stable failure signatures (normalise + hash error text; group failures) +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1665,6 +1669,7 @@ def start_autocontrol_gui(*args, **kwargs): "image_quality", "is_blurry", "quality_gate", "detect_scale", "scale_sweep", "saliency_map", "salient_regions", "most_salient", + "normalize_error", "failure_signature", "group_failures", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 8019fb23..1ebbb836 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2708,6 +2708,21 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: description="Aggregate the self-heal log (heal rate, brittle " "locators).", )) + specs.append(CommandSpec( + "AC_failure_signature", "Testing", "Failure Signature", + fields=( + FieldSpec("error", FieldType.STRING, + placeholder="Timeout at C:\\app.py line 42 (0x7ff..)"), + FieldSpec("length", FieldType.INT, optional=True, default=12), + ), + description="Normalise + hash an error to a stable failure signature.", + )) + specs.append(CommandSpec( + "AC_group_failures", "Testing", "Group Failures by Signature", + fields=(FieldSpec("errors", FieldType.STRING, + placeholder='["err one", "err two"]'),), + description="Group error messages by failure signature (most frequent).", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 39a99780..304a9729 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4348,6 +4348,24 @@ def _most_salient(source: Any = None, region: Any = None, size: Any = 64, return {"found": result is not None, "region": result} +def _failure_signature(error: str, length: Any = 12) -> Dict[str, Any]: + """Adapter: normalise + hash an error message to a stable signature.""" + from je_auto_control.utils.failure_signature import ( + failure_signature, normalize_error) + return {"signature": failure_signature(str(error), length=int(length)), + "normalized": normalize_error(str(error))} + + +def _group_failures(errors: Any) -> Dict[str, Any]: + """Adapter: group error messages by failure signature.""" + import json + from je_auto_control.utils.failure_signature import group_failures + if isinstance(errors, str): + errors = json.loads(errors) + groups = group_failures(errors) + return {"groups": groups, "count": len(groups)} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6576,6 +6594,8 @@ def __init__(self): "AC_scale_sweep": _scale_sweep, "AC_salient_regions": _salient_regions, "AC_most_salient": _most_salient, + "AC_failure_signature": _failure_signature, + "AC_group_failures": _group_failures, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/failure_signature/__init__.py b/je_auto_control/utils/failure_signature/__init__.py new file mode 100644 index 00000000..a9017441 --- /dev/null +++ b/je_auto_control/utils/failure_signature/__init__.py @@ -0,0 +1,6 @@ +"""Normalise error messages into stable SHA-256 failure signatures + grouping.""" +from je_auto_control.utils.failure_signature.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +__all__ = ["normalize_error", "failure_signature", "group_failures"] diff --git a/je_auto_control/utils/failure_signature/failure_signature.py b/je_auto_control/utils/failure_signature/failure_signature.py new file mode 100644 index 00000000..5743eec4 --- /dev/null +++ b/je_auto_control/utils/failure_signature/failure_signature.py @@ -0,0 +1,68 @@ +"""Normalise an error message into a stable failure signature. + +Two runs that failed the *same way* almost never have byte-identical error text — +paths, line numbers, memory addresses, ids and timestamps differ every time. That +defeats any attempt to ask "is this the same failure as yesterday?" or "which +tests fail *together*?". ``failure_signature`` strips the variable parts of an +error to a canonical form and hashes it (SHA-256), so the same *kind* of failure +gets the same short signature across runs — the join key the rest of the +test-robustness tools (run diffing, flake clustering) group on. + +Pure standard library (``re`` + ``hashlib``); no device, no ``PySide6``. +""" +import hashlib +import re +from typing import Any, Dict, Iterable, List + +# Ordered (pattern, replacement): the volatile parts of an error, most specific +# first so e.g. a path's trailing line number isn't half-collapsed by the digit rule. +_NORMALIZERS = [ + (re.compile(r"[A-Za-z]:\\[^\s:*?\"<>|]+"), ""), # Windows path + (re.compile(r"(?:/[\w.\-]+)+/[\w.\-]+"), ""), # POSIX path + (re.compile(r"0x[0-9A-Fa-f]+"), "0x"), # memory address + (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}" + r"-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), ""), + (re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?"), ""), + (re.compile(r"\bline\s+\d+\b", re.IGNORECASE), "line "), + (re.compile(r"\b\d+\b"), ""), # any leftover int +] +_WHITESPACE = re.compile(r"\s+") + + +def normalize_error(message: str) -> str: + """Collapse the volatile parts of an error message to a canonical form. + + Paths, hex addresses, UUIDs, timestamps, line numbers and bare integers + become placeholders, and whitespace is squeezed — so messages that differ + only in those details normalise to the same string. + """ + text = str(message) + for pattern, replacement in _NORMALIZERS: + text = pattern.sub(replacement, text) + return _WHITESPACE.sub(" ", text).strip() + + +def failure_signature(message: str, *, length: int = 12) -> str: + """Return a short stable SHA-256 signature of a normalised error message.""" + digest = hashlib.sha256(normalize_error(message).encode("utf-8")).hexdigest() + return digest[:max(1, int(length))] + + +def group_failures(messages: Iterable[str]) -> List[Dict[str, Any]]: + """Group error messages by signature, most frequent first. + + Returns ``[{signature, normalized, count, examples}]`` (up to three distinct + raw examples per group). ``None`` / empty messages are skipped. + """ + groups: Dict[str, Dict[str, Any]] = {} + for message in messages: + if not message: + continue + signature = failure_signature(message) + group = groups.setdefault(signature, { + "signature": signature, "normalized": normalize_error(message), + "count": 0, "examples": []}) + group["count"] += 1 + if len(group["examples"]) < 3 and str(message) not in group["examples"]: + group["examples"].append(str(message)) + return sorted(groups.values(), key=lambda group: group["count"], reverse=True) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index b84715a1..8a3ddfbc 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7652,6 +7652,29 @@ def flakiness_tools() -> List[MCPTool]: handler=h.flaky_report, annotations=READ_ONLY, ), + MCPTool( + name="ac_failure_signature", + description=("Normalise an error message (strip paths / addresses / " + "line numbers / timestamps / ids) and hash it to a stable " + "SHA-256 signature, so the same kind of failure matches " + "across runs. Returns {signature, normalized}."), + input_schema=schema({"error": {"type": "string"}, + "length": {"type": "integer"}}, + required=["error"]), + handler=h.failure_signature, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_group_failures", + description=("Group a list of error messages by failure signature, " + "most frequent first: [{signature, normalized, count, " + "examples}]."), + input_schema=schema({ + "errors": {"type": "array", "items": {"type": "string"}}}, + required=["errors"]), + handler=h.group_failures, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index c00840fe..552793a8 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2543,6 +2543,16 @@ def most_salient(source=None, region=None, size=64, threshold=None, min_area=4): return _most_salient(source, region, size, threshold, min_area) +def failure_signature(error, length=12): + from je_auto_control.utils.executor.action_executor import _failure_signature + return _failure_signature(error, length) + + +def group_failures(errors): + from je_auto_control.utils.executor.action_executor import _group_failures + return _group_failures(errors) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_failure_signature_batch.py b/test/unit_test/headless/test_failure_signature_batch.py new file mode 100644 index 00000000..c1f93933 --- /dev/null +++ b/test/unit_test/headless/test_failure_signature_batch.py @@ -0,0 +1,70 @@ +"""Headless tests for error normalisation + stable failure signatures.""" +import je_auto_control as ac +from je_auto_control.utils.failure_signature import ( + failure_signature, group_failures, normalize_error, +) + +_RUN_A = r"Timeout locating element at C:\Users\me\app.py line 42 (0x7ffab12c) at 2026-06-24 11:03:21" +_RUN_B = r"Timeout locating element at C:\Users\you\app.py line 99 (0x1234abcd) at 2026-06-25 09:15:00" +_OTHER = "Connection refused to /var/run/db.sock" + + +def test_normalize_collapses_volatile_parts(): + assert normalize_error(_RUN_A) == ( + "Timeout locating element at line (0x) at ") + + +def test_same_failure_same_signature_across_runs(): + assert failure_signature(_RUN_A) == failure_signature(_RUN_B) + + +def test_different_failure_differs(): + assert failure_signature(_RUN_A) != failure_signature(_OTHER) + + +def test_signature_length_param(): + assert len(failure_signature(_RUN_A, length=8)) == 8 + assert len(failure_signature(_RUN_A)) == 12 + + +def test_uuid_and_posix_path_normalised(): + msg = "row 00000000-1111-2222-3333-444444444444 at /tmp/x/data.json missing" + assert normalize_error(msg) == "row at missing" + + +def test_group_failures_counts_and_skips_empty(): + groups = group_failures([_RUN_A, _RUN_B, _OTHER, + "Connection refused to /tmp/other.sock", None, ""]) + assert len(groups) == 2 + assert groups[0]["count"] == 2 # most frequent first (tie → 2 each) + timeout = next(g for g in groups if "Timeout" in g["normalized"]) + assert timeout["count"] == 2 + assert len(timeout["examples"]) == 2 # both raw variants kept (max 3) + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _failure_signature, _group_failures) + sig = _failure_signature(_RUN_A) + assert sig["signature"] == failure_signature(_RUN_A) + assert sig["normalized"].endswith("at ") + grouped = _group_failures(f'["{_OTHER}", "{_OTHER}"]') + assert grouped["count"] == 1 and grouped["groups"][0]["count"] == 2 + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_failure_signature", "AC_group_failures"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_failure_signature", "ac_group_failures"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_failure_signature", "AC_group_failures"} <= specs + + +def test_facade_exports(): + for name in ("normalize_error", "failure_signature", "group_failures"): + assert hasattr(ac, name) and name in ac.__all__ From aee77c4a02206b5029e81d43364fb7bc10082d77 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 01:40:02 +0800 Subject: [PATCH 2/4] Add run_diff: LCS-aligned diff of two run step-traces A run history says a run failed but not what changed from the run that passed. Align two step sequences with an LCS walk (so an inserted or removed step shifts the rest into place instead of mis-pairing) and classify the differences: added/removed steps, status flips (with the new failure's signature), and timing regressions. summarize_run_diff renders a one-line summary. Pure stdlib over step dicts. --- WHATS_NEW.md | 6 + .../doc/new_features/v192_features_doc.rst | 48 +++++++ .../Zh/doc/new_features/v192_features_doc.rst | 45 +++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 13 ++ .../utils/executor/action_executor.py | 15 +++ .../utils/mcp_server/tools/_factories.py | 16 +++ .../utils/mcp_server/tools/_handlers.py | 5 + je_auto_control/utils/run_diff/__init__.py | 4 + je_auto_control/utils/run_diff/run_diff.py | 119 ++++++++++++++++++ .../unit_test/headless/test_run_diff_batch.py | 93 ++++++++++++++ 11 files changed, 367 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v192_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v192_features_doc.rst create mode 100644 je_auto_control/utils/run_diff/__init__.py create mode 100644 je_auto_control/utils/run_diff/run_diff.py create mode 100644 test/unit_test/headless/test_run_diff_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 74f81365..b5e8ffbd 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Run-Trace Diff (what changed between two executions) + +See exactly what changed between a passing run and a failing one. Full reference: [`docs/source/Eng/doc/new_features/v192_features_doc.rst`](docs/source/Eng/doc/new_features/v192_features_doc.rst). + +- **`diff_runs` / `summarize_run_diff`** (`AC_diff_runs`): a run history says a run *failed* but not *what changed* from the run that passed. This aligns two step sequences with a longest-common-subsequence walk (so an inserted/removed step shifts the rest into place instead of mis-pairing everything) and classifies the differences: **added**/**removed** steps, **status_flips** (an aligned step that changed status — with the new failure's `failure_signature` when it carries an error), and **timing_regressions** (a step that got `regress_factor`× slower). `summarize_run_diff` renders a one-line summary. Pure stdlib over lists of `{name,status,duration,error}` step dicts. No `PySide6`. + ## What's new (2026-06-25) — Stable Failure Signatures Match the *same kind* of failure across runs, despite differing paths and ids. Full reference: [`docs/source/Eng/doc/new_features/v191_features_doc.rst`](docs/source/Eng/doc/new_features/v191_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v192_features_doc.rst b/docs/source/Eng/doc/new_features/v192_features_doc.rst new file mode 100644 index 00000000..d717e2e1 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v192_features_doc.rst @@ -0,0 +1,48 @@ +Run-Trace Diff (what changed between two executions) +==================================================== + +A run history tells you a run *failed*, but not *what changed* from the run that +passed: which step was added or dropped, which step flipped pass→fail, which step +got slower. ``run_diff`` aligns the two step sequences with a longest-common- +subsequence walk — so an inserted or removed step shifts the rest into place +instead of mis-pairing everything — and classifies the differences: + +* **added** / **removed** — steps present in only one run, +* **status_flips** — an aligned step whose status changed, with the new failure's + :func:`failure_signature` when it carries an ``error``, +* **timing_regressions** — an aligned step that got ``regress_factor`` x slower. + +A step is any dict with a name key (default ``"name"``) and optional ``status`` / +``duration`` / ``error``. Pure standard library; no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import diff_runs, summarize_run_diff + + before = [{"name": "login", "status": "ok", "duration": 1.0}, + {"name": "submit", "status": "ok", "duration": 1.0}] + after = [{"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok"}, # inserted + {"name": "submit", "status": "error", "error": "Timeout ..."}] + + diff = diff_runs(before, after) + # {"added": [accept_cookies], "removed": [], + # "status_flips": [{"name": "submit", "from": "ok", "to": "error", + # "signature": "..."}], + # "timing_regressions": [], "aligned": 2, "identical": False} + + summarize_run_diff(diff) # "+1 added, 1 status flip(s)" + +``regress_factor`` (default ``1.5``) is the slowdown ratio that counts as a +regression; ``key`` selects the field steps are aligned on. ``summarize_run_diff`` +renders a one-line summary (``"no change"`` when identical). + +Executor commands +----------------- + +``AC_diff_runs`` (``before`` / ``after`` / ``key`` / ``regress_factor``) returns +the diff plus a ``summary`` field. It is exposed as the read-only ``ac_diff_runs`` +MCP tool and as a Script Builder command under **Testing**. diff --git a/docs/source/Zh/doc/new_features/v192_features_doc.rst b/docs/source/Zh/doc/new_features/v192_features_doc.rst new file mode 100644 index 00000000..73116cf0 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v192_features_doc.rst @@ -0,0 +1,45 @@ +執行軌跡比較(兩次執行之間改變了什麼) +====================================== + +執行歷史告訴你某次執行*失敗*了,卻不告訴你相較於通過的那次*改變了什麼*:哪個步驟被加入或移除、 +哪個步驟由通過翻轉成失敗、哪個步驟變慢了。``run_diff`` 以最長共同子序列(LCS)走訪對齊兩個步驟 +序列——這樣插入或移除一個步驟會把其餘步驟順移到位,而非整個錯位配對——並將差異分類: + +* **added** / **removed** ——只存在於其中一次執行的步驟, +* **status_flips** ——某個已對齊步驟的狀態改變,若帶有 ``error`` 則附上新失敗的 + :func:`failure_signature`, +* **timing_regressions** ——某個已對齊步驟變慢了 ``regress_factor`` 倍。 + +步驟可為任何帶有名稱鍵(預設 ``"name"``)與選填 ``status`` / ``duration`` / ``error`` 的字典。 +純標準庫;不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import diff_runs, summarize_run_diff + + before = [{"name": "login", "status": "ok", "duration": 1.0}, + {"name": "submit", "status": "ok", "duration": 1.0}] + after = [{"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok"}, # 插入 + {"name": "submit", "status": "error", "error": "Timeout ..."}] + + diff = diff_runs(before, after) + # {"added": [accept_cookies], "removed": [], + # "status_flips": [{"name": "submit", "from": "ok", "to": "error", + # "signature": "..."}], + # "timing_regressions": [], "aligned": 2, "identical": False} + + summarize_run_diff(diff) # "+1 added, 1 status flip(s)" + +``regress_factor``(預設 ``1.5``)是算作退化的變慢比率;``key`` 選擇步驟對齊所依據的欄位。 +``summarize_run_diff`` 產生一行摘要(相同時為 ``"no change"``)。 + +執行器指令 +---------- + +``AC_diff_runs``(``before`` / ``after`` / ``key`` / ``regress_factor``)回傳該差異並附帶 +``summary`` 欄位。以唯讀 ``ac_diff_runs`` MCP 工具及 Script Builder 指令(位於 **Testing** +分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 446a003b..35482895 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -92,6 +92,8 @@ from je_auto_control.utils.failure_signature import ( failure_signature, group_failures, normalize_error, ) +# Run-trace diff (LCS-aligned: added/removed steps, status flips, regressions) +from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1670,6 +1672,7 @@ def start_autocontrol_gui(*args, **kwargs): "detect_scale", "scale_sweep", "saliency_map", "salient_regions", "most_salient", "normalize_error", "failure_signature", "group_failures", + "diff_runs", "summarize_run_diff", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 1ebbb836..9dc991d9 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2723,6 +2723,19 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: placeholder='["err one", "err two"]'),), description="Group error messages by failure signature (most frequent).", )) + specs.append(CommandSpec( + "AC_diff_runs", "Testing", "Diff Two Run Traces", + fields=( + FieldSpec("before", FieldType.STRING, + placeholder='[{"name": "login", "status": "ok"}]'), + FieldSpec("after", FieldType.STRING, + placeholder='[{"name": "login", "status": "error"}]'), + FieldSpec("key", FieldType.STRING, optional=True, default="name"), + FieldSpec("regress_factor", FieldType.FLOAT, optional=True, + default=1.5), + ), + description="LCS-align two run step-traces: added/removed/flips/regress.", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 304a9729..4c1706c6 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4366,6 +4366,20 @@ def _group_failures(errors: Any) -> Dict[str, Any]: return {"groups": groups, "count": len(groups)} +def _diff_runs(before: Any, after: Any, key: str = "name", + regress_factor: Any = 1.5) -> Dict[str, Any]: + """Adapter: diff two run step-traces (added/removed/flips/regressions).""" + import json + from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff + if isinstance(before, str): + before = json.loads(before) + if isinstance(after, str): + after = json.loads(after) + diff = diff_runs(before, after, key=str(key), + regress_factor=float(regress_factor)) + return {**diff, "summary": summarize_run_diff(diff)} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6596,6 +6610,7 @@ def __init__(self): "AC_most_salient": _most_salient, "AC_failure_signature": _failure_signature, "AC_group_failures": _group_failures, + "AC_diff_runs": _diff_runs, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 8a3ddfbc..a0934ecb 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7675,6 +7675,22 @@ def flakiness_tools() -> List[MCPTool]: handler=h.group_failures, annotations=READ_ONLY, ), + MCPTool( + name="ac_diff_runs", + description=("Diff two run step-traces (lists of {name,status," + "duration,error}) — LCS-aligned so inserts shift rather " + "than mis-pair. Returns {added, removed, status_flips " + "(with failure signature), timing_regressions, aligned, " + "identical, summary}. 'regress_factor' = slowdown ratio."), + input_schema=schema({ + "before": {"type": "array", "items": {"type": "object"}}, + "after": {"type": "array", "items": {"type": "object"}}, + "key": {"type": "string"}, + "regress_factor": {"type": "number"}}, + required=["before", "after"]), + handler=h.diff_runs, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 552793a8..0914842b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2553,6 +2553,11 @@ def group_failures(errors): return _group_failures(errors) +def diff_runs(before, after, key="name", regress_factor=1.5): + from je_auto_control.utils.executor.action_executor import _diff_runs + return _diff_runs(before, after, key, regress_factor) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/je_auto_control/utils/run_diff/__init__.py b/je_auto_control/utils/run_diff/__init__.py new file mode 100644 index 00000000..05cee29c --- /dev/null +++ b/je_auto_control/utils/run_diff/__init__.py @@ -0,0 +1,4 @@ +"""Diff two run traces (LCS-aligned step diff: added/removed/flips/regressions).""" +from je_auto_control.utils.run_diff.run_diff import diff_runs, summarize_run_diff + +__all__ = ["diff_runs", "summarize_run_diff"] diff --git a/je_auto_control/utils/run_diff/run_diff.py b/je_auto_control/utils/run_diff/run_diff.py new file mode 100644 index 00000000..26b2f494 --- /dev/null +++ b/je_auto_control/utils/run_diff/run_diff.py @@ -0,0 +1,119 @@ +"""Diff two run traces — what changed between two executions of a flow. + +A run history tells you a run *failed*, but not *what changed* from the run that +passed: which step was added or dropped, which step flipped pass→fail, which step +got slower. ``run_diff`` aligns the two step sequences with a longest-common- +subsequence walk (so an inserted / removed step shifts the rest into place instead +of mis-pairing everything) and classifies the differences: + +* **added** / **removed** steps (present in only one run), +* **status flips** (an aligned step whose status changed — with the new failure's + :func:`failure_signature` when it carries an ``error``), +* **timing regressions** (an aligned step that got ``regress_factor`` x slower). + +A step is any dict with a name key (default ``"name"``) and optional ``status`` / +``duration`` / ``error``. Pure standard library; no device, no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence + +Step = Dict[str, Any] + + +def _lcs_pairs(left: Sequence[str], right: Sequence[str]) -> List[tuple]: + """Return aligned ``(i, j)`` index pairs of the longest common subsequence.""" + n, m = len(left), len(right) + table = [[0] * (m + 1) for _ in range(n + 1)] + for i in range(n - 1, -1, -1): + for j in range(m - 1, -1, -1): + if left[i] == right[j]: + table[i][j] = table[i + 1][j + 1] + 1 + else: + table[i][j] = max(table[i + 1][j], table[i][j + 1]) + pairs, i, j = [], 0, 0 + while i < n and j < m: + if left[i] == right[j]: + pairs.append((i, j)) + i, j = i + 1, j + 1 + elif table[i + 1][j] >= table[i][j + 1]: + i += 1 + else: + j += 1 + return pairs + + +def _status_flip(before: Step, after: Step, name: str) -> Dict[str, Any]: + """Build a status-flip record, attaching a signature for a new error.""" + flip = {"name": name, "from": before.get("status"), + "to": after.get("status")} + error = after.get("error") + if error: + from je_auto_control.utils.failure_signature import failure_signature + flip["signature"] = failure_signature(str(error)) + return flip + + +def _regression(before: Step, after: Step, name: str, + factor: float) -> Dict[str, Any]: + """Return a timing-regression record, or ``{}`` if not a regression.""" + prev, curr = before.get("duration"), after.get("duration") + if not isinstance(prev, (int, float)) or not isinstance(curr, (int, float)): + return {} + if prev > 0 and curr >= prev * float(factor): + return {"name": name, "before": float(prev), "after": float(curr), + "ratio": round(curr / prev, 3)} + return {} + + +def _aligned_changes(before: Sequence[Step], after: Sequence[Step], + names: Sequence[str], pairs: List[tuple], + factor: float) -> tuple: + """Classify the LCS-aligned pairs into (status flips, timing regressions).""" + flips, regressions = [], [] + for i, j in pairs: + if before[i].get("status") != after[j].get("status"): + flips.append(_status_flip(before[i], after[j], names[i])) + regression = _regression(before[i], after[j], names[i], factor) + if regression: + regressions.append(regression) + return flips, regressions + + +def _keys(steps: Sequence[Step], key: str) -> List[str]: + return [str(step.get(key, "")) for step in steps] + + +def _unmatched(steps: Sequence[Step], matched: set) -> List[Step]: + return [steps[k] for k in range(len(steps)) if k not in matched] + + +def diff_runs(before: Sequence[Step], after: Sequence[Step], *, + key: str = "name", regress_factor: float = 1.5) -> Dict[str, Any]: + """Diff two step sequences into ``{added, removed, status_flips, + timing_regressions, aligned, identical}``. + + Steps are aligned by their ``key`` value via LCS; ``regress_factor`` is the + slowdown ratio that counts as a timing regression. + """ + left, right = _keys(before, key), _keys(after, key) + pairs = _lcs_pairs(left, right) + flips, regressions = _aligned_changes(before, after, left, pairs, + regress_factor) + added = _unmatched(after, {j for _, j in pairs}) + removed = _unmatched(before, {i for i, _ in pairs}) + return {"added": added, "removed": removed, "status_flips": flips, + "timing_regressions": regressions, "aligned": len(pairs), + "identical": not any((added, removed, flips, regressions))} + + +def summarize_run_diff(diff: Dict[str, Any]) -> str: + """Render a one-line human summary of a :func:`diff_runs` result.""" + if diff.get("identical"): + return "no change" + parts = [] + for label, field in (("+{} added", "added"), ("-{} removed", "removed"), + ("{} status flip(s)", "status_flips"), + ("{} regression(s)", "timing_regressions")): + count = len(diff.get(field, [])) + if count: + parts.append(label.format(count)) + return ", ".join(parts) diff --git a/test/unit_test/headless/test_run_diff_batch.py b/test/unit_test/headless/test_run_diff_batch.py new file mode 100644 index 00000000..a705afdc --- /dev/null +++ b/test/unit_test/headless/test_run_diff_batch.py @@ -0,0 +1,93 @@ +"""Headless tests for run-trace diffing (LCS-aligned step diff).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff + + +def _before(): + return [ + {"name": "login", "status": "ok", "duration": 1.0}, + {"name": "open_form", "status": "ok", "duration": 2.0}, + {"name": "submit", "status": "ok", "duration": 1.0}, + ] + + +def _after(): + return [ + {"name": "login", "status": "ok", "duration": 1.1}, + {"name": "accept_cookies", "status": "ok", "duration": 0.5}, # inserted + {"name": "open_form", "status": "ok", "duration": 5.0}, # 2.5x slower + {"name": "submit", "status": "error", "duration": 1.0, + "error": r"Timeout at C:\app.py line 42 (0x7ff)"}, # flip + ] + + +def test_lcs_alignment_isolates_the_insert(): + diff = diff_runs(_before(), _after()) + # the inserted step is the only add; login/open_form/submit stay aligned + assert [s["name"] for s in diff["added"]] == ["accept_cookies"] + assert diff["removed"] == [] + assert diff["aligned"] == 3 + assert diff["identical"] is False + + +def test_status_flip_carries_failure_signature(): + flips = diff_runs(_before(), _after())["status_flips"] + assert len(flips) == 1 + assert flips[0]["name"] == "submit" + assert flips[0]["from"] == "ok" and flips[0]["to"] == "error" + assert len(flips[0]["signature"]) == 12 # failure_signature attached + + +def test_timing_regression_detected_with_ratio(): + regs = diff_runs(_before(), _after())["timing_regressions"] + assert len(regs) == 1 and regs[0]["name"] == "open_form" + assert regs[0]["ratio"] == pytest.approx(2.5) + # a small slowdown under the factor is not a regression + slow = diff_runs([{"name": "a", "duration": 1.0}], + [{"name": "a", "duration": 1.2}]) + assert slow["timing_regressions"] == [] + + +def test_removed_step_detected(): + diff = diff_runs(_before(), [_before()[0], _before()[2]]) # drop open_form + assert [s["name"] for s in diff["removed"]] == ["open_form"] + assert diff["added"] == [] + + +def test_identical_runs(): + diff = diff_runs(_before(), _before()) + assert diff["identical"] is True + assert summarize_run_diff(diff) == "no change" + + +def test_summary_lists_changes(): + summary = summarize_run_diff(diff_runs(_before(), _after())) + assert "added" in summary and "flip" in summary and "regression" in summary + + +# --- wiring --------------------------------------------------------------- + +def test_executor_path_includes_summary(): + import json + from je_auto_control.utils.executor.action_executor import _diff_runs + out = _diff_runs(json.dumps(_before()), json.dumps(_after())) + assert out["aligned"] == 3 + assert out["summary"] == summarize_run_diff(diff_runs(_before(), _after())) + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_diff_runs" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_diff_runs" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_diff_runs" in specs + + +def test_facade_exports(): + for name in ("diff_runs", "summarize_run_diff"): + assert hasattr(ac, name) and name in ac.__all__ From 3fd8b686d9e52117a51f26c866488db99a302759 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 01:49:53 +0800 Subject: [PATCH 3/4] Add flake_cluster: cluster tests that flake together (co-failure Jaccard) Flaky tests are rarely independent - a wobbly fixture or noisy dependency makes a group fail in the same runs (~75% of flaky tests cluster). Ranking tests one-by-one by flip rate misses that shared root cause. Measure how often each pair fails in the same runs (Jaccard over their failing-run sets) and group tests above a threshold into connected clusters with a cohesion score. Pure stdlib over a list of failed-test sets. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v193_features_doc.rst | 46 +++++++++ .../Zh/doc/new_features/v193_features_doc.rst | 41 ++++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 19 ++++ .../utils/executor/action_executor.py | 24 +++++ .../utils/flake_cluster/__init__.py | 6 ++ .../utils/flake_cluster/flake_cluster.py | 97 +++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 29 ++++++ .../utils/mcp_server/tools/_handlers.py | 10 ++ .../headless/test_flake_cluster_batch.py | 83 ++++++++++++++++ 11 files changed, 364 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v193_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v193_features_doc.rst create mode 100644 je_auto_control/utils/flake_cluster/__init__.py create mode 100644 je_auto_control/utils/flake_cluster/flake_cluster.py create mode 100644 test/unit_test/headless/test_flake_cluster_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index b5e8ffbd..3789042d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Flaky-Test Co-Failure Clustering + +Find the tests that flake *together* — and the shared root cause behind them. Full reference: [`docs/source/Eng/doc/new_features/v193_features_doc.rst`](docs/source/Eng/doc/new_features/v193_features_doc.rst). + +- **`cofailure_pairs` / `failure_clusters`** (`AC_cofailure_pairs`, `AC_failure_clusters`): flaky tests are rarely independent — a wobbly fixture or noisy dependency makes a *group* fail in the same runs (~75% of flaky tests cluster). Ranking tests one-by-one by flip rate misses that. This measures how often each pair of tests fails in the *same* runs (Jaccard over their failing-run sets) and groups tests above a threshold into connected clusters with a cohesion score — so you chase one root cause instead of N symptoms. Input is a list of runs, each the test names that failed in it. Pure stdlib. No `PySide6`. + ## What's new (2026-06-25) — Run-Trace Diff (what changed between two executions) See exactly what changed between a passing run and a failing one. Full reference: [`docs/source/Eng/doc/new_features/v192_features_doc.rst`](docs/source/Eng/doc/new_features/v192_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v193_features_doc.rst b/docs/source/Eng/doc/new_features/v193_features_doc.rst new file mode 100644 index 00000000..3abad25e --- /dev/null +++ b/docs/source/Eng/doc/new_features/v193_features_doc.rst @@ -0,0 +1,46 @@ +Flaky-Test Co-Failure Clustering +================================ + +Flaky tests are rarely independent: a wobbly shared fixture, a slow dependency or +a noisy environment makes a *group* of tests fail in the same runs (research finds +~75% of flaky tests fall into co-failure clusters). Ranking tests one-by-one by +flip rate misses that shared root cause. ``flake_cluster`` measures how often each +pair of tests fails in the *same* runs — Jaccard similarity over the set of runs +each failed in — and groups tests whose co-failure exceeds a threshold, so you can +chase one root cause instead of N symptoms. + +* :func:`cofailure_pairs` — test pairs that fail together above a threshold, +* :func:`failure_clusters` — connected clusters of co-failing tests with a + cohesion score (mean pairwise Jaccard). + +Input is a list of runs, each a collection of the test names that failed in that +run. Pure standard library; no device, no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import failure_clusters, cofailure_pairs + + runs = [["test_a", "test_b"], # both failed in this run + ["test_a", "test_b"], + ["test_c"], + ["test_a", "test_b", "test_c"]] + + failure_clusters(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "size": 2, "cohesion": 1.0}] + + cofailure_pairs(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "jaccard": 1.0, "co_failures": 3}] + +``threshold`` is the minimum co-failure Jaccard to link two tests; ``min_size`` +(default ``2``) drops singletons so only genuine clusters surface. Clusters come +back largest / most cohesive first. + +Executor commands +----------------- + +``AC_failure_clusters`` (``runs`` / ``threshold`` / ``min_size``) and +``AC_cofailure_pairs`` (``runs`` / ``threshold``). They are exposed as read-only +``ac_*`` MCP tools and as Script Builder commands under **Testing**. diff --git a/docs/source/Zh/doc/new_features/v193_features_doc.rst b/docs/source/Zh/doc/new_features/v193_features_doc.rst new file mode 100644 index 00000000..222fc3a7 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v193_features_doc.rst @@ -0,0 +1,41 @@ +不穩定測試的共同失敗分群 +======================== + +不穩定(flaky)測試很少是獨立的:搖晃的共用 fixture、緩慢的相依、或吵雜的環境,會讓*一群*測試在 +相同的執行中一起失敗(研究發現約 75% 的 flaky 測試落在共同失敗的群集裡)。逐一以翻轉率排名測試 +會錯過這個共同根因。``flake_cluster`` 量測每對測試多常在*相同*執行中失敗——即各自失敗的執行集合 +之間的 Jaccard 相似度——並把共同失敗超過門檻的測試分群,讓你能追一個根因,而非 N 個症狀。 + +* :func:`cofailure_pairs` ——共同失敗超過門檻的測試對, +* :func:`failure_clusters` ——共同失敗測試的連通群集,附凝聚度分數(群內平均成對 Jaccard)。 + +輸入是一份執行清單,每個元素為該次執行中失敗的測試名稱集合。純標準庫;不涉及裝置,不匯入 +``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import failure_clusters, cofailure_pairs + + runs = [["test_a", "test_b"], # 這次執行兩者皆失敗 + ["test_a", "test_b"], + ["test_c"], + ["test_a", "test_b", "test_c"]] + + failure_clusters(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "size": 2, "cohesion": 1.0}] + + cofailure_pairs(runs, threshold=0.6) + # [{"tests": ["test_a", "test_b"], "jaccard": 1.0, "co_failures": 3}] + +``threshold`` 是連結兩測試所需的最小共同失敗 Jaccard;``min_size``(預設 ``2``)會丟棄單例, +讓只有真正的群集浮現。群集以最大 / 最凝聚者在前回傳。 + +執行器指令 +---------- + +``AC_failure_clusters``(``runs`` / ``threshold`` / ``min_size``)與 +``AC_cofailure_pairs``(``runs`` / ``threshold``)。皆以唯讀 ``ac_*`` MCP 工具及 Script Builder +指令(位於 **Testing** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 35482895..39c53d8a 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -94,6 +94,8 @@ ) # Run-trace diff (LCS-aligned: added/removed steps, status flips, regressions) from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff +# Flaky-test co-failure clustering (Jaccard over shared failing runs) +from je_auto_control.utils.flake_cluster import cofailure_pairs, failure_clusters # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1673,6 +1675,7 @@ def start_autocontrol_gui(*args, **kwargs): "saliency_map", "salient_regions", "most_salient", "normalize_error", "failure_signature", "group_failures", "diff_runs", "summarize_run_diff", + "cofailure_pairs", "failure_clusters", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 9dc991d9..b26d514f 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2736,6 +2736,25 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: ), description="LCS-align two run step-traces: added/removed/flips/regress.", )) + specs.append(CommandSpec( + "AC_failure_clusters", "Testing", "Cluster Co-Failing Tests", + fields=( + FieldSpec("runs", FieldType.STRING, + placeholder='[["test_a", "test_b"], ["test_a", "test_b"]]'), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.5), + FieldSpec("min_size", FieldType.INT, optional=True, default=2), + ), + description="Cluster tests that flake together (co-failure Jaccard).", + )) + specs.append(CommandSpec( + "AC_cofailure_pairs", "Testing", "Co-Failing Test Pairs", + fields=( + FieldSpec("runs", FieldType.STRING, + placeholder='[["test_a", "test_b"]]'), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.5), + ), + description="Test pairs that fail together above a Jaccard threshold.", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 4c1706c6..377cb4f9 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4380,6 +4380,28 @@ def _diff_runs(before: Any, after: Any, key: str = "name", return {**diff, "summary": summarize_run_diff(diff)} +def _failure_clusters(runs: Any, threshold: Any = 0.5, + min_size: Any = 2) -> Dict[str, Any]: + """Adapter: cluster tests that fail together (co-failure Jaccard).""" + import json + from je_auto_control.utils.flake_cluster import failure_clusters + if isinstance(runs, str): + runs = json.loads(runs) + clusters = failure_clusters(runs, threshold=float(threshold), + min_size=int(min_size)) + return {"clusters": clusters, "count": len(clusters)} + + +def _cofailure_pairs(runs: Any, threshold: Any = 0.5) -> Dict[str, Any]: + """Adapter: test pairs that fail together above a Jaccard threshold.""" + import json + from je_auto_control.utils.flake_cluster import cofailure_pairs + if isinstance(runs, str): + runs = json.loads(runs) + pairs = cofailure_pairs(runs, threshold=float(threshold)) + return {"pairs": pairs, "count": len(pairs)} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6611,6 +6633,8 @@ def __init__(self): "AC_failure_signature": _failure_signature, "AC_group_failures": _group_failures, "AC_diff_runs": _diff_runs, + "AC_failure_clusters": _failure_clusters, + "AC_cofailure_pairs": _cofailure_pairs, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/flake_cluster/__init__.py b/je_auto_control/utils/flake_cluster/__init__.py new file mode 100644 index 00000000..376dbb05 --- /dev/null +++ b/je_auto_control/utils/flake_cluster/__init__.py @@ -0,0 +1,6 @@ +"""Cluster tests that flake together by co-failure Jaccard similarity.""" +from je_auto_control.utils.flake_cluster.flake_cluster import ( + cofailure_pairs, failure_clusters, +) + +__all__ = ["cofailure_pairs", "failure_clusters"] diff --git a/je_auto_control/utils/flake_cluster/flake_cluster.py b/je_auto_control/utils/flake_cluster/flake_cluster.py new file mode 100644 index 00000000..d35d261c --- /dev/null +++ b/je_auto_control/utils/flake_cluster/flake_cluster.py @@ -0,0 +1,97 @@ +"""Cluster tests that flake *together* by co-failure similarity. + +Flaky tests are rarely independent: a wobbly shared fixture, a slow dependency or +a noisy environment makes a *group* of tests fail in the same runs (research finds +~75% of flaky tests fall into co-failure clusters). Ranking tests one-by-one by +flip rate misses that shared root cause. ``flake_cluster`` measures how often each +pair of tests fails in the *same* runs (Jaccard similarity over the set of runs +each failed in) and groups tests whose co-failure exceeds a threshold into +clusters — so you can chase one root cause instead of N symptoms. + +Input is a list of runs, each a collection of the test names that failed in that +run. Pure standard library; no device, no ``PySide6``. +""" +from itertools import combinations +from typing import Any, Dict, List, Sequence, Set + + +def _set_jaccard(left: Set[int], right: Set[int]) -> float: + union = left | right + return len(left & right) / len(union) if union else 0.0 + + +def _fail_runs(runs: Sequence[Sequence[str]]) -> Dict[str, Set[int]]: + """Map each test name to the set of run indices in which it failed.""" + fails: Dict[str, Set[int]] = {} + for index, run in enumerate(runs): + for test in set(run): + fails.setdefault(str(test), set()).add(index) + return fails + + +def cofailure_pairs(runs: Sequence[Sequence[str]], *, + threshold: float = 0.5) -> List[Dict[str, Any]]: + """Return test pairs whose co-failure Jaccard meets ``threshold``. + + Each entry is ``{tests:[a,b], jaccard, co_failures}``, most similar first. + """ + fails = _fail_runs(runs) + pairs = [] + for left, right in combinations(sorted(fails), 2): + score = _set_jaccard(fails[left], fails[right]) + if score >= float(threshold): + pairs.append({"tests": [left, right], "jaccard": round(score, 3), + "co_failures": len(fails[left] & fails[right])}) + pairs.sort(key=lambda pair: pair["jaccard"], reverse=True) + return pairs + + +def _connected_components(nodes: Sequence[str], + adjacency: Dict[str, Set[str]]) -> List[List[str]]: + seen: Set[str] = set() + components = [] + for node in nodes: + if node in seen: + continue + stack, component = [node], [] + while stack: + current = stack.pop() + if current in seen: + continue + seen.add(current) + component.append(current) + stack.extend(adjacency[current] - seen) + components.append(component) + return components + + +def _cohesion(component: Sequence[str], fails: Dict[str, Set[int]]) -> float: + scores = [_set_jaccard(fails[a], fails[b]) + for a, b in combinations(component, 2)] + return round(sum(scores) / len(scores), 3) if scores else 1.0 + + +def failure_clusters(runs: Sequence[Sequence[str]], *, threshold: float = 0.5, + min_size: int = 2) -> List[Dict[str, Any]]: + """Group tests that fail together into co-failure clusters. + + Builds a graph linking test pairs whose co-failure Jaccard meets + ``threshold``, then returns its connected components of at least ``min_size`` + tests as ``[{tests, size, cohesion}]`` (largest / most cohesive first). + ``cohesion`` is the mean pairwise Jaccard within the cluster. + """ + fails = _fail_runs(runs) + tests = sorted(fails) + adjacency: Dict[str, Set[str]] = {test: set() for test in tests} + for left, right in combinations(tests, 2): + if _set_jaccard(fails[left], fails[right]) >= float(threshold): + adjacency[left].add(right) + adjacency[right].add(left) + clusters = [] + for component in _connected_components(tests, adjacency): + if len(component) >= int(min_size): + clusters.append({"tests": sorted(component), "size": len(component), + "cohesion": _cohesion(component, fails)}) + clusters.sort(key=lambda cluster: (cluster["size"], cluster["cohesion"]), + reverse=True) + return clusters diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index a0934ecb..95752986 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7691,6 +7691,35 @@ def flakiness_tools() -> List[MCPTool]: handler=h.diff_runs, annotations=READ_ONLY, ), + MCPTool( + name="ac_failure_clusters", + description=("Cluster tests that flake TOGETHER: 'runs' is a list of " + "runs, each the list of test names that failed in that " + "run. Groups tests whose co-failure Jaccard >= " + "'threshold'. Returns {clusters:[{tests,size,cohesion}], " + "count} — chase one root cause, not N symptoms."), + input_schema=schema({ + "runs": {"type": "array", + "items": {"type": "array", "items": {"type": "string"}}}, + "threshold": {"type": "number"}, + "min_size": {"type": "integer"}}, + required=["runs"]), + handler=h.failure_clusters, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_cofailure_pairs", + description=("Test pairs that fail together above a Jaccard " + "'threshold' over shared failing runs: " + "{pairs:[{tests,jaccard,co_failures}], count}."), + input_schema=schema({ + "runs": {"type": "array", + "items": {"type": "array", "items": {"type": "string"}}}, + "threshold": {"type": "number"}}, + required=["runs"]), + handler=h.cofailure_pairs, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 0914842b..34319fcd 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2558,6 +2558,16 @@ def diff_runs(before, after, key="name", regress_factor=1.5): return _diff_runs(before, after, key, regress_factor) +def failure_clusters(runs, threshold=0.5, min_size=2): + from je_auto_control.utils.executor.action_executor import _failure_clusters + return _failure_clusters(runs, threshold, min_size) + + +def cofailure_pairs(runs, threshold=0.5): + from je_auto_control.utils.executor.action_executor import _cofailure_pairs + return _cofailure_pairs(runs, threshold) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/test/unit_test/headless/test_flake_cluster_batch.py b/test/unit_test/headless/test_flake_cluster_batch.py new file mode 100644 index 00000000..e415668e --- /dev/null +++ b/test/unit_test/headless/test_flake_cluster_batch.py @@ -0,0 +1,83 @@ +"""Headless tests for co-failure flake clustering (Jaccard over failing runs).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.flake_cluster import cofailure_pairs, failure_clusters + + +def _runs(): + # a&b always fail together; c&d always together; e fails alone + return [ + ["a", "b"], + ["a", "b"], + ["c", "d"], + ["a", "b", "c", "d"], + ["e"], + ["c", "d"], + ] + + +def test_clusters_group_cofailing_tests(): + clusters = failure_clusters(_runs(), threshold=0.6) + grouped = sorted(tuple(c["tests"]) for c in clusters) + assert grouped == [("a", "b"), ("c", "d")] + assert all(c["cohesion"] == pytest.approx(1.0) for c in clusters) + assert all(c["size"] == 2 for c in clusters) + + +def test_singleton_excluded_by_min_size(): + # 'e' never co-fails, so it is not in any cluster of size >= 2 + tests_in_clusters = {t for c in failure_clusters(_runs()) for t in c["tests"]} + assert "e" not in tests_in_clusters + + +def test_min_size_includes_singletons_when_one(): + clusters = failure_clusters([["e"]], threshold=0.5, min_size=1) + assert clusters == [{"tests": ["e"], "size": 1, "cohesion": 1.0}] + + +def test_high_threshold_keeps_only_perfect_cofailure(): + # a&b co-fail perfectly (jaccard 1.0); a&c only sometimes + clusters = failure_clusters(_runs(), threshold=0.95) + assert sorted(tuple(c["tests"]) for c in clusters) == [("a", "b"), ("c", "d")] + + +def test_cofailure_pairs_scores_and_sorted(): + pairs = cofailure_pairs(_runs(), threshold=0.6) + assert {tuple(p["tests"]) for p in pairs} == {("a", "b"), ("c", "d")} + assert pairs[0]["jaccard"] == pytest.approx(1.0) + assert pairs[0]["co_failures"] == 3 + + +def test_empty_and_no_cofailure(): + assert failure_clusters([]) == [] + assert failure_clusters([["x"], ["y"], ["z"]]) == [] # nobody co-fails + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + import json + from je_auto_control.utils.executor.action_executor import ( + _cofailure_pairs, _failure_clusters) + runs_json = json.dumps(_runs()) + clusters = _failure_clusters(runs_json, threshold=0.6) + assert clusters["count"] == 2 + pairs = _cofailure_pairs(runs_json, threshold=0.6) + assert pairs["count"] == 2 + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_failure_clusters", "AC_cofailure_pairs"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_failure_clusters", "ac_cofailure_pairs"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_failure_clusters", "AC_cofailure_pairs"} <= specs + + +def test_facade_exports(): + for name in ("cofailure_pairs", "failure_clusters"): + assert hasattr(ac, name) and name in ac.__all__ From a1089843a7e926b0c6a794e06bfc5f7fbdff5d75 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 02:00:36 +0800 Subject: [PATCH 4/4] Add step_timeline: per-run step waterfall + bottleneck steps The action profiler aggregates timings by step name across runs, which can't explain why one specific run was slow. Turn a single run's ordered steps into a waterfall (each step's offset, duration and share of the total) with the bottleneck step and a parallelism ratio, and rank the dominant steps. A step is any {name, duration, start?} dict; explicit start times place overlapping steps on an absolute timeline. Pure stdlib. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v194_features_doc.rst | 51 +++++++++++++ .../Zh/doc/new_features/v194_features_doc.rst | 44 +++++++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 15 ++++ .../utils/executor/action_executor.py | 20 +++++ .../utils/mcp_server/tools/_factories.py | 24 ++++++ .../utils/mcp_server/tools/_handlers.py | 10 +++ .../utils/step_timeline/__init__.py | 6 ++ .../utils/step_timeline/step_timeline.py | 75 +++++++++++++++++++ .../headless/test_step_timeline_batch.py | 75 +++++++++++++++++++ 11 files changed, 329 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v194_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v194_features_doc.rst create mode 100644 je_auto_control/utils/step_timeline/__init__.py create mode 100644 je_auto_control/utils/step_timeline/step_timeline.py create mode 100644 test/unit_test/headless/test_step_timeline_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 3789042d..0587166d 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Per-Run Step Timeline (waterfall + bottleneck steps) + +Read why *this* run was slow — a step waterfall and its bottlenecks. Full reference: [`docs/source/Eng/doc/new_features/v194_features_doc.rst`](docs/source/Eng/doc/new_features/v194_features_doc.rst). + +- **`build_timeline` / `critical_steps`** (`AC_build_timeline`, `AC_critical_steps`): the action profiler aggregates timings by step *name* across runs — useless for "why was *this* run slow". This turns one run's ordered steps into a waterfall (each step's offset, duration, and `pct` share of the total) with the `bottleneck` step and a `parallelism` ratio (`> 1` when steps overlap via explicit `start` times); `critical_steps` ranks the dominant steps to optimise. A step is any `{name, duration, start?}` dict. Pure stdlib. No `PySide6`. + ## What's new (2026-06-25) — Flaky-Test Co-Failure Clustering Find the tests that flake *together* — and the shared root cause behind them. Full reference: [`docs/source/Eng/doc/new_features/v193_features_doc.rst`](docs/source/Eng/doc/new_features/v193_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v194_features_doc.rst b/docs/source/Eng/doc/new_features/v194_features_doc.rst new file mode 100644 index 00000000..f8568eb6 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v194_features_doc.rst @@ -0,0 +1,51 @@ +Per-Run Step Timeline (waterfall + bottleneck steps) +==================================================== + +The action profiler aggregates timings by step *name* across many runs — great +for "which action is slow on average", useless for "why was *this* run slow". A +single run is an ordered timeline: step A ran, then B, then C, and one of them +dominated. ``step_timeline`` turns one run's steps into a waterfall (each step's +offset from the start, its duration and its share of the total) and ranks the +bottleneck steps, so you can read a single slow run instead of an average. + +* :func:`build_timeline` — the waterfall + total / busy / bottleneck / + parallelism, +* :func:`critical_steps` — the steps that dominate the run, longest first. + +A step is any dict with a name (default ``"name"``) and a ``duration``; an +optional ``start`` places it on an absolute timeline (overlapping / parallel +steps), else steps are laid out back-to-back. Pure standard library; no device, +no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import build_timeline, critical_steps + + steps = [{"name": "login", "duration": 1.0}, + {"name": "load_dashboard", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + build_timeline(steps) + # {"steps": [{"name": "login", "offset": 0.0, "duration": 1.0, "pct": 16.7}, + # {"name": "load_dashboard", "offset": 1.0, ..., "pct": 66.7}, ...], + # "total": 6.0, "busy": 6.0, + # "bottleneck": {"name": "load_dashboard", "duration": 4.0}, + # "parallelism": 1.0} + + critical_steps(steps, top=2) + # [{"name": "load_dashboard", "duration": 4.0, "pct": 66.7}, + # {"name": "login", "duration": 1.0, "pct": 16.7}] + +``total`` is the wall-clock span, ``busy`` the summed step time; ``parallelism`` = +busy / total is ``1.0`` for a purely sequential run and ``> 1`` when steps overlap +(supply ``start`` times). ``pct`` is each step's share of the total time. + +Executor commands +----------------- + +``AC_build_timeline`` (``steps``) and ``AC_critical_steps`` (``steps`` / ``top``). +They are exposed as read-only ``ac_*`` MCP tools and as Script Builder commands +under **Testing**. diff --git a/docs/source/Zh/doc/new_features/v194_features_doc.rst b/docs/source/Zh/doc/new_features/v194_features_doc.rst new file mode 100644 index 00000000..63a9e8db --- /dev/null +++ b/docs/source/Zh/doc/new_features/v194_features_doc.rst @@ -0,0 +1,44 @@ +單次執行的步驟時間軸(瀑布圖 + 瓶頸步驟) +========================================== + +動作 profiler 把計時按步驟*名稱*跨多次執行聚合——很適合「哪個動作平均較慢」,卻無助於「為什麼 +*這一次*執行很慢」。單次執行是一條有序時間軸:步驟 A 跑完、接著 B、再 C,其中某一步主導了時間。 +``step_timeline`` 把一次執行的步驟轉成瀑布圖(每步距起點的偏移、其時長、其占總時間的比例),並 +排名瓶頸步驟,讓你能讀懂單一慢執行,而非平均值。 + +* :func:`build_timeline` ——瀑布圖加上 total / busy / bottleneck / parallelism, +* :func:`critical_steps` ——主導該次執行的步驟,最長者在前。 + +步驟可為任何帶名稱(預設 ``"name"``)與 ``duration`` 的字典;選填 ``start`` 會把它放到絕對 +時間軸上(重疊 / 平行步驟),否則步驟會背靠背排列。純標準庫;不涉及裝置,不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import build_timeline, critical_steps + + steps = [{"name": "login", "duration": 1.0}, + {"name": "load_dashboard", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + build_timeline(steps) + # {"steps": [{"name": "login", "offset": 0.0, "duration": 1.0, "pct": 16.7}, + # {"name": "load_dashboard", "offset": 1.0, ..., "pct": 66.7}, ...], + # "total": 6.0, "busy": 6.0, + # "bottleneck": {"name": "load_dashboard", "duration": 4.0}, + # "parallelism": 1.0} + + critical_steps(steps, top=2) + # [{"name": "load_dashboard", "duration": 4.0, "pct": 66.7}, + # {"name": "login", "duration": 1.0, "pct": 16.7}] + +``total`` 是牆鐘時間跨度,``busy`` 是各步驟時長總和;``parallelism`` = busy / total,純序列執行 +為 ``1.0``,步驟重疊時 ``> 1``(需提供 ``start`` 時間)。``pct`` 是每步占總時間的比例。 + +執行器指令 +---------- + +``AC_build_timeline``(``steps``)與 ``AC_critical_steps``(``steps`` / ``top``)。皆以唯讀 +``ac_*`` MCP 工具及 Script Builder 指令(位於 **Testing** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 39c53d8a..81bc5b59 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -96,6 +96,8 @@ from je_auto_control.utils.run_diff import diff_runs, summarize_run_diff # Flaky-test co-failure clustering (Jaccard over shared failing runs) from je_auto_control.utils.flake_cluster import cofailure_pairs, failure_clusters +# Per-run step waterfall + bottleneck (critical) steps +from je_auto_control.utils.step_timeline import build_timeline, critical_steps # VLM element locator (headless) from je_auto_control.utils.vision import ( VLMNotAvailableError, click_by_description, locate_by_description, @@ -1676,6 +1678,7 @@ def start_autocontrol_gui(*args, **kwargs): "normalize_error", "failure_signature", "group_failures", "diff_runs", "summarize_run_diff", "cofailure_pairs", "failure_clusters", + "build_timeline", "critical_steps", # VLM locator "VLMNotAvailableError", "locate_by_description", "click_by_description", "verify_description", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index b26d514f..d4d5bce4 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2755,6 +2755,21 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: ), description="Test pairs that fail together above a Jaccard threshold.", )) + specs.append(CommandSpec( + "AC_build_timeline", "Testing", "Step Timeline (waterfall)", + fields=(FieldSpec("steps", FieldType.STRING, + placeholder='[{"name": "login", "duration": 1.2}]'),), + description="Per-run step waterfall: offsets, durations, bottleneck.", + )) + specs.append(CommandSpec( + "AC_critical_steps", "Testing", "Critical (Bottleneck) Steps", + fields=( + FieldSpec("steps", FieldType.STRING, + placeholder='[{"name": "login", "duration": 1.2}]'), + FieldSpec("top", FieldType.INT, optional=True, default=3), + ), + description="The steps that dominate a run's time, longest first.", + )) specs.append(CommandSpec( "AC_scan_secrets", "Tools", "Scan for Hardcoded Secrets", description="Scan 'data' (JSON view) for hardcoded secrets that " diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 377cb4f9..edff6eda 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -4402,6 +4402,24 @@ def _cofailure_pairs(runs: Any, threshold: Any = 0.5) -> Dict[str, Any]: return {"pairs": pairs, "count": len(pairs)} +def _build_timeline(steps: Any) -> Dict[str, Any]: + """Adapter: a per-run step waterfall (offsets / durations / bottleneck).""" + import json + from je_auto_control.utils.step_timeline import build_timeline + if isinstance(steps, str): + steps = json.loads(steps) + return build_timeline(steps) + + +def _critical_steps(steps: Any, top: Any = 3) -> Dict[str, Any]: + """Adapter: the steps that dominate a run's time (bottlenecks).""" + import json + from je_auto_control.utils.step_timeline import critical_steps + if isinstance(steps, str): + steps = json.loads(steps) + return {"steps": critical_steps(steps, top=int(top))} + + def _image_histogram(source: Any = None, bins: Any = 32, space: str = "hsv", region: Any = None) -> Dict[str, Any]: """Adapter: per-channel colour histogram of an image / the screen.""" @@ -6635,6 +6653,8 @@ def __init__(self): "AC_diff_runs": _diff_runs, "AC_failure_clusters": _failure_clusters, "AC_cofailure_pairs": _cofailure_pairs, + "AC_build_timeline": _build_timeline, + "AC_critical_steps": _critical_steps, "AC_image_histogram": _image_histogram, "AC_histogram_changed": _histogram_changed, "AC_changed_regions": _changed_regions, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 95752986..d32d15ae 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -7720,6 +7720,30 @@ def flakiness_tools() -> List[MCPTool]: handler=h.cofailure_pairs, annotations=READ_ONLY, ), + MCPTool( + name="ac_build_timeline", + description=("Per-run step waterfall from 'steps' (list of {name," + "duration,start?}): {steps:[{name,offset,duration,pct}], " + "total, busy, bottleneck, parallelism}. Reads ONE slow " + "run, not a per-name average."), + input_schema=schema({ + "steps": {"type": "array", "items": {"type": "object"}}}, + required=["steps"]), + handler=h.build_timeline, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_critical_steps", + description=("The 'top' steps that dominate a run's time (bottlenecks " + "to optimise): {steps:[{name,duration,pct}]}, longest " + "first."), + input_schema=schema({ + "steps": {"type": "array", "items": {"type": "object"}}, + "top": {"type": "integer"}}, + required=["steps"]), + handler=h.critical_steps, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 34319fcd..ab6d91cc 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2568,6 +2568,16 @@ def cofailure_pairs(runs, threshold=0.5): return _cofailure_pairs(runs, threshold) +def build_timeline(steps): + from je_auto_control.utils.executor.action_executor import _build_timeline + return _build_timeline(steps) + + +def critical_steps(steps, top=3): + from je_auto_control.utils.executor.action_executor import _critical_steps + return _critical_steps(steps, top) + + def image_histogram(source=None, bins=32, space="hsv", region=None): from je_auto_control.utils.executor.action_executor import _image_histogram return _image_histogram(source, bins, space, region) diff --git a/je_auto_control/utils/step_timeline/__init__.py b/je_auto_control/utils/step_timeline/__init__.py new file mode 100644 index 00000000..36cae102 --- /dev/null +++ b/je_auto_control/utils/step_timeline/__init__.py @@ -0,0 +1,6 @@ +"""Per-run step waterfall timeline + bottleneck (critical) step ranking.""" +from je_auto_control.utils.step_timeline.step_timeline import ( + build_timeline, critical_steps, +) + +__all__ = ["build_timeline", "critical_steps"] diff --git a/je_auto_control/utils/step_timeline/step_timeline.py b/je_auto_control/utils/step_timeline/step_timeline.py new file mode 100644 index 00000000..4f677bcd --- /dev/null +++ b/je_auto_control/utils/step_timeline/step_timeline.py @@ -0,0 +1,75 @@ +"""Build a per-run step waterfall and find the run's bottleneck steps. + +The action profiler aggregates timings by step *name* across many runs — great for +"which action is slow on average", useless for "why was *this* run slow". A single +run is an ordered timeline: step A ran, then B, then C, and one of them dominated. +``step_timeline`` turns one run's steps into a waterfall (each step's offset from +the start, duration and share of the total) and ranks the bottleneck steps, so you +can read a single slow run instead of an average. + +A step is any dict with a name (default ``"name"``) and a ``duration``; an optional +``start`` places it on an absolute timeline (overlapping / parallel steps), else +steps are laid out back-to-back. Pure standard library; no device, no ``PySide6``. +""" +from typing import Any, Dict, List, Sequence + +Step = Dict[str, Any] + + +def _normalize(steps: Sequence[Step], name_key: str, start_key: str, + duration_key: str) -> List[Dict[str, Any]]: + """Resolve each step to ``{name, start, end, duration}`` (sequential if no start).""" + resolved, cursor = [], 0.0 + for step in steps: + duration = float(step.get(duration_key, 0.0) or 0.0) + raw_start = step.get(start_key) + start = float(raw_start) if raw_start is not None else cursor + end = start + duration + cursor = max(cursor, end) + resolved.append({"name": str(step.get(name_key, "")), "start": start, + "end": end, "duration": duration}) + return resolved + + +def build_timeline(steps: Sequence[Step], *, name_key: str = "name", + start_key: str = "start", + duration_key: str = "duration") -> Dict[str, Any]: + """Return a waterfall timeline for one run. + + ``{steps:[{name, offset, duration, pct}], total, busy, bottleneck, + parallelism}`` — ``total`` is the wall-clock span, ``busy`` the summed step + time, ``parallelism`` = busy / total (1.0 for a purely sequential run), + ``bottleneck`` the longest single step. + """ + resolved = _normalize(steps, name_key, start_key, duration_key) + if not resolved: + return {"steps": [], "total": 0.0, "busy": 0.0, "bottleneck": None, + "parallelism": 0.0} + base = min(step["start"] for step in resolved) + span = max(step["end"] for step in resolved) - base + busy = sum(step["duration"] for step in resolved) + rows = [{"name": step["name"], "offset": round(step["start"] - base, 6), + "duration": step["duration"], + "pct": round(step["duration"] / span * 100, 1) if span > 0 else 0.0} + for step in resolved] + bottleneck = max(resolved, key=lambda step: step["duration"]) + return {"steps": rows, "total": round(span, 6), "busy": round(busy, 6), + "bottleneck": {"name": bottleneck["name"], + "duration": bottleneck["duration"]}, + "parallelism": round(busy / span, 3) if span > 0 else 1.0} + + +def critical_steps(steps: Sequence[Step], *, name_key: str = "name", + start_key: str = "start", duration_key: str = "duration", + top: int = 3) -> List[Dict[str, Any]]: + """Return the ``top`` steps that dominate the run, longest first. + + Each entry is ``{name, duration, pct}`` where ``pct`` is the step's share of + the total step time — the bottlenecks worth optimising. + """ + resolved = _normalize(steps, name_key, start_key, duration_key) + busy = sum(step["duration"] for step in resolved) or 1.0 + ranked = sorted(resolved, key=lambda step: step["duration"], reverse=True) + return [{"name": step["name"], "duration": step["duration"], + "pct": round(step["duration"] / busy * 100, 1)} + for step in ranked[:max(1, int(top))]] diff --git a/test/unit_test/headless/test_step_timeline_batch.py b/test/unit_test/headless/test_step_timeline_batch.py new file mode 100644 index 00000000..5f8cb179 --- /dev/null +++ b/test/unit_test/headless/test_step_timeline_batch.py @@ -0,0 +1,75 @@ +"""Headless tests for per-run step timeline (waterfall + bottleneck steps).""" +import pytest + +import je_auto_control as ac +from je_auto_control.utils.step_timeline import build_timeline, critical_steps + + +def _sequential(): + return [{"name": "login", "duration": 1.0}, + {"name": "load", "duration": 4.0}, + {"name": "submit", "duration": 1.0}] + + +def test_sequential_waterfall_offsets_and_bottleneck(): + tl = build_timeline(_sequential()) + offsets = {s["name"]: s["offset"] for s in tl["steps"]} + assert offsets == {"login": 0.0, "load": 1.0, "submit": 5.0} + assert tl["total"] == pytest.approx(6.0) + assert tl["busy"] == pytest.approx(6.0) + assert tl["parallelism"] == pytest.approx(1.0) # purely sequential + assert tl["bottleneck"] == {"name": "load", "duration": 4.0} + + +def test_pct_share_of_total(): + pct = {s["name"]: s["pct"] for s in build_timeline(_sequential())["steps"]} + assert pct["load"] == pytest.approx(66.7, abs=0.1) + + +def test_overlapping_run_reports_parallelism(): + par = [{"name": "a", "start": 0.0, "duration": 3.0}, + {"name": "b", "start": 1.0, "duration": 3.0}] + tl = build_timeline(par) + assert tl["total"] == pytest.approx(4.0) # span 0..4 + assert tl["busy"] == pytest.approx(6.0) # 3 + 3 + assert tl["parallelism"] == pytest.approx(1.5) # overlap detected + + +def test_critical_steps_ranked_with_pct(): + crit = critical_steps(_sequential(), top=2) + assert [s["name"] for s in crit] == ["load", "login"] # longest first + assert crit[0]["pct"] == pytest.approx(66.7, abs=0.1) + assert len(critical_steps(_sequential(), top=1)) == 1 + + +def test_empty_run(): + assert build_timeline([]) == {"steps": [], "total": 0.0, "busy": 0.0, + "bottleneck": None, "parallelism": 0.0} + assert critical_steps([]) == [] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + import json + from je_auto_control.utils.executor.action_executor import ( + _build_timeline, _critical_steps) + steps_json = json.dumps(_sequential()) + assert _build_timeline(steps_json)["bottleneck"]["name"] == "load" + assert _critical_steps(steps_json, top=1)["steps"][0]["name"] == "load" + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_build_timeline", "AC_critical_steps"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_build_timeline", "ac_critical_steps"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_build_timeline", "AC_critical_steps"} <= specs + + +def test_facade_exports(): + for name in ("build_timeline", "critical_steps"): + assert hasattr(ac, name) and name in ac.__all__