|
| 1 | +"""Named benchmarks: where a model's `mmlu` or `terminal-bench` figure comes from (96 §B4). |
| 2 | +
|
| 3 | +An agent's `needs.accept-when: - mmlu.score > 50` is decided against the score |
| 4 | +a catalogue row records under `benchmarks:`. This module is the one way such a |
| 5 | +score gets there, and it always arrives with where it came from: |
| 6 | +
|
| 7 | +* `record` writes a figure somebody measured or read — a Terminal-Bench run |
| 8 | + on their own harness, a leaderboard — into the workspace's |
| 9 | + `models/catalog.yaml`, with `provenance:` saying so. That is the path for |
| 10 | + every `runs: host:<name>` benchmark, and for a published figure of any. |
| 11 | +* `run_deepeval` measures one of the benchmarks DeepEval ships |
| 12 | + (`runs: deepeval:MMLU`) against a model, asked through the same |
| 13 | + `(system, user) -> answer` seam a `graded-by:` judge uses, and answers the |
| 14 | + figure on the benchmark's own scale. |
| 15 | +
|
| 16 | +PACT runs nothing on its own: a runtime that serves models (Bud Agent Flow's |
| 17 | +`budflow benchmark`) calls these. And nothing is fetched by this module — a |
| 18 | +DeepEval benchmark downloads its own dataset, which is that library's act and |
| 19 | +says so when it cannot. |
| 20 | +""" |
| 21 | + |
| 22 | +from __future__ import annotations |
| 23 | + |
| 24 | +import datetime as _dt |
| 25 | +from collections.abc import Callable, Mapping |
| 26 | +from dataclasses import dataclass |
| 27 | +from pathlib import Path |
| 28 | +from typing import Any |
| 29 | + |
| 30 | +import yaml |
| 31 | + |
| 32 | +from .providers import AskModel |
| 33 | +from .selection import benchmarks_of |
| 34 | + |
| 35 | +__all__ = ["Benchmark", "Unrecordable", "known", "record", "run_deepeval"] |
| 36 | + |
| 37 | + |
| 38 | +class Unrecordable(ValueError): |
| 39 | + """A figure that cannot be written down as it stands, and why.""" |
| 40 | + |
| 41 | + |
| 42 | +@dataclass(frozen=True) |
| 43 | +class Benchmark: |
| 44 | + """One entry of a catalogue's `benchmarks:`.""" |
| 45 | + |
| 46 | + name: str |
| 47 | + description: str = "" |
| 48 | + scale: str = "percent" |
| 49 | + runs: str = "" |
| 50 | + with_: Mapping[str, Any] | None = None |
| 51 | + |
| 52 | + @property |
| 53 | + def runner(self) -> str: |
| 54 | + """`deepeval`, `host`, or `""` for a benchmark whose figures are only published.""" |
| 55 | + return self.runs.split(":", 1)[0].strip() if ":" in self.runs else "" |
| 56 | + |
| 57 | + |
| 58 | +def known(document: Mapping[str, Any] | None = None) -> dict[str, Benchmark]: |
| 59 | + """Every benchmark the distribution's catalogue and the workspace's own declare.""" |
| 60 | + from .resolve import BUILTIN_CATALOGUE |
| 61 | + |
| 62 | + out: dict[str, Benchmark] = {} |
| 63 | + layers = [yaml.safe_load(Path(BUILTIN_CATALOGUE).read_text(encoding="utf-8")) or {}] |
| 64 | + if document: |
| 65 | + layers.append(document.get("models") or {}) |
| 66 | + for layer in layers: |
| 67 | + for key, entry in ((layer or {}).get("benchmarks") or {}).items(): |
| 68 | + entry = entry if isinstance(entry, Mapping) else {} |
| 69 | + out[str(key)] = Benchmark( |
| 70 | + name=str(key), |
| 71 | + description=str(entry.get("description") or ""), |
| 72 | + scale=str(entry.get("scale") or "percent"), |
| 73 | + runs=str(entry.get("runs") or ""), |
| 74 | + with_=dict(entry.get("with") or {}), |
| 75 | + ) |
| 76 | + return out |
| 77 | + |
| 78 | + |
| 79 | +def record( |
| 80 | + workspace: Path, |
| 81 | + model: str, |
| 82 | + benchmark: str, |
| 83 | + value: float, |
| 84 | + *, |
| 85 | + source: str, |
| 86 | + harness: str = "", |
| 87 | + recorded_by: str = "", |
| 88 | + as_of: "_dt.date | None" = None, |
| 89 | + document: Mapping[str, Any] | None = None, |
| 90 | +) -> Path: |
| 91 | + """Write one figure into `<workspace>/models/catalog.yaml`, with its provenance. |
| 92 | +
|
| 93 | + The benchmark must be one the catalogue declares (by name or alias, written |
| 94 | + under its own key), the figure must be on its scale, and `source` must say |
| 95 | + where it came from — a figure nobody can trace is the one Y12 forbids. A |
| 96 | + catalogue with comments in it is not rewritten: a YAML round trip would drop |
| 97 | + them, and a person's comment is not this command's to lose. Instead the |
| 98 | + lines to paste are in the error. |
| 99 | + """ |
| 100 | + names = benchmarks_of(document or {}) |
| 101 | + found = names.get(benchmark.strip().lower()) |
| 102 | + if found is None: |
| 103 | + raise Unrecordable( |
| 104 | + f"no benchmark is called {benchmark!r}. Name one the catalogue declares under " |
| 105 | + f"`benchmarks:`, or add your own there first." |
| 106 | + ) |
| 107 | + key, scale = found |
| 108 | + top = 100.0 if scale == "percent" else 1.0 |
| 109 | + if not 0.0 <= float(value) <= top: |
| 110 | + raise Unrecordable( |
| 111 | + f"{key} is written out of {top:g}, so {value:g} is not a score on it" |
| 112 | + ) |
| 113 | + if not source.strip(): |
| 114 | + raise Unrecordable("say where the figure came from: a run you did, or a published table") |
| 115 | + figure: dict[str, Any] = { |
| 116 | + "value": f"{float(value):g}", |
| 117 | + "provenance": { |
| 118 | + "source": source.strip(), |
| 119 | + **({"harness": harness.strip()} if harness.strip() else {}), |
| 120 | + "as-of": (as_of or _dt.date.today()).isoformat(), |
| 121 | + **({"recorded-by": recorded_by.strip()} if recorded_by.strip() else {}), |
| 122 | + }, |
| 123 | + } |
| 124 | + path = Path(workspace) / "models" / "catalog.yaml" |
| 125 | + text = path.read_text(encoding="utf-8") if path.exists() else "" |
| 126 | + if any(line.lstrip().startswith("#") for line in text.splitlines()): |
| 127 | + paste = yaml.safe_dump( |
| 128 | + {"models": {model: {"benchmarks": {key: figure}}}}, sort_keys=False, allow_unicode=True |
| 129 | + ) |
| 130 | + raise Unrecordable( |
| 131 | + f"{path} has comments in it, and writing the figure would drop them. Add these " |
| 132 | + f"lines under the model's own row instead:\n{paste}" |
| 133 | + ) |
| 134 | + data = yaml.safe_load(text) if text.strip() else {} |
| 135 | + data = data if isinstance(data, dict) else {} |
| 136 | + data.setdefault("version", 1) |
| 137 | + rows = data.setdefault("models", {}) |
| 138 | + if model not in rows: |
| 139 | + # A workspace row REPLACES the distribution's row of the same name — the catalogue is |
| 140 | + # layered row by row (`resolve._layer`) — so a row holding only this figure would lose |
| 141 | + # where the model is served, its price and its window. The distribution's row is copied |
| 142 | + # first, and the reviewer reads all of it in the diff. |
| 143 | + rows[model] = _shipped_row(model) |
| 144 | + row = rows[model] |
| 145 | + row.setdefault("benchmarks", {})[key] = figure |
| 146 | + path.parent.mkdir(parents=True, exist_ok=True) |
| 147 | + path.write_text(yaml.safe_dump(data, sort_keys=False, allow_unicode=True), encoding="utf-8") |
| 148 | + return path |
| 149 | + |
| 150 | + |
| 151 | +def _shipped_row(model: str) -> dict[str, Any]: |
| 152 | + """The distribution's own row for `model` (under its id or any alias), or a new empty one.""" |
| 153 | + from .resolve import BUILTIN_CATALOGUE |
| 154 | + |
| 155 | + shipped = (yaml.safe_load(Path(BUILTIN_CATALOGUE).read_text(encoding="utf-8")) or {}).get("models") or {} |
| 156 | + for name, row in shipped.items(): |
| 157 | + if name == model or model in (row or {}).get("also-known-as", []): |
| 158 | + return dict(row or {}) |
| 159 | + return {} |
| 160 | + |
| 161 | + |
| 162 | +def run_deepeval( |
| 163 | + benchmark: Benchmark, model: str, ask: Callable[[str, str], str] |
| 164 | +) -> float: # pragma: no cover - downloads a dataset and asks a live model |
| 165 | + """Measure one of DeepEval's own benchmarks against `model`, on the benchmark's scale.""" |
| 166 | + if benchmark.runner != "deepeval": |
| 167 | + raise Unrecordable( |
| 168 | + f"{benchmark.name} is run by `{benchmark.runs or 'nobody here'}`, not by DeepEval: " |
| 169 | + f"record its figure from wherever it was measured" |
| 170 | + ) |
| 171 | + try: |
| 172 | + import deepeval.benchmarks as shipped |
| 173 | + except ImportError as missing: |
| 174 | + raise Unrecordable(f"DeepEval is not installed here ({missing}): `pip install deepeval`") from missing |
| 175 | + class_name = benchmark.runs.split(":", 1)[1].strip() |
| 176 | + cls = getattr(shipped, class_name, None) |
| 177 | + if cls is None: |
| 178 | + raise Unrecordable(f"DeepEval has no benchmark called {class_name!r}") |
| 179 | + result = cls(**dict(benchmark.with_ or {})).evaluate(model=AskModel(model, ask)) |
| 180 | + accuracy = float(getattr(result, "overall_accuracy", result)) |
| 181 | + return accuracy * 100.0 if benchmark.scale == "percent" else accuracy |
0 commit comments