diff --git a/CLAUDE.md b/CLAUDE.md index 1aa9f8a..a2acb28 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,13 +72,17 @@ call → response with metrics. See `claude_explain.md` for detailed architectur `{type: enabled, budget_tokens: N}`), thinking counts against `max_tokens`. The old production value `1536` silently starved the visible text output on complex cases when thinking was on (production is now `4096`). `Prompt.__init__` now refuses to load a thinking-enabled config with `max_tokens < 4096`; ≥4096 (8192 worked in past experiments) is the floor. -- **Neither production model accepts `temperature`.** Opus 4.7 (reviewer) and Sonnet 5 (explainer) both reject +- **Neither production model accepts `temperature`.** Opus 5 (reviewer) and Sonnet 5 (explainer) both reject non-default sampling parameters with a 400, so `prompt_testing/reviewer.py` omits it and `app/prompt.yaml` sets none. Only pre-5 Sonnet models accept `temperature`; restore it in the YAML if you ever pin one of those. - **Sonnet 5 runs adaptive thinking by default when `thinking` is omitted** (unlike 4.6, where omitted meant off). `app/prompt.yaml` therefore sets `thinking: {type: disabled}` explicitly; dropping that line silently turns thinking on and eats the `max_tokens` budget. Sonnet 5 also uses a new tokenizer (~30% more tokens for the same text than 4.6) — don't reuse token counts or cost baselines measured on 4.6. +- **`model.effort` is plumbed but a no-op with thinking disabled.** The 2026-07 sweep (low/medium/high, 21 cases) + showed identical latency and cost across levels with `thinking: disabled` — effort mostly modulates thinking + depth, so there's nothing to modulate. Production leaves it unset (API default `high`). It becomes meaningful + on the `useThinking` path or if adaptive thinking is ever made the default. - **Reviewer thinking is on by default.** `prompt-test run --review` and `prompt-test review` default to `--reviewer-thinking adaptive` / `--thinking adaptive`. It catches factual errors the no-think reviewer misses but adds ~70% to review cost. Pass `off` to compare runs or save money on large batches. diff --git a/app/prompt.py b/app/prompt.py index 8b4b61d..c0b230c 100644 --- a/app/prompt.py +++ b/app/prompt.py @@ -53,6 +53,12 @@ def __init__(self, config: dict[str, Any] | Path): # {"type": "enabled", "budget_tokens": 2000}. When set, callers # should drop `temperature` (the API requires it to be unset/1). self.thinking = self.config["model"].get("thinking") + # Optional effort level ("low" | "medium" | "high" | "xhigh" | "max"). + # Controls the model's reasoning/token spend; unset means the API + # default (high). Sent as output_config: {effort: ...}. + self.effort = self.config["model"].get("effort") + if self.effort is not None and self.effort not in ("low", "medium", "high", "xhigh", "max"): + raise ValueError(f"invalid model.effort {self.effort!r}") if self.thinking and self.max_tokens < MIN_MAX_TOKENS_WITH_THINKING: # Adaptive thinking happily consumes the entire token budget on # complex inputs and leaves nothing for the visible text block. @@ -307,6 +313,8 @@ def build_api_payload(self, request: ExplainRequest) -> dict[str, Any]: "messages": base["messages"], } # Resolve thinking config: per-request override wins over the YAML. + if self.effort is not None: + payload["output_config"] = {"effort": self.effort} thinking = {"type": "adaptive"} if request.useThinking else base.get("thinking") if thinking is not None: payload["thinking"] = thinking diff --git a/app/test_explain.py b/app/test_explain.py index 4045c9b..2ed15ee 100644 --- a/app/test_explain.py +++ b/app/test_explain.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from ruamel.yaml import YAML from app.explain import process_request from app.explain_api import ( @@ -208,6 +209,33 @@ async def test_use_thinking_overrides_prompt(self, sample_request, mock_anthropi # max_tokens bumped to at least the documented floor. assert kwargs["max_tokens"] >= MIN_MAX_TOKENS_WITH_THINKING + def test_effort_emitted_as_output_config(self, sample_request): + """model.effort in the YAML becomes output_config on the payload.""" + yaml = YAML(typ="safe") + with Path("app/prompt.yaml").open(encoding="utf-8") as f: + config = yaml.load(f) + config["model"]["effort"] = "medium" + payload = Prompt(config).build_api_payload(sample_request) + assert payload["output_config"] == {"effort": "medium"} + + def test_no_effort_means_no_output_config(self, sample_request): + """Without model.effort the payload omits output_config (API default).""" + payload = Prompt(Path("app/prompt.yaml")).build_api_payload(sample_request) + assert "output_config" not in payload + + def test_invalid_effort_rejected_at_load(self): + """A typo'd effort level fails loudly at config load, not at request time.""" + config = { + "model": {"name": "test", "max_tokens": 1024, "effort": "turbo"}, + "system_prompt": "", + "user_prompt": "", + "assistant_prefill": "", + "audience_levels": {}, + "explanation_types": {}, + } + with pytest.raises(ValueError, match="effort"): + Prompt(config) + @pytest.mark.asyncio async def test_returns_error_when_no_text_block(self, sample_request, noop_metrics): """A response with no text block (e.g. thinking exhausted max_tokens) diff --git a/prompt_testing/cli.py b/prompt_testing/cli.py index 452e050..5ed0ef9 100644 --- a/prompt_testing/cli.py +++ b/prompt_testing/cli.py @@ -45,7 +45,7 @@ def cli(ctx, project_root): @click.option("--output", help="Output filename") @click.option("--max-concurrent", type=int, default=5) @click.option("--review", is_flag=True, help="Also run Opus correctness review on results") -@click.option("--review-model", default="claude-opus-4-7", help="Model for correctness review") +@click.option("--review-model", default="claude-opus-5", help="Model for correctness review") @click.option( "--reviewer-thinking", type=click.Choice(["off", "adaptive"]), @@ -285,7 +285,7 @@ def _print_review_summary(results: dict[str, Any]) -> None: @cli.command() @click.argument("results_file") -@click.option("--model", default="claude-opus-4-7", help="Reviewer model") +@click.option("--model", default="claude-opus-5", help="Reviewer model") @click.option( "--thinking", type=click.Choice(["off", "adaptive"]), diff --git a/prompt_testing/prompts/s5-effort-low.yaml b/prompt_testing/prompts/s5-effort-low.yaml new file mode 100644 index 0000000..4f7bf41 --- /dev/null +++ b/prompt_testing/prompts/s5-effort-low.yaml @@ -0,0 +1,77 @@ +name: Sonnet 5 effort low +description: Sonnet 5 with conciseness tuning; thinking off by default (per-request opt-in unchanged) +model: + name: claude-sonnet-5 + # Sonnet 5 runs adaptive thinking by default when `thinking` is omitted, + # which would starve a small max_tokens budget; keep it explicitly disabled + # (the useThinking request flag still switches to adaptive per-request). + # Sonnet 5 also rejects non-default temperature, so none is set here. + max_tokens: 4096 + thinking: + type: disabled + effort: low +audience_levels: + beginner: + description: For beginners learning assembly language. Uses simple language and explains technical terms. + guidance: | + - Use simple, clear language. Define technical terms inline when first used (e.g., 'vectorisation means processing multiple data elements simultaneously'). + - Explain concepts step-by-step. Use analogies where helpful. + - When registers are used for parameter passing or return values, explain the calling convention (e.g., 'By convention, `edi` holds the first integer parameter on x86-64'). + - Include foundational concepts about register purposes and memory organisation when relevant. + experienced: + description: For users familiar with assembly concepts and compiler behaviour. Focuses on optimisations and technical details. + guidance: | + - Assume familiarity with basic assembly and common instructions. + - Focus on the 'why': optimisation reasoning, architectural trade-offs, and what alternatives the compiler considered. + - Discuss microarchitectural details, pipeline behaviour, and performance implications when relevant. Use qualified language ('typically', 'on most modern processors') for performance claims. + - Distinguish between specific optimisations: constant folding, dead code elimination, register allocation, instruction selection, loop optimisations, inlining. State which are present or absent. + - Provide practical insights for writing compiler-friendly code. +explanation_types: + assembly: + description: Explains the assembly instructions and their purpose. + focus: | + - Lead with the single most important insight or pattern, then build supporting details. + - Group related instructions and explain their collective function. + - Use backticks around instruction names, registers, and values (e.g., `mov`, `rax`, `0x42`). + - Keep the explanation no longer than the assembly being analysed. Prioritise the most essential points and stop there. + - When relevant, compare with what other optimisation levels or architectures might produce. + - When optimisation choices create notable patterns, discuss what optimisations appear to be applied and their implications. + - For unoptimised code, identify redundancies (like store-then-load patterns) and explain what the optimised version would look like. + user_prompt_phrase: assembly output + haiku: + description: Tries to capture the essence of the code as a haiku. + focus: | + Focus on the overall behaviour and intent of the code. Use vivid imagery and concise language. + Produce only the three-line haiku itself — no other output. + user_prompt_phrase: assembly output + audience_levels: + beginner: + guidance: + experienced: + guidance: +system_prompt: | + You are an expert in {arch} assembly and {language}, helping Compiler Explorer users understand how their code compiles. + + The request is a JSON document containing source code, compilation options, and the resulting assembly. + + ## Core principles + + - **Be accurate.** Before explaining any instruction, trace its inputs and outputs step-by-step. Be especially careful with multi-operand instructions (e.g., verify whether `lea` performs address calculation vs memory access, check `imul` operand order). + - **Be definitive about what you can observe** (instruction behaviour, register usage, memory operations). Be appropriately cautious about inferred purposes or design decisions. + - **Be concise.** Don't explain what the source code does — the user wrote it. Reference source only to clarify the assembly mapping. Skip non-essential context, keep examples minimal, and stop once the essential points are covered; when in doubt, cut. Cover the few points that matter rather than every instruction. Keep the whole explanation under roughly 250 words; only exceed that when the assembly is unusually long or complex, and even then stay brief. + - **Characterise optimisation levels accurately.** If compilation options are empty or contain no optimisation flags, this is definitively unoptimised code — state it confidently, never say "likely -O0" or "appears to be". When explicit flags are present, reference them directly. + - **Handle undefined behaviour.** When code contains UB, explain that the compiler was free to choose any implementation. Describe the result as "one possible implementation" and explain why the code is problematic. + - **Use qualified language for performance claims** ('typically', 'on most modern processors') — avoid absolute claims about branch prediction, cache performance, or pipelining unless verifiable for the specific architecture. + - **Do not provide an overall conclusion or summary section.** + - Do not include internal or system XML tags in your response. + +user_prompt: | + Explain the {arch} {user_prompt_phrase}. + + ## Target audience: {audience} + {audience_guidance} + + ## Explanation type: {explanation_type} + {explanation_focus} + +assistant_prefill: "" diff --git a/prompt_testing/prompts/s5-effort-medium.yaml b/prompt_testing/prompts/s5-effort-medium.yaml new file mode 100644 index 0000000..4a50b0b --- /dev/null +++ b/prompt_testing/prompts/s5-effort-medium.yaml @@ -0,0 +1,77 @@ +name: Sonnet 5 effort medium +description: Sonnet 5 with conciseness tuning; thinking off by default (per-request opt-in unchanged) +model: + name: claude-sonnet-5 + # Sonnet 5 runs adaptive thinking by default when `thinking` is omitted, + # which would starve a small max_tokens budget; keep it explicitly disabled + # (the useThinking request flag still switches to adaptive per-request). + # Sonnet 5 also rejects non-default temperature, so none is set here. + max_tokens: 4096 + thinking: + type: disabled + effort: medium +audience_levels: + beginner: + description: For beginners learning assembly language. Uses simple language and explains technical terms. + guidance: | + - Use simple, clear language. Define technical terms inline when first used (e.g., 'vectorisation means processing multiple data elements simultaneously'). + - Explain concepts step-by-step. Use analogies where helpful. + - When registers are used for parameter passing or return values, explain the calling convention (e.g., 'By convention, `edi` holds the first integer parameter on x86-64'). + - Include foundational concepts about register purposes and memory organisation when relevant. + experienced: + description: For users familiar with assembly concepts and compiler behaviour. Focuses on optimisations and technical details. + guidance: | + - Assume familiarity with basic assembly and common instructions. + - Focus on the 'why': optimisation reasoning, architectural trade-offs, and what alternatives the compiler considered. + - Discuss microarchitectural details, pipeline behaviour, and performance implications when relevant. Use qualified language ('typically', 'on most modern processors') for performance claims. + - Distinguish between specific optimisations: constant folding, dead code elimination, register allocation, instruction selection, loop optimisations, inlining. State which are present or absent. + - Provide practical insights for writing compiler-friendly code. +explanation_types: + assembly: + description: Explains the assembly instructions and their purpose. + focus: | + - Lead with the single most important insight or pattern, then build supporting details. + - Group related instructions and explain their collective function. + - Use backticks around instruction names, registers, and values (e.g., `mov`, `rax`, `0x42`). + - Keep the explanation no longer than the assembly being analysed. Prioritise the most essential points and stop there. + - When relevant, compare with what other optimisation levels or architectures might produce. + - When optimisation choices create notable patterns, discuss what optimisations appear to be applied and their implications. + - For unoptimised code, identify redundancies (like store-then-load patterns) and explain what the optimised version would look like. + user_prompt_phrase: assembly output + haiku: + description: Tries to capture the essence of the code as a haiku. + focus: | + Focus on the overall behaviour and intent of the code. Use vivid imagery and concise language. + Produce only the three-line haiku itself — no other output. + user_prompt_phrase: assembly output + audience_levels: + beginner: + guidance: + experienced: + guidance: +system_prompt: | + You are an expert in {arch} assembly and {language}, helping Compiler Explorer users understand how their code compiles. + + The request is a JSON document containing source code, compilation options, and the resulting assembly. + + ## Core principles + + - **Be accurate.** Before explaining any instruction, trace its inputs and outputs step-by-step. Be especially careful with multi-operand instructions (e.g., verify whether `lea` performs address calculation vs memory access, check `imul` operand order). + - **Be definitive about what you can observe** (instruction behaviour, register usage, memory operations). Be appropriately cautious about inferred purposes or design decisions. + - **Be concise.** Don't explain what the source code does — the user wrote it. Reference source only to clarify the assembly mapping. Skip non-essential context, keep examples minimal, and stop once the essential points are covered; when in doubt, cut. Cover the few points that matter rather than every instruction. Keep the whole explanation under roughly 250 words; only exceed that when the assembly is unusually long or complex, and even then stay brief. + - **Characterise optimisation levels accurately.** If compilation options are empty or contain no optimisation flags, this is definitively unoptimised code — state it confidently, never say "likely -O0" or "appears to be". When explicit flags are present, reference them directly. + - **Handle undefined behaviour.** When code contains UB, explain that the compiler was free to choose any implementation. Describe the result as "one possible implementation" and explain why the code is problematic. + - **Use qualified language for performance claims** ('typically', 'on most modern processors') — avoid absolute claims about branch prediction, cache performance, or pipelining unless verifiable for the specific architecture. + - **Do not provide an overall conclusion or summary section.** + - Do not include internal or system XML tags in your response. + +user_prompt: | + Explain the {arch} {user_prompt_phrase}. + + ## Target audience: {audience} + {audience_guidance} + + ## Explanation type: {explanation_type} + {explanation_focus} + +assistant_prefill: "" diff --git a/prompt_testing/reviewer.py b/prompt_testing/reviewer.py index 243dae7..e9e5f3b 100644 --- a/prompt_testing/reviewer.py +++ b/prompt_testing/reviewer.py @@ -66,7 +66,7 @@ class CorrectnessReviewer: """Reviews explanations for factual correctness using a powerful model.""" - def __init__(self, model: str = "claude-opus-4-7", thinking: dict[str, Any] | None = None): + def __init__(self, model: str = "claude-opus-5", thinking: dict[str, Any] | None = None): """Initialise the reviewer. Args: @@ -105,14 +105,17 @@ async def review( ) # Opus 4.7+ rejects `temperature`; rely on the model's own default. + # Thinking is always explicit: on Opus 5, omitting the field runs + # adaptive thinking by default, which would silently defeat + # `--reviewer-thinking off`. Thinking counts against max_tokens, so + # give thinking-enabled reviews the larger budget. api_kwargs: dict[str, Any] = { "model": self.model, - "max_tokens": 2048, + "max_tokens": 4096 if self.thinking else 2048, "system": REVIEW_SYSTEM_PROMPT, "messages": [{"role": "user", "content": user_prompt}], + "thinking": self.thinking if self.thinking else {"type": "disabled"}, } - if self.thinking: - api_kwargs["thinking"] = self.thinking msg = await self.client.messages.create(**api_kwargs) # When thinking is enabled the response contains thinking blocks diff --git a/prompt_testing/runner.py b/prompt_testing/runner.py index e678447..66cf564 100644 --- a/prompt_testing/runner.py +++ b/prompt_testing/runner.py @@ -68,19 +68,9 @@ async def _run_one(self, test_case: dict[str, Any], prompt: Prompt) -> dict[str, async with self.semaphore: case_id = test_case["id"] request = self._to_request(test_case) - prompt_data = prompt.generate_messages(request) - - api_kwargs: dict[str, Any] = { - "model": prompt_data["model"], - "max_tokens": prompt_data["max_tokens"], - "system": prompt_data["system"], - "messages": prompt_data["messages"], - } - if prompt_data.get("thinking"): - # Extended thinking: temperature must be 1 / unset. - api_kwargs["thinking"] = prompt_data["thinking"] - else: - api_kwargs["temperature"] = prompt_data["temperature"] + # Single source of truth: run exactly what production would send + # (thinking/temperature/output_config resolution included). + api_kwargs = prompt.build_api_payload(request) start = time.time() try: @@ -100,7 +90,7 @@ async def _run_one(self, test_case: dict[str, Any], prompt: Prompt) -> dict[str, f"empty response (stop_reason={msg.stop_reason}, " f"in={msg.usage.input_tokens}, out={msg.usage.output_tokens})" ), - "model": prompt_data["model"], + "model": api_kwargs["model"], "input_tokens": msg.usage.input_tokens, "output_tokens": msg.usage.output_tokens, "elapsed_ms": elapsed_ms, @@ -109,7 +99,7 @@ async def _run_one(self, test_case: dict[str, Any], prompt: Prompt) -> dict[str, "case_id": case_id, "success": True, "explanation": explanation, - "model": prompt_data["model"], + "model": api_kwargs["model"], "input_tokens": msg.usage.input_tokens, "output_tokens": msg.usage.output_tokens, "elapsed_ms": elapsed_ms,