Consolidate benchmark shell scripts into shared lib + merged sweep script

AGENTS.md

@@ -60,7 +60,11 @@ src/fuzzer_tool/
 
 tools/
 ├── corpus_png.py      # PNG corpus generator for libpng fuzzing
-└── release.sh         # Release automation
+├── release.sh         # Release automation
+├── bench.sh           # 4-way config comparison (baseline/enhanced/enhanced+/optimal)
+├── bench_sweep.sh      # Exhaustive feature/combination sweep
+└── lib/
+    └── bench_common.sh # Shared helpers for bench.sh and bench_sweep.sh
 
 dictionaries/
 └── png.dict           # PNG format tokens
@@ -99,9 +103,9 @@ Fuzzer state is saved to `{corpus_dir}/state.json` on shutdown. Use `--resume` t
 - Transitions accumulate across sessions
 
 ### Meta-Scheduler (Elo Arbitration)
-- `--meta-elo` enables Elo-based arbitration between bandit and MOpt strategies
-- Requires `--elo --mc-bandit --mopt` to be active
-- Both strategies run in shadow; Elo picks which one to trust each iteration
+- `--elo` alone now enables Elo-based arbitration between operator strategies (bandit/MOpt/replicator) and seed strategies (ga/weighted/pareto/format); the separate `--meta-elo` flag was consolidated into `--elo` (see `_use_elo` in `services/fuzzer.py`)
+- Enable `--mc-bandit`/`--mopt`/`--replicator` alongside `--elo` to add those strategies to the arbitration pool
+- All available strategies run in shadow; Elo picks which one to trust each iteration
 - Strategy ratings tracked in `elo.json` under `strategy_ratings` / `strategy_match_count`
 - Probabilistic selection via softmax over Elo gap (temperature=400)
 

README.md

@@ -281,11 +281,16 @@ tools/bench.sh targets/png_read 10000
 
 Configurations:
 - **baseline**: no features
-- **enhanced**: elo + meta-elo + bandit + mopt
+- **enhanced**: elo + bandit + mopt
 - **enhanced+**: all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar
 - **optimal**: elo + mopt + replicator + markov ensemble (orders 0,1,2,3) + markov-gen
   - Best edge coverage at -n 1k (sweep-validated: 74 edges vs 61 baseline, 70 enhanced+)
 
+For a broader sweep across individual features and many combinations (instead
+of these four named configurations), use `tools/bench_sweep.sh`. Both scripts
+share common helpers (SHM cleanup, log metric extraction, coverage
+verification) from `tools/lib/bench_common.sh`.
+
 ## Troubleshooting
 
 ### Zero edges discovered (ASan + LD_PRELOAD conflict)

docs/TODO.md

@@ -11,11 +11,14 @@
 - [x] Sanitizer regex groups closed (ASAN/TSAN)
 - [x] Timeout crash detection fix
 - [x] Ptrace initial SIGTRAP crash detection
+- [x] Consolidated benchmark shell scripts: shared helpers extracted to `tools/lib/bench_common.sh`; merged `tools/bench_sweep2.sh` into `tools/bench_sweep.sh` (deduped identical combos, kept all unique variants)
 
 ## Bugs Fixed
 - [x] `--stats-file` eaten by `-A` (REMAINDER) — user must place `-A` last
 - [x] CEM never engaging — refit now triggers at elite_set >= 10
 - [x] dict_insert/dict_replace 0/0 — was missing `-D` flag, now works
+- [x] `report.py` referenced removed `_use_meta_elo` attribute (AttributeError when `--elo` + `--report` used) — updated to `_use_elo`
+- [x] Benchmark scripts (`tools/bench.sh`, `tools/bench_sweep.sh`, `tools/bench_sweep2.sh`) still passed removed `--meta-elo` CLI flag — removed, docs updated
 
 ## Pending Bugs
 - [ ] `_apply_single_mutation` havoc doesn't enforce max_len strictly (allows +1 byte per insert, up to +8 total)

docs/compose/reports/fuzzer-optimization-journey.md

@@ -58,7 +58,7 @@ fuzzer-tool fuzz targets/png_read -d corpus/ -c -D dictionaries/png.dict
 
 # Full feature stack (as benchmarked)
 fuzzer-tool fuzz targets/png_read -d corpus/ -c \
-    --elo --meta-elo --mc-bandit --mopt \
+    --elo --mc-bandit --mopt \
     --markov --markov-gen --markov-order 0,1,2,3 \
     --mi-guided --transfer-entropy --replicator \
     --secretary --pairwise-blend 0.5

src/fuzzer_tool/services/report.py

@@ -903,7 +903,7 @@ def _elo_ratings(f) -> str:
                 lines.append(f"    {i}. {op:<20s} {rating:>7.0f} ({sign}{delta:.0f})")
 
     # Meta-scheduler strategy ranking (bandit vs MOpt)
-    if f._use_meta_elo and f._elo:
+    if f._use_elo and f._elo:
         strategy_ranking = f._elo.get_strategy_ranking()
         if strategy_ranking:
             lines.append("")

tools/bench.sh

@@ -9,11 +9,14 @@
 #
 # Configurations:
 #   baseline:  no features
-#   enhanced:  elo + meta-elo + bandit + mopt
-#   enhanced+: elo + meta-elo + bandit + mopt + markov + replicator + shapley
+#   enhanced:  elo + bandit + mopt
+#   enhanced+: elo + bandit + mopt + markov + replicator + shapley
 #              + renyi + transfer-entropy + grammar
 #   optimal:   elo + mopt + replicator + markov (ensemble 0,1,2,3) + markov-gen
 #              Best edges at -n 1k (74 vs 61 baseline) and -n 10k (184 vs 167 baseline)
+#
+# For a broad sweep of individual feature/combination effects instead of
+# these four named configurations, use tools/bench_sweep.sh.
 
 set -euo pipefail
 
@@ -28,122 +31,8 @@ OPTIMAL_DIR="/tmp/fuzz_bench_optimal"
 REPORT_FLAG="${BENCH_REPORT:-}"  # set BENCH_REPORT=--report to generate full reports
 
 cd "$BASE_DIR"
-
-# ── SHM cleanup ───────────────────────────────────────────────────────
-# Remove all orphaned SHM segments owned by the current user.
-# Previous fuzzer runs (especially those killed by signals) leave
-# segments behind. Accumulation can cause shmget to fail or the
-# target to attach to stale segments.
-cleanup_shm() {
-    local before
-    before=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true)
-    # Remove all segments owned by current user
-    ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do
-        ipcrm -m "$shmid" 2>/dev/null || true
-    done
-    local after
-    after=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true)
-    if [[ "$before" -gt 0 ]]; then
-        echo "[*] Cleaned $((before - after)) orphaned SHM segments ($before -> $after)"
-    fi
-}
-
-# ── SHM verification ──────────────────────────────────────────────────
-# After a fuzzer run, verify that the SHM bitmap actually received data.
-# This is more reliable than checking log messages — it checks the
-# actual SHM segment that was created during the run.
-verify_shm() {
-    local log="$1"
-    local label="$2"
-
-    # Extract the SHM ID from the log
-    local shm_id
-    shm_id=$(grep -oP "SHM bitmap, id=\K[0-9]+" "$log" | tail -1)
-
-    if [[ -z "$shm_id" ]]; then
-        echo "FAIL: $label — no SHM ID found in log (coverage not enabled?)"
-        return 1
-    fi
-
-    # Try to attach and check if bitmap has any non-zero bytes
-    local has_data
-    has_data=$(python3 -c "
-import ctypes, ctypes.util
-libc = ctypes.CDLL(ctypes.util.find_library('c') or 'libc.so.6', use_errno=True)
-libc.shmat.restype = ctypes.c_void_p
-ptr = libc.shmat($shm_id, None, 0)
-if ptr is None or ptr == -1:
-    print('FAIL')
-else:
-    size = 4096  # default map size
-    bitmap = (ctypes.c_uint8 * size).from_address(ptr)
-    non_zero = sum(1 for i in range(size) if bitmap[i] != 0)
-    libc.shmdt(ptr)
-    if non_zero > 0:
-        print(f'OK:{non_zero}')
-    else:
-        print('EMPTY')
-" 2>/dev/null)
-
-    if [[ "$has_data" == FAIL ]]; then
-        echo "FAIL: $label — SHM segment $shm_id could not be attached"
-        return 1
-    elif [[ "$has_data" == EMPTY ]]; then
-        echo "FAIL: $label — SHM segment $shm_id has 0 non-zero bytes (coverage-blind)"
-        return 1
-    else
-        local nedges="${has_data#OK:}"
-        echo "[+] $label — SHM verified: $nedges non-zero bytes in bitmap"
-        return 0
-    fi
-}
-
-# ── Coverage-attachment sanity check ──────────────────────────────────
-# Combine log-based and SHM-based checks for maximum reliability.
-check_coverage() {
-    local log="$1"
-    local label="$2"
-
-    # Check for explicit SHM failure messages in the log
-    if grep -qi "SHM not attached\|AFL shim area is NULL\|shmat.*failed\|Coverage data will be empty" "$log"; then
-        echo "FAIL: $label — SHM coverage did not attach (coverage-blind run)"
-        return 1
-    fi
-
-    # Verify actual SHM bitmap has data
-    if ! verify_shm "$log" "$label"; then
-        return 1
-    fi
-
-    return 0
-}
-
-# ── Run with retry ────────────────────────────────────────────────────
-MAX_RETRIES=3
-
-run_with_retry() {
-    local log="$1"
-    shift
-    local attempt=1
-
-    while [[ $attempt -le $MAX_RETRIES ]]; do
-        echo "[*] Attempt $attempt/$MAX_RETRIES..."
-        python -m fuzzer_tool "$@" 2>&1 | tee "$log"
-
-        if check_coverage "$log" "attempt $attempt"; then
-            return 0
-        fi
-
-        echo "[*] Coverage did not attach. Cleaning SHM and retrying..."
-        cleanup_shm
-        sleep 2
-        attempt=$((attempt + 1))
-    done
-
-    echo "FAIL: Coverage failed to attach after $MAX_RETRIES attempts."
-    echo "  Last log: $log"
-    return 1
-}
+# shellcheck source=lib/bench_common.sh
+source "$BASE_DIR/tools/lib/bench_common.sh"
 
 # ── Main ──────────────────────────────────────────────────────────────
 
@@ -171,9 +60,9 @@ cleanup_shm
 sleep 1
 
 # Run enhanced
-echo "[*] Running enhanced (elo + meta-elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..."
+echo "[*] Running enhanced (elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..."
 run_with_retry /tmp/fuzz_bench_enhanced.log \
-    fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --meta-elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG
+    fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG
 echo ""
 
 # Clean SHM between runs
@@ -184,7 +73,7 @@ sleep 1
 echo "[*] Running enhanced+ (all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar)..."
 run_with_retry /tmp/fuzz_bench_enhanced+.log \
     fuzz "$TARGET" -d "$ENHANCEDP_DIR" -c -n "$ITERS" \
-    --elo --meta-elo --mc-bandit --mopt \
+    --elo --mc-bandit --mopt \
     --markov --markov-gen --markov-order 0,1,2,3 \
     --replicator --shapley --renyi-weight --transfer-entropy \
     -g dictionaries/png.gram \
@@ -211,28 +100,6 @@ echo " COMPARISON"
 echo "============================================================"
 echo ""
 
-extract() {
-    grep -oP "$1" "$2" | tail -1
-}
-
-# Extract CI values from crash/timeout rate lines (format: "rate%  ±1σ: lo% ±2σ: lo% ±3σ: lo%")
-extract_ci() {
-    local log="$1"
-    local pattern="$2"
-    local line
-    line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1)
-    if [[ -z "$line" ]]; then
-        echo "  -  -  -"
-        return
-    fi
-    # Extract the three CI values: ±1σ, ±2σ, ±3σ
-    local ci1 ci2 ci3
-    ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+')
-    ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+')
-    ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+')
-    echo "${ci1:--} ${ci2:--} ${ci3:--}"
-}
-
 b_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_baseline.log)
 e_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced.log)
 p_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced+.log)
@@ -258,11 +125,11 @@ e_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced.log
 p_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced+.log)
 o_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_optimal.log)
 
-# Extract CI for crash rates
-b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:")
-e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:")
-p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:")
-o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:")
+# Extract CI for crash rates (space-delimited for direct display in the table below)
+b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:" " ")
+e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:" " ")
+p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:" " ")
+o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:" " ")
 
 printf "%-25s %12s %12s %12s %12s\n" "Metric" "Baseline" "Enhanced" "Enhanced+" "Optimal"
 printf "%-25s %12s %12s %12s %12s\n" "-------------------------" "------------" "------------" "------------" "------------"