diff --git a/AGENTS.md b/AGENTS.md index b63e4967..7add1a64 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,11 @@ src/fuzzer_tool/ tools/ ├── corpus_png.py # PNG corpus generator for libpng fuzzing -└── release.sh # Release automation +├── release.sh # Release automation +├── bench.sh # 4-way config comparison (baseline/enhanced/enhanced+/optimal) +├── bench_sweep.sh # Exhaustive feature/combination sweep +└── lib/ + └── bench_common.sh # Shared helpers for bench.sh and bench_sweep.sh dictionaries/ └── png.dict # PNG format tokens @@ -99,9 +103,9 @@ Fuzzer state is saved to `{corpus_dir}/state.json` on shutdown. Use `--resume` t - Transitions accumulate across sessions ### Meta-Scheduler (Elo Arbitration) -- `--meta-elo` enables Elo-based arbitration between bandit and MOpt strategies -- Requires `--elo --mc-bandit --mopt` to be active -- Both strategies run in shadow; Elo picks which one to trust each iteration +- `--elo` alone now enables Elo-based arbitration between operator strategies (bandit/MOpt/replicator) and seed strategies (ga/weighted/pareto/format); the separate `--meta-elo` flag was consolidated into `--elo` (see `_use_elo` in `services/fuzzer.py`) +- Enable `--mc-bandit`/`--mopt`/`--replicator` alongside `--elo` to add those strategies to the arbitration pool +- All available strategies run in shadow; Elo picks which one to trust each iteration - Strategy ratings tracked in `elo.json` under `strategy_ratings` / `strategy_match_count` - Probabilistic selection via softmax over Elo gap (temperature=400) diff --git a/README.md b/README.md index 0690d194..dbe40855 100644 --- a/README.md +++ b/README.md @@ -281,11 +281,16 @@ tools/bench.sh targets/png_read 10000 Configurations: - **baseline**: no features -- **enhanced**: elo + meta-elo + bandit + mopt +- **enhanced**: elo + bandit + mopt - **enhanced+**: all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar - **optimal**: elo + mopt + replicator + markov ensemble (orders 0,1,2,3) + markov-gen - Best edge coverage at -n 1k (sweep-validated: 74 edges vs 61 baseline, 70 enhanced+) +For a broader sweep across individual features and many combinations (instead +of these four named configurations), use `tools/bench_sweep.sh`. Both scripts +share common helpers (SHM cleanup, log metric extraction, coverage +verification) from `tools/lib/bench_common.sh`. + ## Troubleshooting ### Zero edges discovered (ASan + LD_PRELOAD conflict) diff --git a/docs/TODO.md b/docs/TODO.md index 4aad46f0..a46da11e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -11,11 +11,14 @@ - [x] Sanitizer regex groups closed (ASAN/TSAN) - [x] Timeout crash detection fix - [x] Ptrace initial SIGTRAP crash detection +- [x] Consolidated benchmark shell scripts: shared helpers extracted to `tools/lib/bench_common.sh`; merged `tools/bench_sweep2.sh` into `tools/bench_sweep.sh` (deduped identical combos, kept all unique variants) ## Bugs Fixed - [x] `--stats-file` eaten by `-A` (REMAINDER) — user must place `-A` last - [x] CEM never engaging — refit now triggers at elite_set >= 10 - [x] dict_insert/dict_replace 0/0 — was missing `-D` flag, now works +- [x] `report.py` referenced removed `_use_meta_elo` attribute (AttributeError when `--elo` + `--report` used) — updated to `_use_elo` +- [x] Benchmark scripts (`tools/bench.sh`, `tools/bench_sweep.sh`, `tools/bench_sweep2.sh`) still passed removed `--meta-elo` CLI flag — removed, docs updated ## Pending Bugs - [ ] `_apply_single_mutation` havoc doesn't enforce max_len strictly (allows +1 byte per insert, up to +8 total) diff --git a/docs/compose/reports/fuzzer-optimization-journey.md b/docs/compose/reports/fuzzer-optimization-journey.md index d3908033..d85cdff5 100644 --- a/docs/compose/reports/fuzzer-optimization-journey.md +++ b/docs/compose/reports/fuzzer-optimization-journey.md @@ -58,7 +58,7 @@ fuzzer-tool fuzz targets/png_read -d corpus/ -c -D dictionaries/png.dict # Full feature stack (as benchmarked) fuzzer-tool fuzz targets/png_read -d corpus/ -c \ - --elo --meta-elo --mc-bandit --mopt \ + --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --mi-guided --transfer-entropy --replicator \ --secretary --pairwise-blend 0.5 diff --git a/src/fuzzer_tool/services/report.py b/src/fuzzer_tool/services/report.py index 3232254e..2aed2acb 100644 --- a/src/fuzzer_tool/services/report.py +++ b/src/fuzzer_tool/services/report.py @@ -903,7 +903,7 @@ def _elo_ratings(f) -> str: lines.append(f" {i}. {op:<20s} {rating:>7.0f} ({sign}{delta:.0f})") # Meta-scheduler strategy ranking (bandit vs MOpt) - if f._use_meta_elo and f._elo: + if f._use_elo and f._elo: strategy_ranking = f._elo.get_strategy_ranking() if strategy_ranking: lines.append("") diff --git a/targets/asan_target b/targets/asan_target index effb1b75..c7ef7b67 100755 Binary files a/targets/asan_target and b/targets/asan_target differ diff --git a/targets/test_target b/targets/test_target index 51d52aad..9fbe068c 100755 Binary files a/targets/test_target and b/targets/test_target differ diff --git a/tools/bench.sh b/tools/bench.sh index 5f615780..c0df0370 100755 --- a/tools/bench.sh +++ b/tools/bench.sh @@ -9,11 +9,14 @@ # # Configurations: # baseline: no features -# enhanced: elo + meta-elo + bandit + mopt -# enhanced+: elo + meta-elo + bandit + mopt + markov + replicator + shapley +# enhanced: elo + bandit + mopt +# enhanced+: elo + bandit + mopt + markov + replicator + shapley # + renyi + transfer-entropy + grammar # optimal: elo + mopt + replicator + markov (ensemble 0,1,2,3) + markov-gen # Best edges at -n 1k (74 vs 61 baseline) and -n 10k (184 vs 167 baseline) +# +# For a broad sweep of individual feature/combination effects instead of +# these four named configurations, use tools/bench_sweep.sh. set -euo pipefail @@ -28,122 +31,8 @@ OPTIMAL_DIR="/tmp/fuzz_bench_optimal" REPORT_FLAG="${BENCH_REPORT:-}" # set BENCH_REPORT=--report to generate full reports cd "$BASE_DIR" - -# ── SHM cleanup ─────────────────────────────────────────────────────── -# Remove all orphaned SHM segments owned by the current user. -# Previous fuzzer runs (especially those killed by signals) leave -# segments behind. Accumulation can cause shmget to fail or the -# target to attach to stale segments. -cleanup_shm() { - local before - before=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) - # Remove all segments owned by current user - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done - local after - after=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) - if [[ "$before" -gt 0 ]]; then - echo "[*] Cleaned $((before - after)) orphaned SHM segments ($before -> $after)" - fi -} - -# ── SHM verification ────────────────────────────────────────────────── -# After a fuzzer run, verify that the SHM bitmap actually received data. -# This is more reliable than checking log messages — it checks the -# actual SHM segment that was created during the run. -verify_shm() { - local log="$1" - local label="$2" - - # Extract the SHM ID from the log - local shm_id - shm_id=$(grep -oP "SHM bitmap, id=\K[0-9]+" "$log" | tail -1) - - if [[ -z "$shm_id" ]]; then - echo "FAIL: $label — no SHM ID found in log (coverage not enabled?)" - return 1 - fi - - # Try to attach and check if bitmap has any non-zero bytes - local has_data - has_data=$(python3 -c " -import ctypes, ctypes.util -libc = ctypes.CDLL(ctypes.util.find_library('c') or 'libc.so.6', use_errno=True) -libc.shmat.restype = ctypes.c_void_p -ptr = libc.shmat($shm_id, None, 0) -if ptr is None or ptr == -1: - print('FAIL') -else: - size = 4096 # default map size - bitmap = (ctypes.c_uint8 * size).from_address(ptr) - non_zero = sum(1 for i in range(size) if bitmap[i] != 0) - libc.shmdt(ptr) - if non_zero > 0: - print(f'OK:{non_zero}') - else: - print('EMPTY') -" 2>/dev/null) - - if [[ "$has_data" == FAIL ]]; then - echo "FAIL: $label — SHM segment $shm_id could not be attached" - return 1 - elif [[ "$has_data" == EMPTY ]]; then - echo "FAIL: $label — SHM segment $shm_id has 0 non-zero bytes (coverage-blind)" - return 1 - else - local nedges="${has_data#OK:}" - echo "[+] $label — SHM verified: $nedges non-zero bytes in bitmap" - return 0 - fi -} - -# ── Coverage-attachment sanity check ────────────────────────────────── -# Combine log-based and SHM-based checks for maximum reliability. -check_coverage() { - local log="$1" - local label="$2" - - # Check for explicit SHM failure messages in the log - if grep -qi "SHM not attached\|AFL shim area is NULL\|shmat.*failed\|Coverage data will be empty" "$log"; then - echo "FAIL: $label — SHM coverage did not attach (coverage-blind run)" - return 1 - fi - - # Verify actual SHM bitmap has data - if ! verify_shm "$log" "$label"; then - return 1 - fi - - return 0 -} - -# ── Run with retry ──────────────────────────────────────────────────── -MAX_RETRIES=3 - -run_with_retry() { - local log="$1" - shift - local attempt=1 - - while [[ $attempt -le $MAX_RETRIES ]]; do - echo "[*] Attempt $attempt/$MAX_RETRIES..." - python -m fuzzer_tool "$@" 2>&1 | tee "$log" - - if check_coverage "$log" "attempt $attempt"; then - return 0 - fi - - echo "[*] Coverage did not attach. Cleaning SHM and retrying..." - cleanup_shm - sleep 2 - attempt=$((attempt + 1)) - done - - echo "FAIL: Coverage failed to attach after $MAX_RETRIES attempts." - echo " Last log: $log" - return 1 -} +# shellcheck source=lib/bench_common.sh +source "$BASE_DIR/tools/lib/bench_common.sh" # ── Main ────────────────────────────────────────────────────────────── @@ -171,9 +60,9 @@ cleanup_shm sleep 1 # Run enhanced -echo "[*] Running enhanced (elo + meta-elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..." +echo "[*] Running enhanced (elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..." run_with_retry /tmp/fuzz_bench_enhanced.log \ - fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --meta-elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG + fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG echo "" # Clean SHM between runs @@ -184,7 +73,7 @@ sleep 1 echo "[*] Running enhanced+ (all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar)..." run_with_retry /tmp/fuzz_bench_enhanced+.log \ fuzz "$TARGET" -d "$ENHANCEDP_DIR" -c -n "$ITERS" \ - --elo --meta-elo --mc-bandit --mopt \ + --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy \ -g dictionaries/png.gram \ @@ -211,28 +100,6 @@ echo " COMPARISON" echo "============================================================" echo "" -extract() { - grep -oP "$1" "$2" | tail -1 -} - -# Extract CI values from crash/timeout rate lines (format: "rate% ±1σ: lo% ±2σ: lo% ±3σ: lo%") -extract_ci() { - local log="$1" - local pattern="$2" - local line - line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo " - - -" - return - fi - # Extract the three CI values: ±1σ, ±2σ, ±3σ - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--} ${ci2:--} ${ci3:--}" -} - b_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_baseline.log) e_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced.log) p_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced+.log) @@ -258,11 +125,11 @@ e_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced.log p_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced+.log) o_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_optimal.log) -# Extract CI for crash rates -b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:") -e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:") -p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:") -o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:") +# Extract CI for crash rates (space-delimited for direct display in the table below) +b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:" " ") +e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:" " ") +p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:" " ") +o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:" " ") printf "%-25s %12s %12s %12s %12s\n" "Metric" "Baseline" "Enhanced" "Enhanced+" "Optimal" printf "%-25s %12s %12s %12s %12s\n" "-------------------------" "------------" "------------" "------------" "------------" diff --git a/tools/bench_sweep.sh b/tools/bench_sweep.sh index b49e7cd4..7817c275 100755 --- a/tools/bench_sweep.sh +++ b/tools/bench_sweep.sh @@ -3,6 +3,9 @@ # Tests individual features and combinations to find the best configuration. # # Usage: tools/bench_sweep.sh +# +# For a simple 4-way baseline/enhanced/enhanced+/optimal comparison +# instead of this exhaustive sweep, use tools/bench.sh. set -euo pipefail @@ -15,76 +18,14 @@ GRAMMAR="-g dictionaries/png.gram" REPORT_FLAG="${BENCH_REPORT:-}" cd "$BASE_DIR" -mkdir -p "$RESULTS_DIR" +# shellcheck source=lib/bench_common.sh +source "$BASE_DIR/tools/lib/bench_common.sh" -# Clean SHM -cleanup_shm() { - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done -} - -# Extract metrics from log -extract() { - grep -oP "$1" "$2" 2>/dev/null | tail -1 -} - -# Extract CI values from crash rate lines -extract_ci() { - local log="$1" - local line - line=$(grep -P "Crash rate:" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo "-|-|-" - return - fi - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--}|${ci2:--}|${ci3:--}" -} +mkdir -p "$RESULTS_DIR" # Results CSV echo "combo,edges,corpus,eps,duration,exec_p50,collision,crash_ci1,crash_ci2,crash_ci3" > "$RESULTS_DIR/sweep.csv" -# Run a single combination -run_combo() { - local name="$1" - shift - local flags=("$@") - local dir="/tmp/fuzz_sweep_${name}" - local log="$RESULTS_DIR/${name}.log" - - rm -rf "$dir" - mkdir -p "$dir" - cleanup_shm - - echo "[*] Running: $name" - python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ - $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true - - local edges corpus eps dur p50 coll crash_ci - edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") - corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") - eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") - dur=$(extract "Duration:\s+\K[0-9s]+" "$log") - p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") - coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") - crash_ci=$(extract_ci "$log") - - local ci1 ci2 ci3 - ci1=$(echo "$crash_ci" | cut -d'|' -f1) - ci2=$(echo "$crash_ci" | cut -d'|' -f2) - ci3=$(echo "$crash_ci" | cut -d'|' -f3) - - echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" - printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" - - cleanup_shm - sleep 1 -} - echo "============================================================" echo " Feature Combination Sweep: -n $ITERS on $TARGET" echo "============================================================" @@ -95,7 +36,6 @@ echo "=== Phase 1: Individual features ===" run_combo "baseline" run_combo "elo" --elo -run_combo "meta_elo" --elo --meta-elo run_combo "bandit" --mc-bandit run_combo "mopt" --mopt run_combo "markov" --markov --markov-gen --markov-order 0,1,2,3 @@ -117,8 +57,6 @@ echo "=== Phase 2: Scheduling combinations ===" run_combo "s1_elo_bandit" --elo --mc-bandit run_combo "s2_elo_mopt" --elo --mopt -run_combo "s3_elo_meta_bandit" --elo --meta-elo --mc-bandit -run_combo "s4_elo_meta_mopt" --elo --meta-elo --mopt run_combo "s5_bandit_mopt" --mc-bandit --mopt run_combo "s6_elo_bandit_replicator" --elo --mc-bandit --replicator run_combo "s7_elo_mopt_replicator" --elo --mopt --replicator @@ -152,25 +90,25 @@ echo "=== Phase 5: Game theory additions ===" run_combo "gt1_sched_shapley" --elo --mc-bandit --shapley run_combo "gt2_sched_rep_shapley" --elo --mc-bandit --replicator --shapley -run_combo "gt3_sched_all_game" --elo --meta-elo --mc-bandit --replicator --shapley +run_combo "gt3_sched_all_game" --elo --mc-bandit --replicator --shapley # ── Phase 6: Full combos (best of each) ─────────────────────────── echo "" echo "=== Phase 6: Full combinations ===" -run_combo "f1_enhanced" --elo --meta-elo --mc-bandit --mopt -run_combo "f2_enhanced_plus" --elo --meta-elo --mc-bandit --mopt \ +run_combo "f1_enhanced" --elo --mc-bandit --mopt +run_combo "f2_enhanced_plus" --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy run_combo "f3_lean_best" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f4_full_kitchen" --elo --meta-elo --mc-bandit --mopt \ +run_combo "f4_full_kitchen" --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy \ --mi-guided --sensitivity --secretary --mc-cem --anneal-budget 500 run_combo "f5_elo_markov_renyi" --elo --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f6_bandit_markov_renyi_transfer" --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy run_combo "f7_elo_bandit_markov_renyi_transfer" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy -run_combo "f8_elo_meta_bandit_markov_rep_shapley" --elo --meta-elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley +run_combo "f8_elo_bandit_markov_rep_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley run_combo "f9_elo_mopt_markov_renyi_transfer" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy run_combo "f10_elo_bandit_markov_shapley_transfer" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --shapley --transfer-entropy run_combo "f11_elo_bandit_rep_markov_renyi" --elo --mc-bandit --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight @@ -178,6 +116,9 @@ run_combo "f12_elo_bandit_markov_renyi_shapley" --elo --mc-bandit --markov --mar run_combo "f13_elo_bandit_markov_transfer_shapley_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --transfer-entropy --shapley --renyi-weight run_combo "f14_elo_mopt_rep_markov_transfer" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --transfer-entropy run_combo "f15_elo_bandit_markov_mi_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --mi-guided --renyi-weight +run_combo "f16_mopt_markov_renyi_shapley" --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley +run_combo "f17_elo_mopt_rep_markov_renyi" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight +run_combo "f18_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight # ── Phase 7: Pairwise blend + annealing variations ──────────────── echo "" @@ -190,6 +131,20 @@ run_combo "t4_pairwise75" --elo --mc-bandit --pairwise-blend 0.75 run_combo "t5_anneal250" --elo --mc-bandit --anneal-budget 250 run_combo "t6_anneal1000" --elo --mc-bandit --anneal-budget 1000 run_combo "t7_best_blend" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy +run_combo "t8_best_blend_rep" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --replicator --renyi-weight + +# ── Phase 8: Final variance check on top candidates ─────────────── +echo "" +echo "=== Phase 8: Top candidates x2 for variance ===" + +run_combo "z1_best_a" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z1_best_b" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z2_second_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z2_second_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z3_elo_bandit_a" --elo --mc-bandit +run_combo "z3_elo_bandit_b" --elo --mc-bandit +run_combo "z4_elo_mopt_renyi_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight +run_combo "z4_elo_mopt_renyi_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight # ── Summary ─────────────────────────────────────────────────────── echo "" @@ -197,7 +152,7 @@ echo "============================================================" echo " RESULTS SORTED BY EDGES (descending)" echo "============================================================" echo "" -sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -20 +sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -40 echo "" echo "Full results: $RESULTS_DIR/sweep.csv" echo "Logs: $RESULTS_DIR/*.log" diff --git a/tools/bench_sweep2.sh b/tools/bench_sweep2.sh deleted file mode 100755 index 1c60ff06..00000000 --- a/tools/bench_sweep2.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Phase 2 of the sweep — remaining fast combinations. -set -euo pipefail - -TARGET="targets/png_read" -ITERS=1000 -BASE_DIR="$(cd "$(dirname "$0")/.." && pwd)" -RESULTS_DIR="/tmp/fuzz_sweep_results" -DICT="-D dictionaries/png.dict" -GRAMMAR="-g dictionaries/png.gram" -REPORT_FLAG="${BENCH_REPORT:-}" - -cd "$BASE_DIR" - -cleanup_shm() { - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done -} - -extract() { - grep -oP "$1" "$2" 2>/dev/null | tail -1 -} - -extract_ci() { - local log="$1" - local line - line=$(grep -P "Crash rate:" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo "-|-|-" - return - fi - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--}|${ci2:--}|${ci3:--}" -} - -run_combo() { - local name="$1" - shift - local flags=("$@") - local dir="/tmp/fuzz_sweep_${name}" - local log="$RESULTS_DIR/${name}.log" - - rm -rf "$dir" - mkdir -p "$dir" - cleanup_shm - - echo "[*] Running: $name" - python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ - $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true - - local edges corpus eps dur p50 coll crash_ci - edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") - corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") - eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") - dur=$(extract "Duration:\s+\K[0-9s]+" "$log") - p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") - coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") - crash_ci=$(extract_ci "$log") - - local ci1 ci2 ci3 - ci1=$(echo "$crash_ci" | cut -d'|' -f1) - ci2=$(echo "$crash_ci" | cut -d'|' -f2) - ci3=$(echo "$crash_ci" | cut -d'|' -f3) - - echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" - printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" - - cleanup_shm - sleep 1 -} - -echo "=== Phase 5: Game theory additions ===" -run_combo "gt1_sched_shapley" --elo --mc-bandit --shapley -run_combo "gt2_sched_rep_shapley" --elo --mc-bandit --replicator --shapley -run_combo "gt3_sched_all_game" --elo --meta-elo --mc-bandit --replicator --shapley - -echo "" -echo "=== Phase 6: Full combinations (top candidates) ===" -run_combo "f1_enhanced" --elo --meta-elo --mc-bandit --mopt -run_combo "f2_enhanced_plus" --elo --meta-elo --mc-bandit --mopt \ - --markov --markov-gen --markov-order 0,1,2,3 \ - --replicator --shapley --renyi-weight -run_combo "f3_lean_best" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f5_elo_markov_renyi" --elo --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f6_bandit_markov_renyi" --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f7_elo_bandit_markov_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f8_elo_meta_bandit_markov_rep_shapley" --elo --meta-elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley -run_combo "f9_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f10_elo_bandit_markov_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --shapley -run_combo "f11_elo_bandit_rep_markov_renyi" --elo --mc-bandit --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f12_elo_bandit_markov_renyi_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley -run_combo "f14_elo_mopt_rep_markov" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "f15_elo_bandit_markov_mi_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --mi-guided --renyi-weight -run_combo "f16_mopt_markov_renyi_shapley" --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley -run_combo "f17_elo_mopt_rep_markov_renyi" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f18_elo_meta_mopt_markov_renyi" --elo --meta-elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight - -echo "" -echo "=== Phase 7: Advanced tuning ===" -run_combo "t1_pairwise0" --elo --mc-bandit --pairwise-blend 0.0 -run_combo "t2_pairwise25" --elo --mc-bandit --pairwise-blend 0.25 -run_combo "t3_pairwise50" --elo --mc-bandit --pairwise-blend 0.5 -run_combo "t4_pairwise75" --elo --mc-bandit --pairwise-blend 0.75 -run_combo "t5_anneal250" --elo --mc-bandit --anneal-budget 250 -run_combo "t6_anneal1000" --elo --mc-bandit --anneal-budget 1000 -run_combo "t7_best_blend" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "t8_best_blend_rep" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --replicator --renyi-weight - -# Final: the very best candidates, run twice for variance -echo "" -echo "=== Final: Top 3 candidates x2 for variance ===" -run_combo "z1_best_a" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z1_best_b" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z2_second_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z2_second_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z3_elo_meta_bandit_a" --elo --meta-elo --mc-bandit -run_combo "z3_elo_meta_bandit_b" --elo --meta-elo --mc-bandit -run_combo "z4_elo_mopt_renyi_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "z4_elo_mopt_renyi_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight - -echo "" -echo "============================================================" -echo " FINAL RESULTS SORTED BY EDGES (descending)" -echo "============================================================" -echo "" -sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -40 -echo "" -echo "Full results: $RESULTS_DIR/sweep.csv" diff --git a/tools/lib/bench_common.sh b/tools/lib/bench_common.sh new file mode 100755 index 00000000..322eefef --- /dev/null +++ b/tools/lib/bench_common.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# Shared helper functions for tools/bench.sh and tools/bench_sweep.sh. +# Source this file; do not execute it directly. + +# ── SHM cleanup ─────────────────────────────────────────────────────── +# Remove all orphaned SHM segments owned by the current user. +# Previous fuzzer runs (especially those killed by signals) leave +# segments behind. Accumulation can cause shmget to fail or the +# target to attach to stale segments. +cleanup_shm() { + local before shmids + before=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) + # Capture matching SHM IDs into a variable first: under `set -o pipefail`, + # piping straight into `while read` would abort the script (via `set -e`) + # whenever grep finds no matches (the common case with no stale segments). + shmids=$(ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' || true) + if [[ -n "$shmids" ]]; then + while read -r shmid; do + ipcrm -m "$shmid" 2>/dev/null || true + done <<< "$shmids" + fi + local after + after=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) + if [[ "$before" -gt 0 ]]; then + echo "[*] Cleaned $((before - after)) orphaned SHM segments ($before -> $after)" + fi +} + +# ── Metric extraction ────────────────────────────────────────────────── +# Extract the last match of a PCRE pattern from a log file. +extract() { + grep -oP "$1" "$2" 2>/dev/null | tail -1 +} + +# Extract CI values from a "Crash rate:" line (format: "rate% ±1σ: lo% ±2σ: lo% ±3σ: lo%"). +# Usage: extract_ci [pattern] [delimiter] +# pattern defaults to "Crash rate:" +# delimiter defaults to "|" (used to join the three CI values, e.g. for CSV rows). +# Pass " " for space-separated output suitable for direct display. +extract_ci() { + local log="$1" + local pattern="${2:-Crash rate:}" + local delim="${3:-|}" + local line + line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1) + if [[ -z "$line" ]]; then + printf -- "-%s-%s-\n" "$delim" "$delim" + return + fi + local ci1 ci2 ci3 + ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') + ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') + ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') + printf -- "%s%s%s%s%s\n" "${ci1:--}" "$delim" "${ci2:--}" "$delim" "${ci3:--}" +} + +# ── SHM verification ────────────────────────────────────────────────── +# After a fuzzer run, verify that the SHM bitmap actually received data. +# This is more reliable than checking log messages — it checks the +# actual SHM segment that was created during the run. +verify_shm() { + local log="$1" + local label="$2" + + local shm_id + shm_id=$(grep -oP "SHM bitmap, id=\K[0-9]+" "$log" | tail -1) + + if [[ -z "$shm_id" ]]; then + echo "FAIL: $label — no SHM ID found in log (coverage not enabled?)" + return 1 + fi + + local has_data + has_data=$(python3 -c " +import ctypes, ctypes.util +libc = ctypes.CDLL(ctypes.util.find_library('c') or 'libc.so.6', use_errno=True) +libc.shmat.restype = ctypes.c_void_p +ptr = libc.shmat($shm_id, None, 0) +if ptr is None or ptr == -1: + print('FAIL') +else: + size = 4096 # default map size + bitmap = (ctypes.c_uint8 * size).from_address(ptr) + non_zero = sum(1 for i in range(size) if bitmap[i] != 0) + libc.shmdt(ptr) + if non_zero > 0: + print(f'OK:{non_zero}') + else: + print('EMPTY') +" 2>/dev/null) + + if [[ "$has_data" == FAIL ]]; then + echo "FAIL: $label — SHM segment $shm_id could not be attached" + return 1 + elif [[ "$has_data" == EMPTY ]]; then + echo "FAIL: $label — SHM segment $shm_id has 0 non-zero bytes (coverage-blind)" + return 1 + else + local nedges="${has_data#OK:}" + echo "[+] $label — SHM verified: $nedges non-zero bytes in bitmap" + return 0 + fi +} + +# ── Coverage-attachment sanity check ────────────────────────────────── +# Combine log-based and SHM-based checks for maximum reliability. +check_coverage() { + local log="$1" + local label="$2" + + if grep -qi "SHM not attached\|AFL shim area is NULL\|shmat.*failed\|Coverage data will be empty" "$log"; then + echo "FAIL: $label — SHM coverage did not attach (coverage-blind run)" + return 1 + fi + + if ! verify_shm "$log" "$label"; then + return 1 + fi + + return 0 +} + +# ── Run with retry ──────────────────────────────────────────────────── +# Runs `python -m fuzzer_tool "$@"`, verifying coverage attached; retries +# on coverage-blind runs up to MAX_RETRIES (default 3) with SHM cleanup +# between attempts. +BENCH_MAX_RETRIES="${BENCH_MAX_RETRIES:-3}" + +run_with_retry() { + local log="$1" + shift + local attempt=1 + + while [[ $attempt -le $BENCH_MAX_RETRIES ]]; do + echo "[*] Attempt $attempt/$BENCH_MAX_RETRIES..." + python -m fuzzer_tool "$@" 2>&1 | tee "$log" + + if [[ ! -s "$log" ]]; then + echo "[*] Run produced no log output (crashed before startup, or the log could not be written). Retrying..." + elif check_coverage "$log" "attempt $attempt"; then + return 0 + fi + + echo "[*] Coverage did not attach. Cleaning SHM and retrying..." + cleanup_shm + sleep 2 + attempt=$((attempt + 1)) + done + + echo "FAIL: Coverage failed to attach after $BENCH_MAX_RETRIES attempts." + echo " Last log: $log" + return 1 +} + +# ── Sweep combo runner ───────────────────────────────────────────────── +# Runs a single named feature combination for the sweep scripts, appending +# a CSV row to $RESULTS_DIR/sweep.csv. Requires $TARGET, $ITERS, $DICT, +# $GRAMMAR, $REPORT_FLAG, and $RESULTS_DIR to be set by the caller. +run_combo() { + local name="$1" + shift + local flags=("$@") + local dir="/tmp/fuzz_sweep_${name}" + local log="$RESULTS_DIR/${name}.log" + + rm -rf "$dir" + mkdir -p "$dir" + cleanup_shm + + echo "[*] Running: $name" + python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ + $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true + + local edges corpus eps dur p50 coll crash_ci + edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") + corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") + eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") + dur=$(extract "Duration:\s+\K[0-9s]+" "$log") + p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") + coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") + crash_ci=$(extract_ci "$log") + + local ci1 ci2 ci3 + ci1=$(echo "$crash_ci" | cut -d'|' -f1) + ci2=$(echo "$crash_ci" | cut -d'|' -f2) + ci3=$(echo "$crash_ci" | cut -d'|' -f3) + + echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" + printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" + + cleanup_shm + sleep 1 +}