From e69facb2dbb69b158dd803409fdaee9969294f01 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:40:04 +0000 Subject: [PATCH 1/6] Plan: update benchmark scripts to fix removed flag and add missing features --- targets/test_target | Bin 26880 -> 16056 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/targets/test_target b/targets/test_target index 51d52aad306975959dd10e24e29c80ad1d75e671..9fbe068cf0f27a95c27b3f54707e020fd49a889a 100755 GIT binary patch literal 16056 zcmeHOYitzP6~4Q+!8Cx)tKkt~l|mH~d9aPKAjM&^y=GjQN5H75lrkB6$KGw$yYB96 zW2bF^Qlh#*MWnX*lS(wI8l(v|p3tQEeBkaf#BQvvVsXaz(M8Cf@5I>#NN1XOY!tgGPy zvBS7R`Uyn2-(G^nloIZzk8g_!4C=TmXBs?*Gi@_5Bkc_fLdPcS&WY}v=zur4LzIAs zC&z@3EfUW#HnNBkFs#dnE{vU?OaZwKPW!Lf>3^ray`rm5xYejIws^DxqP>SnCzn<{ z?hLrh`=q_uuEQe|f1rd{ZOmo+T3a^e(ha$6zC6+}vZb}5wM8oxv}U(q?pJ~i9#cDa zA5c`y1alHb6+i6RsE^B*N;&(#HaWKH!*{>@#>1;8Uw@|k{F|8vCsXXh{bmjOaN_zf zMA+s$Upw~k;&Fs>9>;judVPp-wsbcYEUr+vWfne5e4>Isw_m9q<|^^Wh)>WFPTwFR z5hlsrK*zlmcx4zR+w6bb=+8WE44B!RDq3cmSh`;QTvW7;Av2pNrhmjV2C{iGmwnPw zW?!LbD|X^**s6B+bhmdH&033}Yt}aV`Asd#=v3FNTu8_BS&AyyP zbq@{|^3s~&xmI-K>s3Y9s@naGxu5)mz5N(kn$0fYJ69L3yfQnFulhc*fBek1=rYIE zL5T&PBe;JLFY*e;1u$-Smv^+0Pl@EPjMr_&qXDK z(*YiBbMb`m>>C`{uLO9$50EYec)r)76h$D4Koo%}0#O8_2t*NxBJiI@;BR$nKGcu? zxmG`3^ZSiT>BlDQxN}KA`peo2ZpF^#uMzF6KS7^$Ym*syjS^srX57e#gRIc7x-xabS3fZkQ?520R$1We#$KKM9zI|hFZ>njc=~ew? z+iTQ{vup{uzB{1Rt^Ep_bOI;awi2VCtZ8Q0I&CkbQ+r?YPHmoZwQlWU_PZcR{_|Y_ z&2feexAd_a`s81>>yy(}dhDWp`IfzuDtK8csCBM-=K(#R|KT<+R^VSQfH5T>1@@Zx#yVr)BSZXrN+1E<<*2QZtE>CBm5!Z0>U3{ z+gq-gIzl4XS3iDWI;}sj)NR79nHr&jSLt$d7u>PnUwy7QN>K!&2t*NxA`nF&ia->B zC<0Lgq6kD0h$8U+5P=x|3lm#a^$Gr8nEzk7=Z52?2_GdqL6{@_F5z>8J%q<@I?m4t z`M*p~u_yPb*hqbB)%>}&r|5S8=J`K+o%}z}4xMw|zhc)>wuO%IzdBBmP%W(QTDbg^ zb$2~cdsuB>x%K`{>)gNlSpO&;uaF=9!)15oZ8|21=e{CwTink{YVQb>sfG1V$2%6x ziH}f^n21sofhYn|1fmE;5r`rXMIeem6oDuLQ3U?CB7k*%Sl5R&d%TvA(|iy7g@v@J z&`tAYG3E-fv3_us*my-+DK^#-@_Q90l)rb&DX={*ivqD0Z@ijWtVLDoij;rEqty=t z?-E=txJ(crGId7Q`(bUME)M+u#|di;c^xaKVOfj_eq1_+eowewO3|W9H=+GCV#Dw6 zj(2_(SC^u{MBVcpmip))TGvM0>2=e!4@&t5V&god`u`eW{dc&+MtMn23hoWQG>4t< zU}IhA&W?@;6ZakHE9dQUqG^-1No#CqExTs(Q_U?}V~f_bK??3rox7&ATHSQQK71W` zJ?seyR6-n<5LWW%>lCL|(U(H!#X9DdtaByPO`GVGdE6&Lt8U7~)Jj#K^cY-6US%Cq zA5|FlUa3-Fqzu=C`wsk9UH@eY_YwFPC{wBbvz^yZ+JPX3H1Uam9@%dazbL*!VSIyso_TIrMDiF@FA`r*<79af zo*o_|eleAtmT~C*wnarRiGO{_|GfB*$@n}X)%}Wi?oZhNcf=>eCkXEl@BT0rBq?7@ z56H!Fdg_Pn3$?$$Xg6sETBC0EWewXLR78~Q^1y)BuWqkrH|!y!pI5Y($TM9q26Kfz zGiRjjLa}6+W3T z&v3P^l1-f}4q7&K%P_k3?bwwvQoB11T29~j>D@bab$1X=yC95|E~R?sK4t9e+0(wG z$Jo==wLjHs^zLZyNs%JF`ag-~{j`wZUHR`5yvqNMOZ%;~X`9MDIjVQ&UIV|D7I)Wy zZ@QHlEcEsSkZ|S(0V7>17#TC4=FJ1$d&n!D%^T&CMLQR)Ja1pZYxL6priuP!L>MLhy z2TV3C)QVD3xHDwNEsWMMaAyyg<>meGg zp}Xl{H+lXz4DUF5uFSI&I8ICFR z<1&u~PKYDg51;?DqHh;J%!7e?K+iD?>%T)B#{~8J-y`yUg7*Wd#_cQejL*>#)F(s- zOtMoT;cE)`0+j{zk4XU#ey9RAa6F_RmI7cSgo4lke;U%CmIB~sxpRR89VlLh8SFpi zH9)+7!9RTeyexW*GnEie!LUDU;BQ2a@ly?*U<;K86LjGIKpE~o^ji{M3Gl3E`kxp- zAg)nnAssS}_vgddXBqZ~4ZKO(x_}<@@9_pADZ$UPZrkAhK?OW^(LUzI*f)#)v3&n= zf)3a5`^542Lyz~)GaK1ksYg8UKcNTCA#qTzlH%IvA=*I&(7~QhV&>33=7aU35Bqal z!L&%)7k9r1dI_V?Lmxz3d*6vNNvZIZbZj+oJVwzUJb(E;4t=MrpDytW7^Q`w%F(V@ T>b=YgiQ5M$TK#_ literal 26880 zcmeHQdw5&Lm7jYh*@|Uba-5JSM5al|%km>m;)EFN*iPh<*lz4Vc?wy&wnbz~=n;}Y zn?M4^gs|b&1r{i!K>7Nyg#t|}ZNWeY6j&Nyn}sfgv|F}Gp)9Y`&{Dj=GY?t1mJI3l zZTBBfqM13*IdkT5=iWPaysdTZYF*Qq3Kn(>Bj`Psz%)w2cAv~3X=L@R6!Ba(hZO*} za-5(yiUd>+ioX(VdDpeLzN^<$a<|08+DUm8w+H z)8)BI7{Sfsbovf4AgDA`j(l@vB`32h8?w`SS?@-fDmfi&B=>-ndqB#m)I|&kDpmd@ z^Xbt%WQ+HbRL2bIchzW4pxRkk<{kO&Gkzwm)3a_ zQBN(SV#}pIs@HbyQp5sBdS5bL94MT4EwSVb+kl;zgYrEXmOA;B z9z6NVjh{Ij0V0?FMZmkw%z#gTp}F{90WBBal81k39{h!Q@I`s>@8&6YFb{uc9{j%m ze`1dMeiyhhHiHc{3Zm-o%;)PHh(>&gq(7eY`IxV*dz~+6#ErgCB5B0C*ENTu5u@AR z6E*}bH^rCmM?%TaRwM2UMIaaPhkXOaz(8~p#PSdWu`QtaAqkmKGB01kAR6b3#*!fz z8#IzeAW1d{Hu_Qt!{_fsnZCG@OvNLLS5OE1;cztITg>8yKd4A?BNjz%6Uk`Ys9~@( zFc34*!cl*)M&bJXkzm;HMWUu6QW511Utc)d;}0j4#3m!2AWH{Ai2;8y(C-WM8?bd# zy<*5Ah*2Dn?J(dEQ9S~i{l4B1dm>iZCj1`B-`^bTba*ubpEfWwaxOqZ}DLmYmPV7l}kd4R*;Bbb(}NABS8 zodnZm^2i+V=kR3&)1~@ICx^EZOqawXjU4{K z!&vVgz&zCav2F0B^Spz<_3rrHd!5~_HA6K&_wH_Z5yauizX0n;y&mVRot&5I3);<4 z=-q82JG{$}CMUyP*MO1SHDUOWbJliZeoCg`4{`n_J(SG<%scp=_lb8;^*(Xb;??$h zU-~RL2?{ny1@_@Xz0O%JBk~_QPx#CBhWU_Xsf8Q7I~r=i$-IMaC5yeg8ukJ>JRj9M z97MF=rh!j87iFi*C;1~6fw%!OO{X_)XzK3rUiwZ94dmT@+xE|fhua3<*wL_$v?eD$ zem?Mr`+=p}SiFNzd3Wz1T-)H&-e)WiUG>8z)chzG2HwGDJF@pcGqSF>!PncKc*oN8 z%+A?t`|!5+5hyExW{EQN|t>_UV z9jT$>ci%`d-d%!2JK!CB()+|8EZ$3>q~e@=C+>XNdE*n_!Tme_yRfnTp!2pTE1ui= za?uVw6?X@w~fBmhdXRNa7bj{PH2^<#&y6 zd3Oxij>UpAprXmS_wuJvhm~EaVqURFp^pu}EDJtXmDWs^O{t>8-}w~1;OmIE^1&zD zmcNl)i8d=;dZ_i0KOyuwBhU6aV^6RqXPIavo^io{JTns+@2(w3g?HYbs@jswY?EzC z`N0F;OZQR#IlJkcrgNLlJNGH-T8{xiqhcj)$jNIhutZP}zxG#HmISPLrb?b_&OoEhhYA#{e(ztZlAkcv;nFBokFhBa!q8z!PBF49m$>AUzD&`9H(MbXVT; z-th2!NV}1CBW-wpcz7Sun?4*Keh2A;NQ-fAqMo^HTh}vfvrC&%Txj2;6}kvddkT(1 z--$?7$j#5#Y%p;CcZY|Eh`H3Yx^(K7osNs_+u5npPhPNiHdjOPE%@z$-puFpB!3Qm z7sB4N$<7GIrv3*9XJH(~+k6Hh>6_=y&0FJLse{ zP4@{pp{)4Pvv;~Rqj$>`=)ud_EzH@*qj3+6dtlrH;~p6Iz_b zwWo*Qn*gWwf6;vj1x5eUXTwo~ACQ}^==~an12RMRaui;bbb3)s;SFiO+OsrOpxI=Z zszi#h!*bu1y0@`Qd37JB_H<2=f-$+7OX2OZ!m8f<4K{eyo-b8C-evLnfW%qxMvud< zN=0}d%3)bN!)+dizc1;ZNLZDl>hX_G)&6mFxl3ikt(NKOGVPV=CYfF#({IZ39+~cy z>3*3Wl&QI#{zfxKz>RK^s(#TZK(<}tuaK!kk4n}wH?MHdYcYC4{)oG#&Qt5DuB=%; zUuOAjv*;qZ8(>Z}aCPS9VId-!fv{mN8e2{>&`NRNnINKY1q96y-9mW zAXvp&@Qt=tDoB8*d_7@)q&c?(U?=*B`|2q#pzGSN2V!CZ`tO)DMDSe%r-km8LLK^+f_=tg-ZCj6~fd zVU5kEEg5>PgTLN~Vg%QuolrNd{T7_I3rJkIzW`g;;&%qw>;le07Bky@Fv+gl=_`PZ zAkl7XPG$w~z)chMf<83JWbo7e=^GSP2y>Paz4&Cb&2+Hb2l9`IW!8xPTcEg1w{OKP z^gKuh(;7&C6?_ZiX?np$$WtNjg8Y$UVYZE^#__uSJ4oy>qZB`BoCW$Sbo+i_s2+2X zE+Q(G%nEwYOtbakb~Nw(aDEFYU(TkUq!-)=P6ya~K%s3jQuAj(FVO9cD1bC?2jz;i ztnF_=FVyXJB)0&2w~~|I)PS$n?H&~RW03Zl$n$`&*X@_$TJb7KZzwV=SOOO=*X=>n z{a!GA49cfO74_L)%Gk+z@f@@j)n^i%HXXkV>M44`9#op@Qws`xGoDm7KCcStN z6q2QeK0PhVsV#cJHWWaz5m1t8S=%nC^y>Ea(8I0-_IgFJ%my!;GY8h{9eUaG(0eav z52fk6Z#l0*4?0^fAjQmiK7ey{K2Ge;J}5p{0In(B>|rQi_DZOmMC}~W{}U}fmtdyZ zoXE#?OEZ8HWXti>=99{kh@xMp^DD15A3&X=$>t~poYE~i?DrzurN|l$Q8tz2FVZgp zwHLVy71iP)?bGgt!JGBJqG1*Tz7o{yvpKl@7X1^{Hl@+G1+6V*oD z>oVQG7no)EH4uf%*`7r$uF&maFnk5ra}>owU4q#rLfZ()0^9w2P@UPFP|&fc(D#`7=eZkS!K6La;2g`~tcTfc(~I z7Ri?a#|nEB$~X$1&xmh3FXMNx^c4Fm7^Y>&&BTurEWCLa5!KRSUkLlDThsw;NwZL! zi19GvF=SUeXJT5UN#z|43o^onc7jsw(o6#bAHz3`h=yF68UO)|gq8)&b3xBH31*Xipk_#UM z%DT%zuKtVFf`YAAbKq}Q4dz;RbKrd|%?Z|PIPih>cobm$Ck}jQeS=Kc!+}3p2MD;9 z10P#|N^IA0;Llc?Rjt=^;8W{wV1V@o4ji@KMQk^6;1la2qI{DBpIP5V53&9;2kx^T zfs?H_ao{J`J*cwvW)3`OJwys_;lT5y8;I>z?hM+@SNayZr1drq(?&5i!9YzP)BcM* z@i_Mcn-E2zPiP+#>Pd-u4LbK}6Tn;ev_ySGJyM%VsAnbWm#FS@+EPM2&ryZHmXzPZ zjDOSGi1MOD9S7$g(9R*$OA_@FP_JlFLjAi$MNrB??J`3BheYj1DX(ca66#kHwF~{` zO^y0c;UR%?XnG7zDf$U;4z1v3sIk-ZUS+l6dnRV5nb%=2JO%Y~X!boIJM_Yb=<;JP zTn?nZ4^GfuqUa5Z-lpgiiYB1x^=XKTNM8~4gc8b?P*Fv+)+r%e5zRCubO9-$T(zCz zBAR!KXmZt`r|5Nx{y@=DL}j+A)S1g{(`c1eW}DtZKB%|MnsHW&D zxJ!SNU|J*TpHNhQR@2KVa#K`E(F%&zP}EJ4pP~pw7gKa4MK@A(Cq?&D^f;p00%zHN zs9bzHY%Z8mvqgp;aCEn?1nVt+ad9iZ`XqUPM{T*M`1kjj#zNq|I{e;Y(QQH;{8u zQN!T3v9|I$+;i*r&Z`_81Vcd`vy`rIoa|WcXe(biZ6<1M^Ju3|1F}Lt*|yvfDXp#0 zR!$3+Yv3+0k=n{Lh=nWIiV6)f6=hSwYoE&tn6O+UT^#Qq9Ys#&sIXJnj@iuNIK^SX zSSVJCEo4cgR25i)c~vl%nhZq`b(WCr!pyevdYJCwOckW0Vp62Et^Cw!qE2OY#w%Gd ziAuUE2jmdYv$48k@+UmaDiMuS7*Xqq#AM%N~kJs(KYVE4itLo)guguXAzgPWW%dFc;(^OYcgO0b$2)e|~O@%j3y@YqKQca`6oE2!hDiqJYik}US zH0a@krfGjIEujSS5$z2?W)>z!#&+O$H-5S{OLyAi$Z4+-_(zI}Q=dZYCsD+yO@w;^ zRMJiJc~0F0H2YtVAKf49!|(U_O~8Ujl+y=Dy+M+V%ww5qVT#Nd>P#eVUw+63kCg2 zKZLl^UG#1BhJ?{XUsiVoqA^359`MKfflzV_;|$$#f52c(u^4XGSF`S>HNNHzUELk) z*m{G1cb%bYT`0KT2yTr=q@s=M@QpJ)>)JS?X`O#FOU06Lvir0wCy#0P5@R5i+#;%L z6025D;#CNCCF7w;-_o!V=}Yz>tAA>s4Bxo(ud&g#O7n>OR|fk1@l&`PHngAC-f>1d z+t_0C`cvWL>QET<-rPkWa*wK9P4$QimUP0xqCA0$*dN~K>~el9;G6QP+!Cd zx>40pW{+vevKH+4ibr{4XB-`n{+lx^_jrsM_-C$IWG8nVr;VO+Yje&FO9ElTA16nwHafe8cMX>}YUMsx#NjnS3QscxB^8CyF~ z3v6X;jAUor*c8IRX^Ws2rvh}CMQ4C*Y>RBdtRf0)Yhg4GGxD)dbjI|w<0pZRIF9EK z)+8?SrXf&Me@R`#%_mgyy7_V!M-lVV<*Q>Z6*TRNT zW@HY=+G?~(yRkW$)gI`hd=l=CGNtD1bD8J`d{~%kmd6n-H0O@u&6w+JYN*yNt#Ue` z>ABON>|e{TPHbcAW^s^8XEVn9XhnIK9qUh%?l%pyoLyA;pAU6fp^y)==1g-9#vJE$ z)FoTjlL34vjWT;o>7`<7p-I9Qq~18H?3{I2Dux-2vQZM5&0SNA=59{K<3=Re8OYH% zV;Z+iu4!Oh+F7gPhMX8Mmj_b8C`}xgLJ~%Iw4;k(8Zffk4IBr<#G+$t19^v#Yll{J zdbQl(m+tOUyKP1OxeEj-plx)@6|RQZfhIE)M6YJVcRA#za&Z&}Pf&h!|T4V|YA zT!7|C_2g=gOy1hMd=nfz##d-m)G=5r7B#-jxcrC#Ni*oE(aO?`30i%PS(}>bkrisI zT(yoVAgfRHAIMq#++)z+g-cG(F_`JEWpa{f?F^tt$!=-B-r&g)RvJFOrj!%KD2vDJ zwdpIGxtViM$9$dW^Ci=l0`%dyzoX{@+}6lhA>$J9xpPHkeQTDD>U*s{$Jiy(vz5GJ z()|N=uDhETF084O@9^D~1Bw1bawF1YDq=MB8Cal(0s#H-K!1Zc9L-(X=dSFibywmI zd_!L(#pNq|JKLJwmA#Q@Wlt&;PKF`^FgDA}WJ194WHN+YbRd+h?4`9qWh{!7tPuw? z#RpYok`Bxw_D!Kgs3#Q09aBSpC>S)P%23Q-84C1*A<$bHi%01^D_q+fdQ;&rkp9G$ zNT5F+jbN=%nWggBT3-(OL_dy>a;FChY-M~MfPT#{`lH5WX1k5_)AlZ3?q1W1`6(Jq zg^djx+FJgOxoKooVp=ogxw2w_i6sV|F$VATvW;qiOmjqfif8g}fsNdHP~G|ITIo~2 z5`3kV6bFRy&Q!B(q$_<+7ouBpzJhP?2ZOkJQIn6+c;XB%(Hvw+74@@V70x4z!+(Jg zI!$E^OMt4rK%law4*zL`isVAi^)0QFXZ)eh(y@Q4_;?j>NROu~{XY>8jOO5MK6!uP z3B+P}WcE2CV{@bp@CBkc?k&Pb5PV3`L2yB%H{t2epa|A1RDu@T{5B3~s!RGDxh%LB z&l@r+Bc)-nZs4pm!zYI>m6-c8p{oAqfKe3;g#DXB7euQDw!rkDQPrE;isS2)468~d z;^aSSimFkruc@mFMFQbe&=8Cg7|qy2w`*KiUY5j`Yz~Gw33Va+x5NzYIsVuJh*Sip z@&%DW4BL>&7x2)GS=sw|XOJE6(g{!M%G2j?x+NKp19S490KAUVTiuz5l>z6L_)Zgk zvBazWhX8qKL#*Na!F`PaCi^JRS4SEFJ^0p68yj;E4(5cpj7(0(rBf7Ybuu{`{=!7G=Z_wwMacr!qL8#3v40G~^b zI}g4g5B@xkFCF#fbOE)mPHzj92V~k6h$m}2QCiycgz#->9|Izh#5x*Zh>jd=>Prsz z0$3IpbYy2RD$noq1@X)v;q#|9GhCx$VVpA>^i=2Opp!X6K0iGj^WlavzJ>9%pf8vj z7@&hqGay{CLdh&veuc}t3}=7I>Pw`0d@a7}>gwgoeJG#6HVM+Hq^s9At!wqQwztrU ztv)PTe63!I^tP;LzBOw*RyD2lb*x_9)!OarZd$dr6^t!swl}S7Yleipz~Zq2Uy(Gb z<3s7pQ1gMXR81g9uC(dvI+jr4Y}Oo)F7pX|{)w<-9zffH15$&b2p#$wlt;P}zdB7d z`?y*kJzEkds;WBYJIR$VKYZ!KVzZB4HMxg7?+czp3xm?l6ih^YqI_7>lSoL*)IyYY zJj#~If3B>msQJ8Ed7`a6wbtNI5T&y0!*21MIs2H|bmi~@R7&JOyq2$=nJ2Mj05>>D zlW~#aeoN#cxRdclvG z(u^#?2U^?+`$<6NV&NodL|^tG>*H+fVX6$=b61kK7kw7kWP0G!d2>VvpH(ua9Qf}u!n zROS6WJ$NFlAe`Q9s5~61vV=xoiJK<6zYnDEGw?PaYabERIqE8POT0?d^;*HSr6p57 zhWVL-O4WKvU};U39hv}Q>Q0xR?;L+6MZB4T){!6M8wq9q1G0RT&Xdg8#K42u{w`qY zjkl6l-<7D8jyI!#ohjT-C671xJe-Y~id6FII~A3x?@|P}(4q2_2JoYAK$N`t?n9*u zB(t(#@u@ThI(<{3VD(*zN~@*3*?x&;+mN9)QS#~UT4Y7lcP*-XbNvUU{3@wXeJ7(* z^<53g(~b$V{Pn<*i^)D7)b(4Y>iZi-HuoRe7nLdRmU1c`Qi3uysb@EvDqq3(n&j0wTBUSExBcSs{~ad(F$aH zk;{}<_aQ1>ndTH3bNTyB@~^rDpGrRxU^-TQQ-~Ll$(%o4s1OvDx)q~L&E>xiJo!|) zS#2;>e|N!Lo`O~SHe^dP4D!XojEVZAKSLc-vRRM`GPbBGB$*Z&&!kO~wyST$rK9-gg86~IE z$tL;p8wKC_)iM=HI(DaXB18AHq$_QIqreYIc~y?Uj?5nlcFXc5yFxQ1KP!z98LBG< zmCl#)0y`4FDHV2(pm^v!N_MhG?2Z~bg_A7j363o$0Tamn7mmrRT>t<8 From 6ee1e909d1161a9aa9dae0b644b1c2a3e4438164 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:44:03 +0000 Subject: [PATCH 2/6] Plan: fix removed --meta-elo flag in benchmark scripts and report.py --- targets/asan_target | Bin 27344 -> 27208 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/targets/asan_target b/targets/asan_target index effb1b75842415eaa3a4baf0bc8c2afba35ebf7c..c7ef7b6710d360702e470fd054c9f8a8ca1a3f1f 100755 GIT binary patch literal 27208 zcmeHw33yz^ooBt*huXTjCHce-Zh`IiK&{h4b}TGelHH1BOR;1y;b^zIpQOgEZcQJy z%wbG~GYDhA3?VQHi6=XoV?z?>VjWHai7{lu7_tLSHitI@j769%Upx?4{z z0h!tF+xjPHI1oYVHYuiPL~KwlO&wjDKki#SOY6Uyo@bn zcEDDS6Z9sLfGWZ%ePZg6cneaJn@^b>=^}wIQsxpc$=N6?VRJf~ zC-_vV=;`v@g3e57hpeAUv*pM)t_mUNYMhvM$$IaQsgfggB-bG28l;>`-LgEDDu0qs z=xUJq%xM*96jW-K8<2A5v{i^Pl~NfLx?f5J|C;i;q}*vYSHsL{hd?uxs`5SsIr7Vg z4XT;9$?~RZhu5S(RZ8tmp)wrmt*@&L2P?v%NOHJhxS_tHzRnYmdTKbG>~iCxKDBA< zb_O+Lgj3C_?-#JrN3_|JPW<7Mhu1{^pF7PvB`&l=AR=9M0%Y|gL|Sdupf(&siK0dK>Vi~l=Q;K_bBE^{~q zL@qrS0Pi+41AZJ=F8=$cz*nIHx%ls#g8$c3;O9@F=L=KtKRgA#c?vymPr=^}{_{XH zhXEi;*h03mNl>Vd@Zj_H4@M)tc)}k`_mFf562Z{ zw-JkzJ%ge6pg$29@C618ShTxAG2{>wMh6CV`TG2!Fc>ky1JZwk{!j#%z_8!f7mE19 zp{orBGC7Bad;ExZo=0}{wfoG zzvOo`34Kbxg54&3lUwjLnDBQ zCJQf@a!z?I3!gndypx4jz9#`S&PaFmBswJdNq4rL!&!JW_miIeS@`sr1^%WiJk8%K z+@6J3z9!n;S$H&jIvmNu%PkM5eLV|bkmhFWXcoRO3x6yNkB*iOV_Ep(G?uXwS$Jm_ z{$v(jZXL6zGuasr%y?kN12Z1@f6W7Do#(#m9eLH^J!pHXk}>arv4oyF=^gp8;|XqR zYV}&6Q%lz&b}nvW#7}!tHvYR*D%E#8hiO3^e=Y;lf<69t2Bw8^{CgRg7R2$dWMEos z$M49%v@ni;CIi!=J08oxv_OvcW?)(v$G2r*S`f!KWMEnd$7?b$Er8?arQy`lmpzR2 zITyDwUcc1Ri%sxW6THy`Uu1&oO|ZuVFEhdCn&7!6xYz{SP4N4;Taa!4A1A=o|JFYG z?3Lcp-+4z~e66det$M8bDes}ipQBk)bIyYG%|4HF@r`f_I>4dEb}08AvTdMb%{z%X z=o{b9ZLv3D$9?YQ^Z_YUrH2u4>tAzICJUysnnSuTI;y&yMVPTAe%0q z_l8%Ga77P$LG+IPu!)jG_wGc}KKh%H#$GBXQTpgkBxmNL z2&$jOJNkt8(8zw^+DCulebVy%n}660U;Gh0#yi^LK=w|quEo_p`m6TGUbQqodE*kc zKXv`<2$Z+-y!Hm=9mreHo;@4%9<(l94xQf7wo}z(-jRpN7_0NMzreup9y~x6K0sr_ zd#LC>UeYt((I?OR@Ap!v17pqu-_LM)4|%QL(G%X$$GwleV)5SaI2G-@zx2lEnw+0I z4xy2M&1-6S!FliFJuEaL%j&!aI6k2UX;tNJhpS56~cf zWCyqM$un)lBZ<&)7r^mV|1mo@NISMW4;(-#!j^CSh1=34Y@y@+`txe!qbFAenvWb0h~5@8}C>O5k1L*m~*M zK4;XX!_a1DiS#RFTrh%T79rz3IPwna#vM!+(#WCW$=PH%<;@Q6gk{{p1iyiD-etKUe(EBW&) z0eAu^Ol`1Lsx7p9q62vPRnhc+QmOq&-LIumUq@PnbPOqbJ(U_jGLDp0O1}p>r8AwC3^03QLf!!*%Qr zsnltttk|`&c>X7y1$!L(+4^(VtgKyf9-&G8a$G~OG5dK0;S;z9fIo>;1=4#9u34~o z7TsTID|UTWZz-~Gz14EadW&tuZh1q8ZN&EjT(6u>rTkQf$$X#Gz(=rBdf2 zP4jIB5g^h3f~y?#OCp-UX=Vfz_||1@$hLtTtONUt&`DF_}^I9+2{qjj8haZxZNJe0XZX>j!p@!qWi` z@0a@Us6x@X9r#(s;cb$RoAVr2`Z&{nVx->JHmj_X4b&vlE}8C>>5xqK%k*}c9+Bx$ znU2Zyq)boC^fj59%l#nO(UhB`$u01z|0p)~UY~l8FSj(_V{U0#>t4RSHyKGJ-PN_8 zT2EC)eUgJU`)cYuRdt@~6_UVzWQvZIyIC$kDLQ)veO1t(l5g}KrH7|E!pq>aF^w}X z>O_|%|87v0(cQogP z0MNTqi(}|}+EIaE?rXuQeP1fr1)j15VSb=FhXJtin~D3jdH)q1+i?(x(m3@0m)Z9a z{6&J3^>ysXq6Y}R53uz%Cn~$N@bkz!tWU#KeHjOutThrBcMSyel^kHMVgut= zzd%5^;ZfW{avlk5?EIUFyQbj3L#Yg|*}nwOo7yko9^0oO{Li=TAz~qm zneD4E={?QyRk%9`Ry%MA4cpVt3$GJ$C5S2=1_8ZYm3-rPZ(Y!YSu@00C6RC@I`)|SdBB-ARWhYU|E6iSv z;uq@#@`y-H4c^Ag~f>W(gS((i)wkcs?B;45^;RaiMr zg7my1Gy77w(4#wUL*X~Tj6Z<#CQ(Iw#!!(ey>Kzwit1B{b}hw~MXlBC--BsXpOv6g z5tY|xt`P~cdcANjx&g_0LD`y?<3NilPN2EeCuK297?8iLyd;F{ON4iSfADb&rb zMXdw+&(STG63jH4fz~&4OA7!S&;__OdN9nE5k>FQ;X0tofxke}WOD?;aFuRZ0`*NG zZB}HBhA5j)@?o9wSAs&{YH-DtJkq}4Nf9%^Fxmtm5PvI*UC1l{Cqh`dK)^GK(CIa+bI?l=z5 zkc;Pldm%1u5m9Z_z4qviLO8b)*wsYga<-47gllxi9&oh-yIoN%)FnPH2F$g(^C>`t z83a8}JX|jyPS@$PJ_)m^Ef0cnSZQF+i(%$|eGydxjd%*i&e?%R)PJYDXre=Kl{snR zTV&yrmb>gFB7aOkuI0t3lfKMa@HKE=Z@mUpv&<{q1R%9yEdZBgrDHd0bj~Z#yvo^0 zH9d!h{{?wsVo&({nWcgQg)aJLtFjUrkHApo8ivVL!?2Y=&2=SAt?ojWK;6{_^)mYJyaSethS6n>t}(anSm25WQ&E25G?hUW6<>tkRO@MBKZt( ztaXG@#g=O#X=c&!98dI%U0*DD2o&ftD`kvv3?C z1eQ5og%7O@sGzTwQLDNz4)g9oi|8GI^eZU34$-XRDEpZ94T$IcNTVHIiFF+prkTuo zO98q1UF%ucVLilw_pGm=R@PfN@Va#e8o_!S2i~y8QGoS!4xF{p2Elrm18-SxgT>Z6 zIPgbnDH->79C+L6K|i$K$$`IEXOp(iao`>6yQJ+d4*bb_0Zg-go&)b&o8fNj-5j{j z`W%^a4+kE!evb1K2=x<*dK=20)SU2L-U*J%`=zA( z6lOfFokx_PN|YU~`ZH}cp`MYb7lAsdc?tEbL|uteey&|csQ)QZXHd%XT7*z9NYnxJ zn_p?vhw^?cPz9QP2TaSq6SxA+J_%b*#(+oKgb1o-nY;z zfV_=5%tLU3{uD*eQ}izsy-tya#?ng=<&(aAn*NI@S42hS(`KfKaQQUT6wwM%M7c@_ z#rZVvZ_UB-=HS8#miVHCJEZ<#;Lle&<-EF0<~|%W!@uSf;w2A1&99p zytPy_;r?RLBD%x&eWE#R-=;*giSuIO6Ah!wk*wK2gVxe@+fDev;c{4?0lVe#g{uP(jOwm4yJcwA? z6VR0R9lhvXg=e$wfJ|dC@hm`f1v3}p5{`R7&O4%WhALv<=vfazQBxuD@lEa+>EjV6 z=Z`>YIZ>aJKYaPKGh+H8?Q1P~R>@Z6^Xzp;uCqkg$|ADaa`?oxjMwdp;No>lvHZPI z<|1XPE%*aFu51h1t$>s*rkv&Qv3&($Bf?gZ6_&$C_Z9w|{0ONL2nv6u(udgWvReS4 zxP7c}ul%h&!Wppck)|QGK(dVOvjo_pg?AJGbw{sfvs-1A=1TrKC*TXsU3wI-eq9q< z;0mH>Hr8HNhx-sZzHfsS7J|`^b<9${w%`*5YYN)S)-70s$-?H*)-M3ET>pe^O+lo% zrd(UMAXuh>+ioJYmt_&*pS9&0WXenCgV(W)7cgs$M!Gn@fOOA zRh_TuM_lC=>C~Gw>DL7l$n(f@?$|WLIijlEJEE!!m5B5h8BRtnD$$k}mlaqRFI?=J zhlzu`pSLej&*e2QpoS&07Mp5RKs9oMhq{F1DVL!1)p=4xbr{TJ=bKQN-rdl?Uqo!cpf`IhK$LTK7a+7IT>sFLen-qxNO0`)DuQ*iarS@NyR?2%3 z`9yTw3p|YXY3>Me?>bbpVlp?@GTA9HSyygA|0Fk6(GaUf_vYf%gKDy?yncZWi)*us zAoXI%VZdd&xdmdGjz?0~iQcuP2ZvqL+WDBvD~Hc$TkA-hik(NfvTj0T>!RpVifmPEmadioi&N9Tpq1JjFWKfW-KsrrE2YG7!sZ&W73v#G zZSy*8a~);2qWc`>wt0?5TNy{*Xe)A*IYw;mmuw65=2BbHX4|}^pcQSg6*?{>E%qI@ zD!V1@v1yh$`ex1M(wEKA523V@mu(J|IG>ol1g=7RrA>o2*9x1achA$8l{!x7%i#sz zr?*yD`+WEYI+VZ%(Xr}=_6WZA#^=#n2K_@9<3sRhtUb`yQr9-fb~S@dk-dh0Xp6r; z6ks1-#cq6l9qfaWNisE6N^f@{I%EX96R}XFzurGIgu4}OU7G_}PZp_DA{k4!Mq7lv zb-nm_y$$7=s#i0uP|%<7Lx|V2n?8Tv9ygll<9BY9sBOSMuKKP zYuVo2)47FhGx$gKSvt0ag4>MX)zOF{6ztl9@6hQ^QF-GvrUd{omy!!XF)$k}SqanYb;fziL!KbXUaX+2yQ zhvCp5I-+RRmS`jq>Q6?Kamt}FjaX;r2GO`U(Zk)cYE^ZeyxZ-r7>p0Z6T6Tmk`be^ z-vD7=zK+UOCtK!Zdwxo@P2d!_Vs=M|N8* z7L9E&;&Fe!u_4*lhlzvw{l3W~W>Sr^`|KK;PntVAI$PQ>VWy{MI?VzFhDDu*Vg^6a zV<vKso7NqjpBO5}Ac#dHzW=A<#&4XrImo}(UXfU<-aS%rbx;?L)a78TU-`f(6 z?h46CE@sfcnN-VZ3%L+NMr^ksMzm=@n{M82P;>G}ojum1dqdl{&65|h+>6rcK-lod z$PwF&L?}jU6}r<#8Zn|9$ZD`nMxrZb><(efv`5e*lYFRk1=z0k$ZkxgLa@D+(XzwH zUp~>X(zC6f#%5;>C%y>H@jZT=yz)_&wAH;hD<(pI!TB#3%X01jIIxG_s?8dVe4#^? zaL9}7>!yOH-WqgvSL1LR6MJs&)yXwB-Nl(|JYVnLgc0|#-e@$O?!4K%0q!U1A~E(l zjJ|}h6?=x5A4`AZBr~!HR!tS!WUDcpn9v^RZ+z|Pi87_;vMZQqQQpPPHJic_t+dok z;>}v(s;jBit!;8>)2h_vPYiVMMVakt8x|X{t`>~n$%>}jm~GoE-ESH)Iky1H|9sfe z9*%rCVg57MVA?TTM_saQ8yUcd#U!(*m97`l1Pyp!FxR#)yYZ@P(0DD$Sj>ndx&k>| zJ#A4J%8ePyN>_a&PAHOH-4_oegHamN7~FBAC)(M~CvA+=tp*MvVq)hssd^I(JgtSRHrhv*K|`{Ep3B$vr9tZFqWo`{&-?LU+||j2z#Hd&4X9L zxyjxS@@`GtRMQna#m0m1KW3F9EpOhsEE=cf@kMPqKL0?wVd? z%=OGYtMXm0&zDHg=;*vLe`oJi*xt#(oi)>c;E>O*Z}o+fI$c}uN9+sfNk=Yx^y_C* z#Xdy$j}PFmF?YJlU`1nmeM3X@#cWa!&8|qM;kI`Ba=Wn&BQ+XKhK=pp+gtyNQJPt8 z&AoHV9w(MgAI?m}ZoH4}Qkys$E$PXQ$sH}rY>%n#@+dZb&>H_*2Y!h2eMExsA0pKk zW`2fzz|Rl*RnG4KH2Q-_`0Wnyj3q%SI+=@zZmpP4xw$gE+~LWe}(;zs-W<_WI(Ufh>w(oq*!Mr1%dQp!wHO zN_+y`hno>%pNIQ5pnSmhRUbsJ%>Dh<2WVANtTGvokxR+Mxc{drsiHSjEun)(LC7ubwHe{v=xZlUD_;Ae3 zf~SwT6KQ3);6WS(H;-|j6KO7fdg`s?3Aro#a5~`J65nLP)7?$tSMQ+$=yRBQB8;aHJnRA9E%Vt4-NG(G4HOa2k9qhC@TRWD@n7S3*Q7_6_W|z~q7!0z z+sKu}Z$IaHCa`i~9QYhf%dEItp6lGxD#GjMq|a$o8tJt?!aJH2Mx^b)=c*Uol6PgD ztE!|fpMrnS6nJ{wkMs-w-^^0j#- z(%ZU?`8IWQZfNfCb#C0)-PYslY2MJ$2FBJ)w>EESZ-Indhxw_x6OZW?B)({89kNSR zRoj}JxALYx-nSFS*akxpI*~UhHcosC#f8*KxD!tV_R+l-armt=cY;l|z-FI@+oX=Y zr4w#t{XW#0z&;-ePJ1#KkNO7uksuBwmPY~me7*5_+7MxbI-@sTXr0>6q9c5W(}7i$ z&8G%WIFVSKL2U3FY?ICcPM3^Rer0W^K603EQkf^dcM!MOktSjy#m#+@;~TOe=852* zx(Ay{5BBr8DTN0`85G}`R^Z2vKs9e@#|`^_#@PpOmz;Tmdm|`9q<9>}DblD3fDg2o z5%!aS%ngMT%!4k57+Ft$RA%BvfO!(Q>BT&}2YO;r?hlVKAcxmL5ZV+3X>n&+3^k!r z%S-b94?a{s2%MG~(P;>`}{1HfwxY5A$n-&ZnfomOzYN#1-8 z{}RP0Q?+g@H0}Sg%UAnPmAVz9OwIE2S{RNwh^qRl?}k*W_Gxx0Z?@kLJRa?*>#x2m zQt4^wFtU-dN}itSQwZTAm#gyCcSi>Buzd`Do3)X?N_i{mM_^Ankjkd#x$O6rJ&LkQeI#)@gb?OD}&-WC8y9d zKbdhv`?ZvBEK-~zHS^NvW@a35o^z4lzsuC&&Ai*DkUzRX$nP=?;x8F fRrE$1g4)LjXtJ+qn$0Ee^$7|3dlcEh1hRhzo+wv` literal 27344 zcmeHwdwg5fo#(k%lC4;_CEIy92}DklkeB60oR9>P*s-0+&BLC=fijRPvUF{$ktJ1+ z#G!?lCShYjy5ZHK476ExO8Lx(UDDE~EiDiz4NRd-7^W%gur$+_!5L^53Z>9eyx-q> z$XC~rDc$+ZXaDd`bkF%c&+q)s?>z47_(1!Xjk=~Wb=cTC#-R#gAPjRH+3FC<5Mrkj+Lk6cD$MnQ{HNy zf~|7O&Mql;QZiW6Hd&rZRe65}IoY(5W8y@G$97p>q4pb;?VwVM6FR6|(m}DdY1>Yw z53$pPGt;L6CU+Cv_(91dJM?c{^3zA|cxcW8hpLk=HosFBjr6Q+s*8qeqmfubLw1He`0!IM639{dZy<>NnE z0B?t)eEb^=@P`WE$%kJ2tjEFv`j>&xYh?!PDZqaMw0w46D1cvDKu>Q0{F?>zxC-!J zUjW};0Dl~K+4WG;*zt8_E7)vyx>?{!JeogX!$9DR4^7vN3J!Jfk+H;v0yYXUlQIIqUX&R~8tF_jrg#N)AQ+9tLxBbcYeEAFMNArr zIO-dX2g4w79@d*Q$XH%bj_`7pG=gD8%EXil0)5eVPcWKNl)XkWMdl4eQUk$ss6P%`X!9Z^W4um%u$Yc-o@l^zbD7TVr+S0M1HPGl=-jvNY z`c_cOY9IN}1`ZuRaZvsN0BU^u)-7Mc@L^-8BrY>6GKGQXaVc^Y{$@_j&O-_wvf!J&g6~cX{#J=UVZq-i@oK)M>c}b~$Jpuh zJmF)fF0%amv~)H zGT~(?5So3;gfBRGan8xpqz5KFFzJCw5Bx`Xz<=-!r~jzq+jWfj51&r!qi6jGpLG5( zt9o?Bk3kw;@GQ>mxy{HC+)wli=SOiYc!FS>x-OjM@WTYt)Oz7GhrdZMO<@;Ka`^8F zre*1c6CA#mV4BJ<9Ov*|1kg&_{#N-#~27ZM!4iC~(dFYM;<7YL>)??M-cuO*nK z>I=;rP7+L0--UV(|H8*u?{UEVX!v!<@bjPc5C6h{@Kz>;-CjYkk;y(EEJn3P!Z@9Ayaia>g|f5dll+B!Ant@r%M~pJUvbt3!UxT021$MDM?kH2ASdFs%7HZ*$REgY1$^St(3$~%#_zy0>x{)3NDxpw!B zFCldNM-P)R4^ve5kColeRX^t+ern{`e;OSHUAIQIRKsH)*YGY9mm?w3$w41X3qohyU4TQCpuQYlJ+5V%2%Fm_kZ_y^5_^A zaiZda(Eg*XgJ&3feP)LYB2u6jxH61SZwy`Ca!t#wmQU~cp*gB=cq2irbL^fW#4V5J zSC&GAq8a#+FZ>a1%trG;!sP^X5@Uo6==-QCh6@Jtd;}a~_-(Rlk<51-JA4|&K0JyN z(EyK;ts|pQR@d?ws=xpWuzRJ?-JIg$*f{P0=-4NsDja{Tuw5Hz5D zb_tpO)W|Tb7M>j`M|R|UI4Pd3k|)D?NRH-@oEd`(nmi^wFzJCw4@`Ps(gTwonDoG; z2mXKc06iC&=xFpdt@1W_o4lbljjOyXn~+=O#bb|Xbj|Wc2yWQ9sb(VikD@hrmp6JF zSFH4|^!8+W*EBV-CB7}K4GfQ|`iyj{jz6UlfG-4P+HBipxIAWbLm=^gMn~yFzWGgY zrO}0SH{jK8jgFp0>V9W*^gX1VzZ)I(Vv!p`+DwaG9O|CEcI|ei4SKZMrNz$UTCs=l z^v=Nubk8G3dF1D3Y(D73|Ke{%nXKHiv3%wy-L8GkA+~>h$%y1G*b z$zOwC5VoJArf4bm+^Dyf6>a{a?U?;$$H5}o86CC}-y!_mZ_7F;zHM2)S`B<;%LDjb zg1XN_n$@=xM1Vwp7QY(MNoSTG5OhM>@uPe8YAil!s`96*%FrO7{8_fPQC9%*AsLo`tLp*jT3y5E*NmogIzjC zWQHE>(($6C(}P+%ej@Ex?5Ve|l1%TD=_#3>k?A>^{+moydHu~Qmzy?w1+3zvS(4TJ zed>Kai5{11YHeNPUDRguM1nDILzA!3S6|z(da=y%%Uuy7`5Ryv6zZ%3Q77mZIuV_OoOTF$v~Ox|3TedCsCN^hNqa~jSj}If&uFKlf)seFwiD(%ntKQUn-?JN zn`bjp!BQAJMC$hsxRa~v;AcP z775@JSD}{r5)Lp=c?LaIUnU^jaw&foJyG{bSYz|)l?=Vn^%#`O!!z{|)QxJt0H@lKC8&A=nc4OieA)*R-6X@VNh;YR3Xe+ z%JkC9&^EKcLPOzqh-L1W{=1;KLU&$^Fnb21b6E`}z>2;KaGSeRoE zs&SF-{O?GdFryqlZaXWoL0^sTJOd2XVq1@<0L=(QQC`9q+W=+0&oK$?d@xiKs2_!H1eb*B@_UBG@#$w_Z&z}M?e9}4{* zNKabGi-2FHJ8!^T@ghjCC^9Qr0T-^;onh4delWcY$_GRh^*K|{*kyX@0<;y?XDXaF z3qKQeonCYtm8SYMfYYjoaEobEm- z-X#FfjBfTVC}7T7sGCad9Mk^=ExwRora9cmCv;mYfHGvO@Y5EP%1el%e^%!+ueKOK zlcLGyCerynNd9S1U&!UiR{S;iZE6O^U8kQx068doK(~Dd6&rr)oNYHI3lg7RIZ zfw>1zi<|X1R7bS&dDLfGE!tTBjqagw5QnE~8jXufZG24gRy|DQS^;?$UksJ{O1tYp za2~LC!BX3d%1!{IONs$_Y)hQ?pyXNf9otg(UUI}N8XcAuw;}6Xnhy3cTP+7lJvC5W zw;nuCQN2A^K(KxuOd?S8WMFE850xZP_soL&#;?G01cpngv?kk^q2YG;=`Q@V&rxZd zu)=m0`TqgrBZ^=nTWn;6U|DJVN9Z~W@~h)nBwq#`Yn&}8<2~?vNPI)Qj9 zK)OMv!#GWO7B%?3b{0M^{(-g-mFxP&J18>=B zOt62E1HZLjgaYjUl>=|vUm+8YbKrOO0RnE}z`OPz5ZkRB_`RJ*Rr_rm_`v?JFu;C0 z2i~*On}_y0IPkuG8BxB(fe-DkBSP$d!+{6v7vN<3ogDbS{Wz*@|1t-DY(Gy5?&82R zI)JKC1naJnt8)TV&9 z_(u};4n?Fkhfq&T)K5{}A8RWK^$bT9|8Gh81J#5*0%!=d>>n>Sq#l2Bo~D-9f1TB~eEaH$T@XhKkP%luOeSa7xMdfpcj^Uxped z4exb!2fkZkP8xY#&f;~bmrHXV2ic_;KSa}yvv@U-`jc>i{yd#tq0{ShdY?{H(DZsW zP9>zTgd(Aga%EIh39WU?2v-#onmzQ9G#BP=?*&GOQ#3vMDHaw7P%|VK;`l)U~|!olA|bXicMWMHqk`v zn7fc|Lm<b_M0_{{o^tS~Wy?(Xd5? zKaF28*`ly8zQ%Q#YqhJR>hkJ22m*&sTVD-ijeeP9wJTQMSfgED9j?;AU1TA3RGEly&zc$y zGBp)5!RuVe3z)K6BV8QtA{`}e=Bja0*{=D_`tXvgXhPg&Cms<=) zuXUG^?ZV8Cs#P%E!OC*zf@nM$TrPG>eJ*~Dv&FqFDcUFC$S zI?Q&}%oOTtiplb67Pg`&+&P_F4sFs~IWCz|hk~f8GgbYFtHvgsdXpyoTAf3lL6&pJ zW*KIQs&em$s?Js-(qm*e89ApyTTou*vdx`6*Mqw>7p7eMhB2;LV#mzH6 z&x6M0kfzjlC>F?;QkO;?+@ha*U@YvaSiyUJb1@k*J}iC>sjkh{-OeOYUm)<;bRtfD z2C-j4Cr)i49DN{8x@o%1tGkFM*W2);%daQ#dkw!S7kWICr3Sm)VyGGYYwOWMXky)jp=snoTHKw=hF_)K}V^+q0%v9i(|U8%2D=!v&J#Q zxzKTOFm&t4T}IryTV~wy4jc*{126HHYUl zM}q4`DZOLGZi}~y z!kc>VeR?~DEw)Z96b`0?5aLGfpfB8ari>Q)zI{h1o-l;zp)QQqi8R_~@MOMD7{Wv^6-$GKqAO z?B1N?Nyb9qR>0~6&I2Dp?=H-E4pA|`s11w8H7RB z#x|=(G@@b20;6wRa3GHn6MDEQ1;a@L#dB*smX7pg;+YiX(27Q~vvY$8kFoaN9_i~( z=d?JsbLjK(Qzn**^u>&@7gZf+_JoF9+GeEE$vAIp(*_P{9*2ocsy{%HFYMgJ_ejVw zft!b@PIkp3DTHEpt1%Ey?w7qX5T%aBd0J@ih9IwltXO+88BcCCQmJ5{u_4pji(WyE zdHwh>7*`{6L#(Dw(z0bsXKOpUHp&xwS0GOXMn#&pnVM#6+HU8PE)_H7Mf=ZGz6QeHXXiexglzcn7;6OlbPX;3tbtL21+TnZs0 zxz`Y3YU!jCjby9Tfb(&?#OiD`v~S-$e#mXiJ2I>YMU7yR9I@RLlkF1w53Q_i0$c!?Zv1f3hQWNG!8TJ zaXRn7k0q+*pniX(F8;!Z+ zcuiw{Lj%>itz8ZWG(2|&)BRic?8Nr84~p$6Kz=p+O=Lrf3{jcHTOy6y1z^**FPvb2l`i zxmz>Iq!CMZh4M7cgvMPe*EFy$>#U8~xJP!5Kq{08$7$fe5Rx*w6cZn<@+~Ej-acx*SV1RQbwCG>QpuV=$H88QT}Z zGcc=<%^t(Dp=iZ z8Uenhlmo>$izkfQ?2Kk@=KRAkUnd3v>FiX17)}N|dp?898aXPMrihOmE6nw6xO80X zwfB6?kjRc!a>k_V2Ye6UZC$#wp-H}B;jJA=^{3K%kft*+V{M;-1!^P&FqjPWuN7O@ zd29Q;wVjRLTD%{zwl9|9^0mEP9j)Hl-dMc0CliULBQXIOgL0Zo2{@TfN05sTMAEgr zv^JafUwq?Y=HQBaFSS!@#sQh@XFFSpzA6t%jvmFJtGQJK#T=PkPT)#B8+gO}# z+Y!hgHSHLm;^9ox*txT#?JpRc##SYkHA8`!6&*}0F=(eZc(0f3Q43@mBeFw0lUECD z?9zkk&bK?0yRz}E*Z9{%a7KzRy3(vSbM3Wa=QSQ&4VR9E)<=XCU8?gH{n}tSjMb(e|oJ*av`DvE1Trb zjEH60tWlYfz!U)Qxj}LsMNO7K&!VhG3Myc*Y*OjN`fr zvZVIsaxjcls0-{*?@t(1k4y|(C5Dl~)dcQq@JtxCAyfFk-8g(t_137B0}l&v3$QA? zf2vn9;4Wrf{)>UvvFmPqg=Cb8SK@bD@XIA$y*Db^SqCy2_J@43SzyRMI_N7ejZM$) z*T<1KeI7q5SG|v_@JEo*@OGx!4^ZL1j!eG#ezySr4Dh61z2`bhD*CC!tM_5&OMD^! z{0ejpJ9>KNNk&tW&z|3MdIdY_H8CRVpxdXcI{SX9qR!^{8O-_~cv~JyPFlK zNa+=iO4N(Vc2Q={haUN4Ek}T!5PEYk0$5*we<$H}H&gHRD$%P8@b4>tr+r5&^r_kR zf$;!=A9RB`pPeTQ;GYHFD`m~e2=Ep3cCaMkVKP5{4}3oTA8>s6_$QX!&r;Zw%m&_* z$N!7C|4ocCyAJqyls5dI2S0xf&i_kUvjb%5?&tkAFb{{6>y1ANLr03He8-C!op$ zGTjkMrW<^5S|s&E0`$5R5UDg)hx)Si^C-HESS@yAP>AFf+(7 zt0kh?H#Y36&(A@du0{euy3ZECrD<|MHiX5hG7!!T4A4fkCJ1v}B%Q;`=S1_w%@Nt} zlLM(tPoOPOUthm^bpYiP*df6N=-#-!Wovt&eOnvtz#G7FC(!PfNPpXQ7TC0J6a(jr)0i~Ztixzg2dr#0CVZPR8zSRB!ToNHoCq;@YG?9zp+Q)}gs}Dx(0k#$7<>qmBrsIIpQ1=)hH8A5@5&^`cV>y(&M}ueqJU0X+Mo z;;2|)=r*qp&B%PT@I}uD&gjYXK|C1i!+0XWNUS%m^1+@S+(T9nc4IeG9u8GmLL)Yk zmI3WA1L=DkyorD{kvP=8_$u{Eyh_x3tzde8$1I<~I8291)jCRGX|0xfv;f2qoh`r6 z-(^rz#A`liefciFc~SPCl;x{*kz~dT82qr>zXMpz4%zb6cS0&vfA;`y)o?qNJUy1D z<7)hGS^y3oL(Qm?Eq$2Wr<7Oi$GnnNrbIU%UNeCFDg3B>C9keeRN7|YxBCA{ zi~M_DA)wMZq}+5UzbVvn$e70u>)-eADn^-F%YPYo@~Q0h+3O1RcQmZ!DOjbiL$=%` zuf9V(UJE1+YkMg^Y{a{Z2wVq(aV_R zVkNKkn?GGg+<3QBB-yjFTcxECHp{E;97m-5U{+FOl$=VZS>$&&3%=d;G8IYo?9Jvx zhOTo-SJwV!fj=$fRXGAXR(}P1W%-g_p_!7Oo5hF>)s+sFE|&5FJ9hrLRM<6!;yER! z(5ni_*EI-^5Z=f$A6EIc0`j-66H@dMnE9~EZ!aLfqE$%I*KOv*0D?1K;AC4@nIX(z#`QsIdf`Z;hg>}g+Q|$Kj#Ca_X|}#sQHJ+M-@9lLY$A# c`)oKiTZpP$L8WW({PHgZ1jl}hfCXg#7w=)phyVZp From f7f424edfc7e23b6b6a4983d7b10a5eddc3aedd2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:46:20 +0000 Subject: [PATCH 3/6] Fix removed --meta-elo flag in benchmark scripts and report.py --- AGENTS.md | 6 +++--- README.md | 2 +- docs/TODO.md | 2 ++ .../reports/fuzzer-optimization-journey.md | 2 +- src/fuzzer_tool/services/report.py | 2 +- tools/bench.sh | 10 +++++----- tools/bench_sweep.sh | 15 +++++++-------- tools/bench_sweep2.sh | 14 +++++++------- 8 files changed, 27 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b63e4967..b7967841 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,9 +99,9 @@ Fuzzer state is saved to `{corpus_dir}/state.json` on shutdown. Use `--resume` t - Transitions accumulate across sessions ### Meta-Scheduler (Elo Arbitration) -- `--meta-elo` enables Elo-based arbitration between bandit and MOpt strategies -- Requires `--elo --mc-bandit --mopt` to be active -- Both strategies run in shadow; Elo picks which one to trust each iteration +- `--elo` alone now enables Elo-based arbitration between operator strategies (bandit/MOpt/replicator) and seed strategies (ga/weighted/pareto/format); the separate `--meta-elo` flag was consolidated into `--elo` (see `_use_elo` in `services/fuzzer.py`) +- Enable `--mc-bandit`/`--mopt`/`--replicator` alongside `--elo` to add those strategies to the arbitration pool +- All available strategies run in shadow; Elo picks which one to trust each iteration - Strategy ratings tracked in `elo.json` under `strategy_ratings` / `strategy_match_count` - Probabilistic selection via softmax over Elo gap (temperature=400) diff --git a/README.md b/README.md index 0690d194..19ce9d09 100644 --- a/README.md +++ b/README.md @@ -281,7 +281,7 @@ tools/bench.sh targets/png_read 10000 Configurations: - **baseline**: no features -- **enhanced**: elo + meta-elo + bandit + mopt +- **enhanced**: elo + bandit + mopt - **enhanced+**: all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar - **optimal**: elo + mopt + replicator + markov ensemble (orders 0,1,2,3) + markov-gen - Best edge coverage at -n 1k (sweep-validated: 74 edges vs 61 baseline, 70 enhanced+) diff --git a/docs/TODO.md b/docs/TODO.md index 4aad46f0..5afcfa4a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -16,6 +16,8 @@ - [x] `--stats-file` eaten by `-A` (REMAINDER) — user must place `-A` last - [x] CEM never engaging — refit now triggers at elite_set >= 10 - [x] dict_insert/dict_replace 0/0 — was missing `-D` flag, now works +- [x] `report.py` referenced removed `_use_meta_elo` attribute (AttributeError when `--elo` + `--report` used) — updated to `_use_elo` +- [x] Benchmark scripts (`tools/bench.sh`, `tools/bench_sweep.sh`, `tools/bench_sweep2.sh`) still passed removed `--meta-elo` CLI flag — removed, docs updated ## Pending Bugs - [ ] `_apply_single_mutation` havoc doesn't enforce max_len strictly (allows +1 byte per insert, up to +8 total) diff --git a/docs/compose/reports/fuzzer-optimization-journey.md b/docs/compose/reports/fuzzer-optimization-journey.md index d3908033..d85cdff5 100644 --- a/docs/compose/reports/fuzzer-optimization-journey.md +++ b/docs/compose/reports/fuzzer-optimization-journey.md @@ -58,7 +58,7 @@ fuzzer-tool fuzz targets/png_read -d corpus/ -c -D dictionaries/png.dict # Full feature stack (as benchmarked) fuzzer-tool fuzz targets/png_read -d corpus/ -c \ - --elo --meta-elo --mc-bandit --mopt \ + --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --mi-guided --transfer-entropy --replicator \ --secretary --pairwise-blend 0.5 diff --git a/src/fuzzer_tool/services/report.py b/src/fuzzer_tool/services/report.py index 3232254e..2aed2acb 100644 --- a/src/fuzzer_tool/services/report.py +++ b/src/fuzzer_tool/services/report.py @@ -903,7 +903,7 @@ def _elo_ratings(f) -> str: lines.append(f" {i}. {op:<20s} {rating:>7.0f} ({sign}{delta:.0f})") # Meta-scheduler strategy ranking (bandit vs MOpt) - if f._use_meta_elo and f._elo: + if f._use_elo and f._elo: strategy_ranking = f._elo.get_strategy_ranking() if strategy_ranking: lines.append("") diff --git a/tools/bench.sh b/tools/bench.sh index 5f615780..7b464dd9 100755 --- a/tools/bench.sh +++ b/tools/bench.sh @@ -9,8 +9,8 @@ # # Configurations: # baseline: no features -# enhanced: elo + meta-elo + bandit + mopt -# enhanced+: elo + meta-elo + bandit + mopt + markov + replicator + shapley +# enhanced: elo + bandit + mopt +# enhanced+: elo + bandit + mopt + markov + replicator + shapley # + renyi + transfer-entropy + grammar # optimal: elo + mopt + replicator + markov (ensemble 0,1,2,3) + markov-gen # Best edges at -n 1k (74 vs 61 baseline) and -n 10k (184 vs 167 baseline) @@ -171,9 +171,9 @@ cleanup_shm sleep 1 # Run enhanced -echo "[*] Running enhanced (elo + meta-elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..." +echo "[*] Running enhanced (elo + bandit + mopt${EXTRA_FLAGS:+$EXTRA_FLAGS})..." run_with_retry /tmp/fuzz_bench_enhanced.log \ - fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --meta-elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG + fuzz "$TARGET" -d "$ENHANCED_DIR" -c -n "$ITERS" --elo --mc-bandit --mopt $EXTRA_FLAGS $REPORT_FLAG echo "" # Clean SHM between runs @@ -184,7 +184,7 @@ sleep 1 echo "[*] Running enhanced+ (all enhanced + markov + replicator + shapley + renyi + transfer-entropy + grammar)..." run_with_retry /tmp/fuzz_bench_enhanced+.log \ fuzz "$TARGET" -d "$ENHANCEDP_DIR" -c -n "$ITERS" \ - --elo --meta-elo --mc-bandit --mopt \ + --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy \ -g dictionaries/png.gram \ diff --git a/tools/bench_sweep.sh b/tools/bench_sweep.sh index b49e7cd4..5b294cfc 100755 --- a/tools/bench_sweep.sh +++ b/tools/bench_sweep.sh @@ -95,7 +95,6 @@ echo "=== Phase 1: Individual features ===" run_combo "baseline" run_combo "elo" --elo -run_combo "meta_elo" --elo --meta-elo run_combo "bandit" --mc-bandit run_combo "mopt" --mopt run_combo "markov" --markov --markov-gen --markov-order 0,1,2,3 @@ -117,8 +116,8 @@ echo "=== Phase 2: Scheduling combinations ===" run_combo "s1_elo_bandit" --elo --mc-bandit run_combo "s2_elo_mopt" --elo --mopt -run_combo "s3_elo_meta_bandit" --elo --meta-elo --mc-bandit -run_combo "s4_elo_meta_mopt" --elo --meta-elo --mopt +run_combo "s3_elo_bandit" --elo --mc-bandit +run_combo "s4_elo_mopt" --elo --mopt run_combo "s5_bandit_mopt" --mc-bandit --mopt run_combo "s6_elo_bandit_replicator" --elo --mc-bandit --replicator run_combo "s7_elo_mopt_replicator" --elo --mopt --replicator @@ -152,25 +151,25 @@ echo "=== Phase 5: Game theory additions ===" run_combo "gt1_sched_shapley" --elo --mc-bandit --shapley run_combo "gt2_sched_rep_shapley" --elo --mc-bandit --replicator --shapley -run_combo "gt3_sched_all_game" --elo --meta-elo --mc-bandit --replicator --shapley +run_combo "gt3_sched_all_game" --elo --mc-bandit --replicator --shapley # ── Phase 6: Full combos (best of each) ─────────────────────────── echo "" echo "=== Phase 6: Full combinations ===" -run_combo "f1_enhanced" --elo --meta-elo --mc-bandit --mopt -run_combo "f2_enhanced_plus" --elo --meta-elo --mc-bandit --mopt \ +run_combo "f1_enhanced" --elo --mc-bandit --mopt +run_combo "f2_enhanced_plus" --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy run_combo "f3_lean_best" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f4_full_kitchen" --elo --meta-elo --mc-bandit --mopt \ +run_combo "f4_full_kitchen" --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight --transfer-entropy \ --mi-guided --sensitivity --secretary --mc-cem --anneal-budget 500 run_combo "f5_elo_markov_renyi" --elo --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f6_bandit_markov_renyi_transfer" --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy run_combo "f7_elo_bandit_markov_renyi_transfer" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy -run_combo "f8_elo_meta_bandit_markov_rep_shapley" --elo --meta-elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley +run_combo "f8_elo_bandit_markov_rep_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley run_combo "f9_elo_mopt_markov_renyi_transfer" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy run_combo "f10_elo_bandit_markov_shapley_transfer" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --shapley --transfer-entropy run_combo "f11_elo_bandit_rep_markov_renyi" --elo --mc-bandit --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight diff --git a/tools/bench_sweep2.sh b/tools/bench_sweep2.sh index 1c60ff06..13013592 100755 --- a/tools/bench_sweep2.sh +++ b/tools/bench_sweep2.sh @@ -76,19 +76,19 @@ run_combo() { echo "=== Phase 5: Game theory additions ===" run_combo "gt1_sched_shapley" --elo --mc-bandit --shapley run_combo "gt2_sched_rep_shapley" --elo --mc-bandit --replicator --shapley -run_combo "gt3_sched_all_game" --elo --meta-elo --mc-bandit --replicator --shapley +run_combo "gt3_sched_all_game" --elo --mc-bandit --replicator --shapley echo "" echo "=== Phase 6: Full combinations (top candidates) ===" -run_combo "f1_enhanced" --elo --meta-elo --mc-bandit --mopt -run_combo "f2_enhanced_plus" --elo --meta-elo --mc-bandit --mopt \ +run_combo "f1_enhanced" --elo --mc-bandit --mopt +run_combo "f2_enhanced_plus" --elo --mc-bandit --mopt \ --markov --markov-gen --markov-order 0,1,2,3 \ --replicator --shapley --renyi-weight run_combo "f3_lean_best" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f5_elo_markov_renyi" --elo --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f6_bandit_markov_renyi" --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f7_elo_bandit_markov_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f8_elo_meta_bandit_markov_rep_shapley" --elo --meta-elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley +run_combo "f8_elo_bandit_markov_rep_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley run_combo "f9_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "f10_elo_bandit_markov_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --shapley run_combo "f11_elo_bandit_rep_markov_renyi" --elo --mc-bandit --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight @@ -97,7 +97,7 @@ run_combo "f14_elo_mopt_rep_markov" --elo --mopt --replicator --markov --markov- run_combo "f15_elo_bandit_markov_mi_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --mi-guided --renyi-weight run_combo "f16_mopt_markov_renyi_shapley" --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley run_combo "f17_elo_mopt_rep_markov_renyi" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f18_elo_meta_mopt_markov_renyi" --elo --meta-elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight +run_combo "f18_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight echo "" echo "=== Phase 7: Advanced tuning ===" @@ -117,8 +117,8 @@ run_combo "z1_best_a" --elo --mopt --replicator --markov --markov-gen --markov-o run_combo "z1_best_b" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 run_combo "z2_second_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 run_combo "z2_second_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z3_elo_meta_bandit_a" --elo --meta-elo --mc-bandit -run_combo "z3_elo_meta_bandit_b" --elo --meta-elo --mc-bandit +run_combo "z3_elo_bandit_a" --elo --mc-bandit +run_combo "z3_elo_bandit_b" --elo --mc-bandit run_combo "z4_elo_mopt_renyi_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight run_combo "z4_elo_mopt_renyi_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight From 6112ed296ef073e212b04a4daac653a9014aceb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:02:34 +0000 Subject: [PATCH 4/6] Consolidate benchmark shell scripts into shared lib + merged sweep script --- AGENTS.md | 6 +- README.md | 5 + docs/TODO.md | 1 + tools/bench.sh | 149 +++--------------------------- tools/bench_sweep.sh | 92 +++++-------------- tools/bench_sweep2.sh | 132 --------------------------- tools/lib/bench_common.sh | 187 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 233 insertions(+), 339 deletions(-) delete mode 100755 tools/bench_sweep2.sh create mode 100755 tools/lib/bench_common.sh diff --git a/AGENTS.md b/AGENTS.md index b7967841..7add1a64 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,11 @@ src/fuzzer_tool/ tools/ ├── corpus_png.py # PNG corpus generator for libpng fuzzing -└── release.sh # Release automation +├── release.sh # Release automation +├── bench.sh # 4-way config comparison (baseline/enhanced/enhanced+/optimal) +├── bench_sweep.sh # Exhaustive feature/combination sweep +└── lib/ + └── bench_common.sh # Shared helpers for bench.sh and bench_sweep.sh dictionaries/ └── png.dict # PNG format tokens diff --git a/README.md b/README.md index 19ce9d09..dbe40855 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,11 @@ Configurations: - **optimal**: elo + mopt + replicator + markov ensemble (orders 0,1,2,3) + markov-gen - Best edge coverage at -n 1k (sweep-validated: 74 edges vs 61 baseline, 70 enhanced+) +For a broader sweep across individual features and many combinations (instead +of these four named configurations), use `tools/bench_sweep.sh`. Both scripts +share common helpers (SHM cleanup, log metric extraction, coverage +verification) from `tools/lib/bench_common.sh`. + ## Troubleshooting ### Zero edges discovered (ASan + LD_PRELOAD conflict) diff --git a/docs/TODO.md b/docs/TODO.md index 5afcfa4a..a46da11e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -11,6 +11,7 @@ - [x] Sanitizer regex groups closed (ASAN/TSAN) - [x] Timeout crash detection fix - [x] Ptrace initial SIGTRAP crash detection +- [x] Consolidated benchmark shell scripts: shared helpers extracted to `tools/lib/bench_common.sh`; merged `tools/bench_sweep2.sh` into `tools/bench_sweep.sh` (deduped identical combos, kept all unique variants) ## Bugs Fixed - [x] `--stats-file` eaten by `-A` (REMAINDER) — user must place `-A` last diff --git a/tools/bench.sh b/tools/bench.sh index 7b464dd9..014ff052 100755 --- a/tools/bench.sh +++ b/tools/bench.sh @@ -14,6 +14,9 @@ # + renyi + transfer-entropy + grammar # optimal: elo + mopt + replicator + markov (ensemble 0,1,2,3) + markov-gen # Best edges at -n 1k (74 vs 61 baseline) and -n 10k (184 vs 167 baseline) +# +# For a broad sweep of individual feature/combination effects instead of +# these four named configurations, use tools/bench_sweep.sh. set -euo pipefail @@ -28,122 +31,8 @@ OPTIMAL_DIR="/tmp/fuzz_bench_optimal" REPORT_FLAG="${BENCH_REPORT:-}" # set BENCH_REPORT=--report to generate full reports cd "$BASE_DIR" - -# ── SHM cleanup ─────────────────────────────────────────────────────── -# Remove all orphaned SHM segments owned by the current user. -# Previous fuzzer runs (especially those killed by signals) leave -# segments behind. Accumulation can cause shmget to fail or the -# target to attach to stale segments. -cleanup_shm() { - local before - before=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) - # Remove all segments owned by current user - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done - local after - after=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) - if [[ "$before" -gt 0 ]]; then - echo "[*] Cleaned $((before - after)) orphaned SHM segments ($before -> $after)" - fi -} - -# ── SHM verification ────────────────────────────────────────────────── -# After a fuzzer run, verify that the SHM bitmap actually received data. -# This is more reliable than checking log messages — it checks the -# actual SHM segment that was created during the run. -verify_shm() { - local log="$1" - local label="$2" - - # Extract the SHM ID from the log - local shm_id - shm_id=$(grep -oP "SHM bitmap, id=\K[0-9]+" "$log" | tail -1) - - if [[ -z "$shm_id" ]]; then - echo "FAIL: $label — no SHM ID found in log (coverage not enabled?)" - return 1 - fi - - # Try to attach and check if bitmap has any non-zero bytes - local has_data - has_data=$(python3 -c " -import ctypes, ctypes.util -libc = ctypes.CDLL(ctypes.util.find_library('c') or 'libc.so.6', use_errno=True) -libc.shmat.restype = ctypes.c_void_p -ptr = libc.shmat($shm_id, None, 0) -if ptr is None or ptr == -1: - print('FAIL') -else: - size = 4096 # default map size - bitmap = (ctypes.c_uint8 * size).from_address(ptr) - non_zero = sum(1 for i in range(size) if bitmap[i] != 0) - libc.shmdt(ptr) - if non_zero > 0: - print(f'OK:{non_zero}') - else: - print('EMPTY') -" 2>/dev/null) - - if [[ "$has_data" == FAIL ]]; then - echo "FAIL: $label — SHM segment $shm_id could not be attached" - return 1 - elif [[ "$has_data" == EMPTY ]]; then - echo "FAIL: $label — SHM segment $shm_id has 0 non-zero bytes (coverage-blind)" - return 1 - else - local nedges="${has_data#OK:}" - echo "[+] $label — SHM verified: $nedges non-zero bytes in bitmap" - return 0 - fi -} - -# ── Coverage-attachment sanity check ────────────────────────────────── -# Combine log-based and SHM-based checks for maximum reliability. -check_coverage() { - local log="$1" - local label="$2" - - # Check for explicit SHM failure messages in the log - if grep -qi "SHM not attached\|AFL shim area is NULL\|shmat.*failed\|Coverage data will be empty" "$log"; then - echo "FAIL: $label — SHM coverage did not attach (coverage-blind run)" - return 1 - fi - - # Verify actual SHM bitmap has data - if ! verify_shm "$log" "$label"; then - return 1 - fi - - return 0 -} - -# ── Run with retry ──────────────────────────────────────────────────── -MAX_RETRIES=3 - -run_with_retry() { - local log="$1" - shift - local attempt=1 - - while [[ $attempt -le $MAX_RETRIES ]]; do - echo "[*] Attempt $attempt/$MAX_RETRIES..." - python -m fuzzer_tool "$@" 2>&1 | tee "$log" - - if check_coverage "$log" "attempt $attempt"; then - return 0 - fi - - echo "[*] Coverage did not attach. Cleaning SHM and retrying..." - cleanup_shm - sleep 2 - attempt=$((attempt + 1)) - done - - echo "FAIL: Coverage failed to attach after $MAX_RETRIES attempts." - echo " Last log: $log" - return 1 -} +# shellcheck source=lib/bench_common.sh +source "$BASE_DIR/tools/lib/bench_common.sh" # ── Main ────────────────────────────────────────────────────────────── @@ -211,28 +100,6 @@ echo " COMPARISON" echo "============================================================" echo "" -extract() { - grep -oP "$1" "$2" | tail -1 -} - -# Extract CI values from crash/timeout rate lines (format: "rate% ±1σ: lo% ±2σ: lo% ±3σ: lo%") -extract_ci() { - local log="$1" - local pattern="$2" - local line - line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo " - - -" - return - fi - # Extract the three CI values: ±1σ, ±2σ, ±3σ - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--} ${ci2:--} ${ci3:--}" -} - b_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_baseline.log) e_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced.log) p_edges=$(extract "Edges discovered:\s+\K[0-9]+" /tmp/fuzz_bench_enhanced+.log) @@ -264,6 +131,12 @@ e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:") p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:") o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:") +# extract_ci returns "|"-joined values here; normalize to space-joined for display +b_crash_ci="${b_crash_ci//|/ }" +e_crash_ci="${e_crash_ci//|/ }" +p_crash_ci="${p_crash_ci//|/ }" +o_crash_ci="${o_crash_ci//|/ }" + printf "%-25s %12s %12s %12s %12s\n" "Metric" "Baseline" "Enhanced" "Enhanced+" "Optimal" printf "%-25s %12s %12s %12s %12s\n" "-------------------------" "------------" "------------" "------------" "------------" printf "%-25s %12s %12s %12s %12s\n" "Edges discovered" "${b_edges:-?}" "${e_edges:-?}" "${p_edges:-?}" "${o_edges:-?}" diff --git a/tools/bench_sweep.sh b/tools/bench_sweep.sh index 5b294cfc..7817c275 100755 --- a/tools/bench_sweep.sh +++ b/tools/bench_sweep.sh @@ -3,6 +3,9 @@ # Tests individual features and combinations to find the best configuration. # # Usage: tools/bench_sweep.sh +# +# For a simple 4-way baseline/enhanced/enhanced+/optimal comparison +# instead of this exhaustive sweep, use tools/bench.sh. set -euo pipefail @@ -15,76 +18,14 @@ GRAMMAR="-g dictionaries/png.gram" REPORT_FLAG="${BENCH_REPORT:-}" cd "$BASE_DIR" -mkdir -p "$RESULTS_DIR" +# shellcheck source=lib/bench_common.sh +source "$BASE_DIR/tools/lib/bench_common.sh" -# Clean SHM -cleanup_shm() { - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done -} - -# Extract metrics from log -extract() { - grep -oP "$1" "$2" 2>/dev/null | tail -1 -} - -# Extract CI values from crash rate lines -extract_ci() { - local log="$1" - local line - line=$(grep -P "Crash rate:" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo "-|-|-" - return - fi - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--}|${ci2:--}|${ci3:--}" -} +mkdir -p "$RESULTS_DIR" # Results CSV echo "combo,edges,corpus,eps,duration,exec_p50,collision,crash_ci1,crash_ci2,crash_ci3" > "$RESULTS_DIR/sweep.csv" -# Run a single combination -run_combo() { - local name="$1" - shift - local flags=("$@") - local dir="/tmp/fuzz_sweep_${name}" - local log="$RESULTS_DIR/${name}.log" - - rm -rf "$dir" - mkdir -p "$dir" - cleanup_shm - - echo "[*] Running: $name" - python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ - $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true - - local edges corpus eps dur p50 coll crash_ci - edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") - corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") - eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") - dur=$(extract "Duration:\s+\K[0-9s]+" "$log") - p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") - coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") - crash_ci=$(extract_ci "$log") - - local ci1 ci2 ci3 - ci1=$(echo "$crash_ci" | cut -d'|' -f1) - ci2=$(echo "$crash_ci" | cut -d'|' -f2) - ci3=$(echo "$crash_ci" | cut -d'|' -f3) - - echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" - printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" - - cleanup_shm - sleep 1 -} - echo "============================================================" echo " Feature Combination Sweep: -n $ITERS on $TARGET" echo "============================================================" @@ -116,8 +57,6 @@ echo "=== Phase 2: Scheduling combinations ===" run_combo "s1_elo_bandit" --elo --mc-bandit run_combo "s2_elo_mopt" --elo --mopt -run_combo "s3_elo_bandit" --elo --mc-bandit -run_combo "s4_elo_mopt" --elo --mopt run_combo "s5_bandit_mopt" --mc-bandit --mopt run_combo "s6_elo_bandit_replicator" --elo --mc-bandit --replicator run_combo "s7_elo_mopt_replicator" --elo --mopt --replicator @@ -177,6 +116,9 @@ run_combo "f12_elo_bandit_markov_renyi_shapley" --elo --mc-bandit --markov --mar run_combo "f13_elo_bandit_markov_transfer_shapley_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --transfer-entropy --shapley --renyi-weight run_combo "f14_elo_mopt_rep_markov_transfer" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --transfer-entropy run_combo "f15_elo_bandit_markov_mi_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --mi-guided --renyi-weight +run_combo "f16_mopt_markov_renyi_shapley" --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley +run_combo "f17_elo_mopt_rep_markov_renyi" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight +run_combo "f18_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight # ── Phase 7: Pairwise blend + annealing variations ──────────────── echo "" @@ -189,6 +131,20 @@ run_combo "t4_pairwise75" --elo --mc-bandit --pairwise-blend 0.75 run_combo "t5_anneal250" --elo --mc-bandit --anneal-budget 250 run_combo "t6_anneal1000" --elo --mc-bandit --anneal-budget 1000 run_combo "t7_best_blend" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --transfer-entropy +run_combo "t8_best_blend_rep" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --replicator --renyi-weight + +# ── Phase 8: Final variance check on top candidates ─────────────── +echo "" +echo "=== Phase 8: Top candidates x2 for variance ===" + +run_combo "z1_best_a" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z1_best_b" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z2_second_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z2_second_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 +run_combo "z3_elo_bandit_a" --elo --mc-bandit +run_combo "z3_elo_bandit_b" --elo --mc-bandit +run_combo "z4_elo_mopt_renyi_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight +run_combo "z4_elo_mopt_renyi_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight # ── Summary ─────────────────────────────────────────────────────── echo "" @@ -196,7 +152,7 @@ echo "============================================================" echo " RESULTS SORTED BY EDGES (descending)" echo "============================================================" echo "" -sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -20 +sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -40 echo "" echo "Full results: $RESULTS_DIR/sweep.csv" echo "Logs: $RESULTS_DIR/*.log" diff --git a/tools/bench_sweep2.sh b/tools/bench_sweep2.sh deleted file mode 100755 index 13013592..00000000 --- a/tools/bench_sweep2.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Phase 2 of the sweep — remaining fast combinations. -set -euo pipefail - -TARGET="targets/png_read" -ITERS=1000 -BASE_DIR="$(cd "$(dirname "$0")/.." && pwd)" -RESULTS_DIR="/tmp/fuzz_sweep_results" -DICT="-D dictionaries/png.dict" -GRAMMAR="-g dictionaries/png.gram" -REPORT_FLAG="${BENCH_REPORT:-}" - -cd "$BASE_DIR" - -cleanup_shm() { - ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' | while read -r shmid; do - ipcrm -m "$shmid" 2>/dev/null || true - done -} - -extract() { - grep -oP "$1" "$2" 2>/dev/null | tail -1 -} - -extract_ci() { - local log="$1" - local line - line=$(grep -P "Crash rate:" "$log" 2>/dev/null | tail -1) - if [[ -z "$line" ]]; then - echo "-|-|-" - return - fi - local ci1 ci2 ci3 - ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') - ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') - ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--}|${ci2:--}|${ci3:--}" -} - -run_combo() { - local name="$1" - shift - local flags=("$@") - local dir="/tmp/fuzz_sweep_${name}" - local log="$RESULTS_DIR/${name}.log" - - rm -rf "$dir" - mkdir -p "$dir" - cleanup_shm - - echo "[*] Running: $name" - python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ - $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true - - local edges corpus eps dur p50 coll crash_ci - edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") - corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") - eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") - dur=$(extract "Duration:\s+\K[0-9s]+" "$log") - p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") - coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") - crash_ci=$(extract_ci "$log") - - local ci1 ci2 ci3 - ci1=$(echo "$crash_ci" | cut -d'|' -f1) - ci2=$(echo "$crash_ci" | cut -d'|' -f2) - ci3=$(echo "$crash_ci" | cut -d'|' -f3) - - echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" - printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" - - cleanup_shm - sleep 1 -} - -echo "=== Phase 5: Game theory additions ===" -run_combo "gt1_sched_shapley" --elo --mc-bandit --shapley -run_combo "gt2_sched_rep_shapley" --elo --mc-bandit --replicator --shapley -run_combo "gt3_sched_all_game" --elo --mc-bandit --replicator --shapley - -echo "" -echo "=== Phase 6: Full combinations (top candidates) ===" -run_combo "f1_enhanced" --elo --mc-bandit --mopt -run_combo "f2_enhanced_plus" --elo --mc-bandit --mopt \ - --markov --markov-gen --markov-order 0,1,2,3 \ - --replicator --shapley --renyi-weight -run_combo "f3_lean_best" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f5_elo_markov_renyi" --elo --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f6_bandit_markov_renyi" --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f7_elo_bandit_markov_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f8_elo_bandit_markov_rep_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --replicator --shapley -run_combo "f9_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f10_elo_bandit_markov_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --shapley -run_combo "f11_elo_bandit_rep_markov_renyi" --elo --mc-bandit --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f12_elo_bandit_markov_renyi_shapley" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley -run_combo "f14_elo_mopt_rep_markov" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "f15_elo_bandit_markov_mi_renyi" --elo --mc-bandit --markov --markov-gen --markov-order 0,1,2,3 --mi-guided --renyi-weight -run_combo "f16_mopt_markov_renyi_shapley" --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight --shapley -run_combo "f17_elo_mopt_rep_markov_renyi" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "f18_elo_mopt_markov_renyi" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight - -echo "" -echo "=== Phase 7: Advanced tuning ===" -run_combo "t1_pairwise0" --elo --mc-bandit --pairwise-blend 0.0 -run_combo "t2_pairwise25" --elo --mc-bandit --pairwise-blend 0.25 -run_combo "t3_pairwise50" --elo --mc-bandit --pairwise-blend 0.5 -run_combo "t4_pairwise75" --elo --mc-bandit --pairwise-blend 0.75 -run_combo "t5_anneal250" --elo --mc-bandit --anneal-budget 250 -run_combo "t6_anneal1000" --elo --mc-bandit --anneal-budget 1000 -run_combo "t7_best_blend" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "t8_best_blend_rep" --elo --mc-bandit --pairwise-blend 0.25 --markov --markov-gen --markov-order 0,1,2,3 --replicator --renyi-weight - -# Final: the very best candidates, run twice for variance -echo "" -echo "=== Final: Top 3 candidates x2 for variance ===" -run_combo "z1_best_a" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z1_best_b" --elo --mopt --replicator --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z2_second_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z2_second_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 -run_combo "z3_elo_bandit_a" --elo --mc-bandit -run_combo "z3_elo_bandit_b" --elo --mc-bandit -run_combo "z4_elo_mopt_renyi_a" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight -run_combo "z4_elo_mopt_renyi_b" --elo --mopt --markov --markov-gen --markov-order 0,1,2,3 --renyi-weight - -echo "" -echo "============================================================" -echo " FINAL RESULTS SORTED BY EDGES (descending)" -echo "============================================================" -echo "" -sort -t, -k2 -rn "$RESULTS_DIR/sweep.csv" | head -40 -echo "" -echo "Full results: $RESULTS_DIR/sweep.csv" diff --git a/tools/lib/bench_common.sh b/tools/lib/bench_common.sh new file mode 100755 index 00000000..a1c5a271 --- /dev/null +++ b/tools/lib/bench_common.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Shared helper functions for tools/bench.sh and tools/bench_sweep.sh. +# Source this file; do not execute it directly. + +# ── SHM cleanup ─────────────────────────────────────────────────────── +# Remove all orphaned SHM segments owned by the current user. +# Previous fuzzer runs (especially those killed by signals) leave +# segments behind. Accumulation can cause shmget to fail or the +# target to attach to stale segments. +cleanup_shm() { + local before shmids + before=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) + # Capture matching SHM IDs into a variable first: under `set -o pipefail`, + # piping straight into `while read` would abort the script (via `set -e`) + # whenever grep finds no matches (the common case with no stale segments). + shmids=$(ipcs -m 2>/dev/null | grep "$(whoami)" | awk '{print $2}' || true) + if [[ -n "$shmids" ]]; then + while read -r shmid; do + ipcrm -m "$shmid" 2>/dev/null || true + done <<< "$shmids" + fi + local after + after=$(ipcs -m 2>/dev/null | grep -c "$(whoami)" || true) + if [[ "$before" -gt 0 ]]; then + echo "[*] Cleaned $((before - after)) orphaned SHM segments ($before -> $after)" + fi +} + +# ── Metric extraction ────────────────────────────────────────────────── +# Extract the last match of a PCRE pattern from a log file. +extract() { + grep -oP "$1" "$2" 2>/dev/null | tail -1 +} + +# Extract CI values from a "Crash rate:" line (format: "rate% ±1σ: lo% ±2σ: lo% ±3σ: lo%"). +# Usage: extract_ci [pattern] — pattern defaults to "Crash rate:" +extract_ci() { + local log="$1" + local pattern="${2:-Crash rate:}" + local line + line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1) + if [[ -z "$line" ]]; then + echo "-|-|-" + return + fi + local ci1 ci2 ci3 + ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') + ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') + ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') + echo "${ci1:--}|${ci2:--}|${ci3:--}" +} + +# ── SHM verification ────────────────────────────────────────────────── +# After a fuzzer run, verify that the SHM bitmap actually received data. +# This is more reliable than checking log messages — it checks the +# actual SHM segment that was created during the run. +verify_shm() { + local log="$1" + local label="$2" + + local shm_id + shm_id=$(grep -oP "SHM bitmap, id=\K[0-9]+" "$log" | tail -1) + + if [[ -z "$shm_id" ]]; then + echo "FAIL: $label — no SHM ID found in log (coverage not enabled?)" + return 1 + fi + + local has_data + has_data=$(python3 -c " +import ctypes, ctypes.util +libc = ctypes.CDLL(ctypes.util.find_library('c') or 'libc.so.6', use_errno=True) +libc.shmat.restype = ctypes.c_void_p +ptr = libc.shmat($shm_id, None, 0) +if ptr is None or ptr == -1: + print('FAIL') +else: + size = 4096 # default map size + bitmap = (ctypes.c_uint8 * size).from_address(ptr) + non_zero = sum(1 for i in range(size) if bitmap[i] != 0) + libc.shmdt(ptr) + if non_zero > 0: + print(f'OK:{non_zero}') + else: + print('EMPTY') +" 2>/dev/null) + + if [[ "$has_data" == FAIL ]]; then + echo "FAIL: $label — SHM segment $shm_id could not be attached" + return 1 + elif [[ "$has_data" == EMPTY ]]; then + echo "FAIL: $label — SHM segment $shm_id has 0 non-zero bytes (coverage-blind)" + return 1 + else + local nedges="${has_data#OK:}" + echo "[+] $label — SHM verified: $nedges non-zero bytes in bitmap" + return 0 + fi +} + +# ── Coverage-attachment sanity check ────────────────────────────────── +# Combine log-based and SHM-based checks for maximum reliability. +check_coverage() { + local log="$1" + local label="$2" + + if grep -qi "SHM not attached\|AFL shim area is NULL\|shmat.*failed\|Coverage data will be empty" "$log"; then + echo "FAIL: $label — SHM coverage did not attach (coverage-blind run)" + return 1 + fi + + if ! verify_shm "$log" "$label"; then + return 1 + fi + + return 0 +} + +# ── Run with retry ──────────────────────────────────────────────────── +# Runs `python -m fuzzer_tool "$@"`, verifying coverage attached; retries +# on coverage-blind runs up to MAX_RETRIES (default 3) with SHM cleanup +# between attempts. +BENCH_MAX_RETRIES="${BENCH_MAX_RETRIES:-3}" + +run_with_retry() { + local log="$1" + shift + local attempt=1 + + while [[ $attempt -le $BENCH_MAX_RETRIES ]]; do + echo "[*] Attempt $attempt/$BENCH_MAX_RETRIES..." + python -m fuzzer_tool "$@" 2>&1 | tee "$log" + + if check_coverage "$log" "attempt $attempt"; then + return 0 + fi + + echo "[*] Coverage did not attach. Cleaning SHM and retrying..." + cleanup_shm + sleep 2 + attempt=$((attempt + 1)) + done + + echo "FAIL: Coverage failed to attach after $BENCH_MAX_RETRIES attempts." + echo " Last log: $log" + return 1 +} + +# ── Sweep combo runner ───────────────────────────────────────────────── +# Runs a single named feature combination for the sweep scripts, appending +# a CSV row to $RESULTS_DIR/sweep.csv. Requires $TARGET, $ITERS, $DICT, +# $GRAMMAR, $REPORT_FLAG, and $RESULTS_DIR to be set by the caller. +run_combo() { + local name="$1" + shift + local flags=("$@") + local dir="/tmp/fuzz_sweep_${name}" + local log="$RESULTS_DIR/${name}.log" + + rm -rf "$dir" + mkdir -p "$dir" + cleanup_shm + + echo "[*] Running: $name" + python -m fuzzer_tool fuzz "$TARGET" -d "$dir" -c -n "$ITERS" \ + $DICT $GRAMMAR "${flags[@]}" $REPORT_FLAG 2>&1 | tee "$log" || true + + local edges corpus eps dur p50 coll crash_ci + edges=$(extract "Edges discovered:\s+\K[0-9]+" "$log") + corpus=$(extract "Corpus:\s+\K[0-9]+" "$log") + eps=$(extract "Avg eps:\s+\K[0-9.]+" "$log") + dur=$(extract "Duration:\s+\K[0-9s]+" "$log") + p50=$(extract "Exec time p50:\s+\K[0-9.]+ms" "$log") + coll=$(extract "Collision risk:\s+\K[0-9.]+" "$log") + crash_ci=$(extract_ci "$log") + + local ci1 ci2 ci3 + ci1=$(echo "$crash_ci" | cut -d'|' -f1) + ci2=$(echo "$crash_ci" | cut -d'|' -f2) + ci3=$(echo "$crash_ci" | cut -d'|' -f3) + + echo "${name},${edges:-0},${corpus:-0},${eps:-0},${dur:-0},${p50:-0},${coll:-0},${ci1},${ci2},${ci3}" >> "$RESULTS_DIR/sweep.csv" + printf " -> edges=%-5s corpus=%-5s eps=%-8s dur=%-8s crash_ci=[%s,%s,%s]\n" "${edges:-?}" "${corpus:-?}" "${eps:-?}" "${dur:-?}" "${ci1:--}" "${ci2:--}" "${ci3:--}" + + cleanup_shm + sleep 1 +} From 12881206d5108196dedc22d43f1a3db34796a87e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:04:32 +0000 Subject: [PATCH 5/6] Address review feedback: extract_ci delimiter param, log-existence check --- tools/bench.sh | 16 +++++----------- tools/lib/bench_common.sh | 14 ++++++++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/bench.sh b/tools/bench.sh index 014ff052..c0df0370 100755 --- a/tools/bench.sh +++ b/tools/bench.sh @@ -125,17 +125,11 @@ e_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced.log p_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_enhanced+.log) o_collision=$(extract "Collision risk:\s+\K[0-9.]+" /tmp/fuzz_bench_optimal.log) -# Extract CI for crash rates -b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:") -e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:") -p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:") -o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:") - -# extract_ci returns "|"-joined values here; normalize to space-joined for display -b_crash_ci="${b_crash_ci//|/ }" -e_crash_ci="${e_crash_ci//|/ }" -p_crash_ci="${p_crash_ci//|/ }" -o_crash_ci="${o_crash_ci//|/ }" +# Extract CI for crash rates (space-delimited for direct display in the table below) +b_crash_ci=$(extract_ci /tmp/fuzz_bench_baseline.log "Crash rate:" " ") +e_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced.log "Crash rate:" " ") +p_crash_ci=$(extract_ci /tmp/fuzz_bench_enhanced+.log "Crash rate:" " ") +o_crash_ci=$(extract_ci /tmp/fuzz_bench_optimal.log "Crash rate:" " ") printf "%-25s %12s %12s %12s %12s\n" "Metric" "Baseline" "Enhanced" "Enhanced+" "Optimal" printf "%-25s %12s %12s %12s %12s\n" "-------------------------" "------------" "------------" "------------" "------------" diff --git a/tools/lib/bench_common.sh b/tools/lib/bench_common.sh index a1c5a271..b1eb28cc 100755 --- a/tools/lib/bench_common.sh +++ b/tools/lib/bench_common.sh @@ -33,21 +33,25 @@ extract() { } # Extract CI values from a "Crash rate:" line (format: "rate% ±1σ: lo% ±2σ: lo% ±3σ: lo%"). -# Usage: extract_ci [pattern] — pattern defaults to "Crash rate:" +# Usage: extract_ci [pattern] [delimiter] +# pattern defaults to "Crash rate:" +# delimiter defaults to "|" (used to join the three CI values, e.g. for CSV rows). +# Pass " " for space-separated output suitable for direct display. extract_ci() { local log="$1" local pattern="${2:-Crash rate:}" + local delim="${3:-|}" local line line=$(grep -P "$pattern" "$log" 2>/dev/null | tail -1) if [[ -z "$line" ]]; then - echo "-|-|-" + printf -- "-%s-%s-\n" "$delim" "$delim" return fi local ci1 ci2 ci3 ci1=$(echo "$line" | grep -oP '±1σ:\s+\K[0-9.]+') ci2=$(echo "$line" | grep -oP '±2σ:\s+\K[0-9.]+') ci3=$(echo "$line" | grep -oP '±3σ:\s+\K[0-9.]+') - echo "${ci1:--}|${ci2:--}|${ci3:--}" + printf -- "%s%s%s%s%s\n" "${ci1:--}" "$delim" "${ci2:--}" "$delim" "${ci3:--}" } # ── SHM verification ────────────────────────────────────────────────── @@ -131,7 +135,9 @@ run_with_retry() { echo "[*] Attempt $attempt/$BENCH_MAX_RETRIES..." python -m fuzzer_tool "$@" 2>&1 | tee "$log" - if check_coverage "$log" "attempt $attempt"; then + if [[ ! -s "$log" ]]; then + echo "[*] Run produced no log output (crashed before startup?). Retrying..." + elif check_coverage "$log" "attempt $attempt"; then return 0 fi From 3ebd126e6d36fbb2900f0fed6f09b37e9bd67b67 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:05:11 +0000 Subject: [PATCH 6/6] Clarify no-log-output error message wording --- tools/lib/bench_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/bench_common.sh b/tools/lib/bench_common.sh index b1eb28cc..322eefef 100755 --- a/tools/lib/bench_common.sh +++ b/tools/lib/bench_common.sh @@ -136,7 +136,7 @@ run_with_retry() { python -m fuzzer_tool "$@" 2>&1 | tee "$log" if [[ ! -s "$log" ]]; then - echo "[*] Run produced no log output (crashed before startup?). Retrying..." + echo "[*] Run produced no log output (crashed before startup, or the log could not be written). Retrying..." elif check_coverage "$log" "attempt $attempt"; then return 0 fi