From e671eb2731b648f9a83d94494705d221f8ec4082 Mon Sep 17 00:00:00 2001
From: Kamal Gurbanov <kamalgurbanov916@gmail.com>
Date: Sun, 31 May 2026 01:48:34 +0400
Subject: [PATCH] Fix KeyError on category-5 (adversarial) QA: read ground
 truth from adversarial_answer

A clean checkout cannot evaluate any model on the category-5 (adversarial)
questions. data/locomo10.json stores cat-5 ground truth under adversarial_answer,
but the code reads 'answer'; 444 of 446 cat-5 questions have no 'answer' key, so
evaluation crashes with KeyError: 'answer' at two sites (task_eval/evaluation.py
scoring, and the cat-5 distractor blocks in gpt/claude/gemini/hf model utils).

Read cat-5 ground truth defensively (answer -> adversarial_answer fallback);
zero behavioural change for categories 1-4. Adds task_eval/test_cat5_eval.py
(no network/keys), proven fails-then-passes.
---
 .../__pycache__/__init__.cpython-314.pyc      | Bin 0 -> 189 bytes
 .../__pycache__/evaluation.cpython-314.pyc    | Bin 0 -> 15048 bytes
 task_eval/claude_utils.py                     |  11 +++---
 task_eval/evaluation.py                       |   8 +++--
 task_eval/gemini_utils.py                     |  11 +++---
 task_eval/gpt_utils.py                        |  11 +++---
 task_eval/hf_llm_utils.py                     |  11 +++---
 task_eval/test_cat5_eval.py                   |  34 ++++++++++++++++++
 8 files changed, 68 insertions(+), 18 deletions(-)
 create mode 100644 task_eval/__pycache__/__init__.cpython-314.pyc
 create mode 100644 task_eval/__pycache__/evaluation.cpython-314.pyc
 create mode 100644 task_eval/test_cat5_eval.py

diff --git a/task_eval/__pycache__/__init__.cpython-314.pyc b/task_eval/__pycache__/__init__.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9d04550d97e709d091430e8eb1d86c4923ddb91f
GIT binary patch
literal 189
zcmdPq<K<!ig03*>ED-$|M8E(el<}Dj$Y@{)Vo+w#Wb|9fP{ah}e+Eh3iqa1)PAw|d
z&rZxu%t<dTN=nSjFVindEly1=O3nx<%FjwoF3~RlD#_1F%*jp7)laP`Ni71h^>gx*
z^K<j#i;I%=OA?E-<5SCkg7NX0d6^~g@p=W7w>WHa^HWN5QtgUZfmVSWQw(B!VrFDy
IEMf+-0A_?S(f|Me

literal 0
HcmV?d00001

diff --git a/task_eval/__pycache__/evaluation.cpython-314.pyc b/task_eval/__pycache__/evaluation.cpython-314.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ec38e4d6832387283a3055697741403d2be95cbf
GIT binary patch
literal 15048
zcmch8YjhJ=o@bR*l4VJ@UVh=HY=g1D*fG$-Ji^m95Q8zYk&}*NjVzUoK$b*R8L$I8
zn(di0CfylKhYrpRlQ<J*+DvD+$=Nxx^gPZ9Js&1>W<E$M3p*9E%l2?~PxezV?3u9p
zVfS~dR8rZ9klx;%>r?-_x{v?8b?^UuuQwJKnJJ|24_2M}U+XC9pUDq388VqiCYquW
z)G+0x4^x8jrUhF5trK+eTQBJG)*WUJ83aR}(#9whpiF<*bf{1$%#@i0Gs>95mP1y-
zDwi1!7ab}VitDJ?i|eVanIQ$+utBhwQuUPASWbBhUUzKOD^-HiYeGq3ITf)BuGgKp
zK3iz788sGC6QP9?ueF>KO1(vZW!_@Ia<2{0;<W=-cpZRiyiUMMuM4osTLM_^Ed{Lc
zuJM)u4=vPs%kjI;TLI|y(v6g-@+&ftUJorXN5Z^-cfo<!L{wloWDR^U#<9p0^~E?8
z^$YCyILieypC!L^nIDfjkO^R6U#A4xOJN~xg1(H3VVMQSs}l@fJq9&;3%raxl>)Cp
zF!?B<&}+mrOp>iXJRXm*1F_R=G(5#}Q_k0h;%^+e2+&Kw(TlHdoGN;47}@WW-v=(f
z9>hjwQvq^VRO_Svj!c3Y%2xELGt8DyWFu%F)yHUxGT9xp6&sH_<1^~?n++P=rl(tJ
zYGq<v*`h`BQ`6F_Db8fIExpXP%vboRDEU%3c@0=O<J3;`K5mTPVRj=0qS2TT5Mms^
z!`&N;vhEn?X2;{gq?>02cXKQnnRE-8CFVV?W{GZ|(t9p6OSFgV_h=+A#`j`33>-Vg
zo|g)OvGI5~!b*kvj`SVc=kGmnR5JD+>-7)zOXkA|dyjd$4(#jik_t}r9zJlavr8&G
zdg92y;UkB8QRbI%#DSOL*~lnQcB@GmYCFNR9N%_2Fdm4EO>iTDXzXko$MS4|3!d!b
zVyD=k&=yCNSTqnBXUE&v^B_Is+afVcC+6q5V4D!&Py5-kC?sGaAcSMl*7&4k^83Tl
zu;BMg6|3l}RjqMh8vGuB1ohZVRn%O&`1Zx6s=D`gU*A2y_2)Z3+L^R&yXP!lbT*`%
z4L45w^z{#3PdhjN#fksu|Ja{)c0M-fE6mf(A6OreJqTv_nB*xUVes4Zs8*IRAR@VV
zCFs-k$W;00tr=3bQz!Hjx>jmJ*A3!4v!z$p141mMYN<FA*ZXwqGd)6W6je?67)^UO
zY2T+xInAJ|6za<oPOeg|!#E9@aSRv-gtY><^Le<@zhq_q30)n&BCZbuKApQs_j(u(
zT`0*k8jgm-*kZ{r#>FP$k|7d12k~R%N=Y>%VE?6piD)puvmuEdmkP!MLhvNZOGcTJ
zd7iLpQ&SVjpDU9wsucb-5vVmaXva?hB&em@hW8`yMiy(gq-wX^>RhPZE;@IlYPZii
ze%y}NbV1U&<3C%RpSO32rQ1I)pWk<F?8?~N;cKxgvAbpKKQ5o{PL^)}+DO^Uri&ix
zsSWL7Y5UFcIl0+8;hS=O`;x76y6DNWg>vnD@~hJAc-nb8hJVkzpJM*9X}^KFv$kTt
zneou_qCg&Zd(4EIqk(Z2)NJ<q$77+12+3Rh{xcJSNT$Z-_m74-UWkOFY&7Qgb2gym
z2yU*NfVBWpso&2-EQEvpfFN+;kqLq2fyv?b2O|NV4~2t5kc~wAej@8Nh3G~IbFaLv
z91(UhAVcuA0N?yRSwQ=`#Nnl)nnd?9V=|ckrNn3`Tc!ZGHEIW0QfEk<u!#IT+JQ`h
z@=?ROj^CxI5X!5m<Mc_|OXnoxP*!@kRHAt%wb6~#P2E?dPBL<=Fu_GVI*uTgn2}g4
zg3mCH#yAj)JpTrTF0gPJ`G{rYwxD$r9-b^yg8CQAPTL<6@DcOTz`rxgS@{cs*(snQ
zSp$s2Jo+cOmk)|WJFz%ahgY|vMBfVD9a2A@^67KD-o2Ni&H+2;^BKMR6}<|41*2dO
z!sGcQhL=&X)KNXKbn>?v%1VR5^^h@SeULOg_bWW>W1vV3QwRHH8SEBLf)B?|x&x62
z@+=fa+#QMqd1RnQSa*1o6i>QO2F|k3IFs&}T!1VO#^qb3f=o_gg2H*8tHYq(9=%k^
zMkioC1O!$x2I6rx8j{Q>13Wf`^n=zZV9y7L5g-++e9Cd(Luc+q0Esyn7I=yFOL|O3
zM#_`jkQqPA`&LMX;OPuXj)-}Aj|0dgsQbmGGvRcxJ8^Ky;+%Q$tv!j(`<9|a=L2We
zZ09>|myFZ&l8scnG0oh!*xuZ=WVK)JxzsajSg_W9ZY{ZQvR<}avRr+6_S}3}Ea|vw
z+Wyc+xf;H9QdWDSN1+z7)9P*$$|0-%XstZ{x05;{YOWOxAq3i$*Jivl6fD!M47QUB
zqR9y1T4={Z0{$i>zm#cYHXBXI9%qQQ6%&JPq)v~@DV{j()8!*HAGp!0Q>_oQqU=<6
zoTB1-4VElP6BU&gi+%+beU31PsiCY|%FRYomc`wov8IM;t#!0D!=RWMNG;9`whwmZ
z#UL++0%)q^6^a_eDpgZG&<1t*BL*>1wLP4|7G6Erf|+VPKuC&EeSok5NH{9EgR!U(
z2uFDkeTMGcJS1y$taSj}=f)u29tL7NN~BPP7_@{0U{?jAW2~f)uu+dep>4t)+)h$y
zR2V^`!#rVJS*0nGmuxpjfV}V1$Rwx-meQG!XsJqcE?G(zEj1}i&1`7CGij+2E$xZU
ze|D~!Hr{ho&3229#%be{qjJ&Fm~u4Eb>HetIvPdCc2Zq7bLM*EZ0L_XH-hs`AD&KE
zw%^*9uKeCz$BWbZptEKJH~QzxK78#@>ym5Q@7mhGD6e{Cpq$$t7ErFrhgQn!$k99|
zB_{?@rui3;`3{CS6o9&cSZ-R<_jY%BbdqHv8V<%nY$zZEq{8en-lP@ICnVgYEDnSi
z#Sw524^NDS1a;3+wrFvGW^vDz&NHHAQ<B;A^h`5sK|R8LqYv!0EMuS`le{1cb$)Tk
z_Hrst6IUj#BaTd5?==82K3zSS36CBRgC{FxRcX8WO-y%*Zjq=JUJTiH!q$%n=YN;Q
zBYuo3*nv_v<s2Gm4qOOCFF-`&74i&?NVFT;u?QF?JwGwR^&qEM<BDFiaqM{PEb9+&
zun;0JtUTG}&Ir=hU9i0a5q@{;s;0dJ6*0O%kG7J_lW$HY*R<Z!-x?5YJCddyPp>(H
zw(<w=l{<O!t~qu>E56W@B-K_`SCUW**M|yF3v((Kj>2D!N5aC>-**n0gZ1<AK#+y8
zbpG4EZ&gz-&}R$;Xokz@FP)!d-Z#H%{v)esTbDGg`<4%kplg8EO52<*lh5;1KPv|B
zIiIQSqY)CI;XA`?w&z<69Y=O!1v!yb+=54T&Ueub5ON7>fa-<;cgE4L+YWlqh(HT$
z2>LvgK(wRBkr97XBlSIfnC7a0;U)zIu4{ck(gjZvJspfpgjk;7Z*nITGAi6P#zxum
zac;NN@mvCm!=TSmRkr6K5b_lO-+YCcCd+nz&^bGDYb@3B(lzeK+=rb{WR?E|QzboP
zuB0Jh_3&#Yqau1G3&AQV-XM1ZSU~o&cpWBaiI!+mOj*g2p<~iPTKAo1OYrf{0N+5o
zI7_dNE|xW?%9=%|XS!g?=D2+B(zyjPU$JbrXxp7M?S7gz$P8rSZ$Rc*0!Jlt7z1SC
zLF2^IUQN_;p;Y(mnafVptLVuW%Y!Z1T$iUVP5tmf(o`kS45GGxW8)Dpg~uS#JV$7q
zBOAbl2q2>>dWZxTlr6Te9*;YPR<b<VhYTz%Q{m;pONBFyGn{CuOzd0M=?puUO|+qD
z*{Cz@A-U>@M#@_C?w*@FKc2XK{6Cx*H|`ay_KBALNoK#YWMrAtL<FL&O|BxNWu}~j
zjX%Sn8Oq8K$P<ERRZ<sPWjsxL`n$C6I7ER%lugph3&9021m^>QNHWJcmN-_3Oh{JZ
z*+xTtftwIcDuk#gFUbLuHz@dJm)0+b+qfwD`|<F503cT#^>Z6U$EL)~_gsy0M?_b9
z;>cqxp>29<(N>qT)rq!x(b50~-m<u1S8Bs9al>x0X-~?sC&}!2dO?ZFEHCIE%Gfnl
zTvla$mj!L5eRL}%6|w8Q`fZuzJqOHOBix<rat}it4pUm!CZBA}ZbfG6A=(*9$40vc
zAa$4sY$mDTEX$3=cvdP!tO&Y@^`8q1C;cOc3J}*9(F8(@92r>Bj|a|63}pT|j4UFD
zxp7iV_er!@(sj3I%&RO}n@3nq$gKV(X^w}-?qc=t<yrlEwvw4sGedJn=62s^K4Fu=
zu?5@7-#L`nH{I~SQapWTreWszPu5-AaAm{WEuy6^$<#f)4O!b`KpsZ34K!jpx=aiR
ztQ*(m=yFh}S|W(Iz{MxVDq<`;m4X}9Sc@*4p-UYnknbTRbyF`dg;cbH7A4)7posCN
z43F#Cai04(=rDzcUk5<Usp4N86>Ts7;^?iB1v}{RuD$&iM^m<!MN3bT>3Q;*?d3-d
zTE1Db&a}c^CZQRb+G?@0`Pui|RJ~>?pcJqaWDq&!qat5+A+S7%7RX^UJ$hx$x}1mc
z|9MmfqmnI_Q_khrlw}?Gmh9~Cg@z$0;!H0?A2Hchr7Wjpdc}=C<FM?@DDjhFh$PsA
z#+EDxQq&NM*T^Y1la}l_7`5PAdgSJ*?O__$`3kgS_)OYggXgvW*^03WeT7;w%>)NQ
zu+Y_n`^S`-e!a(IQaGB3Q%M)+5k&?=bA$qj89Oc+N#vL1xlWXL%vwG_CpmJ$*RMng
zC6|`&)j}g8%Ht5wCPXIuJYsE9A$kVGJdxUnEJ4sjB;9B`Pn=IR(x7TX(`b7}XuO45
z!ZN%B0Cw5*((O~?ro)Syj-@soTiA45tnj8b9iOMKJKryVw|vg=UL}4enbKs1_g>qc
zPpo3w(Z#m@R9pW-+X=DKmufpPUw?h```g~#HrM{%4*X0q<;hCly`qw2X_HvA{-gED
z&0XS#?sQRiqWc~RSzLSR%1dwW5p9i$LrVp=nfBkk__@u#WOvP+{7LZ*S90BpV*R$X
zeH#hX*=IW58onM(HhfR4eKB41VxoJ=Y?%(fwQ05~ZFbMSnqIdpUD%P>kC>*lIC1#N
zayhl8pMD}$`glSD*K0cwtt_wYG&3t$oe-JK>H<jfS;mbwt7J5+?SwVj@NEpKi7tbk
zeRO_5O_hSc%w*&z!QA;BF&m(^2yw|N-|-K;Is<SUPf!zx=^;YD9;{>3u(TevD3|Mq
z<fd}WJZQ`-xwET*FZ5*Oxjqi8>bK-!6($=|r!qOY2N;Ovj$-l%oXad@AVzUq(naGE
zGtLI0iY6eG>M77NOB`l%)>1)yZH&8sHrz!3o`hyHRIm8jKO$8G5Ip=(01*FedSS8Y
z#Z=RaxA=voouYGBs%ht(<Jz_>+uqrZY@#RW+_luab+LIzs(HulEep+i#FD+K<~?)m
z*9NZ)zVj-wiKEGqy$>u!m$$vSEj1KgaGe57`d+zu;7*TN(RZik)~V|i&^l|{@ARZ9
z`tFt-NBw86Q=;W`k~#fk8R3YRo=7EopZuz#?}_L-#W!MRS1b0fWA4xu`!_ImUMSwb
z)qH1n$^N#IUpg(w|FX>hxPpMQmeHp&0m}!$)FC>K;|t>6tCANkzb>D7sEIgbx3w>;
z12CUVQt@cWl;}2QC>!6(C-;3>Ih@mNQ2p6b5}J6<tg?qp|BADss}K_D%jzF>4Jc|5
zK@^;75`lpfHw+lOMy-|TEzthTvO$jck#kc+FoKk1q(abm)o^fYC~__urgrLUs4xvJ
zLbM8;9fKD^ZB`B#AVeqW182wno7#J-fJD66P3;mr>M2B&PHy@Rs%1Aar--=UA!Yv-
zKr)Z98Jm%p%s!%ux;QRI&aJW*CnsYdhBo(mq>E8D@YMsWoLgm;$BSf?$$Mye6%T(0
zAVGb}RDEuB{P19+Yspb^t>Q|>0yKtMZ0lKUJDzGgzR=b$Rt`XQtkf9&smcM-aU${Z
zlB?`m$CZwS_I;nUi0#J~+fSt0Pb{<#iq)^A+6U+NUH85}`0n7`@%O%upQ-9sMAxf{
zBVRLmgKNoAw&<u&5d-)WCh4db9bMB#q7<Anp0uTUwm)sDPcrpSmTgqk3FwFFSH5{{
zqO4_)DcazYY|gLsc>hnH9JT#L?E#bN&Ys!>MW$b_a~*I}f9JFusMr6Un+9CLg#WsB
zF<ENv4YJmR5FzOlH&@3+xH%MjKx)<k$QJO&7e!mw#}{uouUconf7jmf@x_#_OSE(+
znQj=sUEFV@;fm-Bk)E<fd_WTvIWtrh0dYEK+vD6Q8wbtD$ZDU}=S{<iL6Q?DpAO+~
z2yK{<#ORh&9}#J*647F+c8ME&hE=CkIEU)f7<y`Fv}3dknZ-~rRHU{U9Lt?CK{d)L
z1fhmAEIk7D*>;$t9_Lkggu)OaPfGF?^pQZxs-&)gI=cEs%nFN7%>5w}{*{wYufdje
z478@`Fr(_^xGC3fs7%GsN8(Vc5{d4rqiXsW4qn=#gVTyQ3inrEVYjK?UakwTDaL)_
zf?G1iCWQEe;3<-iPbDUX!!N0jV*?@LyTY_Rh1(9~;5!iF6^TOv{agnDodhTv9bzIH
z;$}#}6#$$N8^h62+|N+X4-5)2d+tolA0&5qq+;b@!H-jiF~lF>QDanje%zlDWIqHz
zAsgitJC{f%IVtR5A$Kfrn}Zm`+{bA9XL$He0T7+A*NOJ}#Nm5Z`%KxaOSIN3TGywn
z>qV<)z9DVhl<2x&UXf%<mP*SLy<gZ%r<r>s5cTHN>{`)QH_a?LN@1ebk^_qJYgJdO
zW(Ve)|9Dt*v`ibx0mtZEc+v90XO<V1tkrXL+FGBqR^K<3Et+alrkd-)xu&1Ce9-b?
zt61B9*R=VegL0HTE}@EDNmt{I1M`LtUrxJPZ_#4WW?0D<`&H+)@*kJa){2hid4x<_
z$mySBdKYYK#Xrv|BYYR`5#f<}w;9}&puk}=mx0?1c#Qa6kmKiEEXq!G>AL?ibE<pE
zSD4TCgtL{uJWHinx-{nkDT7z?@n4SD$#GVQA>1}#j$4jeG<K<3rP#Q|SGe+=c~~#^
z1G|DtE%`V<chbBnjHBUadBp%!{)}5z&NgmcJ+8Ny5O}Cskq+N33kLsDT@Up;>Zqgd
zJcZHrT#Q+Xk{9FnQyA?s#$#lTUW3otmz6E*_vXHsv>b!vKuhEETZS+bA{Fy}_i)xs
z!&f#wqfOSpL>tEtxxgSNUtw;AhO*M@B<(Y;JTM(1fn}vf_IOo34u!tJ6%5Qw-AVMN
zVT!5E@%9yxZ|$O$8%P>Vg<exW3Po_@s@xv4tfmI9ti-16v5LNk-LNWov)`B1FTejh
ztT3~SGCmpf+L<rCX6=|Tra{MYpr|-rG*qV!I8?9ZKou*#3;1jqd~1R4f9K^Hi@Fah
zziH*zKA8tFgQkq<r;FRve(L-A>#Nmhar@EZzvefTkZ<_5T_L-TIfAw`(Z=6+j`?-J
z&#r!R9b{+^^iuwITK+)0Uu5)tmIu7nzD9NLa&OH#W_d?bmgRz{C5^Wz?(pe-4j;1x
z*8wu&s-mn9Zqw#G4%yj|HLK(dgb=dk16&x&Gjp<U3gYt1Sd5#L=$%s)(U=?RTfU}*
zaFaW5HV}>kMk4G~5iW7M$u$EM<IbB2Y8*FmaShJIo(+fCXpoiYe(pnjo<#4M+W5vr
z_wjvhM6YmlhIeN!M6#isZd3$<!3i!9oOEY`WwYqy(Q|EheMR)!S5O%wa&SX_ePtjo
zC$|kRNe>4H*Y76dEHM~(0Z})BlPwYMGU^rU<SmjT0RiPe$`HpL2!Xp!3OWHKGwz*n
z0#9P~<a%i=AaFk*m74)PPDy_zz>}}S5}jrzCF3}YphXZDh;SJVacId*GMO;kilY+6
zuptpRImi8bg0!1}p94tDSS%9CSvV5SN+zD=Nm!4^!NOU|tRVGceYjib!6Q_p+_+F}
zAB_Q2E>G|bf{kQHGMkCJ9$XTg{+9H<B8Ru(9N8SWT;y=U@2XgX0N;|$j({AoeeSv7
zF<skzsbHG^+DzF?<g;Yc+}fWuf6$!VxJPW*E86zKI<nYin4c8m*x<(21<U%S(lwX5
zACy+imc0GyboUY}&1p;3?3o3Nd#SYQ+R&Auxkj<HY3@wA)U#OHmMU$-?ZfHr`;Jm#
zQxRv)uJo8(urx1OU6*?=^<M9o>lUlFq^(;~U+kROkS?m3b1f9DRqHp;j*3;A($?11
z^)}DE@8?4|hkiD^VB2~7x0Bvi7QMqM@383ot%W_mwPY{9R(z#+_DtGde_xY-V0Fx_
z{ZY?7XEib2Z1x$)Tm04F?E1GO3$})&Y2(~M(X=sXYq)Q5&74Z(%0_3}()>~Xt&&@(
z((U`iP5VDNp5D}b-{zW~N|$)jwhi;`Y1^hnTSv;)aoe?Md+9UVOG_KuZtqCH&~<ks
zKJ8#?WA|KOcI5r8*)C*fOv$Q)_i7q$Y)SSHCSMu3+aDCyhf@8)PrW}k-89{D{jBI#
zAl=+?+nH|O^@(Yr`4H+->qBBAE7puIjGaoJjwRz$>C<nBV;9mj7p|Be$jAm>NxmAq
zI}j4rv#Ej5rz7*B+o#f74y9WUFEk%UNoqYSHjaulV+$uwCnNFXnK#mr3*yO(>6(i(
z<|S*{qO~Svt(iTwX!YFB{t7Hw-49=-sgj1rurCefq<Pz-d3VaZ`)~M9w*2+^e{y{~
z_>a}1*|)$9E?J7n-5=*n=j?{0Wo?pK`((M4sy+l)e0}K2uhz3NP5*tp<A}ldY2k(=
z=Hh?YSb4-&oZm-fWD3sNw5kR0|J+A~#Q05pRCuE+ZcDDzvhf1kres=a*|LQ|tA_RN
zH6EkMQHj^Es=R=dYvQ!Oo)4SH2G$(56!c~NS;&Y?QrA`0cIq2GETpRJ!)EPN_^>v$
zU+!>AtMN5HsA(wMZwOjlNqx}UtJ8CQSQGar=pkztjR3_}fl+d67-DoH9>;%!@X#Wk
zp&<gyF-Ue?4q{Ovk`nzSacObZ&3dfLrD5VQaz7<to`9PK5W`}Xhs*srDbx73+|NkC
zP69q6;2#K(MLm)7n(66qG9-&}`J5&z!WIA3DU0t>a?UQ_4keQNbM*fP5C2~PzLOiP
z$n)pBv9do4U)9ktZG?Y06P|eu(poBQBl6op4vrTr^-}2z)7`5a+?qK@+S-t`);!z6
zts)L?EoHVPZSEVcxf35&rfnPNkBfzEiT%%TaEqop62}z(mc%90RDcZ8|Bv>r;`<eG
zDAHn8Qt5Y99EyfVjFXQ{EDC4X-G3nSIWaHf)CWI$b$VVb>bZQb%NNSlaP!%x?KyZ<
zd4lqqC|15cOx`{@-u!hr9VEVz)1S+++1SAUCC6tiTuuMN-%w(h2QT@|#45(+i###|
z%%6|%J3s}B#W+WD#7b0*%2mv7QJA0by;P{4wDO3WW#Z&~K0&~pC(U)<BZ@R2kxcS`
zA(^CzgA!RBgkYA(#wKtra8f>3{!eIH0uJFHAOnH$?0A;liBnV={F=hR<XALSq)Zjp
zowMEInvDylmZf|}&x&r@2=hC^{6FF@pL_G%%p@-U%(aV!>%jYk#d!+zN4$z=w&r49
zICPKTd*%J`)XR576%jo`N)H3zzicMsL}C(IAQ^B@9-ov&v}6g!A`$$v7O|<7V|iVS
z=O#%<;zr=oZbUfUDlabKefj7_vLET{Jvea4KXAy~)qm*Nkxs-$N65c?DT;zzc~UgP
zev&&yK$HM-y3XaC4LNW>N6JV%Qntfok<O8A<X$6S4FN>`Nv2)O|IX~@{wsbGDJY*@
z;(UUpzo4vNP?j&KqAw`tmsGh7>%O20zoe?aq#8drIDg<z=<d5pl9c^1Q$m-`j6MdL
zl>LuZ^4}s&l&3>%-Y(z3U#_O<)@74}K0q%w8|l6Da+85}FWYJQ2)%5w(#6ZQ3>~4D
ji_LVyawSb4q?gSlbj9P=4tm}4S$Yj!^0))P<!S#PvWi+^

literal 0
HcmV?d00001

diff --git a/task_eval/claude_utils.py b/task_eval/claude_utils.py
index 44f964a..3b930cf 100644
--- a/task_eval/claude_utils.py
+++ b/task_eval/claude_utils.py
@@ -151,13 +151,16 @@ def get_claude_answers(in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)
diff --git a/task_eval/evaluation.py b/task_eval/evaluation.py
index 8f597dd..0537e69 100644
--- a/task_eval/evaluation.py
+++ b/task_eval/evaluation.py
@@ -196,10 +196,14 @@ def eval_question_answering(qas, eval_key='prediction', metric='f1'):
     answer_lengths = []
     for i, line in enumerate(qas):
         # line = json.loads(line)
+        # Category 5 (adversarial) questions store their ground truth under
+        # 'adversarial_answer' in the released data, not 'answer'. Fall back to
+        # it so scoring does not crash with KeyError on the released benchmark.
+        gold = line.get('answer', line.get('adversarial_answer'))
         if type(line[eval_key]) == list:
-            answer = line['answer']
+            answer = gold
         else:
-            answer = str(line['answer'])
+            answer = str(gold)
         if line['category'] == 3:
             answer = answer.split(';')[0].strip()
         
diff --git a/task_eval/gemini_utils.py b/task_eval/gemini_utils.py
index 99204db..5ed7b6b 100644
--- a/task_eval/gemini_utils.py
+++ b/task_eval/gemini_utils.py
@@ -164,13 +164,16 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)
diff --git a/task_eval/gpt_utils.py b/task_eval/gpt_utils.py
index 64d65df..e59a1c2 100644
--- a/task_eval/gpt_utils.py
+++ b/task_eval/gpt_utils.py
@@ -243,13 +243,16 @@ def get_gpt_answers(in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)
diff --git a/task_eval/hf_llm_utils.py b/task_eval/hf_llm_utils.py
index 7d16cf6..9795514 100644
--- a/task_eval/hf_llm_utils.py
+++ b/task_eval/hf_llm_utils.py
@@ -252,13 +252,16 @@ def get_hf_answers(in_data, out_data, args, pipeline, model_name):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " (a) {} (b) {}. Select the correct answer by writing (a) or (b)."
                 if random.random() < 0.5:
-                    question = question.format('No information available', qa['answer'])
-                    answer = {'a': 'No information available', 'b': qa['answer']}
+                    question = question.format('No information available', adv_answer)
+                    answer = {'a': 'No information available', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'No information available')
-                    answer = {'b': 'No information available', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'No information available')
+                    answer = {'b': 'No information available', 'a': adv_answer}
                 cat_5_idxs.append(len(questions))
                 questions.append(question)
                 cat_5_answers.append(answer)
diff --git a/task_eval/test_cat5_eval.py b/task_eval/test_cat5_eval.py
new file mode 100644
index 0000000..f1a278a
--- /dev/null
+++ b/task_eval/test_cat5_eval.py
@@ -0,0 +1,34 @@
+"""Regression test for category-5 (adversarial) scoring.
+
+The released data/locomo10.json stores the ground truth for category-5
+(adversarial) questions under the key 'adversarial_answer', not 'answer'.
+eval_question_answering previously read line['answer'] unconditionally and
+raised KeyError on every category-5 question, crashing the whole pipeline.
+
+Run with: python -m task_eval.test_cat5_eval   (or pytest task_eval/test_cat5_eval.py)
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from task_eval.evaluation import eval_question_answering
+
+
+def test_cat5_scores_without_answer_key():
+    # A category-5 record exactly as released: no 'answer', only
+    # 'adversarial_answer'. Correct model behaviour is "not mentioned".
+    qas = [{
+        "question": "What did Caroline realize after her charity race?",
+        "evidence": ["D2:3"],
+        "category": 5,
+        "adversarial_answer": "self-care is important",
+        "model_prediction": "Not mentioned in the conversation",
+    }]
+    scores, _lens, _recall = eval_question_answering(qas, 'model_prediction')
+    assert scores == [1], scores
+
+
+if __name__ == "__main__":
+    test_cat5_scores_without_answer_key()
+    print("OK: category-5 scoring works on released-format data")