From d03cc57bf761abca72090fb9ee88e97213c6fafb Mon Sep 17 00:00:00 2001 From: linlin Date: Sun, 26 Apr 2026 01:53:13 +0800 Subject: [PATCH] feat: add data loader and fake test data - JSONL loader with capability/scenario filtering and Pydantic validation - Sample-to-EvalSample conversion for eval scenario (strips Gold_Trace) - 6 fake samples across 5 capabilities with realistic research tasks - Each sample has complete 7-tuple structure with gold traces and negative cases Co-Authored-By: Claude Opus 4.6 --- .../data/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 172 bytes .../__pycache__/generator.cpython-310.pyc | Bin 0 -> 27369 bytes .../data/__pycache__/loader.cpython-310.pyc | Bin 0 -> 3666 bytes airbench/data/generator.py | 442 ++++++++++++++++++ airbench/data/loader.py | 147 ++++++ data/fake/c1_knowledge_compression.jsonl | 2 + data/fake/c2_math_causal_reasoning.jsonl | 1 + data/fake/c3_task_planning.jsonl | 1 + data/fake/c4_gap_mining_innovation.jsonl | 1 + data/fake/c5_engineering_experiment.jsonl | 1 + 10 files changed, 595 insertions(+) create mode 100644 airbench/data/__pycache__/__init__.cpython-310.pyc create mode 100644 airbench/data/__pycache__/generator.cpython-310.pyc create mode 100644 airbench/data/__pycache__/loader.cpython-310.pyc create mode 100644 airbench/data/generator.py create mode 100644 airbench/data/loader.py create mode 100644 data/fake/c1_knowledge_compression.jsonl create mode 100644 data/fake/c2_math_causal_reasoning.jsonl create mode 100644 data/fake/c3_task_planning.jsonl create mode 100644 data/fake/c4_gap_mining_innovation.jsonl create mode 100644 data/fake/c5_engineering_experiment.jsonl diff --git a/airbench/data/__pycache__/__init__.cpython-310.pyc b/airbench/data/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3fae94dbee7893359110d2ac30750ce8b498bfba GIT binary patch literal 172 zcmd1j<>g`kf}bznWXb^P#~=xZ i%!mP*5fdMunU`4-AFo$Xd5gmaVxk?${$h|h91H;3t1F`b literal 0 HcmV?d00001 diff --git a/airbench/data/__pycache__/generator.cpython-310.pyc b/airbench/data/__pycache__/generator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d99b8d8e592aa97a90fbbec01504e842c6358c7b GIT binary patch literal 27369 zcmbV#31C#`ou4FhAr9j^joTBgbhOU4+ikPX)$X?Hx%Rv_GrP&U+cs_2PU`*r|NrkjJqhBZ z3r3oG?|a|(zpwBAJ^R+J%SgbVijRI#_E1Yg!h7_^{^xFd`5OKOzL=Vjpd~mHv_vhb zJkgnm->b@#DpomHCDOIz^5lvXX9|B#DNn6PbEZ|KJJTyxJ6Bg^I5R5NIM-CHb*`;g z=UkVV;8~}odQuLp*V43fZMBx6ttm~?*0!W->q--}^_LSptF^nd4Y+o9X`*%y{@ttH zr`_L@;z`#Y&>qx2p*^I1vLzYc9@ajk{ekvr`c|6c$^YFKFKZmH){a_O9@AdYsuTvt zwI5iR{Zp;l`g}q=sW2lx)o7;xbFEgV)niS;twC$lnzYk^s9E8`d@4Q(JE%+vuqI<>R-bWS_3@aWRIwH~e4#;pt5MTN)bv`bo_g-5@3 z+2Ypg+JN zvaU^Nlejj8Ytss6;a#HhUf?_fobS`Zig&XL=Q)g?XPg(be+EqdIdFa(|9+_bvBLOY zX#Z0C6YXDF7#C@Os{Lz)@w3{$(SBrM{Ab!b+K(0Iexm)be>(SOqZGp_v?T>G!qwg0C5C9r%K*WO!!<^8}i0xTcUbj4Xi zVHw5fn3B*quKg7-{A=wO!1tHh-zYra*Z#Zqw>CZeAKL%a{+EU4ueASd;rV~G|7+p% zciIP*RDP)a+QRKO+W)i0eAJ$(CCE3@M0`ucza%XWKUb9|;AgUl9X_YvU#ixGpJ{wA z-F9y^u4eFaYxv$;xmRlP`9Wa6-g6gzZt&cVpZ9p~#n1aZ_v7aSo(IcPoS(p_^_XQH zes93<_4s`ce$zAe;r9moegMDk#@L7O`xE$gujiBad7qYaAQ`{!$G-mbrN{3maaX!uEGsVy9(B0A znj_$T2{+sU$Jh5gSG31*pbXJ+pHzB(iYe4c*$M9y{y9H4V3x3i|cmz%C-G|cZp|r&{MH^ z*B)PKSxJG{>kGPrbboD;rxc%F@@#VlJd5l0hRQwL{AEFppKhk^V?-7+Tdawr8bcI4|KU(-t$9Nab7?2)*bEx&wz@!K0NPXNFdG;6PbPs62Rd zF*V=|`Aa;`j_0egW^}Xt*B2UQgE_ZD6m6y9;^nr6g&9SqroG|j1zE^!Z z^BIrB6Lh%CH$LXb%gOz6R&^HtHhLm5H4vG)9u42r zXRf?geKIRK^c4b_v5)sw|Nl-A`aGV-b4GK$UOyA7zoCzviA+^v31hBT?;X;|tK(hm z05xIhcrz%*RIham7x$&?A^9HX4pq*Xp&C z`ar#1lw`qJO|#K88oe|a8?6gHMZkw^Wx3espojz@kIlRR9>;F>N1I2$yvS6GG0~{E zHRM*f!qepK(3*r-F7ka=KV|Y-+AS=bWuF{Vuz8@+JK)V9YwI%LQz+LVtIp{9)I@h=Z zflx&yX=K2;K2YMT^tfL1g}j>Ef7H1qCB3D~GB+`-rTmfVx%rkYnAU>})7w##B`b4j`#F}gL**2_bv<^n5P8%1% z?^BW4F=3Dh4;F85v5_IL2&;rPtU?M?-EJo!OwfDPXEJg(I#w7LtZq^5{n>_( zw{`E&Hi~u;Wjls$=fv%@08>G7&m1ydYk@`(I+ri%voqkz9YjOP3~;+}5p)G^c1Zzx z8F@g0&4R@y0m{y6prTJNM5Y_26zfJr=CdiFmAKqkY zZKEZT5I8Y;cDamSK4siGZSrWSU%z--A80WyoP?{wxY(I<#QCN^AQzddgME&Uz;84} zTb1*M9gR)gSVlBU<@6J9A7}KT@yK+0{BoUs=@JRj=yY_bI(oLv(g?AbrdhczYK0j~ z5AdJAc`7T}xz1f#>G!!y4!XSV3XgL`@L-wuFkHC{S4%trSkeP!h(TPwP_Qx-bguh; z2%%W;s0&8a?+M)ngD%>xyf|!SA#A15aK>o4EUpv!9lHuE*A45bH=HBK$}Rw2o;+dQ z4bt6nBk{}U35mKEnJ?Np21_Udh?)HjxGOe8S*O|w5HMcfCY}dqDWlvo0-Ix;AC+ed zciL4(h-xi}hh#MebiHt=1M*@D5UnO*%jAeYdlK`J(QO;mhp$AY`)CGJCH49_VvP9S zo*5Wy(Qr&oj72PSW5_twM#!}TgG>74TfnR!6!h)<hOc#WzN#FHwMTqB}Ysj?B(t zfym5hqox+Kgo?$636aDhKw59NFPViqv-B}#NPvrY8RLAPJ_buUrq_-~X5N5Jjb59H zpSuZwM56^-&0B3FMCE})5M!Nv&EYh0*7sD`y@wyt)BJg{W&bC7nU zr8Cxwz=C96rlsq%Z9q*#67#bbN@QoUQ4BLXFQWMGa&ripz;{3e-WaDZ>Fo`Xa2Vu{ zEg*avR_HK?XgQcmF-**Ffs+_BMm`ZL9$B~$?d&StDfq!|!Q`?zY=yTZOF7%A*A7~& z$1NeDae5ZclZ@iU7Jxug*3QQJ=VczaN9z%>sRRpbA-qjwz6TV8v51WfN5eIb>hmM9 zu^OTge~%8Ijdl)VKBWnh<77UBz8w=rXOG@97Vn-nNx34HHf zBNeJ=MSG^iV+$`tu_O|Cx4HxW!N7_N6cvfX+P2fA=gmv;dbs&X0TrLYws?0N90?xM zo2%t6P#`nedO3RYve9!B)|zyvdp>saQe^(Lj1fIIX*1D4#N=QfO3Dl)vT&RUB7p0h z0JP`>m-JeSOR)sF2x5a*G?G~XXrnz%wDeT1(KQBpfxr$+l;-oFzkbTO?x4H8JX8WJ zNAY2(h{IwP-a;h{D%{7AG4AysJUv3e>9;-Qm6j!@;Ai@hxI*v|ewK``EknUlq35lz z7g4@`^|aiFr;!ZeI2!U(&d=5+CAbZ_v5M^KxZ5Qb$6_ zxegu>-+~?&;@+d7Qve34jGRSc9Y75`jW7_&S^V-@5Keq2)+K|)VVxaaByIZqIeKdL zJPad^M_5ubN8@WJNoML>VDxZZV&h17Qm>u`%=%0RDRg5C!Z@PJ%K!wH$hdit<{0WX zE-Xk;hbIW>xB4*|ri{$JCi$3^Ktd8D;p)hA!+X^y0Hb`i!-(tZA_t&^XiGh@2i9cl z3>J58IN&Y`!j-w)B_+scOOA%}0S-(qc%#?XA;gCxCZ9Fi37`R>I2YJMeFWj>sNQ}> zhLz$m{Q1dKS*sS4*@dprJb@B_8D|^L)FYm<(t|S5(`ve}h zP(!ArH@u;bG#i~aL09K`%!FJYfNGw~;K9)S7`#Hs4Ymo+y8}Ug2srt{XNh%quq39E z(ufK`cbqAxGWc_OfVH%~P#fw!tkHU&s70Pqojo6FoWyEDaTO?=lLAPI16xRo?f&;Ok zYVi-Sf=O(9eNGY;+P(e(866m!z5JS_L%)Op6 zFG_MH9#p!5jy)c?-;1cwaR8V(o(WaBy^ft8kM^RwjjItOTE4U4;1mr@usxzFGYYBKk3VBpJ$7Gjoc{ zBmn-6EkGtRb(3g06{euE`HazUKIFh)7%uYuvolnZ;P6w*GnvK3&mn%y3Vn8I(7;up zpim0q$kbmssW03LxoH&FCm43F42gPBWvKm>a6;8<%o>u=BsNJApWubT!T<@w=_`Y3 zRHFv@8ta&-gGE|O&ZcHes*h4o*hYSaU9DCBSxHbx42Z2tNwpJ^@L9!uE(m7oS59s( zevV`>diu@i8>eN>kP}G8Yla9f8#fUfQph=cp5%<3Gorxg+z1wc?~BzF*_TofW*=~Y z$tMXI8!~EJ!2s0CLS(Cr=0=%Y%8o)6DiVDUi4RqXF5f^x>Ap7yY#sgjYuEtNKhE0$@A)qFnW(^sl9pgq^X7pPw*B zZV{1%ohTi!7TK`_pk!c7zG856q#7oO%n?swuwXNM$Yr+qyc$9kABqJaI>YLQ!?3}+ z8AO)^O2ojv7=h9^jasi5SW)#Tr}9+Ku0DaJ2qBR?A{jRxk)(*I&$!tqL~0QU59-r5 zV*ULR#Hu-YufWoGh8~q5yX=wZR5?Vu9aTz#@S7M=bvvF_7;Ik|V+wXF4subz&Qshp z&h^2p+)i)0#p*$3-sTJ)w1AzXG?1+e*h8P0>w3>ccv@=?zJ=$M8nP(jCS*0!1TKrd zPYs(t35$VlA-T~pYBZm~9esA(_HzcHP}$7xp+Kya=EkhkgO#XYi9y-BM`R{CbOR2V z0|jgr^kn8|DeJ@P0|1an(p@f=Te)rN_qS&J#p{*7e9F06!l+VrW#}8Q#cT#pj_4Vo zyaEPea1N{$PFrb=(FniZh}cM6W4vZg@-BKcytg zcydXI{yRpYO5*q-hHyj(PM`qTq6~(V4ixN$Z$@rN$;wn1x*~+orzf~ZN+8_<9?S+? zw5Qx%;dbPal(H|2Ha3VUp}eI$fauuDCfKdtF`v_GhiJb|g(~bdF}o`Ha2@2Lc&ZG`2i>r>+04|pgKw0c#;MXv@kwg(2^)4^Y%Hj$ZWK9 z>VxMfHfk6XGB%+$hB!JJ?m|IOWqIteqv5((6IGN}WYNy_65mUReM>!(1aVj^fti&q zm3DX0pFe%dxuFCRI4ZsYpI2p0v|}e)E3&*GwovKjOVO?AqxDxFp&O_-pg?%G(0vI;u8k_xgu7X6%3R^gr z689@1mdd3#Yvo)@F57K_N}f~*OUaW2SCUe3!OBm7W3xMtH@oktz*f5lKRDAx_T)Q& zjD}OKD<_eNLo%5Iv6~n4wh80xDE2C8x0H9lJI4?d-+UbfN95G++{irUdc}MAIE@yG1h=rQhoT1Q$@B!gRlL5*7=?$XtxF7pb*6xR z{sJ5yv3*AW}gYi|9oFlHDt8T0xC@^BDFv z8g8(V#C(u-@k~Z%SM=(c$n+HnuYnlt6s#b>r0FQh)Yb^yWqqY<%V6SbU&rRK#|z}C zXg=E(+PiGZ@lI??Qb2v81I&}D@DoJGMbe(mF{-Pdn!s1$pddx}mh+LbDcKj7#0!Q^ zL~vO)MTHoMh6o9=2~Zez5F2Qr$8iT)jm%Fg3CARpDI`9&og{kq+=LzFtcXhDGCWj> zhfPt_+peP?jUp!!2deZUghT&94R-gdksVlKD}q8Y!5JmYK8DE{aF8jd>M54>D;EDK+Z z1JMDAAUA;Z82q5m>&ZUo3nEE@A;dK#OVA*-P}bHTSGx`qt(Z$Z8v&5DwFncxGwVA1 z*gLPg4)eUo?nYZ^k*&CN$bknBy*;Uh9im}}vdBxr0W$A&Mw|euy%FyiK@|c}lY|Uo z!-A$F`m++m`=^mBQsBsszza#e0?&s2MC~NAJrSaFlJUqR`+}ZINA44j74j`}Jn~3J zv}1h5Ev%ohL~F-Z2N8jchaxy%0p+XAt66VPE)B_YY{CP?m<+KUnC`X(8YGxh6hIMV zx)IwI85s)PyeAygD^fcM4vF{zQ2P>FiL8em*z^!{h#n(IEpe^ohA>X1rb!u?C0GGH zm;vaSYYt``u4U%^xH{{*uVioEly|&}E>_`7F20~z{Yv)RbM#0yJ(Bg^{rnK!KE8DI z_!72=GFDg(=pQh&(91`vVX>e371|U>ncs{m(GO4okXH_E%{~6zSDro2lH_=MatZuH zkI{fbHbSqM7WCQUKH~5laO_9VSGl9u?|V@-q}j>68Sua=!3mf?0Wd6+0H(Hm>BGGA zVPf|02_txWyDb-k{m3Jb5xbSu$1kGPw_-T!wsY)Lu&aYK&dxeqwtb9f&eilg^i)r7!Ju?X{T~@mUMYL z%MwF;q zB@2W+qeuwI5ESRp#t8+U;#8I2ei#>2xDE?LeQ5)#^vgVUCX?7dJ!qQL+0p&i0y-lX7AQsQh5am|mgjfl4MIoiG(opm0!A1xoWNl0aVJWUv> z(&$w*3%u%Ig9{vj7i@YO6{^^1X9(vs(7s2#gdhlYd+|0EvD?;#Dq*se{A5BiPwtit zL5VkDN#3ib9Y&Z4)EcC>260yAw*7lKf<``%-SH^|7~IO%qWTl95&@(#-noE&3aC9= z>WpwdcKu4V1jDgkfN$k}Y5&k+%>QTyX^z1dtzl z8?(eq z>-3m9R$UY8I-}Pi;Tc7t&1xN@)`yWMec%+OSxbGZc1EjkSejC4ISH37zy&VV*Glz+ zrT&%aMvA++R}lMhaNYPSG*TjHHLRKiA)yQnZaa&?)*`|pnvIzwoDt6BV*K^ic-J-R z#OiNgETmK+5~H`)>1}VJ6B(ii6+{Zg+NOrgEMRM2&^3Bn^rn6Sw;$!xdE@LHcNtqd z7j~E-t+3nXt_AH~i#}isayO}Ma4}^C&n;=T(X(;^J!(C98^J>3^flqVBu}`Uzft&T zmjTmi^lYYQz;>mjtRi@3i+aWq1zh{mMy2#D4A2dc2i*DcM(V~M>9$J*6j)lb1U!kC zfrP0iQ2b@8jPG~QR?HImG;A-m6S5gx;;ghjGv!R9szAJmz_ok22ns+%EY)~ zVa0R^mv-;jCp?xS-Mfv0xm&(~Fq0UB2A^{iVoYH|VK}ih@owWl-WLF1hVf>uqpT|SQ{5NDbRL{?E@6F|&%CZ+aO$i>0xZxg%{@1$FPsBy5Yc=k zM<2Kba*)fwCf&IS+R|>IeoEei--O-uO?1R$>*G)4K9NhCD*Ajae48n1-GzmFGYj4R zQtEhgY{$_CKf1|}BIP`>y||d@!38_D2Wy$8;n($sE)?y#yVbg?==MZEALuuPX4F;Joh2ZF-s-Bf;wy6>*9*SLlL!Oi2 zdP^T|m2qD=a#r+I$rC%UDB6B(tEd`JWTD?UF&eGAm9ZIg;FcHi9wyk&?EPA9&SMU0 z*5(h8DcDhTg%>=(jbvEw=>b736E(uq#_~B2&~z2G7oXVkH72F@R>pepT&Ne9n|MQ4ysq$RR5z52m~M z(W__=Q+o=+<*cP1Wc>_EDX>c2xtm#T8qgAwk59leS~JYI8i_dK$Seno2P&T=Gyt-c z*`D@!4^SJHx6J74M-+=CkqSuVfc1T*gM?>!_`v!kTMn2}85&@)-Q%g;SBc&VOuzYu z=g47aKAyAd=`7MIZnnlmvZ1^W74E}$g+s2s5wZjCs6&-Lmz<|@v0&|CEU><{H@pD5WfRECCGTN9-(yInR zSgG1xeWp`3vj7Lp!V+uE0yxlM1$$rn&gwv;{Xu_*V zna>dE5lr=5G{W7~2iY)D`e#3)Kfib?bbtag&h(_RN(ga3Cna7@NT&v+c~Z4RS{pJZ zj8uMUqjM05GsFQ)i|vw==HQ9kQac4*%eS%Yb4O4U=cHAi8^;E+vO=aMAcaJtD)FIt z5n+nOR4j<@Uu>_+NCYlOIPI=f3dUklsH0{4BmqTzc4!Zegx{pCBNUg>nt5RmGL1q7 zo{$YWOF)JA^xV@#H*OaMxr_@7v?{j3&Z-@8MH#nul^q zAFMmId{Qk}f*OfsBxiu>b*?$!aRAV8YOB8-hMxxr+|X(JnbWizJ$C7gSu--hhCcCSH%dtHUn$5x0LS?X}A~MRo)A zG%FhaXkpnq@t2kQ{5-8#EkPe_rZyI2*0Mo1{S@-gd`e{R1(+5`g;j_u^8NSWI=V|& z!EZU!O|RojQ{F_L&SNAklFL%eU+WqKRFa()$qN-$`3wZP9Nx<|_j^p=A-g~3m}5zm zPs*bCoT*A&C-mh#6Ve2(sYuR)`NiDI5JW>H5GL1vSTfmF=PG1E&NO9vksoj7{PcoEIAu)*U3P zP$^urKhGEGT%#QA0OV+6r(-mH)(koj?V@g5vTXG;I37zGBzN9YcQJNntFPh_84Pjd zmLPSclJ|*dmyk}6a1si`Dwyk*vQHvh3dUzZw8~0e0s)e&A}$mCK7sv zQr2Mh!~DI0tXSn$-KTbXv}d=IgOD?UU&1pOhy4Z8Ejj8reiGeqa+)YQGpZVg`Jy1L z2ybnDDU>jcq#vNE1O~DO@t&ImQDq3_$$BcApp7&2v|U0FPoh}53HE3cS|$)u+Hzn! z|KR#)@11+;xH(L6U^_Qe0{lHJE63*)SCI`iO;DBaVmthX;xG=!asx1~2%im{cx*k7 zuWH+!StmrGi>5>iHmrM!bG_9dA+Z^IqrIcl9q|^KO!OXrCI}y5?Q?qX7|Nj-BfZbq zvXWC5HUD=T1CL^%Tc05Ry-h!h( zZ$ds~w;Ge++>2g!OOn(!9GgK~6e)mEEA%t9d&PZr%V>%x`OJ>l%hzq^l{5=UHlYs> z=x_BrN@roNqA_Ec`l)g3ll9iq(b;Q{Dt<1Uh+YlD7l|k#qD(0GB6TPDODv4!*0l=9m3$PA8rUbj%U1mXSJOK7Z-dCV<7IzA{c8#hiFvu)xK zNLZL>vXDF?&Q}IgR@el>Q)eHYWx7c1_Bf# zhE$H`91}1o?9|y5>f%B7pft~0a{_mnR#BXfIm%se1|%$IP(u#B9{c@BqU9kmoIgBg z*Cf~~MIFXg6a=a+wnpv6$rohU82#Jmp@>&cM?vs>{QO(`e7^*bJhRwBGL3xnS%9{9 z0CBycl-Fd0L`8~vT+NqeRwWD_^QiKRFp#d!)rs|EsT30k5kvX~e2w7&83+(40$-so z?7&1+mm%B{#?eO{6{lt)^il9I66ZG5Lx(9^Mh7P!t|0xtif&vw%p{zLAk#wJ+#`*| zpe*%_g=eU5S5{<26uy9u)F7LV5*9zINW-xLek>PNJt7v8O zcR)erQnTX~352*} zexfns#*nodsy8H%bfAa5y6vt$g5xtoG9`h)5i6#FXQTW{Y03rVPY`D0hD(qKNX4f+#1EICG_#6Z1@(by$JaN%rmW*B+W zO954C?HZCQ4#fp*b}&jG;e|L(jXB!pS={-%!d7lc3cw61!9v{AYEcq z1Xs~I#n|OafCfCI5-4b@ktXnD77z+?0aN-bWO+$vfracTEQ26kOXpb;EGbAS2RdO) zU8A~A8?}Kjq93OqOSI$H14N;THEm)c?8;D7urIqbgrh~;?W}^OM7}CguT%$ymu;8~ zCbjGsmrPZWWVx&h%Vi$e1!kHGWR+i=t&Vq8D?x9%g|=b^M#lt>-^rZ6O{`U@#v-_L z4=J74KUD4--`XTLSkA86C7t}s3h89Il(p>g4jHt``xMH9s$?ORb>zLUHze#5m@oHa zcM@~SSOh`HVZ8cPJdK*UkQAbaRvlcix~K#N2ng?f;wT>SKM`<=aRrX0QS|BxH3rz+ z<|tUBa9+cdOXXmB7gy9Jv&iSo-o1M}%q4b}RWFxvW*o!_qx1pvr3@;DFV_Kz&)Tn$ zfdaQ>17i9uDvUv?8roNQd-3O_V{;2a6_aUn9uPJf+sg!48=d)7M5pclJBvLSD-Dxs zBnw04zHNo%F-SC)<1G-0vl4|6s5M_Uj?LknHle@@prwi?>VU;8WaPSXwVdDyGKxUU zH3IMuGF%uDxL~YM<&fxJ8e~9`<+kv&bfv2kFkyNfhazqdAK}e*e7ufGm?;vsoUL}v z5RtU#EWEgbT2NxsfM7vI@bu!0YWN07X6X-qmH9&3+rNA&^i8BKsM9Dk zK#vaM^Q{ue;6Vs0_3S|Z&YXwULT|dkaTykOrr~(02k&n(t5lTwnW>oSRQ~x>s4AIT z|8ks^nxa+0$Zud!XG!l5dP)v@xf$190OP}_wm6P(kuum_^a3foe+TDU*>Xv`ygUn+ zFh=bUqHcBXQkV5onTr6j*A*lkM(JlWRja5c0U0c-0V)Wg&spG0Fk=+}SP-b{z(ZAY zgj|%42#!6t#v~RKLuC;uYKgQW1{HbZc31g`B_IY%_wQ7yF#Dfb%Qm?9INzq?b@3^DdcJiKUS>lh8#8VTL- zY)cEm+y`tN62K-VqGSapkSUSfw+C-;1pd3AY z)R4z-Ba`EHGzk}~RngK$2$>n0hMtdHVk4N_-@rDXUQ;GL_?v0JiHjRp)&f(Ol9p*~ zabXCrP0up8ASIQ!klOP*%>-O-(4a8NdLLBk5F z!o=sEzUa+95eV^!`sgeg$*k$g-1EsbJELX1y?-O#-GQwrI-<)H$}2Er?|?}bzc!BC zM952B9Pa@FYYo;aRv=3c9Hp>tAJmC184Y6Z1vIo?*>jy?dq#VIpqV=lBPXJbC~HGGp_Pk~M668BMA zfnGttt9DH5XVD3`Z2mlqxm_M;2^5#c&e%ACQ*%(kO>kn=me}zD- zY>QcaE_U@3AtOGE8A1gL)Tf-0J6HsJr1dyHPU_1=hXA(gGwgPmgrcDf4`Igzy*G3m z1mwi44y13A)!1Ahm*zGFLLfoCtla-@L&Gb%o@{hWA5@NP?_!9%<^5JW+QJJU5>$DfgSe<I0}OFz8s|EBAzq?w;!ils5k(oyseMl5hOGz zz#PIc$UD|RIXP+!n`Z*HnE+!smNzC@fKk&l3$f#+faB>Ea!l8DY%ic+a_$14Sj@FV zTk2sDC^{>Q+M^i}8K>I;0YSs%L}`#8cymml31^i@`(5jwLH=?hxGJzKYAVOzi$zD$0&vq1!bA| zALs&l>C%EM%lkv5M1+(F&&WFaaeB??f7*w)wZulo^f~Iy#EEw0f7!%4?PO8{nw&ag0u$KmJ+yhKi9O~52i>Bl63i85+o zyo3+-3*9L=i@@Hr*oiMYi9EmwM2yp2Y`Rmfxji z$tDtL&Z?C#3|z-)Z2=u8-4y8jN<>8y(H&rjSvfGJDY=oDds?s(vNk!{0Zgog6%M?$ zM8NTsN^;t+Q<;wC3H`OO#)2MEfg4?9nEm|?2tnlFuDKU}67OU}fsWp$Wjl^+j{;;b zj~hOg_~=^d%&;@Z>dbK7SBZB*(<>^MHHgqLTPxa7p}f^2a+lwOv$7iAU8XuzD0@~6 z#)+7L7D~3zOR^#0E9YbBF7tRYr`gg>#l*1ACU2oMdmT-~VGVjgD5nYiA+9iomghPv zXpX#>&g@#wOnNv>?_olX{Q$fq0Y|m`;~c=%J0Pi zOMcIOQSy7!_sSaiYFBxQJjnPErH39B6eY(ZhYqVL8V&`AT+pFYrl^d^`RQaD#mmV! zS(&>v1#@m}fekqJX}oUhu-A8_+@s-4Ikg_44nyw`-*^ZI)XQ-{&qYU|@g69CYcbw; zj_a^T>iz?0*5SS-^T3An+-4W`s8p2EVB1>()cwzZsyN18=JoLFxou`e27IuZUVgrb zUdO(eA#eD^>! zB?+&5Uv)ewnAEqbKVf^qcfRpL((zTtlY_}ss~$=?o>G-qmGW@H@zm0U<7rjNxWE2* zdN4&>UzSjnK9Ja&czktLT3>3_>i)#Ugzta0D)msBmaL^{sVzxu4;;^^O2ZRtFealf zy=rye>iz^R?ZE^d`@}aBwDjbJ#k#;GhaxLH=CE+-d}Hj z&HVnXq{TF}WFn#nET(&2Mo*T@ci2xvTueN2EF}~?ko`DbD4z}oj$`Q=e5mjV$FYZ(zxiIhuHK_L@D9fh5;t8-$Xe@9qJ>f_4ig90 z6ztjK+E-9myl2P0#T0r8Zea1=Z3V>zPw(Edd;hmxJ9qEd;VLXB_7h7NQ{9!79%d$+^bwK7xigeoy~mI=zOU-s2{pXs|QQytxV^ovHH8nRMWwpH7?bj}JEOd*TJN zVEKaiKM#1ptzO*!0?F`Daf0L8a z6W1rMP27Or_a!EM^r?^5CVw*VlSzI$e!rORf-)kAc4Z~{spOgU2a9Pgm*y*Rxtz&F z2`9K5JXl`#B9q*Cj|tI6ZU%IN{eG&(Iq$M=1WM@j2L5DR^{3Dess#Ge=tnyJATIdn zf4k$n|JkDFzPV?|_FX$%+ny^d-n(PpzTM9iIWxB9n(TDmUs$mJ8P~Rg=l2!taqZnv zfdA9M?xI~7l4lKBx4&TDv##Pj1x4yszIE&VT?NIi!rgS;wY#Y3xo;HgmjyOkLmt>s zv}I|azc9Q+QhX-03R`lEC(Ir*cc-z5Ji>B--7m5-8ETl3vvX3YIzdOBcK zbE_3TO#W%|A177az6LP59h2X&#%<953^;ss+1zPFrGLoWOq~zXS1(b=!r;ePW!th< zemgII2L?i=JFM|jtWmIRjejsE>%ob&>1)A>wMk1qlRixTN%BXjY7J27L&8=5CV%+p L)oUNhz!QHDLuxqY literal 0 HcmV?d00001 diff --git a/airbench/data/__pycache__/loader.cpython-310.pyc b/airbench/data/__pycache__/loader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10abcc9d7b8ce2dc6f94abac708a937bfcd8460a GIT binary patch literal 3666 zcma)9TW{RP73R!xxm;?sy2MGH#I6U@4IEAp~@0W(j@#)j4FjX!_m#;xB(X@6JzcH{T!?+O{} zP$XF>Bem*z@1m27)xA7R6R{m9Q7-m(BNd4##EB$MBN68!6dU1Sm_#e$qcDj<0o%fz zFivwV^ls$cJdk0kv2rd<(@cjt&eD7xWBX>PckylhA;lCr3-47tVDy$!nZlI!W=*xnOYbdk zSutEXXW!fipj=#1Ix5m5m8up7eMa|A9_qL8zDbsT5glz6)*i#l@iy?c$q^R9R7}}h?X^mK5b3aVus#^-((6NP zkcd-V&L!DS+z*UC%61y<1pP3N^0FO4zCgx&56ql56W((%RpxV zaq^zNG%FM##ThJY+yp2NAGlL zYF+GD&(i4hQWEF-1LF}NR2E*DQS0s7wN*I?P-K%fj8$KUjmfG<>=5kKe{Q1w2wq+2x6whI?GI!CNkwmpFkk*OIAiEL{x>gyZ`B=Fn}RfsT%RBnc^)Q=gGe~2|Pbr3UKox=)( z3m7zuVXJ1tnX^?#>oa%mTsHoA`OY0t@d|_*B}vU^#hJLWAA|TbqH?~C{=@kKNKA7+-s@Nlx&hiMH5PGam=|QML zsDZ*~s=Z~P?h(Bj`X+#?&_Pg5*}q-i{@w)Yf#e1|a}lRT(<-_^R2tN-uYr(P$--BK%#c=q{UZ7pu2ZQSOpKx@4v=#p zGi`@)l8ZX|z$BELkQb`6rmWAd5b9biWiOyrML z6PH&>fbg?)LbzTk4H)-4^(?It#2R!hTLAu1qH~{b@A@w3H2_n-tHS+&Mx{;gUpAtT z`zB|Yj8QdD5p|Dv4NL)X1><}jjr&8Fb(zf<`8nnyCve7jmo4J^4CCK&&i~{73EyTH zyB?i8=WyooMb_c`AKth2#la~7o&2Ch+Q<^|&zVC)f}6fBpa1pAwn^5c%RP^DlsoVP5@|dOxFv68DTXmkiHz zFqklYKpTER%`efEZ5b;dccu<%e$Nz7wG}DfxWISkSdA=uVoUyj9Wi9p&Y_Jw)YML4 z7YzBUaojK&>AcA}_BgGi2T(s!tpbxmQS&sxAc{9ss3>DmR`sJiM|5Iij&c;eOriD# z8Ym$t=M3T$aD(D?5(qVToC#_$X#yVO*465IqsLxr66kS`dQ7R8NJCC_jc@NH*%o1d zrr0H?g0-5!L!1sb+ zdxT;<3Iaubq None: + """Generate all fake data JSONL files.""" + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + total = 0 + for capability, raw_samples in ALL_SAMPLES.items(): + filename = CAPABILITY_FILE_MAP[capability] + filepath = output_path / filename + + samples = [] + for raw in raw_samples: + sample = Sample(**raw) + samples.append(sample) + + with filepath.open("w", encoding="utf-8") as fh: + for sample in samples: + fh.write(sample.model_dump_json() + "\n") + + total += len(samples) + print(f" Generated {len(samples)} samples -> {filepath}") + + print(f"\nTotal: {total} fake samples generated in {output_path}/") + + +if __name__ == "__main__": + generate_fake_data() diff --git a/airbench/data/loader.py b/airbench/data/loader.py new file mode 100644 index 0000000..0fd7d0c --- /dev/null +++ b/airbench/data/loader.py @@ -0,0 +1,147 @@ +"""AIRBench JSONL data loader. + +Loads .jsonl files where each line is a Sample, validates against the +Pydantic schema, and supports filtering by capability and scenario. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Union + +from pydantic import ValidationError + +from airbench.constants import Capability +from airbench.schema import EvalSample, Sample + + +# ------------------------------------------------------------------ +# Conversion +# ------------------------------------------------------------------ + +def sample_to_eval(sample: Sample) -> EvalSample: + """Strip gold_trace from a Sample to produce an EvalSample.""" + return EvalSample( + meta=sample.meta, + prompt=sample.prompt, + constraint=sample.constraint, + logic_annot=sample.logic_annot, + neg_cases=sample.neg_cases, + eval_dims=sample.eval_dims, + rule_criteria=sample.rule_criteria, + ) + + +# ------------------------------------------------------------------ +# Core loader +# ------------------------------------------------------------------ + +def load_jsonl( + path: str | Path, + *, + scenario: str = "train", + capability: Capability | None = None, +) -> list[Union[Sample, EvalSample]]: + """Load a JSONL file and return a list of Sample or EvalSample objects. + + Parameters + ---------- + path : str | Path + Path to the .jsonl file. + scenario : str + ``"train"`` returns full Sample objects (with gold_trace). + ``"eval"`` returns EvalSample objects (gold_trace stripped). + capability : Capability | None + If set, only return samples whose ``meta.capability`` matches. + + Returns + ------- + list[Sample | EvalSample] + Validated samples in the requested scenario view. + + Raises + ------ + FileNotFoundError + If *path* does not exist. + ValueError + If a line fails Pydantic validation. + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Data file not found: {path}") + + results: list[Union[Sample, EvalSample]] = [] + + with path.open("r", encoding="utf-8") as fh: + for line_num, raw_line in enumerate(fh, start=1): + raw_line = raw_line.strip() + if not raw_line: + continue + + try: + data = json.loads(raw_line) + except json.JSONDecodeError as exc: + raise ValueError( + f"Invalid JSON at {path.name}:{line_num}: {exc}" + ) from exc + + try: + sample = Sample.model_validate(data) + except ValidationError as exc: + raise ValueError( + f"Schema validation failed at {path.name}:{line_num}: {exc}" + ) from exc + + # Filter by capability + if capability is not None and sample.meta.capability != capability: + continue + + if scenario == "eval": + results.append(sample_to_eval(sample)) + else: + results.append(sample) + + return results + + +# ------------------------------------------------------------------ +# Convenience helpers +# ------------------------------------------------------------------ + +def load_samples( + path: str | Path, + *, + capability: Capability | None = None, +) -> list[Sample]: + """Load a JSONL file returning full Sample objects (train scenario).""" + return load_jsonl(path, scenario="train", capability=capability) # type: ignore[return-value] + + +def load_eval_samples( + path: str | Path, + *, + capability: Capability | None = None, +) -> list[EvalSample]: + """Load a JSONL file returning EvalSample objects (eval scenario).""" + return load_jsonl(path, scenario="eval", capability=capability) # type: ignore[return-value] + + +def load_directory( + directory: str | Path, + *, + scenario: str = "train", + capability: Capability | None = None, +) -> list[Union[Sample, EvalSample]]: + """Load all .jsonl files in a directory. + + Files are processed in sorted order for deterministic results. + """ + directory = Path(directory) + if not directory.is_dir(): + raise FileNotFoundError(f"Directory not found: {directory}") + + results: list[Union[Sample, EvalSample]] = [] + for jsonl_file in sorted(directory.glob("*.jsonl")): + results.extend(load_jsonl(jsonl_file, scenario=scenario, capability=capability)) + return results diff --git a/data/fake/c1_knowledge_compression.jsonl b/data/fake/c1_knowledge_compression.jsonl new file mode 100644 index 0000000..44eabb8 --- /dev/null +++ b/data/fake/c1_knowledge_compression.jsonl @@ -0,0 +1,2 @@ +{"meta":{"sample_id":"c1_001","capability":"knowledge_compression","sub_task_type":"paper_analysis","difficulty":"medium","source":"synthetic","tags":["推荐系统","论文分析"]},"prompt":"对论文《Neural Collaborative Filtering》(He et al., 2017)的核心创新点进行抽取与批判性分析，要求覆盖方法论创新与实验验证两个维度。","constraint":["分析需覆盖方法论创新与实验验证两个维度","需指出至少一个潜在局限性","使用中文输出","字数控制在500-800字"],"logic_annot":{"prerequisites":["了解协同过滤基本原理","了解神经网络基础架构"],"causal_chain":["MF局限性分析→NN替代动机→NCF框架设计→实验验证"],"assumptions":["读者具备推荐系统基础背景知识"],"scope_boundary":"仅分析该论文本身的贡献，不涉及后续改进工作","uncertainty":"论文的实验仅覆盖MovieLens和Pinterest数据集，泛化性需进一步验证"},"gold_trace":[{"approach_name":"结构化分解法","thinking_process":"首先明确论文的研究背景：传统矩阵分解用内积建模用户-物品交互，表达能力有限。然后定位核心创新：用多层神经网络替代内积，提出NCF框架，统一了GMF和MLP两种范式。接下来评估实验设计：在两个公开数据集上与多个基线对比，指标为Hit Ratio和NDCG。最后审视局限性：仅验证了隐式反馈场景，对显式评分场景的适用性未讨论。","final_output":"【核心创新点】\n1. 方法论创新：提出NCF（Neural Collaborative Filtering）框架，用神经网络替代传统矩阵分解中的内积操作，增强了用户-物品交互建模的非线性表达能力。\n2. 架构设计：框架统一了GMF（广义矩阵分解）和MLP（多层感知机）两种建模范式，通过神经层进行特征交互而非手工设计交叉特征。\n3. 实验验证：在MovieLens和Pinterest数据集上，NCF在Hit Ratio和NDCG指标上均优于传统MF方法和item-popularity基线。\n\n【潜在局限】\n1. 仅验证了隐式反馈（点击/交互）场景，未涉及显式评分预测。\n2. 模型复杂度显著增加，但计算效率分析不足。\n3. 实验数据集偏向娱乐领域，对稀疏场景的泛化能力未验证。","quality_score":0.95},{"approach_name":"对比批判法","thinking_process":"我换一个视角：先列出论文要解决的问题，再对比现有解决方案，最后评估其增量贡献。论文目标是解决MF表达能力不足的问题。对比 NeuMF 和传统MF，核心差异是用可学习的非线性函数替代固定内积。但要注意，同期已有AutoRec等工作也在做类似的事情。","final_output":"【问题定位】传统矩阵分解(MF)将用户和物品映射到共享隐空间，通过内积计算交互分数。内积操作本质上是线性加权求和，限制了模型捕获复杂交互模式的能力。\n\n【创新贡献】\n1. 提出用多层神经网络替代内积，理论上可拟合任意连续交互函数。\n2. 设计NeuMF模型，将GMF（保留MF线性特性）与MLP（引入非线性）通过拼接层融合，兼顾线性与非线性建模。\n3. 提供了预训练+微调的训练策略，解决深度模型收敛困难。\n\n【批判性分析】\n1. 与同期AutoRec(C Sedhain et al.)相比，创新增量有限——均是用NN增强CF。\n2. 负采样策略采用随机均匀采样，未探索更优的难负例采样方案。\n3. 消融实验对NeuMF的贡献度论证不够充分，GMF与MLP的融合增益未清晰量化。","quality_score":0.85}],"neg_cases":[{"error_type":"hallucination","content":"该论文在Amazon Reviews和Yelp数据集上也进行了广泛实验验证。","explanation":"论文原始实验仅使用MovieLens和Pinterest数据集，未涉及Amazon和Yelp。"},{"error_type":"template_copy","content":"本文提出了一种新颖的方法，在多个数据集上取得了优异的性能，具有重要的研究价值和应用前景。","explanation":"空洞的模板式评价，没有具体指出创新点是什么、在什么数据集上、什么指标提升了多少。"}],"eval_dims":[{"name":"factual_accuracy","description":"事实准确率：引用的论文内容是否准确","weight":0.25,"scoring_criteria":{}},{"name":"innovation_identification","description":"创新点识别完整性","weight":0.25,"scoring_criteria":{}},{"name":"critical_depth","description":"批判性分析深度","weight":0.25,"scoring_criteria":{}},{"name":"structure_clarity","description":"输出结构清晰度","weight":0.25,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c1_r1","rule_name":"创新点必须具体","rule_type":"structural","check_description":"创新点描述不能是泛泛的模板语句","keywords":["新颖的方法","优异的性能","重要价值"]},{"rule_id":"c1_r2","rule_name":"局限分析必选","rule_type":"structural","check_description":"必须包含至少一个局限性讨论","keywords":["局限","不足","未涉及","未讨论"]}]} +{"meta":{"sample_id":"c1_002","capability":"knowledge_compression","sub_task_type":"literature_review","difficulty":"hard","source":"synthetic","tags":["大语言模型","文献综述","RLHF"]},"prompt":"请对2023-2024年间关于RLHF(Reinforcement Learning from Human Feedback)的代表性工作进行综述归纳，重点比较不同RLHF变体在奖励模型训练、策略优化、对齐效果方面的差异。","constraint":["综述范围限定为2023-2024年公开发表的论文或技术报告","需至少覆盖3个不同的RLHF改进方案","需用对比表格呈现核心差异","需讨论RLHF当前面临的主要挑战"],"logic_annot":{"prerequisites":["了解强化学习基础(PPO等)","了解大模型对齐基本概念"],"causal_chain":["传统RLHF框架分析→各变体改进动机→技术路线对比→效果评估→挑战总结"],"assumptions":["讨论的论文均为公开可获取的"],"scope_boundary":"仅覆盖RLHF改进方案，不涉及其他对齐方法(如DPO)的独立讨论","uncertainty":"部分技术报告可能存在实验设置不一致的问题，对比结论需谨慎解读"},"gold_trace":[{"approach_name":"按技术维度对比法","thinking_process":"先梳理标准RLHF流程：收集偏好数据→训练奖励模型→PPO优化策略。然后定位2023-2024年主要改进方向：(1)奖励模型改进(如RLAIF用AI反馈替代人类反馈)；(2)策略优化改进(如RRHF消除单独奖励模型的依赖)；(3)数据效率改进(如条件RLHF)。接下来构建对比表格，最后总结挑战。","final_output":"【RLHF改进方案综述(2023-2024)】\n\n1. RLAIF (Lee et al., 2023): 用AI反馈(如GPT-4标注)替代人类反馈来训练奖励模型，降低了偏好数据获取成本。在Helpfulness和Harmlessness评测中接近人类标注效果。\n\n2. RRHF (Yuan et al., 2023): 将响应排序与策略优化统一到单一训练阶段，消除对独立奖励模型和PPO的依赖，简化了训练流程。\n\n3. Conditional RLHF (Yang et al., 2024): 引入条件奖励模型，根据不同对齐目标动态调整奖励函数，解决了单一奖励模型难以平衡多个对齐维度的问题。\n\n【对比表格】\n| 维度 | RLAIF | RRHF | Conditional RLHF |\n| 奖励模型 | AI标注替代人类标注 | 无需独立奖励模型 | 条件化奖励模型 |\n| 策略优化 | 标准PPO | 排序回归 | 条件PPO |\n| 数据成本 | 低(AI标注) | 中 | 中 |\n| 多目标支持 | 弱 | 弱 | 强 |\n\n【主要挑战】(1)奖励模型泛化性不足；(2)RL训练稳定性差；(3)人类偏好多样性建模困难。","quality_score":0.92}],"neg_cases":[{"error_type":"logic_gap","content":"RLHF是目前最好的对齐方法，所有大模型都使用RLHF进行对齐训练。","explanation":"逻辑断层：(1)RLHF不是唯一方案，DPO、Constitutional AI等也是主流；(2)并非所有模型都使用RLHF，如Llama 2的部分版本。"},{"error_type":"false_innovation","content":"本文提出了一种全新的RLHF方法，通过引入多智能体辩论来提升对齐质量。","explanation":"虚假创新描述，将已有概念(多智能体辩论)简单套用到RLHF上但未给出具体技术细节和实验验证。"}],"eval_dims":[{"name":"coverage","description":"文献覆盖完整性","weight":0.2,"scoring_criteria":{}},{"name":"comparison_depth","description":"对比分析深度","weight":0.25,"scoring_criteria":{}},{"name":"factual_accuracy","description":"引用准确性","weight":0.25,"scoring_criteria":{}},{"name":"challenge_insight","description":"挑战洞察力","weight":0.15,"scoring_criteria":{}},{"name":"structure_quality","description":"表格与结构质量","weight":0.15,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c1_r3","rule_name":"对比表必须包含","rule_type":"structural","check_description":"综述必须包含对比表格","keywords":["|"]},{"rule_id":"c1_r4","rule_name":"至少3个方案","rule_type":"structural","check_description":"至少覆盖3个不同RLHF变体","keywords":["RLAIF","RRHF","Conditional","DPO"]},{"rule_id":"c1_r5","rule_name":"挑战讨论必选","rule_type":"structural","check_description":"必须讨论主要挑战","keywords":["挑战","问题","困难","不足"]}]} diff --git a/data/fake/c2_math_causal_reasoning.jsonl b/data/fake/c2_math_causal_reasoning.jsonl new file mode 100644 index 0000000..a739b11 --- /dev/null +++ b/data/fake/c2_math_causal_reasoning.jsonl @@ -0,0 +1 @@ +{"meta":{"sample_id":"c2_001","capability":"math_causal_reasoning","sub_task_type":"mathematical_proof","difficulty":"hard","source":"synthetic","tags":["概率论","贝叶斯推理","数学推导"]},"prompt":"请完成以下推导：从贝叶斯定理出发，推导出高斯混合模型(GMM)的EM算法中E步的后验责任度γ(z_{nk})的完整表达式，并解释每一步的数学依据。","constraint":["推导过程必须逐步呈现，不能跳步","每一步需标注使用的数学定理或性质","需说明指数族分布的相关性质在推导中的作用","最终表达式需与标准教科书一致"],"logic_annot":{"prerequisites":["贝叶斯定理","多元高斯分布","指数族分布","Jensen不等式"],"causal_chain":["贝叶斯展开→联合分布分解→对数似然→E步推导→后验责任度"],"assumptions":["读者了解概率论基本概念","GMM模型设定已知"],"scope_boundary":"仅推导E步，不涉及M步参数更新公式的推导","uncertainty":"EM算法只保证收敛到局部最优，推导过程不讨论全局最优性"},"gold_trace":[{"approach_name":"逐步展开法","thinking_process":"先明确目标：求γ(z_{nk})=p(z_k=1|x_n)。从贝叶斯定理开始：p(z|x)=p(x|z)p(z)/p(x)。在GMM中，z是one-hot向量，先验p(z_k=1)=π_k，似然p(x|z_k=1)=N(x|μ_k,Σ_k)。分母p(x)=Σ_j π_j·N(x|μ_j,Σ_j)是全概率展开。关键点：分子分母的指数项可以展开和比较。","final_output":"【推导过程】\n\n**Step 1: 贝叶斯定理展开**\n由贝叶斯定理：\nγ(z_{nk}) = p(z_{nk}=1|x_n) = p(x_n|z_{nk}=1)·p(z_{nk}=1) / p(x_n)\n依据：贝叶斯定理（条件概率定义）\n\n**Step 2: 代入GMM设定**\np(z_{nk}=1) = π_k （混合系数，即先验概率）\np(x_n|z_{nk}=1) = N(x_n|μ_k, Σ_k) = (2π)^{-D/2}|Σ_k|^{-1/2}exp{-½(x_n-μ_k)^T Σ_k^{-1}(x_n-μ_k)}\n依据：多元高斯分布定义\n\n**Step 3: 全概率公式展开分母**\np(x_n) = Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n依据：全概率公式（Law of Total Probability）\n\n**Step 4: 合并表达式**\nγ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k) / Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n\n**指数族分布的作用**：高斯分布属于指数族分布，其概率密度可统一表示为exp{η^T·T(x)-A(η)}的形式，这一性质保证了EM算法的收敛性（E步计算后验时，指数项可线性叠加），且在M步推导中可利用充分统计量的性质简化求导。","quality_score":0.95}],"neg_cases":[{"error_type":"logic_gap","content":"γ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k)，这就是后验责任度。","explanation":"跳步严重：缺少分母的全概率展开，未说明推导依据，直接给出了不完整的公式。"},{"error_type":"hallucination","content":"根据中心极限定理，当样本量足够大时，γ(z_{nk})趋近于均匀分布。","explanation":"中心极限定理与GMM后验责任度计算无关，这是错误的数学引用。"}],"eval_dims":[{"name":"formula_correctness","description":"公式推导正确性","weight":0.35,"scoring_criteria":{}},{"name":"step_completeness","description":"推导步骤完整性","weight":0.25,"scoring_criteria":{}},{"name":"math_justification","description":"每步数学依据是否标注","weight":0.25,"scoring_criteria":{}},{"name":"logic_consistency","description":"前后逻辑一致性","weight":0.15,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c2_r1","rule_name":"逐步推导","rule_type":"structural","check_description":"必须分步骤呈现推导","keywords":["Step","步骤"]},{"rule_id":"c2_r2","rule_name":"数学依据标注","rule_type":"logical","check_description":"每步需标注使用的数学定理","keywords":["依据","定理","定义"]}]} diff --git a/data/fake/c3_task_planning.jsonl b/data/fake/c3_task_planning.jsonl new file mode 100644 index 0000000..f4a4699 --- /dev/null +++ b/data/fake/c3_task_planning.jsonl @@ -0,0 +1 @@ +{"meta":{"sample_id":"c3_001","capability":"task_planning","sub_task_type":"research_planning","difficulty":"medium","source":"synthetic","tags":["实验规划","推荐系统","A/B测试"]},"prompt":"设计一个完整的推荐系统算法改进实验方案：研究目标为「在保持点击率(CTR)的前提下，提升推荐多样性与长期用户满意度」。请规划完整的实验流程，包括离线评估方案、A/B测试设计、所需工具与资源。","constraint":["实验需基于PyTorch框架","离线评估需包含至少3个多样性指标","A/B测试需明确样本量计算方法和统计显著性标准","算力限制为4张A100 GPU","需考虑冷启动场景的特殊处理"],"logic_annot":{"prerequisites":["了解推荐系统基本架构","了解A/B测试统计原理"],"causal_chain":["问题定义→基线选定→离线实验设计→多样性指标选取→A/B测试设计→工具资源规划→风险预案"],"assumptions":["已有用户历史行为数据","已有线上推荐服务基础设施"],"scope_boundary":"仅覆盖实验设计，不涉及具体算法实现细节","uncertainty":"多样性指标与长期满意度之间的因果关系需要A/B测试验证，离线指标可能无法完全代表线上效果"},"gold_trace":[{"approach_name":"分阶段规划法","thinking_process":"这是一个复杂的实验规划任务，需要拆解为：1.明确实验目标与约束；2.选择基线模型；3.设计离线实验方案；4.选取评估指标；5.设计A/B测试；6.规划工具和资源；7.制定风险预案。注意约束条件：4张A100、PyTorch框架、3个多样性指标。","final_output":"【推荐多样性改进实验方案】\n\n**一、实验目标**\n主目标：提升推荐多样性（ILS指标提升≥15%）的同时保持CTR下降≤2%。\n次目标：7日用户留存率提升≥3%。\n\n**二、基线模型**\n- 离线基线：GRU4Rec + MLP排序模型\n- 线上基线：当前线上排序模型\n\n**三、离线实验设计**\n1. 数据集：内部用户行为日志（近90天），按时间切分train/val/test=8:1:1\n2. 待测方案：MMR(Marginal Diversity)、DPP、对比学习多样性正则化\n3. 多样性指标：\n - ILS(Intra-List Similarity)：推荐列表内物品相似度均值\n - Coverage：被推荐物品占总物品比例\n - Entropy：推荐分布的信息熵\n4. 准确性指标：HR@10, NDCG@10, CTR预估AUC\n\n**四、A/B测试设计**\n1. 样本量：基于效应量d=0.05, α=0.05, power=0.8，每组约需50万用户\n2. 实验分组：对照组(现有模型) + 3个实验组(3种多样性方案)\n3. 实验周期：14天（含3天预热期）\n4. 统计显著性：p<0.05，使用Bonferroni校正多重比较\n\n**五、工具与资源**\n- 训练：PyTorch + DeepSpeed，4×A100(80GB)\n- 特征工程：Spark + Hive\n- 离线评估：自建Python评测框架\n- A/B测试：公司内部实验平台\n\n**六、冷启动处理**\n- 新用户：基于热门物品+类别多样性采样的兜底策略\n- 新物品：基于内容的相似度计算接入推荐池\n\n**七、风险预案**\n- 离线多样性提升但CTR下降超阈值→调整多样性权重，重新离线验证\n- A/B测试期间线上指标异常→触发自动回滚机制","quality_score":0.9}],"neg_cases":[{"error_type":"logic_gap","content":"直接在推荐模型中加入随机采样来提升多样性即可，不需要复杂的实验设计。","explanation":"逻辑断层：随机采样会严重影响推荐准确性，未考虑准确性与多样性的权衡，未满足约束条件中的指标要求。"},{"error_type":"template_copy","content":"我们将在未来工作中进行A/B测试验证。","explanation":"空洞模板，没有给出任何A/B测试的具体设计方案。"}],"eval_dims":[{"name":"task_decomposition","description":"任务拆解完整度","weight":0.2,"scoring_criteria":{}},{"name":"feasibility","description":"方案可行性","weight":0.2,"scoring_criteria":{}},{"name":"constraint_satisfaction","description":"约束满足度","weight":0.2,"scoring_criteria":{}},{"name":"tool_rationality","description":"工具选择合理性","weight":0.15,"scoring_criteria":{}},{"name":"risk_awareness","description":"风险预案完备度","weight":0.15,"scoring_criteria":{}},{"name":"statistical_rigor","description":"统计设计严谨性","weight":0.1,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c3_r1","rule_name":"多样性指标≥3","rule_type":"structural","check_description":"离线评估需包含至少3个多样性指标","keywords":["ILS","Coverage","Entropy"]},{"rule_id":"c3_r2","rule_name":"A/B测试设计","rule_type":"structural","check_description":"A/B测试需包含样本量计算和显著性标准","keywords":["样本量","显著性","p<"]},{"rule_id":"c3_r3","rule_name":"资源约束","rule_type":"structural","check_description":"需说明算力和工具约束下的可行性","keywords":["A100","GPU","PyTorch"]}]} diff --git a/data/fake/c4_gap_mining_innovation.jsonl b/data/fake/c4_gap_mining_innovation.jsonl new file mode 100644 index 0000000..eebbf2e --- /dev/null +++ b/data/fake/c4_gap_mining_innovation.jsonl @@ -0,0 +1 @@ +{"meta":{"sample_id":"c4_001","capability":"gap_mining_innovation","sub_task_type":"research_gap_identification","difficulty":"hard","source":"synthetic","tags":["跨域创新","知识蒸馏","联邦学习"]},"prompt":"识别「联邦学习」与「知识蒸馏」两个领域的交叉研究空白，提出2-3个具有原创性的研究方向，要求每个方向包含：研究动机、核心研究问题、预期技术路线、潜在挑战。","constraint":["研究方向需区别于已有的FedKD类工作","需从方法论层面而非纯应用层面提出创新","每个方向需明确与现有工作的差异点","需评估每个方向的可行性（高/中/低）"],"logic_annot":{"prerequisites":["了解联邦学习基本框架","了解知识蒸馏原理"],"causal_chain":["两个领域现状分析→交叉点识别→空白挖掘→创新方向提出→可行性评估"],"assumptions":["研究需面向学术前沿，非工程落地导向"],"scope_boundary":"仅提出研究方向框架，不涉及具体实现","uncertainty":"创新方向的原创性需经文献调研确认，可能存在预印本已覆盖的情况"},"gold_trace":[{"approach_name":"双向交叉分析","thinking_process":"先梳理两个领域的核心问题：联邦学习解决数据孤岛+隐私，知识蒸馏解决模型压缩+知识迁移。已有交叉工作FedKD主要是把蒸馏用作联邦通信压缩。还有哪些未覆盖的交叉点？(1)蒸馏过程中的隐私泄漏风险→研究联邦蒸馏中的隐私保护；(2)异构模型蒸馏→联邦环境中客户端模型架构不同时的蒸馏策略；(3)反事实蒸馏→用联邦数据增强蒸馏样本的多样性。","final_output":"【联邦学习×知识蒸馏：交叉研究空白与方向】\n\n**方向一：联邦蒸馏中的隐私泄漏量化与防御**\n- 研究动机：知识蒸馏过程中，教师模型的logits/softmax输出可能泄露训练数据信息。在联邦场景下，全局模型向客户端进行蒸馏时，存在隐私放大风险，但现有FedKD工作未量化此风险。\n- 核心问题：蒸馏通信过程中，攻击者能从蒸馏信号中重构多少原始训练数据？\n- 技术路线：(1)建立蒸馏隐私泄漏的理论界(基于互信息)；(2)设计差分隐私蒸馏机制，在logits中注入可控噪声；(3)在CIFAR-10/100上验证隐私-效用权衡。\n- 潜在挑战：噪声注入对蒸馏效果的负面影响可能显著大于对普通训练的影响。\n- 与现有工作差异：FedKD关注蒸馏效率，本方向关注蒸馏安全。\n- 可行性：高\n\n**方向二：异构联邦环境下的自适应蒸馏架构搜索**\n- 研究动机：实际联邦场景中，客户端设备算力差异大（手机vs服务器），需不同大小的模型。传统蒸馏假设固定教师-学生架构对，不适应异构场景。\n- 核心问题：如何为异构客户端动态选择最优蒸馏策略（包括教师-学生配对、蒸馏层、温度参数）？\n- 技术路线：(1)建立客户端能力画像模型；(2)设计NAS-guided蒸馏架构搜索空间；(3)设计多教师蒸馏路由机制。\n- 潜在挑战：搜索空间过大，联邦通信开销可能抵消蒸馏收益。\n- 与现有工作差异：现有异构FL用模型切割/知识共享，本方向用自适应蒸馏搜索。\n- 可行性：中\n\n**方向三：基于因果推理的联邦反事实蒸馏**\n- 研究动机：标准蒸馏传递的是教师学到的相关性知识（correlation），而非因果知识（causation）。在联邦场景中，各客户端数据分布不同(Non-IID)，相关性知识迁移可能失败。\n- 核心问题：能否通过因果干预生成反事实蒸馏样本，使蒸馏知识具有分布不变的因果性？\n- 技术路线：(1)用结构因果模型(SCM)建模客户端数据生成过程；(2)通过do-calculus生成反事实特征；(3)用反事实特征进行跨域蒸馏。\n- 潜在挑战：SCM的准确建模需要领域先验，联邦场景下难以获取统一因果图。\n- 与现有工作差异：现有FedKD传递相关性知识，本方向传递因果性知识。\n- 可行性：低（但创新性最高）","quality_score":0.93}],"neg_cases":[{"error_type":"false_innovation","content":"将联邦学习和知识蒸馏结合起来，可以同时获得两个方法的优点。","explanation":"空洞的模板式创新描述，未指出具体创新点、未与已有工作区分、无技术细节。"},{"error_type":"logic_gap","content":"联邦蒸馏不需要考虑隐私问题，因为蒸馏已经是在传递知识而非数据。","explanation":"逻辑错误：蒸馏信号本身可能泄露训练数据信息，已有研究表明logits可用于成员推断攻击。"}],"eval_dims":[{"name":"novelty","description":"方向原创性","weight":0.3,"scoring_criteria":{}},{"name":"research_depth","description":"研究问题深度","weight":0.2,"scoring_criteria":{}},{"name":"technical_feasibility","description":"技术路线可行性","weight":0.2,"scoring_criteria":{}},{"name":"differentiation","description":"与现有工作差异明确度","weight":0.15,"scoring_criteria":{}},{"name":"challenge_awareness","description":"潜在挑战识别","weight":0.15,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c4_r1","rule_name":"非FedKD类","rule_type":"logical","check_description":"研究方向需明确区别于FedKD","keywords":["区别","差异","不同于"]},{"rule_id":"c4_r2","rule_name":"方法论创新","rule_type":"logical","check_description":"创新需在方法论层面而非纯应用","keywords":["方法","框架","机制","理论"]},{"rule_id":"c4_r3","rule_name":"可行性标注","rule_type":"structural","check_description":"每个方向需标注可行性等级","keywords":["高","中","低"]}]} diff --git a/data/fake/c5_engineering_experiment.jsonl b/data/fake/c5_engineering_experiment.jsonl new file mode 100644 index 0000000..c9b9f60 --- /dev/null +++ b/data/fake/c5_engineering_experiment.jsonl @@ -0,0 +1 @@ +{"meta":{"sample_id":"c5_001","capability":"engineering_experiment","sub_task_type":"experiment_design","difficulty":"medium","source":"synthetic","tags":["实验设计","消融实验","模型调试"]},"prompt":"针对一个基于Transformer的文本分类模型，当发现模型在验证集上的loss在训练中期开始回升（出现过拟合趋势）时，请设计一套系统的实验方案来诊断原因并进行优化，要求包含消融实验设计。","constraint":["方案需覆盖至少4种可能的过拟合原因","每种原因需设计对应的验证实验","消融实验需包含控制变量说明","需给出实验优先级排序及理由","最终需给出推荐的解决方案组合"],"logic_annot":{"prerequisites":["了解Transformer架构","了解正则化方法","了解过拟合诊断"],"causal_chain":["现象观察→原因假设→实验设计→消融验证→方案推荐"],"assumptions":["训练数据和验证数据划分正确","模型架构基本合理"],"scope_boundary":"仅针对过拟合诊断与优化，不涉及模型架构重新设计","uncertainty":"多种因素可能同时导致过拟合，单一消融实验可能无法完全隔离各因素贡献"},"gold_trace":[{"approach_name":"假设驱动诊断法","thinking_process":"过拟合loss回升可能的原因：(1)训练数据不足/噪声大；(2)模型容量过大；(3)学习率过高导致跳出最优点；(4)缺乏正则化；(5)数据分布偏移。按可验证性和成本排序设计实验。","final_output":"【Transformer文本分类模型过拟合诊断方案】\n\n**一、过拟合原因假设（按优先级排序）**\n\n| 优先级 | 原因假设 | 验证方法 | 预期判据 |\n| P0 | 学习率过高 | 降低lr至1/10，观察loss曲线 | 若loss回升消失则确认 |\n| P1 | 缺乏正则化 | 加入Dropout(0.1→0.3)、Weight Decay | 若val_loss改善则确认 |\n| P2 | 模型容量过大 | 减少Transformer层数/注意力头数 | 若过拟合延迟则确认 |\n| P3 | 数据量不足 | 用数据增强(back-translation等)扩充训练集 | 若过拟合缓解则确认 |\n\n**二、消融实验设计**\n\n实验基线：当前模型配置，固定seed\n控制变量原则：每次仅改变一个因素\n\n实验1：学习率消融\n- lr∈{1e-5, 3e-5, 5e-5, 1e-4}，其他固定\n- 监控：train_loss, val_loss, val_acc\n- 预期：找到loss回升前的最优lr\n\n实验2：正则化消融\n- Dropout∈{0.1, 0.2, 0.3, 0.5}\n- Weight Decay∈{0, 0.01, 0.1}\n- 交叉组合，共4×3=12组\n- 预期：找到最优正则化组合\n\n实验3：模型容量消融\n- Transformer层数∈{2, 4, 6, 12}\n- Attention头数∈{4, 8}\n- 共4×2=8组\n- 预期：找到容量与泛化的平衡点\n\n实验4：数据增强消融\n- 无增强 vs Back-translation vs EDA vs 混合\n- 监控：训练集大小变化对过拟合拐点的影响\n\n**三、推荐解决方案组合**\n1. 首选：lr=3e-5 + Dropout=0.2 + Weight Decay=0.01（低成本高收益）\n2. 次选：方案1 + 模型层数减半（如果首选取效不足）\n3. 兜底：方案2 + 数据增强（如果仍有过拟合）\n\n**四、实验执行注意事项**\n- 使用EarlyStopping(patience=5)防止无效训练\n- 每组实验重复3次取均值，报告标准差\n- 使用TensorBoard记录完整训练曲线","quality_score":0.92}],"neg_cases":[{"error_type":"invalid_experiment","content":"直接把模型换成BERT就行了，不需要做消融实验。","explanation":"无效实验建议：未诊断具体原因就换模型，且BERT本身也是Transformer，同样可能过拟合。"},{"error_type":"logic_gap","content":"过拟合是因为模型太好了，只需要增加训练数据就能解决。","explanation":"逻辑过于简化：过拟合原因多样，增加数据只是方案之一，且未考虑数据获取成本和质量问题。"}],"eval_dims":[{"name":"cause_coverage","description":"原因假设覆盖度","weight":0.2,"scoring_criteria":{}},{"name":"experiment_rigor","description":"消融实验严谨性","weight":0.25,"scoring_criteria":{}},{"name":"priority_rationality","description":"优先级排序合理性","weight":0.15,"scoring_criteria":{}},{"name":"reproducibility","description":"实验可复现性","weight":0.2,"scoring_criteria":{}},{"name":"solution_quality","description":"推荐方案质量","weight":0.2,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c5_r1","rule_name":"原因≥4种","rule_type":"structural","check_description":"需覆盖至少4种过拟合原因","keywords":["学习率","正则化","模型容量","数据"]},{"rule_id":"c5_r2","rule_name":"控制变量","rule_type":"logical","check_description":"消融实验需说明控制变量","keywords":["控制","固定","仅改变"]},{"rule_id":"c5_r3","rule_name":"可复现性要素","rule_type":"structural","check_description":"需包含seed、重复次数等复现要素","keywords":["seed","重复","均值","标准差"]}]}