From 290d8e40dd7cf2e56a93db25cca50c8282a4cd16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Just?= Date: Mon, 9 Sep 2024 09:17:40 -0700 Subject: [PATCH] Handle parameterized unit tests. (#604) --- framework/lib/formatter.jar | Bin 5012 -> 4966 bytes framework/lib/formatter/README.md | 6 +++ framework/lib/formatter/expected-output.txt | 1 + .../cs/mut/testrunner/Formatter.java | 17 ++++++++- .../cs/mut/testrunner/junit4/ParamTest.java | 36 ++++++++++++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 framework/lib/formatter/README.md create mode 100644 framework/lib/formatter/test/edu/washington/cs/mut/testrunner/junit4/ParamTest.java diff --git a/framework/lib/formatter.jar b/framework/lib/formatter.jar index 3de1e053607d2a885edc7ce113e48c33a64347ce..9b2760eab3370590d79864355762c7d406b601a8 100644 GIT binary patch delta 4245 zcmZ{oXEYql+r}3`EE~&OLDuS>=q*U}M2WTp(V}Jb-rFLwR>Ug1RwqQ%h_YHDdhazv zCxVdZDO%)B{?B=z=R9wj4>R{UbKTd>e3&!$Z}5B@yjKi#$Swf^fXe^?MxWL3*J|_Dc5%Gw8kt;0IpOp9gA0 zR8-leO9jLOTwU7Q(G+f~SDoFdxhra#I(TcToKpyeT3X9dEt3YP&uHh^C&yA+CX4nG zf6m$b!hBcL8hz?uE>n}g8v785+*SJFzp;B_p}^eDujT~d1F ztLtEKu9S?s5;qA-sQY2AD9(|kZiO*$Lav&`jI@BkFEvTY0_T0fi_FS>sC=-h1nS&j7ps* zd=TG6rZIrdV%W^k*V%eU;(oG)jsP7Yl^nSvE%(Z{3EE$3axp;Ok8`MrVd5$PIxO&- z773rJTWTO|kq#5}MGkZE&OT6eguir6o5+!Glng|@>8GY(PQ%t1Rzn0OYPPr~S0q9d zG#M^f8%V8JfbdlCCAb+GO?hTPHvF*b{2}Rqh^@x^@w5VETIkchH(d_W8A?8C2jh9f z;e#nDwSn~A!c;uv-Oe9Umec#;yrJ3OL)WPoFq&l#pNDXy6=yF>6p4_*9KfXI<)0gl z8be^_vm3z%io9I)$zrIpiHAKyZ|0@Xm6e59@tm~fj#ey{BED#EkVUY9?EpeoA~n%Y zH*;bfwDGK38&c4e3VN0@`qgeZD)}O+{bp2?Lj;p=8aEE&TRfYODp|YT$XSgNj?Mez zwqnuuF6YrBuJBH-&aa6(J0%li-mMF+)-|)2UuNhGU4%{FbsRz+7D*jsVk!6GAX9$^ zvFG+e^;>e$WZc1~rl#>&D&6fybe>C))q`+{ujAF@)qTlMp$=|RjI0V1E%w6naT@cz zbG_O8I+bfTB8t{F=|58}u|xdMNn075Oau~B5paZ+ef*GLbv*$_3)_ISrT7_C3|WI8FM5iN_#=fm=BFu1}yc<`+cRchoycKAHVPI^R}`s zF5ZYp!r^!W=s)`33YZ;sE#D;tu;|@mFUO5cejest=q@f9`5*Tpt-G>0VRIFci)K(Y4GY7Ib!yb4cvikeVt8uKge`LQcADaY zqn#37VFOv)H){2%_Ods+J+)Kot=7*?&va!?43&gwKWA#(2-*u&kT}h!3$|Q>3c96Xi%vz+_ZO$iK6DQx-l;aGD^Q$*d?PWK>Lrr?NNW&`NGM z39>{gA`mrYDJr_9uszKss$wiLF%n(rp4?KlbIR)`l!rSbGl8ti`pC;H@H~3+ElFG7 zrv_IPzW%zvf_duFrXZR~RbEQ*w-->Py^+vU=t7jWPeAE05oD zEK46ZFRY7tLF;@i+6@-piu|PCrmTFvYS2qVXr}vS-F+x@>BmH0sPj-KoteI58}QYu zZi<&{OBAnUpT{cf6?ZBvxEF9Km2_Wun|SI+rEmbwYd#mbJv?s=W(nxeuO!g_oO)@f zvQXK9BJYAqCsX_c7Sz!U4bx|5hXcr9fxf%zM~}`y6WL}g!ZrNwt+AwMVtxvk5e<%d z0ZgnXZS|H6_-xR8cDNYtf(c;onlXtffSY8QzonR_m;xp-eJkvNd8jcrkaRW_x?G~v z*1G~$v2z;7jxbA&s8B+6X|0y>A6PuGb2LMNC=WQGSoCc#%mitycnGqpIcCwHvdUiY zf{VB`U2Als%~(3--WTYi_4v-`AV|9ta)#!fwe6EkpxL_}P(=&ctKrs{&4EnP=>S*T zib(&>aoBfsCbby~?r%G6(}wo&G&>q77M-fW=-F#Ur?eD&T(s4jwowlM1eaEzZAN`+ z=h2O?+T)~9e;-QMY^d2%ZRaa>%HmS_{M#7K(mNAni}XhK{J=qG+c_YHVQfj@F%V;| zx=DHfR&o^zF~#p>r>4$|;cjfGZtKU*giP4$!rAk z`6{-3P#S{WioTuPW|-X;yV};~oQ{%<^?o~aHea6J&YDeC46VGOzLt5n zcVBC36UGZ4u+|vMgf3g@nYd3{uH*W0$*oRa$Rvx%8K+oqrT0@7Wdxmp`)K_AIc={E zM0g>2oouzLTFvpV*7pTb4J`-1lwCb(UdC%+I<6t(Fi0Ksg&|9kRx#1i99F9X!$T{59zt9HpcG0GjKtH2N&z zNKer!z!^%YHge{H3MpI_k~f;M_yUtPEau?q%0wab*r`7UBz*qDnNV(~9%adFXIkpL zrCVyv?S9t5yHC2kZ%1BWtvNM_Nfcl`L;ZItbQ$oc;?aRg!28Jo0BPJfP!pw;^8Ll6 zEp;teSu3i3s|Ns0tu=U7@8%qjZ;sT%&G>0cKzJn$wx0-jV47B<2Pv_=`llQW`(EcC z#@7xWU65a}!5Q|l3SCL3wdvx>KL!@i8Z58Lo!8kh%7dew^^bAl+D z&vWjniVY<)7k%=deAKa2Nn3=We`@Zn3-I}df_Ko z`b|NGs$Of5X{$Ul;GHzfRp#C*-RQ&F4-B0(>5hDLbZO_~K$jnJi0fP2j51vYHbW2Y zW5pf1{HY54KWh%uk)yoN{1vpYB7(NmDleE-PpD_ajWR`2?2JE;=NwmUnrSG$KoXln z=BO>GA|>pYuGyQ(?bIL!#&S{^N2*f$bZ|E2rR6a;=CVACWw{vyVn*DQaBjllQsd*} z&F=~!=MIUv2k+%9v3Fs`@*`ePeX`ftoA42Xb!!FlNy;Wf1gha4X*uRu)S3WPe~lT< zLu_8yWhsrR;;VsybkJ2#kO#2N(_dEC1#^4l(JyTK*IrA469)F+qW61E-bVVuM9YGx zM9YK7E6c{l+Qwj8PIAW&s_5SeFL5cO5#w+Zor zOYKd@G(0YesE$eJ*QGJ$doc(??&Z)o9dz1OYWRh4+D+rPiL}OHn8Xs*c>3380M;8K z7A40_X~vd4DxD#HW@_#U+K)YnnX0)Vp%x z{GeC&Xg0z)of(G&xwShNPoF038I?^@A#Mc9cTBUoq6lw@=C=)yCo%5&0dH)bL;HcT zZge4;+YOeZ4lx-IoZ|3I5RF1r<`i~)Am6faJXd#E7uf41RpQmuvAdU zi~ICz`mP5H0uQYhIeN8ndn^2UTwjF-84B|^-_hqT^MB*8S>^F#kRjdMP&L9u3i^?_ z)Obr9hvIj(@IdG9wiF1EpoJCKdggwCGhj{($MED-gmO`(fvO~Ilqj>{8kV*>pPK*N ztP0QiBl3{$v90~bT}7U5YO=uV!xa&-aK1$J0)raQYf~p_A0eg;_|NoJfoxfv>WhcE zGRog*qT+gzqTtiSW;3fD(fX))VF2(k-NuO%SX4&2x`Geo8Y8SauIocdE?XG zeC?o(_R*tDY4%VhmF<1N@Pxlv9%T;Kp5UkNi9Hq!7R5n}2chN!+@l2Zh4;dp|T zhNBK{*et1~Vn-P%EZD|~L}&wBCQ& zGP?Xc`QpA-lZHwJ#$W)`nvw)pQATI)%Zyt;W&`tBZU=_xyw55X9yKyvhvi}BeoB^G*%>PBqf8jG2u%puP zH$VSl_8(AgqGKWXh?}N^v;9v4`oFS&_kRrNa1eTq|D|F4mqrDLpy&CEw!`RINc?eW z^kQs(z6kzi?%$qbf2RNdg;BM?HQYEL0#l@6bP6L!O9&DJfk_Pz0bw*KC|ydYNHe-YC8SFw zRa$xV@AJIR8~2xU{my-^>%PC8>ulak;icBsCME$85Ks^hqya(^Nz}Z+)ZZv-XBA^X zKtSMKl6-P9Yv%^tt`snOrb>FI8q0Hs zSSqj?-AwSGsfmB4jcgGc=TAda0-=x?D%DE7CAAe0CFk?c3m5qAInOf{H*Kn(Yc zkQ=uErolWn`+}i2gM{iHOSEQd=3S={qSeqw$YkptmQIUh#CQm7ch@Ex}J`axsyi&(3pdvqz$Uokys#9J#HK z+>_&+0Nqbb$Dd6I0#^4y?tPKiDJUCYWrY?_i|Hgk)F6Q?K$U=OlqfnjniXXZ1&(4p zH?&rFG*<~;ef=Vn0{*0U<0}e8Jr;RT5gdx_f-`mU9{5t^K3z6CtwRP{kHYCXoe!I7 ztz(vjMhDWzq7R~I-!XTk7wo0Of_cIj5&bLW9B=t@$F!Z_F?MRpvUk1T7$QA8il=>t zfpj(=cmSDT!Rp;ZL@-h_+l`KEYbi;CbEQhW_XCw*prB{FL>ye{*QKozIaho~A1t=1 zeWH&^>v@QUWxaBrg%-NECFPrL*4-78BwDSybMT@f^+yY7@L@v)(JwZ6<)4nk_l6cM zhuLMG+pVE>Ip$os0>&TI9JRgWJDt}*#8il8OPA1P^K3%-;%h67=X5sWtOxTibSG*Myfe=7f!*H_(H4HnFOuNRcs@Q$0d~D>+eN8K zhtv$UZ1G-+G#^G?2KktG#3eUA`j6^B7{!4n4t*@%Mh!k){e`;p5GS`2!URJMrWG*z zKuzY_!cS9i63$dvE8vnN5IHq(_8p+AcZ2bbLs6ZvNLJl~c}N_Lvt!)#z2vvH1eCl` z@a4IoR-_|CC*;wO*Uqw^a@HeL+kKRdxN-bk{$K-AiPdL{tnJV8}h)O>NLrHHFmz+@zOZ7_8;vLzlZ1R^Sz|FJ^LWlS@Kh+i#*f}HE$rJl(3Bk-kvx#D!r7v z%t@Izq`WlDQ{8B)O3U{@&59pud!N?_3<-}Q*M_JT*Yh%WUBf_&iY^mnb%Ia2Vv=XrW{UP~f67Z@C zl`B>0l-avk(Yq=(eA`YRpL@t;syLz*SoGbf+s5w_IK};IBmfP@%;tyS^)g%yUAL~5 zVotI>#kWSn7R?zj%Bu3kpN5!wm*)H&OR_MsiAUo~{wWmkO{aiWlQ;yU<2EkPmE1*H%!Uc3wvqTjjWLc#$ zT=!hGByGZAmogy2iS_7fX0q#9=G_H3rUK{T2iRfqxS~{PO!=LWWP4G)uYs5TWQw+`ySJSlBvj=dB$7d-|9k4z*&kqx%IBy=Z*_P<4wM=r^eJlKv z=rdW70j4AbFl4QLDvCPe3BPb%XO-Gqqo^o`*piyPwCS`5qQm z8<3y7<~jN^_+4~6VHTYXS)3@KAw9nXP9;K`grv964sY^ujnUD*^Yx8`qfD-08GHgB zYbnB&bx=AtygXAYoo5=;Q^Eyv^8O;&^n;r^(HymKcMDh$zP=Rr98#T5(`lycCzx{4EqqD!xz z(bP3y$6H2a4bTxnbccw{nC#5*jjc@{3~WH<2xc)5@<=XsgG?4Ttp+92o%rF6l??OG z*#w26sY7JUwp;H-aZDqy2KmTiCHK{o0DO&^=r`ehXWlO#t9b+K=No6*Czrl+&hna_ z@87`R;^8rTf+W1^<^8$Zm3{Ty3b{PVWgexTd4eAw3sXW9hmd$-ko4bm%pgH*00Q;&gN-Nf2HZ!4Ue%T%4SDD5HY1@yf3 zv7E|$(5AA}5GA|{X*v7KqS6*#Vn(aS|NQWZ@ZT#E1;L-ar{8?Ha+j2Vz>|`IK<4-2 z<>=ri8esR(#nsc<*9(_NNRQhGuwmM6?rIlzr-?`jC0ooDT_-lCpPVDAPqnwGaum&* za>&xvl(3T^F7#}Q5cG^`S`Ihez=cwfDzLPpzoRenFkx5GU+U0t&tXFNRm^!8%! z=i-mWlhB`ApZy3tnz&yQMier4i6kt7k<8z16EjYHzT&Fb$rYFxzeIqkSE?{7If#kU&q~D^`FGu(DxcUcL~wxxK;LZ3BW1z%VzS9#Q(Jv1b)F23K-<`l zBHiLFqTM9AF?Pvo*RJ`=_8)R9Yev5quhb6Xx$^)&~`80c1{-HX$Ya_;3m*}1t zv2EHph=Y8gTD@?r##p2$$P_t4M`aSWGTsz~FtK;zETg)2^I8Mt5c5uXPVI~&Xf`>` zlbd4D(M=ta7DXjl_H=$dG1)}EgtcCix4Oyziw(v`+U?{AnKRtcdQ*nMyBl%safZN@ zRn@XRcEk(ECJ{3dk&B))Pm`xMNWXH!yvdeWo?XU8E zmv8Bz$T%^a$w_fe8e!9?jn1@Nyn208!k~{o$EoC)QW{^nVyho~A}=n_y~dEUi8krG zkGAT&jyA%Et6|gbnBm=8%^^4}j>Ts16kwDh0I{L`@~*Ry$}hm=oGQ*s zt_tOVq_mX$h#3;yV%bDkmp&X86xJHp)#c$>DnjqLud}xy{df)2)SO2OGdJCB#(|E! z_w15{v!KOKL=146!XP^rAiaw~V1c_y@{L3{)6EPMlB2VjP839vEuCpxheEUZN$fh) zq_9P0+Rmdec=aIVk?ss9Uh_qCJbF^@e!u~zjXoNL$<;KTM-J?X&#-{`H6rVYjvBK} zOyL_yYVr~pRh@2KiRmR1DbkJ-JEt!56L!v8;RK?kH-#!jv^zs-`Uz{0**JiG&l6C$9ev(|pPqg!%`wQof@pTkmK)eDdC8}ck*K!9ro1(9W#o8=CqPL2;f92TJ*6W&*5>dSJlE#Z4=&IqG z(p=0z8N9hH>dT#;pJputG>@G!kMn~}zjDLO2^KEi&U)P{n&$wed`nqeT{2J)kMDiC zDV3HJP%>ie1>N}Rp63WoSA}?@5(CM7SQGYshXNH)F61x8fZ; zSNT)w&t1F%t4^V&wEbG_au4;YdenW&NAu&x+d`9>zGMkDrE~O$Tmsx6iv)tbOo|xR zP!+3j*F4z$dCQyQLvbIV@aK_x5q|JgQf!$5mykxC2dfse*oe}K2yC-dmP~qs3-yoVq(yE zyq7u>xyLEL`aw%d=3cc-TmT?syH9!DNUs4caoN^$C1xP~vhF*XfLz03kA0Ke`8EzE zkLR*wdFkZ*@B;WC;a7XFrTt?R-ll{hs&%S#T>XkvUz?DKp7_7xIUxe~iB{%+2Xq`K z2=+HZTY{j>jsLQa|JnG*UU2asDB(w33FvPS#|?s@L}Y}xq~E@B4x;+o4*)}nr2lZt zI5_xUobGSV?GML;O9TUM{J(_$`?-Igg0Rv*LayI|{rW$`|7&Ug5w-yS5$*#Rar$($ KWS6vmuKo)#=fSZ6 diff --git a/framework/lib/formatter/README.md b/framework/lib/formatter/README.md new file mode 100644 index 000000000..dd14ae09b --- /dev/null +++ b/framework/lib/formatter/README.md @@ -0,0 +1,6 @@ +# Custom JUnit test formatter that outputs executed and failing tests, following +# Defects4J's naming conventions for test entries. + +Run ant test to compile and test the formatter implementation. + +Run ant jar to build and deploy the `formatter.jar`. diff --git a/framework/lib/formatter/expected-output.txt b/framework/lib/formatter/expected-output.txt index f781b1a47..5b8491922 100644 --- a/framework/lib/formatter/expected-output.txt +++ b/framework/lib/formatter/expected-output.txt @@ -10,6 +10,7 @@ --- edu.washington.cs.mut.testrunner.junit4.EmptyTestClass --- edu.washington.cs.mut.testrunner.junit4.InitError --- edu.washington.cs.mut.testrunner.junit4.MethodTimeout::test2 +--- edu.washington.cs.mut.testrunner.junit4.ParamTest::testWithParams --- edu.washington.cs.mut.testrunner.junit4.NonJUnitTest --- edu.washington.cs.mut.testrunner.junit4.NotPublicSuite --- edu.washington.cs.mut.testrunner.junit4.SimpleTest::test2 diff --git a/framework/lib/formatter/src/edu/washington/cs/mut/testrunner/Formatter.java b/framework/lib/formatter/src/edu/washington/cs/mut/testrunner/Formatter.java index 8d07a5705..bc5dd2f0d 100644 --- a/framework/lib/formatter/src/edu/washington/cs/mut/testrunner/Formatter.java +++ b/framework/lib/formatter/src/edu/washington/cs/mut/testrunner/Formatter.java @@ -79,13 +79,26 @@ private void handle(Test test, Throwable t) { className = test.getClass().getName(); { - Pattern regexp = Pattern.compile("(.*)\\((.*)\\)\\s*"); + /* + The expected format for failing tests in Defects4J is: + --- [::]. + + In JUnit, a test's String representation is: + (). + + In JUnit, a parameterized test's String representation is: + [](). + + The pattern below extracts only the method name and class name, + stripping the [] part if it exists. + */ + Pattern regexp = Pattern.compile("([^\\[\\(]*)(\\[.*\\])?\\((.*)\\)\\s*"); Matcher match = regexp.matcher(test.toString()); if (match.matches()) { // Class name will equal to junit.framework.Junit4TestCaseFacade if Junit4 // style tests are ran with Junit3 style test runner. if(className.equals("junit.framework.JUnit4TestCaseFacade")) - className = match.group(2); + className = match.group(3); methodName = match.group(1); } } diff --git a/framework/lib/formatter/test/edu/washington/cs/mut/testrunner/junit4/ParamTest.java b/framework/lib/formatter/test/edu/washington/cs/mut/testrunner/junit4/ParamTest.java new file mode 100644 index 000000000..8a7f2a797 --- /dev/null +++ b/framework/lib/formatter/test/edu/washington/cs/mut/testrunner/junit4/ParamTest.java @@ -0,0 +1,36 @@ +package edu.washington.cs.mut.testrunner.junit4; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@RunWith(Parameterized.class) +public class ParamTest { + + private Integer testNo; + + public ParamTest(Integer n) { + this.testNo = n; + } + + @Parameters + public static Collection params() { + final List list = new ArrayList(3); + list.add(new Integer[] {1}); + list.add(new Integer[] {2}); + list.add(new Integer[] {3}); + return list; + } + + @Test + public void testWithParams() { + assertTrue(false); + } +}