From 342b23f23ec01b0aa148631a09cdacb964dd7983 Mon Sep 17 00:00:00 2001 From: Jithin James Date: Sun, 14 May 2023 20:46:43 +0530 Subject: [PATCH] name change belar => ragas (#15) * readme init * new logo * new logo * new logo * fix height * change belar -> ragas * fix ci * Update README.md * added quickstart * lint changes --------- Co-authored-by: Jithin James --- .gitignore | 1 + Makefile | 14 ++++----- README.md | 38 +++++++++++++++++++++++-- belar/metrics/__init__.py | 18 ------------ docs/assets/logo.png | Bin 0 -> 33956 bytes examples/quickstart.ipynb | 34 ++++++++++++++-------- pyproject.toml | 4 +-- {belar => ragas}/__init__.py | 0 ragas/metrics/__init__.py | 17 +++++++++++ {belar => ragas}/metrics/base.py | 0 {belar => ragas}/metrics/factual.py | 15 ++++++---- {belar => ragas}/metrics/similarity.py | 2 +- {belar => ragas}/metrics/simple.py | 2 +- {belar => ragas}/utils.py | 0 tests/benchmarks/benchmark.py | 12 ++++++-- tests/unit/test_simple.py | 4 +-- 16 files changed, 109 insertions(+), 52 deletions(-) delete mode 100644 belar/metrics/__init__.py create mode 100644 docs/assets/logo.png rename {belar => ragas}/__init__.py (100%) create mode 100644 ragas/metrics/__init__.py rename {belar => ragas}/metrics/base.py (100%) rename {belar => ragas}/metrics/factual.py (98%) rename {belar => ragas}/metrics/similarity.py (97%) rename {belar => ragas}/metrics/simple.py (98%) rename {belar => ragas}/utils.py (100%) diff --git a/.gitignore b/.gitignore index 4779bb12e..ad3a19fb2 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ #.idea/ belar/_version.py **/tmp.ipynb +ragas/_version.py diff --git a/Makefile b/Makefile index 27a1eb38a..fc5ab0fa8 100644 --- a/Makefile +++ b/Makefile @@ -5,20 +5,20 @@ help: ## Show all Makefile targets .PHONY: format lint type style clean run-benchmarks format: ## Running code formatter: black and isort + @echo "(isort) Ordering imports..." + @isort . @echo "(black) Formatting codebase..." - @black --config pyproject.toml belar tests examples + @black --config pyproject.toml ragas tests examples @echo "(black) Formatting stubs..." - @find belar -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \; - @echo "(isort) Reordering imports..." - @isort . + @find ragas -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \; @echo "(ruff) Running fix only..." - @ruff check belar examples tests --fix-only + @ruff check ragas examples tests --fix-only lint: ## Running lint checker: ruff @echo "(ruff) Linting development project..." - @ruff check belar examples tests + @ruff check ragas examples tests type: ## Running type checker: pyright @echo "(pyright) Typechecking codebase..." - @pyright -p belar + @pyright -p ragas clean: ## Clean all generated files @echo "Cleaning all generated files..." @cd $(GIT_ROOT)/docs && make clean diff --git a/README.md b/README.md index 648f37e4b..3444dc88d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,36 @@ -# BeLAR -Benchmarking LLM Augmented Retrieval +

+ +

+

+ SOTA metrics for evaluating Retrieval Augmented Generation (RAG) +

+ +

+ + GitHub release + + + Build + + + License + + + Open In Colab + + + Downloads + +

+ +

+

+ Installation | + Quick Example | + Hugging Face +

+

+ +## What + diff --git a/belar/metrics/__init__.py b/belar/metrics/__init__.py deleted file mode 100644 index 70fe4a3e5..000000000 --- a/belar/metrics/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from belar.metrics.base import Evaluation, Metric -from belar.metrics.factual import EntailmentScore -from belar.metrics.similarity import SBERTScore -from belar.metrics.simple import (BLUE, EditDistance, EditRatio, Rouge1, - Rouge2, RougeL) - -__all__ = [ - "Evaluation", - "Metric", - "EntailmentScore", - "SBERTScore", - "BLUE", - "EditDistance", - "EditRatio", - "RougeL", - "Rouge1", - "Rouge2", -] diff --git a/docs/assets/logo.png b/docs/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..22e1ffb2ffdb2ed59decd31d7a1f9f18206a1131 GIT binary patch literal 33956 zcmeFZg;UgD_&>V4tg!UbAPv$DA|oF`td*R!v+G?eghZsI^75PTJ7c^wD@4ue3T z`Up($KW;JV55YfB7ab*8NMRSvUkHQ&q9QM&=V`i@ik%WQaM4=VCa&7{BiLDAFHBXU zxl}bf$iX2wStsddx`O``0g_-7<3b%>Jc+UslPu(Ut7jo zNDw6Q+JnHOLy4d`P=S?acjEr{+Q6#V;ZAthxAYbjfzU#PYTJJMe0>eKP>g8HYx@Mx zvXDp1q9VkT)mi>y88}cHjQ@BRk{Q^?qJQKW=YLpM3o%W3-8KvXltk%BWM-F*uF8Kb zLl)BdAGX163WWrrDc$ZQtNw>s!4YBp|3~!yenda8p~-jbqXPqBpN-4%`g64#j}l)0 zhc_vs_@LcE=!OPas*`#xqZ-e48J^Ys3e(>Fp*P+pYODX{04}gJTxhN1Y4!ZV0!wpR z(x`$Urgc+|+OY5cz7)U}dX(yQ8O?qr>spOh=BV1^H$z^es(1XyW2ZkO-|kvO&d=9a zsis<>Ihbtz!;zp$a2N+XorM&GW}l6o>AKej=wQR|Bho4B->LnF*Q{W}jlnO|e>6UM zcg+|KlW-UxTnjJs5MsT0-B0_85kW|l0Lnh6@8&hnNBLsEQUp&%PRaa=kGbZ}xy(pT zMzjUGDl&)VnvDWHi4m#b$qZ>2abH-kZAnXxAyyXB3=vM%W4hPwr|1ut zFSFB>)z_J*UJ7zAShHSx&()LP)nh|1{TBJ6npHOW^e`8M^i@X+= z=B;bS4&@I}Mu6Wk-IX1=ecb`F;Y`fH!L`JBA4go<3quB+iRH@um)WhqT{lE5QE1AQ z`;QE#GhTag0D?F)3uzV#mEcqxZIi4~X9#>5uYUb{I%zO`z4GOymw}qs^PzkKDk}(_ z#2?zqXH?gG1uX=7M*(BU6s4y04ZFVjB)A15oCb;Eu6D6A5brH0To!JH_p3NI;r;c! zC^K-wEtszyhf{U5t(Fv{K~UUC^V+VU3vi1oi)cytJ$`%rjaG0AHt<`-2*TR!dbo%s z4$Z!@*I&vEfoo3=KoW(1xw`W5nX&rU9a~G0VNkXnDqt2h&-cg@o70h`_|@zG*au9N z;Xt-tN+7-ALj^Wi$+D<*+4bF$g50~s))x3XE=%b$F21B!s9uoM=DPDBkf7@DcW>$0 zF0;A_v@Wj-wm|bd=)8CB#vmb}0+7FDr$hT*;1pMBLPnsT=#G^9wa-nH1PtRsbr$mN4rSuru&FClSd}K z^d`JPK_;|~To(+@5Nq})q1Vm_MG|0$1NB-ZNZ2D<<#`QxD4xpX#dSyd!r7E3_X%Dv z_kS*g4Nkg|%l$X=K-Kiev3|hsWZ~$w3m^Gkv2msEd^j*YMfr_Lum2$(tb`bbQ?VYm zw-y$iP&XQDvumS%%g0RYv+?t(KLHK2WbBxUI+mE%9U+}0G!FT8UlrrMz^YK6l#0q{ zrV+WD%HbzB_3yS|o7DR{!3?raMBzA2PVvmyDl~h&3tfo97N^`5Fo=i5Zhp3F^ z3f^TkQ;nv&?hJv-4CJ!)5t`I$gSxmp+uh498Cm6A?81D_5HH_caeI zQEMF(Tn>Ko={U@WecW`T+Y&~K@hrg7=9*ccQwVue#Iolurf%g!@v;~hNrU%`YVqb% z0??1PXl8HU>&IPhcNHFZtMj$W9VR?(n0@W9F_LTFs58jBKe>=7*$io2K4Q3ja;;a} zv(k9V@cmoNBdXyVBa(JHLyf%q+(zuhqx7U01<@8#*Ow=a4VK4#R+}__FwOVXyz%GK zrLIn%^1<8S-i!-!>oZ*73Z;J*yzZf>v~OMygIekgJF*b83|^a8#i#l6t$b2)a@Fup z9KEZvQ&|pgru277!=S(3_5R0tWnm>q80pSet1t`nf7*N+1Ki@IbX~k2c)ZQ^u=D@~ zgK3;ce+j_{5DA~Ie}Lxd1Bm4P5Oz#M+vXH-2brYLwq6!I`0gwnc6q+(e|mZ7wUlam zwn?ISS3C0Vj`D1;{Pg+Zlyut;U)>o~I0c`Ev2oug^&9@}#HT8{SPGroew#-V%)x&S z*2f4>#>(xJ*G9|K2&B)PKHvY6v~7^7_~zw>{}PeTu5L++p%X(Pt-uqDwOK#7qR}(FZ_G?kZ4yNQ^;ctIb z(xvnI^XG@REVY8OYp$*tk{2c>CMIapNoh?FeJm*@SX4QS^LHShLLq>u5sLee+jsk8 z)a9FI$G1ANJu0E%OKCn1Z}VEKDjABO&nbe_^RQa%e7`g4v)TShJ^6v5%K>|f0caPp zT$Z|`lae@dQk=tt$Wzye;IN@UR- z+-<}#h-PSwv z?Sd~(T#d?~85MJ~F)7iy#Xj)?#XhdrC5d0jW9z0 z&KHRojdw52>9Y9E%7zB!Um6~dyL7j|!Y0A%&Q53X#5HT2TTFH&_mmO%&Ia5@QqF{& z9do#$x$~A5XTarY-DQkKaMhdx0@-t@V(CF%|mEehaP&QT3J~suMU2R zi@VtEmZlDTS?GFkyws5*=HmLEF!f>Rhod9S&<|BXHhfU?3$@VEJb5p^Rr=ts2OASy z-)w#h!XXhKr~EG^w+47k$_rdq`r9d~4NSeV_44#Mw%iDw-r#^A^$3w-_@cX*m{k~- zWg!m)9IO7aW-CEbTyM=HtZZ$$h=fyOi&m*T4X5YSmX7@%x>eXwFhSg9Pxe<^GwI!D zp8Sf6D%zfFrLMj-*Q9vh6tK5BH68^MNt2P3v>?**-4CPNJO zBV^H7twWBY#65`ctJ0Tspi2{o7KyG-wE_5GPr=d#_9>CAG=FSpn`0CHP{tX}Q zSM&434aM5`Ma@7bY4xO7fn`g04!sd z61&I=;``o3-J`7Zdea_!&>&O?v@oF2dkmLVSX{hxK?nx(e#Bu!fk zxAN2-I)2Jh;#yR7R z1KAhrq6_(M=iIELsVvK~?M`c1XVKQkN%PLTyWK?MeDXI(>HB-tQAI`+w&JQHj)z|C zo;H)$aejfc-YvFfh|=(pmOM6G9f7Dqyrsl+67ru3qL3}ggnim7++uE#rS!YAtmf%m zFNao<45-A0C}7_&Fw|CzpfsmZDw%UoV??cs-s|+&7C$YD*0Q*;#EXmmY_UW?$Iiv2 z=Dt3fP3)g;|Iqc1g5tDJoPZW=!VD`e6FCh@N>&PvEMnfJsKjNJ7pPY!+!|pgLPR5x z(@ShE#Fmr1m3khRkPV-<4pTFPh4RU*d3Ycb{FOQ!QALR+^oJYaIV0;q zB0+2z5f}&NU@>tBiLpWW(C?U)Zkm!yxUV@mu*+)Bl0z!9A33+PeEO1sGz-h^$MgJv zf$nbjF|041@*JCrvniTzQaa*H&bu=c2b%z z5riK#)gc8Z{V!j1`x_x)3<&-awzmlhY$D@MFMZFpTMy_Xf0EpP#fk&%!h+SLGX$P^ z;Z1{>CcG$By-N=t&+t#Yj_)rRtKj+W)~ZGMnX&@VEbSjMQ) zB^sfxcy0DzuBf=UN2WW^D@!4qd2xE5#T zA>CYjfyJ0X{gRVv-!8i(Hrs@U9x#tpI39kx8Nf?@=rK3i)SKa~i*OnYO1!K&fGHzB zye9Og-KC%8dwuNKPAqI%V)JUwSMTZXB;gj}^hbVA&TRyR(%LV4K+y}5j%kZ zeAHHk^b^Y8NfeDU1dSc2j^;X(V%$E&{~huHy%FV<@{w=7%Yxg%$?r~Qd%+viQhZzm z?!a2oTg9tE47#R|WPCA`s$!R80Aa@@W2#B9s#uNKwaykugabcdDfhH*9cooy@e{BCeyXwk2B zUneLbq%d2*+r>@-1kM!v&HkS)HpAD#Bha0X6^UXm~kopjh9&p|a ziI=V|I@rcEG_;r-8V<{IIoD@0HKw&a3TT?Z@tTfn3QV zAz>0;fidW^g+kX;Rmd}<f|g$Y<_>}kx1jS)w;P*r~^&?U8>kpyCL2=PThW7I!#WJL7c7IoCbv{s^0=C6u|N+ ztZRFkAknR#wL<0>NuSI?DLT`>^7(JZI3wDg?fi;00o%AclLylS%7 zp?gOnMEVr8u?WE&?+Q5Y&hYDl4#~-I1^&^{?Eu2H;#(6BKn$=AWqPhDDjg2Wtfj|l zmJvZVhZ_?M0a$cz=OQJ#b~lE9|GO2~ zKtmaym5e)C;hCevGZS&jQ!$=VV3hdb>({ThBqbldSEmhbc|*-sNk=pZ%Ok!^`cp{b zwJdsevg^IJlC$Y$LqF%Z=GRSoR-P66+rPu{{$Z=15{b6t5Kg+RQWrCruU%H$r5j3= z8C35-}@ME|tz0uiY+t(zlrI^_mXK64QTvs?q7s&emqC#X{8k8>?`^oH*pSUrgmj zsoUycpF~pggAp4Atl2ubwwjHq)mS!+Ox6_j&BZ+HgUBj}2+EP(d`?edpQ1kqU3M62c0mhJ?HiW%kL2<$7e;hufkQ^Uk zh+XS*82?_#HffQE1EUK`NOyPmsCy5gtkE8ROvuDON#xhL8HuG?g5~BMrc(n6_1J3C zCNsJ`nu|*K+n2-jT0P?_w_(Z4WK6{~Cf52!fl6w{HKp5i)Vea0kidJS=P*9K}gB15x%{}~^x8G`KDpEa%1g7zeh`M0~ z*C$pUV{BtR9W40#{V|h=yu1Sz z36ZVs>C}Gi`kenDHK$hy(R0f3u=HFdi2ENR>U!0k72m3f&|L)0!oj`bxC4srxn`ze zzJNw-Lay%VOq}Y&s*TaeKB53%|6kS#4{g@mz25WI?&(}r)3Ri*PcoyVh#!zF&X3BJ zk_BQ(!U*WBP2mY|q7gXWiI-bHH~M#fuX$WF;PO$n?lmM$H)rlFQ;W`pY7X~}e|KRK z7S?y-DHzl1?`Z$>g+W@A5%qlYfxE%SXA?L=lAON2}McU2$e7hB2+}QVYGWRv)wfkHs z#_(U|qtF8`jnJ@z2QT&q1g(FOl6|u3%~I*jR3c<44>W_PK`Bf`s3=(NS^7ZG~5L>_r#^GNY+EEk|oFshR^@Jo1G@x+Nj?A z#ggzK1LKiSavYaIe0vNVH5~#pUk_d1btVcl-WR&#FwP!A>z;VU#|EXN$ zT7J!hKBepRlIJXd2^LD(gI3(9lF7N1)&r5T2}0O1a`A9nCWM~;MO6?wcjFb=xy)ahIqgH7`ar6mrLMF^>{8n@aP(F>GixY) z3^(~7Q5ReflYH!LM&QGIJJ{X8qr$o?-TLvg7L()Sn{+G;J! z9ISt-EoPlWORQ>XzOrHk=+%u5Um3RaU1n^1P2anX)aj+)_GOIUvY7au9RDn^6QH@Z zDDKZzWqHHwUFJ@~XAN@qXUw8zrw8i`ppYrwO`NMSD|{3W)OGB>PU$3|#ANC=px+`nU!`OC3R3!L=q$vOZ%Z^()(rLSc}5=>=QC62k>p^WzM2Z^E*vw+ ziDIDC=g?s;p8P?{GV9Fa5e&k+(=GDpAF`3i<&6?;u`uWzGXXn2#4&dny z%+G)Avnd|dyYF!&v$vzVo^RwR&&P(uEuHNBW$wPa8xdDfxLZ$RHdtVEg&GZ&2R?>a z^kcK7n587(%i_c!)8+6`{Kho9;(xPZ0()pjSJO|JGdkx zTdfB2^m{?kQt=6Pe&(}T%a$tP;jXVl)3`PF%g7+Ee49TALR}2|Fo49ksk*^!2lZQ~ zLLSjs-%v*^pP|0uOl?N3MX$V0M+3|+COze&!S}?*1|HsUv%MmXBFGxM0{{(HoCpd_l7-xu+6_1?NJsiY}#9bC>`&blMnY6&+j9+Q#)FWi6nKFptv1dy*~h1Na&;8GI?mWLhk9s*%6G)`xV&uI`PLYW zFXVs-l?$ZaMnf@t9iLJLLG&$9TCW}!VFkpAN@}c<#(VtgN|ne4t)6OAC1}r8U)u)g zE}BYV&A?${$$KCxVOear^aV$!Wh2vH=1tw7o;Q^*!3+5q*S7qlu zypj1zN;n`>(&GQwP`@&b7R;23=BQ{%Ea_hZm?wX{G7s zu{hxH?;hj*pcAF`_Puip4tjvu#k4|aWlAFaDa~^e*_7U$g^n8{ zNtG%{O@<749&JRZaTK{DVKX}S(5YZ-DFlhMwjN&nX{C?1e3hqH88%dwJPA)jFv-1J zX4tm|?#M`7Cw=S|^ zqj#ZLEUsj!I3wpaWDvbcRwmEL@KiFXK#~+=wx^@yS!NeuW;8n3e2L-S%b)k~LQmhJ zGhSdXCTA#8-+pZNfF+0vV=m>Y_5`tU|Avg-D@eWt zDlubpl?Dm+JTnXx#S$hXC;v4QLSLBW4bWldxi=!5C1kA3>thuZ<9@TZ85LiS563<}kHUc(37pI28_dYjPfl?D0p!6g;e z9prJWhhsalRg$!1mjEZg6M4Z`1GVW&;ke3`kv6Y>u{zj3?)&t{cYnYjZAd~@7X6+Q z>8?pT@Dd~7+=&}F?$_x+NNS#Y=G2SBiMB@`NzZ&AfFQavXyPbFCypQwO%KpkBZXxQ zW~z+FRQxF4`{DR`JzfW|_)P?%=e>G6Mh4YC1Ckq@eY4T8q2^QBckD}F;&E=s^L`#K z#$>KO7`2mpy^L4%5@4atGDQlzL@5R>j{U2o!i?#+1kfUVG~!=ff9$^G7N5f)r~k^? zTVV7pOE*V@^Ip&|O!!+(spF>Ssnu?)tVimK|KlKbnv+#|(NY;G9NY@=ryT!)wzo=f zS6-ZGVa5V_RXVqjRg17Ns3OJ~UTFm4Uhxa?{6i%`8>Et&g;4>&|^ zf4QRTK^kG+UHrXEszMrswz7b1NGqYWK4HKw4(l$lPaY|)3>AHdpb}CHc`|h8cVy29 zhuLh?>)vm+ni5g`j;{d173#D^4yei65c{}5eCevEEdOf1Vnvv^bXb@{a&_NLiLar5 zn*!M)=(A0cz2A`pnbcRP=$Z20-eK|9# zTwL$j0`884-`a4AbGPK-@Ka5d_j5P7wXkatH$NhgKQ$C${vUSJSOph|J$@=Phhcki zz^AKdA5=GHeSk>0egZi{_2)5R3yh{ANdstp$mNB)D@KA}N@0vYcoDoK0|4@5CU;?7u&fACtQ0&POn z0dIR-+uP30d+Q6Y%Frf`wfbrP7hc_Y^lpsxLT?#)5UtHl_kf2!t?!3oV7BFIXU5*9 zVj_u51>(UgP2IF;TassNXQCx9EGsKZ`2d6MSL}!n%VF(}cx5fTD`x&K({voNo*{tF zjXk_X9yKIODe=~C0^jlTOtWCB^DmB_5!*q>#^*YilQDKfg=C=t_u8?2L3QH{Km_mL z5LvWsE`||O$_fDcFL>N=^N?@j<3HW@gE~t03i`@EiIqjMSAUsT)qFpOYN>r}Z@$-q z?wc=|^Fwl&s0gwR#*7=$8)X0%9^i62|11%8!?Wi-cY|PnDDSKA#-C&sm8$nVZr=4Q zzJk1~!m(@5EhEJgcXoE}KEVotX!2u~BU9p9-+R$=$aHR>B3xCo3i z@yP;Cw;)Owt~m=G33j*|pQ<-ZM_d13hq9X9!hxD<5jaqZPo2t;!JPiRF59l~yCSP* zTEQ=x^;d~N_=of`{gb!!rHJQm6iFURj5R^tr21as6nC*cUSFfNgy0JD%=kOyIeAh&THE zUM{LyOVUl#C~gA%zr~FNG1eXNDxen7f1j25mKbOidm|;3;f4nKy-K8p#Lz0&r))31 z3pu|J0BsFF(`+9+m0lh@tfqxSYL0`OM?r7fVWR5EX9q2c1_kPHad*IvsGWx2e6jL5 z=PtHlZY?7nnbY6wgkO7q`^pl6e|(ViDL-&FMDb%U*NCy?8sz>sUQF%jIf(;|TOC~4VG7QBn_N68;tn0n`PV7~IzYsxt>ML=u1rT#7B z&0M76pC2lkniOZeZl6HX>i`MED<~PiEpH-0jWdZ3dPkfk!^mwSg;iiEUUg|ki#E99S0NgEkR1p&s zSqglglw?a5foLT7Y*BEV!U^`A0yf=sGT|}xWIda!uu}GuMrvMJL@w0_5y!L?moBl< zWB>w5%+^yNMHr_+!h6@JA))S7r&svEX!6D3Lm7ziC(kSU&-y*8sVUEruIVqm!NH+_ z%SZXe6YWcpz8Wm+bV{=*3Kt)ZdR#_^+#zf)g5(>RVW@HP2J9oQ`vq@pekVQI46m5< zRvX*KU5`8bj0PD{4HUW7-D#4>b|^*^*Q6F@)Y#coTC<;2A>+CqTQTKl!dtV|ptUE- z^*J8!KV^3%%i?3kw47P_ILq#h2c*1k6ndiVm^oKv{i8QF4I24(TP79miVIawy8R9y zvof|MkCWjvM4@A{qEPIi;F&>q>gQ)8U%$Go-v12ZUzY{b6Odj86_J9e05Q_2nklAj zJ>DF)jEdW=yEHYy(mv}CzS9#7%C#yV)uQp(V;-yVSA?p+WB=roWtF?|!YErhVlQ_S z>)GNTZzo<+|5<3X1LGC=PgZFzq!3xELXQ$B{wvSwiA2KOlCSe3uiH z(c-sAgbLnqh#9;MLbIiL%5MBPb*NCk{o)6JXLeTdi+;VgO?#4*K-BHp7D^;Z!Y3qzR>KH3(Tasdwe#|yc20HLzIQh>t!?oG*qj0_- z%OfFmREqK`t@Py0Rgt92#IV2`=3uEGe?KL6ZdG4wdU<(WfrvxyvyZ0%d&NHhuTYV$0dmKuDcY-7GWh?xP~p+S1?t&q-nKo0B~It3x7%!uy>1pLG;4@hBNy zV&W5@IeuEcQgqi!?Pn>xFS`A1+$e%-8v7Z!4ij`j7SdI9vF_+^NeGXCxS!;xC);rQ zO;nz&I*`kwGcwS>c8YNmxVr+z1Od^zw?1Y7qSR9P>ujk<-)z*6!GV{?6S>V*F5(TP z{#7M~SzVqT7Wtp6QP{A!3tZ07=8ftyM9M!HMndX1Mo^I?PWb10I{hHWo%K6E;TT~F zpOZa`5bAmhIK?(jXQgA_x-;iF=J0m006;ua+D5o#i{e6u&s%-?*sCw^pHgN#wzz0~ zuzcrl(xp}r!y~FQz{#|4x6*Htf0$e1GAMku$6N!iscac|u5g@g4c5e+Y0+W@7i1HT z9v}H&9N3r$=uUwceDIrR^*GN5`Eeo5C6+P~tDECEuqyHfnSBH*9ra)-r*DDgg|Jl{ za_#|lj0@~vpz4!C>{pW4&*V)I7XzfiM5A%`1)A%;Hk z-3}u9_`t&(lqYb5#{yz(PM~UkLZ&-=3ofq$VVBAnN5I6V&SfZI{7U1`f!^e?5-zibGp114OH!FR}Z7=(J zdn4?MGE#^E+tjv}1>Oz_t+yr)-&a`m}h29G(8|u>2L@L#&60VW2N6ef&puu;@`y9&YT&&*uRf6V=rISDF^V z>f2dS;EKI?W@$STNr_f@=6b*$1}&jxc}iX2A^=i@{Uf8YU`q%#A0)q9o{>KAl2Mf9 zqAYrr74AuP3)j}kr0l76Hf~}L90tndlyUbl`_Uu$hltQT3cV*7uvRz-N~YhUU4NXu zIf3BXYoUgze0n^@(b`>7Wf(A4~FgKP~5Cb@JZiYbxS2JgRnd#`k2rZ zqJ0?1Y5!n)O&Sgr+JvNTUhpHJW}%+EK+Q9-FQohQ5R~|L7GHr1D>x0N+Fm81c%^kQ zW>;iz(UEIUt!*Qw@WwQ3Xu8L8bvi))wO-t44{;*0CT0#D1mwKhK0#8az$OeLedxp-ERJGRVN>VHCA}BC#vJBwbv87J?3JkM8pD2k4zsfF}WwWC8Z1 z0f4>k+=cRepC=XrKxM;dY|mh_Eb4JJ?=V?+lAfkbXF1(`cpCKQMtAz4#*zh)XkD0w zAkYdgkPpm(%RgPKpTnT0Z*(s#mqqcF-ng`|)yx(}6T)f8*wb9Gq3mp-xHDHZS>eO# zQUX>Y5dUHz5{b-UTO>W3nVWiAK)ffg=vaEgg?{v_ zRXefuLqbMKAZj~ncY?m@xad}TrN%zM;A27NenjO`hZo18t2kQW=;(8&ZkG)xM$iI1#tt&?9W%N>)G{XlIRF4 zh`#Zq zZw`h?yFrMN)15pZ{1|seWYV*-(%;zQrts6)tel)xUe=uh4*0E2iwA_u3+wEnnCKn7 zR@8F|w7}(0iJyO;LpxI+CsWImmENgeWq7=aC($Y5F#edPHr%)qB82(orm$d({^n%u z6`E@^ay)f;H6kbXlPuth0zFy&dL`oS*Vwgr!{UPyqY}vup6@Dme}x8u>_J0YQ23;6 z96A%N9ST75YiAc%*gQp+!%C3185!8vT4R`L0a{u*?svL=zdGtzl>x;#t;O%)x2l0T z`jSv;c?|r~evop&VPH&_%Y__wVESt|cdD5Qr=K?{ns&tXBJexskZyJZdDavU*LHLf zp$`-C?YI6%l>bQ)=#1NrLRQo!R_ds{gxG|$D9u#gP;~;Jf}fc*;CZ;-~9_xhGWBgaqlqeHc5OX1vk#vSmhK-hi95+LHRG zp0+`b(v7uvTNZy(z-+j*M$$S1(PFzxx1??M$h!{R5(|an)kl|{pa|^sPJJ@`QyYQs zAv`g>EKqT?YFXpEXNR7Hj2zy1N51bT#STQ}x~`pOvZzBW8$EKkAHgl(*hcj~;iPwn z7=FSdoWfkIO=5F99$H>JZ?Xn0144_hbG5aS( zK3>=ZtVHD2Z#4jr$l7oQfDYG;5TEZQ#;h^QSL>7(+4Cd{{zQQ0HX}slESP|Hh4jn9 z5;AscHk_|Kf!ojf%|z zraWKp&>Ili*xGggWyq|ZWT2jC5Sn3MQF%qs()?k&_volq>T@Lt6-hV!&perw$mmEy zGhxv373+rSwdYqt%qgJ6Z48)Z0&5kvsGe7l2dg3$O^EfEl410a(T^+~XfW)-UNxnO zy_b0DbjHBl`cd^}j0_P*hB)CBNNwuQ+r3E4ax_HzT%J zlu>%}XolBNa8f6Sn|zr{QG-&?fKHJIXnFcgHTrRffdHt#z)06bsH8*cAM348+42qz zSY+d%&pP(?BoOPM0g^n=IR?z>BsuBkp4_@aB8lmCwhysN^e^=8I8GM4h`=mEHxf!e z092^d+$U%gNvK^0GQDKUf6UY;y8$EnX2o!{pc%DhtQaZugj6b0!Q{u71{}OU7uA;E|4`^?c*dN zr}ZPy9J-&{;tWimROBbtQ3h1vLQS!~&QJDZ1T0(RozIRrFq;$*%6KV21uqAXvu7!( zmuug;nZ(jMZ|Y~OlF5-`ci29Kb(Rvl<+1Q9&U${@GD|XJqMG;`3G(wN*t^jkkr_L(5VP@XNr* zZQ2@DtsW|Cfvj*q`siR1(B#(UKk`~2`+jk<`Y*K1^1`R%A--)Z_e6=atMO9zD=fzN z1a3RXqW=6yl<+8gj<>pAF-6`)8fsFtm}Cd1#zSJxUWrgP{+1=@bv(lI)y){-4~N>w zqEuDMs4lf$d^R~38PXGC8;Bbp8H5qeeH%_uy| z7n>!$Ujxce-eLL6!vMAUZ4FSH7^x4@(M%u;#(^m4{H*Ti<$ID<)eRaj(}7m(dAkv% z!HZZUOajVvyF<)S-8$4(Jwavy$79)gAPMIK$W^{O9F(uI04-GV5!2>l=#oWKOhUA! zi1p7ZW|X$-MQa~*E;7OGX{7yXh?F1($QBm!mqbt%YT>7($E>;tt?_2ZPJ1TJI>Ra? zn(6R0lqcI7!w0%gYA{s(eT=S<;kbja*8SGp!Tsn_zZiQT8$ytenMpShNkRV~WwDk5>ct%$O47Uk0wDBE!wvLbn3JR5ETWM+> zXh=qjK!DfoBKB*T#mA>yIaJcAxmtf|10Ja0>K( zl@Y~2aJCp^h+!i4!dxrCkOMO1V#h*c4S;a_^Qwi@T9R5Md`GORxJgKU%ND%v%NgON z9vvX*vnT2%0{ z%^Bo;uu?rxIx>kKP&NbI-}9nTGBEvUxCsU%2aTFp*n-g0dwY_Hum>@Trv=JyX;fYq&*%kb^l)Y6`hp(MyFVdb1 z1r->UG*mvH2DqER-C@7Wa}SY4*zOttVjPPt3qS*{$v|23FbtoLTS(uRC@`BA(F zxtdDk=zs6ZhX>|Lr0y0|e{!kd?3#qUHr=uq5(t&82O_p?fUXQ;Wj3Z5JbF=90}2m< z4~v%AFaRyNRu|5Ff=n2E#e4zJJm11G{O+7Rn730~gt$gws9JjFFX|k8;+-#pX$B+k-$&)?71jHDg=gf!PdMEyzfTkp<+n zL50b)l@4&J`leln4te1_(2jXI6eAnJ`(L<&i@$PV(oPi$4PE5X^>SN`MRyI zY6%AV9jsp<6x^uz3E)+gc;1J3btnZE8uwxIKy))(ic=s{_(d`h?NkzuUyT|Ucp(10 zHtFAM?}OLU6yb4yv2|<3L1ZAE03Y6QcD!3ga0GfyrNw*s_qAAC>Ml;~#5OADO7$7R zA~ufL?~=Q1Y4xs?fi7oEQ{=OGCk0>aUUbdLO8#iGDH)H%LagqUFg0TgS6SiVm*I4z zSy1$wyPQ{J(t`oE8U-xhUE+%cBJ?U`$V(+sMNS$iQ!-z$Q#Q)?!>hMU71Q%Z3e)JP8Q-FC^l(neh22% z1NjYas4#K=9>}}OQ$Jq3G?p~k2V#8O1g}Mrr8{vsre~E+5u!_vg~M1gbR%iTWsQYL zADcQwxPakh$0c3FR^4W8kNyx$O)u@`h_29ceb~u2J+UBC93p2lCjWp>S+K-n0 zG1Z90MK59A6?M&Kz=l>Si1EMT5)7Od098~BUWb(Z!LQ8e=s$g#GWYRL*`Xx;5crJ;<#o}Rm z>bEAH-79Qamt9aBtT5S?x2>n@YJ`rQ&veaS4&(kyfPD=I{M`4e>hr(Yr3V_1frt09 z*6R)@eR{vU*xK9UMI+P4)Ke>N+@e469(`-<%&ux5vm3!(1;20vLgGdfE+yPPFAT zBmj^dy1{rbWOr^ZBwO@(a*xeizXT47zfhCFvF{Whn%}zBr@+ zXTd3410>MQ4xL*wjn+U>rfP2@OnxJZ7N`h)#w?_&uSVd*&0?%N-jCdKHZ_?Zum7N_ zgXi?7M+5_O#E(fS;MpEuHc$a~=}iPLMRB0un`|i7;D7G{k}?j|+=xPAD`&-jZ4V%> z#zq>+)52nqNImT?S&=8dGj=Ds4}T06>N?OsB^oMuubd-s27MlGTL# zp^aD$UAREdVBy33igb&?=Mx{h64|S$M})Z=d|fX?j`%DVRooHUypKv)cXF%j8~AaKz_q$9OrAuxr9o-;uNN-_Jb zX+9fbmLad(r2-^<4wL~#QC78rsetI=d3EPWuYB=XyDR&ZhvGnIXrG&uKYIrzER4aN z6em6bn&PX8iPB}yMg*Ts zRF&Pwc}@gle-(c>!u5v_JN)ta@lz23z9f(q6x`nx9tniZ5ZGW+tIe7ViTRK?b!1W0EMhLFR~Z7iVX<2;cHO86(d*f^AUT*{g$Z zEdW_zIxJwuAMkvEzu^;qLzyFyo=mU)l|wH7vOtKt>de7Tv{Y5f{6f*~^^(QkH&xs( zC3VVn3SaK%;PER^RN+-g%YPTrI(b_kNRR9P5OkmIR&$(iJQ`6g#YmdZc5Ukw;Vz#5 zyEFt^NSw;m!s=2VT{t+Z^9wM_not9DyxP=Es-u}Pl2fOuRqRMb(--^te@DLMEJv2T z`|yFUa)34BN_}std-Wv^qLr3l9*t+0B5|`PD@*U4fzn`H9$5@ME%>?@&-&A60J4Rs z&jqIM5eyy+ghhLW+zA4sc}S`%paL4TO}pX^o$%>mBw@7Tu5WMP7Bp{_*dC)cDnY83 zRik4B0oQj^;@lc@_q%PHmm*MMW|lc`)*hxk0Jx2DM~`*NmEUj%LG;!E+ZZ>*R_E|J zUR#p&t?=ix&cxc`Sg1UDjS)Xf;rx->6%s*e{aA4M0UkhY=rKW@`K+7si_GAGx#>H~ z#h!N4!;RO;8$icbF>T!b#pL%~Qzb{v)ogBZ@@k9xT3YkN$*U6L?BaCOcueE}CbWmH z?iw~14P>IaQsnzajjXs($ z7)<6_$fg8|XSaS$>llroe^=AlGp z@3Zy!eBbZy`uz#-+tt-|RrcO%ul=lNJ;Qz9OYdHHnxtA1zrH6EJ>RhFo7H3l$u%?# zK`PSN&s-VR@LM%|$_{|-1O@R=H9Tm0)8j)MPIT*%^mf{S9(y}r9i0A(scmos{n2I- zw~NZIlkY@g>Ue0AYIUa{xygNMibw72S0D8;qw8NXNjkfnS7*iSQW$Qme<^b`H1`M_ z@%tG6V&j4RyL2NZkEuZG8Dk5fYm^BuAMRILQ+={scFl0QIAujE$gTUzuIsL#XT=Yg z$$bCaZg-ckFjPL-#_{+~Okq-qh#^$f?APYMc(Z?a@E6wD6GCEdlCKji4==g1_}Wu(OWyU}ucKFnCFlDpf!iF;3D|)cI;k zoU>i;!JiwdrxT@K75(TjwlQH6hh6=4`)hS%TfrjRO7>#7XgKk~M%%_URE{rQ;-SeK zpG)MILp$}(heGdDx#!#jjix9>DA=2s)E_n$rvl`9&6n~#){Wl3d*^0<;!86E8<5`H zUiqKF)Py6>5*JUISjc^B&tMZ&GvbRxCOAG9W*wMe$;3XN`d5xl^?B*SMZ93cfKfdz zYTbmbo=i6FI*{ARmTJqXES97(wiL2gyA2(`KUiI@IEu_y_8o~IK3(;F1Nq6tv>j2_ zPaBJWb#c6cf(RT{Pi2rGc=dcLY3fOcF!X-(6S}0pD;|R640&hh=Ibk|g`%RN;AC}v& zJt?BvTVztM3iax&w+4C$h_o#=U&47Pt4t-u!ao<`@u@od+sDG^?6aUm=d3ln*j3vF zCofoT^-V0)&5z;->^;}xi=^$IraaIwZk5whnlcr!ZtKe+H*Zx7!7?}B?QT50{?Dp zpbsuRN=#(n8C`}v+s`9VRmEQB%o<58SDmtDXrW#Csb3*B)F{)V=@ANt5hRf>MLbt*|~z(c3#IIXJt){GSf zpydhmL0kMS;-1r5ED|?TbFI3FLevR(BJc1z?>d6=_@&I8AsbX<1`-J{G&OW}bv1}j%=&rV z;>KqFs+q_d`-#F~)VFX7J8!-rn083Kk~}5j=F1#Es@^HgVQy^2JloD!N7xXYSFbPw zxxVGiPuL{H2<{F#G`(bcPe0~};256uIa0+8BnhZBEcRa^_tjWK$&5Bps{%)YxOvsy zzCBa^)(DN5+I!HAF>%wSaK(R)?4)%?C9svTTRNY*l6NN5WRJ!+H{TgyAZU+uzahc4`pIu@;%LMhzlRXyyg;$XtOdjRo6}>AN|lq- zZsh_Nr)t`w-6!uKsXYX3Qc(lEp16$6=z^9+5IeQg$g24pf3N;J?QWv!MYlL;4u`B| z-$BZQJRV?CPWFvH_nF1eo~;zho|)eYLUvXS_d&p43x9eSAbbfVXz~h-Xg^mcH0Je` zU%${qADU#6`ah*BkwoBkCQ?>m@R@(|6JRp;048HB-ZP99wDw4XFi#+`gS{1Vc!o6t1}Rxhu&qM5shswHRY1;&|6Jj3Pz`aKGSfkt7FwF4ZO;*| zG_LFD)Q@mUhV@M$Y~~?E*`g~$##F`UlAckyV^XOTWs+Wp(h10^YNfa4Uw1g--)_9q zn1CGrsXdn~tZqG@p7gzr=4Orxt($c0Q2tVZg51Q9%fbktwZb5xXKr4L40~`Jo^8u| zM(wYW$WSMYPb;L@1y3-pimZXA$){j8x{g&q3U$I7SagaBt6KhajC7ipBPZCa?udoM zoeFY!oYt`~8HSYX%)oXIDC&BA!SIV#gij;%gp0c}1|uMLI>~;9i}aA$t)T3NR%WVt zS(nW6dz0UdN}YmdqcZ?mB-N3%HNlIc#wb|@`YQiD8+Pq*mSPsg}sPJB{<26pH3NE@QPUWK?0t|VN| z>8fDg7<5>?pEEOKDCZV2We0GinmAwEjJ$U z_+;$BcS#GHn~jVW2+K}9&4LxGC#k+#;ixk=ck4F5@!d)`F?(f0&LWk?*g~`^qw;y7 z93RSh;|_|F9(cY7WlSG}p)_U7yj~n8I~?PTJrt|*9J9AmG7u?&MQ`^MHl&Fjl>$bR?pF!-;4Ej(B;0Uu)(L@-%Mm zv|RX%N3^dV$mRsx$0w49nmBG2__?>qIC7+rLM;M~xLFTcAHkGd>d(^`Fn)dYZhfQ0 zTaN*<>hFPT`iAB_0c-QWZof3<0-$!?@95}FU$s9CmMfO`zHa?@C%Gy|*|~S@!e<~i zBmj@g7@_8X7gE@O{%PXb8M(cU_^cN%_+L*&A`O&WfM{hhkJUBzzs~O%&)(=cxV^#M zf*QpAWd$5mTsFU&1-3i-(iowEsgd~rjI1{T=+B120L#&D9vbK&g_7|I2tfbrvp8uO zUCC{9WnEcQO-7J}XVd<*`S|zk-a$PjFR+)>u z%@)WWsZ8}eUdTM%9V4KXB4~U#fBhu!+4(pnXj0fgs1Bk6EV@7Cdms;XU%D-iBP7Ue zPM@guc^8yy0-9yVjT2_y#eSL9Qvm3DM8@@#qcKICLO6A6EH}_k(75#Q-8;`>3Z5J% z086Lr(&}05t8!xLjot0-ce=zvVUAgBsEV$ZJzD-Ac6t<1GWw`=V~RRY33YTl$q@BZ z2EZ!^o+ks6g(QA$X$NueUxT-tvqxjySAgBqda|B!OuqMaNiP3Ec4;FwJN|hj31+8> z!H=oaoYRPNE!c~@+G0LFWTxSAsr_Nyy9bZyO*u}eB?r6mAu4aFs@)0*{_7Z1T@)kH zUrC#0Ie?ChhYLZ4-b1tvn1o6tLh{#z#&(yk_rFkqL{BeurMs2YwT=q`0SnP(LLI90 z3YFa6_RSmND9=>Gi|k?a<7|G3Z+kgGyoDZvU)Y1(g}%#ypB_uWwne!WSHXBPD*prE zPwl_DKx~l$G8byVrYvCKn3jg1$43)DL;or+Jw~4ir5U?jeZ1{eE9xpTJesf>Btp-R zDFc?-Qz0?f=6$w3WfndbG0(<7$BEP}hZVi>b(F-#jPX*UuTh&xx(hRKytLp6#A`kj zzZ?fjEydA#CWpc>=fBK^Nxr=+`IR|<3-$PW%R!7b{iQecCrh$8u0W%3;PiW!BQG%{Ll_jxbgsVFgw`!XKW!SG_)}!l z^F9q*s`VX(^f*co%QNibu&}wKRy@~p&*E{)60+MKmKX#FQKgvEO@hEM_H~m*7H<^# z^JT1vT+dskrRThFalAt((+{>(dx#8&2h_gOs)NYAZk>{s?|Eh>X&I9WpgZRPkS&u? zIAZpk(I*~7(hXv-E;}Rz%t#uaHPM%uQJGfE3S3qov8G6DfT&v4Px4Y#I>cD7Y}s2p z|3k)mR&X7E#h}oH=d9y^*46(1yv1_}490rp?^PDnWW^gf&paYQSN*=c4DDTwD`JFS z@d9^%HMYr8%ZH{O0KD0^yOf>pBQ{Q=aVWe-*n_%ltF^rD=1AI>kp{N>A+r&6b> zWhF;F6WldUE2~UC<5OO^DB6H9$X58dRR3P0m9fK$(3osOaCg`L4KzpM@+p=M_|og; z8+4bv@>{uDSZgm|e>Rg>VY{BL%?=X(xGzI~_$C3_pG^#>Q7as`iJaM)fabzV?t<&K zU=?Pgyoj&k&D!1$lv*9oJrRJX1Fg65H=li(LJg*<~HV@Hc<^b6i&fJ7Q&e(XRyexed`-;AWla7h~qtjr?` zGwWBl)&_YsZN_~FH`07_ZKAfJh&^_}0EFs(1f)zf?{Ms`MGl4ThO43L5u-~A z&y+~$^!1<#Ci3FL3aBVFo;)laPm5X#=j;iQXTtQptQnL+_!YejPw9<)2_x{TOYkzH*!laYHv8 z`gShTY=)5}M4zxP_g!YX^45l#;$D# zE@dl+rBvIvS6fy2+c=vjRy05#<~uaTzf+6UX#peA@1Br9mM!)Z?VwK`tGgM06rb20 zITyRCxU9AB(qHJPX8<>%6S`&91Eh2E#P(xDuuQ|V$pX>T?1eS%{zPuh~E@!$o! zmN|8-77w2cv`1>lTf*+DY$>%fwRC=oHEUv{zrDt@IsPRd*rW5VQ>5$flW=>ji}0*L-qn z3BzReokF1=j@fUDJ?DdOJ-?qIl?x@NiVEMC%vN!!7l~5OGk=g4X_;JWWG&PjPstjT z;Ff^V=<49`>%$>5irTu8a)Dqa|2UL+f)fjEq)RoPGtd60@>K@KOueG_fdFh_?@sD9 z;(xIK%{$+{mo{31CLk;P4Rl!6an^Cvv@${h=VZ3|_4YZs{rVPHgw5_P&S*a}+s^pg zv?22^@}kIzv@)NeZbf?wPYixr?przPou-~DDj!fe?h`jh4fyTsH-fd~R>C2R)l;#X zT#p)NZ%dpIMq&6B#SqKQ*FKB^@gc$8r+2dC{@j_l_HMf001A?4yA4e!{3)uMe0rOq z?!wC}NR9DQST*^rMnlNo#$%W1BI3OnF{mNp=L2^p4l!Ux;`IeN>G;0Iqr$u$~*TgJ$L7P*uNc_(Q=h2#v z%LdGlo{BBaD5p9d5i;QhWXuO^S-b?4ffplkDAAN+oVH_}U+RNu`3g(e>5b}2tFvgg zN(R=qjCAwt22Gst=gxs z7e-54Owxs&l4*OwPiR&J%uV(t#zxl)7JI3ctfkP zE3vwoXj}$f4#+B+_IScuGSar8Sy?aGWSSDehNk<8+_%(bz&K3Jl!VPm7k#_{TBef- z_XMhgyh0r88rOy+IfAd@?o}~q`GobN=}0rK7kzg0hM<@B;6)U$cim1>+!i2?#a`~_ zb{Yx0q^Pj+e6C~tz#2b0-&U74ja#HXG&{~x&xMhW>APkeYU(lim2C6s&Sf#Cck60` zvfr7aHcxtU9KK*~gqnD+JlQ0veaaVY6?o)La^W=~h2(I&{1&)!mrg+|=~U$A(M>11 zL3gf2!u8`QM^G{|kt3I<;Wg>ME&6DEUB#D^XXF_unO?VxT)rbk+d$tHUK1>Gu9YYy zY7Ff^+LZu~CEm9ONpaOrC=IWU+ioJ1Hvoqpq-7h!?jM3--sg8?;i!{^ULWtS#nz!O8*+kkT*9;ze_Gib0 zT;rG0l~nusyIkjN_Oq-EQ$v4qyCrW_?W8VEiP}6Nc)vl{v(?g*P_xymVIKCNl3N5I zrkepcYF#s#)%PL0y5smWANaM+PDIqU6e@JTz*{BdG1rbpWxe+#5>zZumsw^4U2hxX zltr6dS3${cfQk1^-UMTt0N%I6W3ofrd7Ntj*~Igo3u94+78V&J{UY7qK8tKI(UG@# zZo+a>UQ8DeoSbw<*gyvp@%Bwc=e^rJGyYPO9h$GR4o$=1j7%QS&}CLCO?9U^{AmUR z$0`$p+Z=cc@kRPbhv<*P)R1G&>CKl+{AK$K)DJCY2Od8(6fY%&{p3}XJRbg~W%;Tv@YPt%YG)l3|kWJJ?Y`pQ1 zk2)~}eKnnzB9-p^dc)j)1$OXAo=3Aj6Lr(M`8X+K6yVs-_7x0oK+EW^xoUcRd~-Jj zGLN%yN;Dj@b#K$qs?U|UNSmMeGw2-_xp3S)6BjBO;!Xvu&f0HR%v0F+Z!E?G{o_6= z9_33x1KLt6R$=nT?o=HWg#@x{hqDLCo4&>;;8C8RTjqTJf;sf2Ga4 zY2!m(lt5ll1#?<4B*|7O zUDu!C#}Y^Cr$qO@k4kvY8~H?FHj!flwjB&n%4bxjYUQFY62e&P(m8QfH@+XOKX;PN zz3DTnjDOk0U)RK{{SeBX9^hWQ?1Yd0uIvQ+p`tm{*s)+Y3 z25abD4eD$ohdp<<0XxP=EA!{xcP@O=?dh*%6f~m8<7%&@NxCLOC!*@C?)VDuT#`pd z%$(P5*)W67qPsSo0v(odH7q&;gVmW@^fU(Z2erw`5mV470`ua$LuM=WEPJn_Ke2@t()rR6E(j7(wb|FE- z=vQ*;MIi0rsC+ysHkGhgpU(lj7u#qX!i=bU23tr?A0%RH-*}4iX@<%*P}DYmC|y`^ zzgbMoLc(di<^yxhwwZXytYpgbQb{0hb+;PVMgg^A#~o`9kMWZz!Px%( z{_^?Q877PiAl%-~KNy|~h8%uN^ZkP=_fo$DZPz+O~#)CFUNw-t+uH)tTEwi%@6 z*f#PV)12%HsS4l=E=Tkf0{?taYOQ+PGv3A#B6u!k&f(&610p8^RCHdb)7B%@>jw`W zJS$X%W;4R41y0t>!d+|8*k9orv@4n4Rycec7dXaG{gg8wcVOP+QS~wdM0;1dlrDnf z0ZlSm&;C5U+~W@E?h#@XV(+W(xF4R5aLfVXZj080FcKvPY-UsTP(I2IYQx4QOt2P9 zb{?JE%SblAk}SB>bNJS|HkuM_)WhRSs0$ND%(mL|Ex*w7zSfPQa)SQV%Xdfa>3rhE zk%8b@sF>_8k#Xu9BURcDckCicSR?ZH#vX=|i^=YC8Tci6w&6&l(k3(a*K{t-hkiHu z{jK}AS}CNCmw;5b?QYYS7i@^F7nsBjh(7uL4ceVwDq^0iffnZjgm|-etI7`yKn*0k zV_!u{t2+W4M621ofe}a)L;x1aRWa@L8uf9QOuvI-AR-vtg|k9mGTcq|^2a@jiJ!U6 zTY?BwJ*VN=1$6ttzy&XA^ttBIV#@~7P244NslD*0DNW`|z6S~T&(H{x4}o0wbR2JE zq1>-XsknAi*wGkPUf|tCwW82Fi}h`vXGGQNu8dh-^qX^1-Lw?|oW9=}A8;O@$L&4d zKdFJhgB}&b;JfCNOSPWY zZsPz*LcK2lS$gD#AMAlrme%)pby?;CaQ4QY43?sM;+>HmJ7imb&nDs%y zmJNx}KVnMn#zpESFL8q+!HxbYo6x2z3cu!U4{!p**zA`qwrJ0!Uh;d#LAOJ6Z z^lWsn$;yxkiv9P19|uITpvs$$-9`Q11T2E3xV6P!Bwh-}lewj#To^^w+^eowwZ6Q| zGsovNh`svfx<+j2@Z9-LI9C1uS?8D+5_DW*N{^LNJ8+6E|g3G z8SH=&9WVM`@AH*0K+Nh_G{pnik?nJ+cWX5-3je6e=wX$$AZZNR7CZHCzWot~HYgG1 z6`v784uv7mAd&$iGp;?({F-l!N&|er_On5WJzDKxaW!P7mko-_n2t5*WUD@q%E>Ns z@gVWY)(L$xT&Y_<%d_(;DY=MpzVFptnwm!G-s3IAl6b05PY|PPQnP~U?$9SGMA6E% ze&b}Fo6$|=b@op_iN4URj?*$To9~jRVV0`OW*a=WZA19J^f9ZASe~pvq&JmJm@g|# z-ny_ddGx{7w_>5mDQ66jxat4B203P9V`Hdi83s5h?mTpcy4;HCpFN%igX+S4*#`09 z8f&J0xdpn={ zK+gUDJ)?;p12QEEkUltlKjSt=Xz%Fgvq@7#z5u5S*ah#RJ6Kw1ven|6#lnwAHx2!C zQzLY)3Z6G9Z4I(>lmgAwx=r6>xhB`su79;-f#YUi*}|B(YRSx35CJ+fOjgetBlA|Gj7 z5#XWY?0X*i=GAk1$-EV+H(MzG>^cQhF9tfvQ`}N^{STQ&t#D|1D?TOR(N0#@CLD-l zl|}b&y6G_pF3WRq#F+Xea{ig_%Q&XtGy6wZxdST^6mqD1har2TQ+LE@ff&r^MdwQXl8rlwV9r*c=4045n%K3$J{* z-2ILs>1|*;m3hKpzHh2D&UD^A3wL$KWZ&OheDl6}CxfNU`ESOa)YQ$T`M2=_?b9czE?zbrM86?a>+q-dY!a(;pzYp=nPNq-nO z!Qu%hv@7Yw9g_X497_%nv&x%f>kT0W}7BC_Gl8WEG?8~ z*3_!4P5%n;bMY^unPy#oNnyvZmh_-6dc|eBYIv8ke>br20qf&e1^J0LHoNqkW$8s* zfJ($kL}9wI0Da&Q61q~W5*S#Qm{J)i{dx_}7lp#$%5I{=P}Ht&JAaQ&t`rL=pZ;cP z_nmit%8G#fy__vpNiLBe#5%JkfMXdQK5ezG*EL<@;i{_)cw& zX6%DJ@|>!BBnZ3g-$v05xku1)2*bYrKkpz2!}?kKeu$2fW{lM2xb>@uT?CdNzH@AY5c{CTis>=~hoZA*XX_Icfkf4haELxfz$GI#J z+Qs;))H|qJiT^g5-Q8Ubxp)y99TQ^@^}wM)gv$O7&cP@f%;tpu8uJEotCT|*aD-mP zROj1Hc8B;eVYUYzixQP6(O8Ue=F_M0$lf0>{W}UqjQy>{_HYZ~+Q11kpJdk%>Wrb8 zd-#9fUKl2L?G$nK#PV~}5bYlCL#edD-<&kdcj9~CVfP3@L2rHy-&mzWpfJLdlSX2z(0n56vA%3wm;f}2IY>Z-Oq_IR5R)7`07J~cV)6Gn-ixG^IdqmO<0-CL zXW=lo&)jbFpT(*dDu(S~F683Z5e{LU-^z_g@?*5G;oOSq-yH(wB8Z+LU|#ax2$Lgg z1B=7>QsCX*5I)|wc(8Uc>6O6iB1Ps*wFl>MoL1QcFR$`gX0eNTd3nKS)!p1m5P3D= z-d&qqUU=c{?_XnQZ-3o9{=5(&{0t;{pG;EzeE>P!vLZ?ogAr!R+4{yK{_6wT3k?8* z3Yu*Go#W!kzMy`Gjw01&{5aUJvK8{wijEc! zzBc#;j!@u+o$kDv4mC0fnGaF&sCY7rFk{YzVdFlF8;N;42ZkASni57zyf}_b3;@rc3$#w#z^9+BRrd@+!oX9)CxDDubRz8YU(u zAC*6N5P1OUD9o~xS#C5jgc|FWST--3yFHyCF_ml4X(ldW@T}(8&W1@7QQ}J#OyIU2 z!5Nn$L7~Trt@wD}bK0toQPg*NtU}mpVNcfMhqAU@+1!ne+R{=T2Ng%ez;WA(L{6d- z27QFePB{L4`QDdLKRfhs_hfvCf8w{t>zpXbhJH+7u=wTEp@7A-U6EQnU;BlLl~n+m z&G$fAByC~*(H!)>c@Ro;{nznv8>ryN{>SR@>zCo~FVW}}${&R>33bPBwu#B4WNa%>l`YGv=2#90EmlzN^NhX$gC&nZ(Ne67X z7pg1d4rtM{OP-W}ao0J3zVQi@67!g3<-P^TN7MHk$^-LMvlq zQyQ*pkJx6#Hd*NCIJ(?Cr9r=(nsNwM=2H@A_)P*P@o>3RW)Tw|9T##C!rQlRr}{$OIA|sKa*km_DR1_uI4^u)GR)hx z_rAQwc$nS8C`M7yIx#VE@r-`Vd+a~|o4U_Z6O@X9IF1tQPrrl628}O<2eR`L$fRvOmB5OIjwYK(^ii#@s?h!A^Rm#YoAU4nU z@&51{gaAfP+_hq#WG}8>^~^=}QxAX7U(YSv*_2KR*Zjt5Xu0`{$AX1iTc4J8M?~Gvdfrku|c!wI3Q8(PfuT4DBq7pID2cr z?`YyM!gSzwE_)b;t4%J7+$C5|uMR4Aq~d2d{NCxHn^0i4q0ZLRt>P7@BEC7BFSAt^^Hu`D!ub)b zTO*ycXZv>X6SjsfDR)K;?i!wxrkAKp2^OYm1;;CiyN@drZ~FJC91Bf({Mb400X}L0 z$Y4lgC~C;IF=va5QW{Scfi9|nGW-A6}Fyy1;;#9}8?V6Qd@PqufCDT=hJN}c& z$Ol>nqnQZWTBz&<09c@qecm2&D$vZ#%(J+oe_ndSp8XK8XM!eW@A|IiY92)BBWi7# z=l1XKfWe|O{M^sNgfhT{Myc?@EGQF>MW&>u3xcw6h0;!7bro#=zn7b7S-*|tSCXe+ zbEHMB?lcUYwg#+IayqUz&>WGZqLb)FW&yjc1`f1g@xW_&k{9MSx~O(ikQv=Tcg@l9 zK}*b$PC$T90-T|uTR>2-Yn@3Bg6siSO@j)E1^`%YY6k_6);kolLQxSB6atZF91oC_ zw*&a+2A1RBEboC0EjI)-#p2x_qpXQ@E>hHrq+%?1s~>=9N`ku07un*}ROag0y!==> zq!qBs4!i;I&Aw+-T3Z#tJ&FcT8-^e>>D=Ptzj?`(kDon%ZVyK#c!12ZuW4BDzNs-$ zV8>0S7);Kg^UX819#X(EJ@p&e{Ruv@ciplxFQsnf;3|!NrFfPYc4u zf^czvC|rCzv;I{GRL`>O>P9SI{5Y~Q4P168eD;@ozh>A_#iNda;Vvt~w>=>&tcXI_ zV#TTaRof#*UqtBEx`=f3j9V&RxsJrbz9*o3AUMlg2lM0K|7P^h%t-a?U&A4*6$)?; z$#8d#5k!WZV-oP`w?Qo;&UEN#Dup%UP~Mp(3^PI z4Dxg5q!?S_!H}qM67ZSm`-j0bAtvBep$GrGM1F|v94AQl{YHSuaB{#Kksoa`A0_u- za|chC^$s+E1(AaUINm4c?t3HO;}8qqVH!M6B%T06jVn0~9yk@BQ(T+^`Eq-AcM^n; zkEXxxrO>cSbgh36*`EZH&4mWrtshE6V ziVO+j5P8tcm&1Skh012ut5+Wm^LwRRnwgD|??IHI3~@Hk#C~>1;B+1@p^2(ElU>K|&Q`fdL>MwDni$9D1@jFe4CU9E_5-Q^J|dW| zL=45", "setuptools_scm[toml]>=6.2"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] -write_to = "belar/_version.py" +write_to = "ragas/_version.py" diff --git a/belar/__init__.py b/ragas/__init__.py similarity index 100% rename from belar/__init__.py rename to ragas/__init__.py diff --git a/ragas/metrics/__init__.py b/ragas/metrics/__init__.py new file mode 100644 index 000000000..e2518e77f --- /dev/null +++ b/ragas/metrics/__init__.py @@ -0,0 +1,17 @@ +from ragas.metrics.base import Evaluation, Metric +from ragas.metrics.factual import EntailmentScore +from ragas.metrics.similarity import SBERTScore +from ragas.metrics.simple import BLUE, EditDistance, EditRatio, Rouge1, Rouge2, RougeL + +__all__ = [ + "Evaluation", + "Metric", + "EntailmentScore", + "SBERTScore", + "BLUE", + "EditDistance", + "EditRatio", + "RougeL", + "Rouge1", + "Rouge2", +] diff --git a/belar/metrics/base.py b/ragas/metrics/base.py similarity index 100% rename from belar/metrics/base.py rename to ragas/metrics/base.py diff --git a/belar/metrics/factual.py b/ragas/metrics/factual.py similarity index 98% rename from belar/metrics/factual.py rename to ragas/metrics/factual.py index 3a9769280..3928f5b25 100644 --- a/belar/metrics/factual.py +++ b/ragas/metrics/factual.py @@ -9,18 +9,23 @@ import numpy as np import spacy import transformers -from transformers import (AutoConfig, AutoModelForSequenceClassification, - AutoTokenizer, PreTrainedModel) +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, + PreTrainedModel, +) -from belar.metrics import Metric -from belar.utils import device_check +from ragas.metrics import Metric +from ragas.utils import device_check if t.TYPE_CHECKING: from torch import device as Device from transformers.models.auto.modeling_auto import ( MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, - MODEL_WITH_LM_HEAD_MAPPING_NAMES) + MODEL_WITH_LM_HEAD_MAPPING_NAMES, +) MODEL_MAPPINGS_NAMES = [ MODEL_WITH_LM_HEAD_MAPPING_NAMES, diff --git a/belar/metrics/similarity.py b/ragas/metrics/similarity.py similarity index 97% rename from belar/metrics/similarity.py rename to ragas/metrics/similarity.py index 8c38137c9..f4435ce04 100644 --- a/belar/metrics/similarity.py +++ b/ragas/metrics/similarity.py @@ -7,7 +7,7 @@ from numpy.linalg import norm from sentence_transformers import SentenceTransformer -from belar.metrics.base import Metric +from ragas.metrics.base import Metric if t.TYPE_CHECKING: from torch import Tensor diff --git a/belar/metrics/simple.py b/ragas/metrics/simple.py similarity index 98% rename from belar/metrics/simple.py rename to ragas/metrics/simple.py index 116225776..81fdad42f 100644 --- a/belar/metrics/simple.py +++ b/ragas/metrics/simple.py @@ -8,7 +8,7 @@ from nltk.translate.bleu_score import sentence_bleu from rouge_score import rouge_scorer -from belar.metrics.base import Metric +from ragas.metrics.base import Metric ROUGE_TYPES = t.Literal["rouge1", "rouge2", "rougeL"] diff --git a/belar/utils.py b/ragas/utils.py similarity index 100% rename from belar/utils.py rename to ragas/utils.py diff --git a/tests/benchmarks/benchmark.py b/tests/benchmarks/benchmark.py index 50d63beb5..dc8812f5c 100644 --- a/tests/benchmarks/benchmark.py +++ b/tests/benchmarks/benchmark.py @@ -5,8 +5,16 @@ from tqdm import tqdm from utils import print_table, timeit -from belar.metrics import (EditDistance, EditRatio, EntailmentScore, - Evaluation, Rouge1, Rouge2, RougeL, SBERTScore) +from ragas.metrics import ( + EditDistance, + EditRatio, + EntailmentScore, + Evaluation, + Rouge1, + Rouge2, + RougeL, + SBERTScore, +) DEVICE = "cuda" if is_available() else "cpu" BATCHES = [0, 1] diff --git a/tests/unit/test_simple.py b/tests/unit/test_simple.py index a247ee597..0ea853d21 100644 --- a/tests/unit/test_simple.py +++ b/tests/unit/test_simple.py @@ -1,4 +1,4 @@ def test_import(): - import belar + import ragas - assert belar is not None + assert ragas is not None