From ba4298059e969ff483e9ea38ea7416e325398b68 Mon Sep 17 00:00:00 2001
From: PNRIA - Julien <julien.rabault@irit.fr>
Date: Tue, 3 May 2022 15:09:03 +0200
Subject: [PATCH] add encoding label
---
Datasets/index_to_pos1.pkl | Bin 0 -> 255 bytes
Datasets/index_to_pos2.pkl | Bin 0 -> 350 bytes
Datasets/index_to_super.pkl | Bin 0 -> 168867 bytes
SuperTagger/Decoder/RNNDecoderLayer.py | 177 ------------------
SuperTagger/Encoder/EncoderLayer.py | 67 -------
SuperTagger/EncoderDecoder.py | 36 ++--
.../EncoderInput.py => EncoderTokenizer.py} | 2 +-
SuperTagger/SymbolTokenizer.py | 56 ++++++
SuperTagger/eval.py | 10 +-
SuperTagger/utils.py | 2 +-
Utils/PostpreprocesTXT.py | 26 ++-
train.py | 104 +++++-----
12 files changed, 160 insertions(+), 320 deletions(-)
create mode 100644 Datasets/index_to_pos1.pkl
create mode 100644 Datasets/index_to_pos2.pkl
create mode 100644 Datasets/index_to_super.pkl
delete mode 100644 SuperTagger/Decoder/RNNDecoderLayer.py
delete mode 100644 SuperTagger/Encoder/EncoderLayer.py
rename SuperTagger/{Encoder/EncoderInput.py => EncoderTokenizer.py} (95%)
create mode 100644 SuperTagger/SymbolTokenizer.py
diff --git a/Datasets/index_to_pos1.pkl b/Datasets/index_to_pos1.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..005a0bc309f087dd5b4af75a12d59239b9b723b6
GIT binary patch
literal 255
zcmZo*nfiqR0&1sdcr*4eIR{ViX6j)L2=Wj2nBvXc!|WFjFvXjtht<V31SG=R!x9kW
z>Ndrjt%o^4+hvM3dk<@Xzn^o+6mO0m#(*i_oIT9WK0#Bwxq6rbg8Zj=bN8?YXoFci
zJ*<u{VIV_zdzk#3r+D-Au!MQ~22Anh2kG&j;w{j_7&gUQ5G3s9HpN@0hdC@DXo|OR
z50h)i6mJobs^BT!qCHH`&QrX_dKf*Yc#HQiyMQDmKxzY~cuV%M1qAsAx%y1;mg-@4
fbn%+vE#1Qu7CgmU2E+}U;w=kuhF7@9lu|tavtCO8
literal 0
HcmV?d00001
diff --git a/Datasets/index_to_pos2.pkl b/Datasets/index_to_pos2.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..ab2916fbf6a280de8ecbffdf6108dcad81cce6a2
GIT binary patch
literal 350
zcmZo*nHtQ<00y;FG`tymm;-_Wrg$^;aD=%ASrupI7EJMGhHx@-3(}@|v-B|g`TI`s
zX6<1P2=$xd&DO&n5ae&=>E|-Vn;pV-arK?z%>iKt_y<q%=Imki_V=6O&DF#1;u>P*
z7!)$ao4beE(Isq(H%||HK&YQpa%RaCZ(gV#*Ptohd=N_uic*WGc=Lmt<v+z+poiHn
z)OU)vAVe}VFD-M5w-AJrR$5Xz#akG{NzTtpnc^)1;S`r96-@CK1sNV-m6BRA#apb0
z+0W5;inn+Vv!`Fk6mJQToWE6&tIrf~Nr<L`f}+$Z-cmixj!r>Syrp}XgCl*Xc*}rX
e?KQ<)7UC1I@8n<_Qj4Z|%R@Msxdo|HO7#F9gKS^`
literal 0
HcmV?d00001
diff --git a/Datasets/index_to_super.pkl b/Datasets/index_to_super.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..606848f7652155e0048c281320e36f46886661e5
GIT binary patch
literal 168867
zcmZo*oyy3-$N&PhQ#8C8dPMU|a|<f<lJkpF^>RydN-`6RiV`cQ^sp5tC*~v;O_@BU
zhZU@3N)KyFNo7GQNQ5cWWJ+g8`;?$58s5x3j5bsJ{QSHHyji?CN~R<^3nVizFyt^W
zFvK%3FcdK`FidHi5;Vn|F@rgStp`PG1{ZFPm2izrV2#Y7Mg|B=AvyzKRtBU%t<+#(
zU@%}{V9;S;U_f^UNDQ00Jg9mQt;xW^0KyO_GsA4QM7OyFW^)NP6G^eT0BSqP98h2(
z`vBx3kgGv7$W<^5aUBcHb%yA!3xT;V1Y{~GDv(Vj$92WfZ~?ghVmm9$c2jiQQ((5I
zP}6o$V#;J-U;u?h8is4wV6HJocTEn=H96FF4JhV7I1!r6OE6r;K7g(QrKMB`1_sQS
z<AAw}%Bk8Anvy{_f>JFgyg{x&PJ1vpP%MJN4wRl?7?Q3yVXmWcZbx?=$R(&YbHQvT
zF-3#?iJr$m{sV;phz8jLViThdq!tnm+)&pTV}wHmEJasfGZC8@$Q2+LfM}2nAT};_
zpjZLPgTe=Yp5uYK&=4aX`M_N0gUuFfVj!1+Yz5IU7lQIr1p@;Ehz5y4!i5)VtC1Oc
zPS1eZnt{y*Y+@kWko^m?1>_23wS?0RAJj$E3I}X1g1HprLP$99Lv1CsRz&t2tfqvS
z4)QxF-a)p2*r1Tdi~#|d%|_@k5CMyU2yC`s6GOKdTZ#j@1m-G;-vwbVpdhE>idT?r
z$S%dD4irWZ*9bvfV~kM`*TBM~2H8YdsDs4N%SVtHwz7m!ngF>P6h@%%D`#L}$Y5Y#
zD1_EW5LXIAU1>x{n4!B86l)+GK{O~FKx~l938@3Ag}6op>KbE=Ts#35W)narqK5@Y
z4BZw`*nlw16`)oEs7^+%caj(w81fky7(l5RSsdmAkjp?AWCs3l6otB+oP3AR<sg@U
zd;xO>G9MCVVlbPH(eqslEX-m+;X`VeA=^aQ|Kc!LP$3<GLK<cx%-<j%pobH>I*?jO
zm`FffV~El2bAg443&=!b!UWwGWS2o~lZ4quMvX_vPaxZ{m$y<dTS@dIC|*IPBKx0k
zX(kP}jVTnviz=`HRY5ioR+zyo1JQ`+kpUaV9BPPBH0r=i)4^vN_5hKEno4RG1i1~A
z#n1x<*$*&zm|BouKp2t$<e)Z_TGhi$BgSS>G6dNO!Vp{Kp|%>4nDI%q6;yeE3Qv$e
z%r2G!)D^}^eSb(dyaN`O9k}8W7OKedu%w7Ah8z>1umZUR6b>LZG3r2SK^PKlicl9D
zTA&x09xxYs;4&4Ti$O62auJ9I*#^Q88<nVJBcZ|qVWTq4Mly03%#WZ@2C)&Qs=!Re
z=zM6vLQaEF$iYGf5;CeV6ERYc1<XVXVob!AMj<w+!EC^&m~3D+*br+2C^SG#2#_B@
z7!fk+Fk3JhE)FnT97wSRW)mpJL8TqSCJh?c1hNHrPyiBgnlPI%8uSJ*KN%466UgnL
z#ubQ0_(2P1GKr}Z6y~5D1~LuPRBfoK#5QGVRnkC0PY322jOrQt&=VoofKmj=@1QUM
zg(QegDD+`!LFyr{(uKOp7$Z0Iz~ZzA*;JUTKqld`k=%hpP@V<(3WPzq3X}>#F$;1n
z3`1P42X(a(MoPrqwIs&Xu+k0WE0FI%uEAZ)>BC%rQBEbm!YzT+aKq*TP*{L4w(x+2
zg#pwh#MWNe!U7c1$lV=~|3TpZa}jFj8-i^ly6J~*A;@r$fe_~#fem90HNmKS55NNQ
z04N~Qod^;`Z><xX@UZCxc@^emkn2EfV$^}uf-p!uD55|ZlypFPK}9G?4&qZ|s83BX
zlFSX5Pj7&9(Zi=8SAtv&iVKiy(GxmM4&-_e2Jt~|Ll1FKs1Y&?q#qIjCeRQt!ss-h
z&XZ6x1VAnXxgJD=Tmr%%KFHM|4B|svX9{&4q461dx(*aKpsbH@r5VhX7)4(LEEP3?
zVv63b1lft2-p!#dHNlAOBQTd90qLT)OF^*;ieXUN28A04gZLo3(9<umB?c(<U^5qF
zHYD{}Kz(b9(fPUp^X(Oo3x=X^L8%bEgaG**gdx7Sg!vw$8F2^Z`#VF~_n^23VNhNK
z#US=(i4`;yj4_&cQ(&PmWuQX=l#)TY0Yro14}>9cXbp9}2}a4X0_OS^gTeKncmVkT
zl)rJMOo)$cpgtOm@qw#VgoqDYsO!zq<HJ{h5gH**3XBX4^o|oy`bTaog6yYuxeAJ3
zkh!2z2bAYP_Q667eT*I^4ocCWv;hi1WDH3QcCgSyZ+-b{z(Z4GG&C7tp$Q5}5C-Ki
zPz)f4D2NRSReM;dqL=%=I`B}{84Xpep$bahuoe?2L}3^bvJTLYHNyy57I?_A4APK=
z<wQ^}##Ks#<Uk=ny&Ucc3o-QC*Ovz#Vmu=(#6US1gz@KMCuqnKT0=swk{_0SKsf@E
z8=ME33qfHD!Uz|-KwU`hzAeZVpu7aiXCT*rFftz!ey&j0(YtSp&vl?QiMsZ{4dzOW
zK2!^|!Q<QlDl_Pn4nbiGawYba67Epfnqu^C-@sh^hSshHxeio<f^rP<R2(P;fb6DL
zy9{IxC@w)5)EWTA0QNqR2Q*YnFoyD%z(QpS$YmfCK&?91R31o-nyw`!R6wBz3Na83
z%0D0s$v2)b-(XDR?11@Z2gv1v%{QQU0{IA*g0Y1L#D`u`ADW@3VqXz>ttmo>S`(DQ
zu(=ZC3t~eU6n}(#4l@sA7bqoz(jctOOJ2{y8ybS<=qcV;1|EVkBRm8_u>!)NSO$eU
zD2E|qNLuuPh2#MDctNfNr4LYg!d9vhliNXhK|ThhH(a?8BnL{{AiF@JLQGl&nF+!m
zH-hxy?m_rMLzB=#3}Spk?l>vPl_1}NLIXV{L1G}EfYJdZ{{05Ii$SghVT6nQVJ=4R
zrFo-HpAJ+Q!%`k<J1YR@YV@9xH~q#b@uduytFgDc0->%q!DuC}fz`=tKskffVGc?O
zAPma6AiI#;L$I8U-l74ed5|242BjhphNS%<n6KzK4+?Sx$Oj-7!NMHG2Vqiu1=53T
z4z^qm@o6yBr-Pxs1i2c7@%a`Mx1f?4UrQBRNTP=TD0D#S5+n|?4`e1~EsqdbNTTNi
zUln-ELxoN)4^SwAFvw@15W|)mK;j@@4@5|U+y=s+atGu_P|icfkhB>J4N*et&57{|
zC3Pmqr63G)DabDL(1fKokPjd(4;vyb2l)zw5iSphxg4Y9n*l4&XMpkzC>#i-9$JPu
z$kiZ>oPr=>9|7|LdLPPr4$KF02AdB+?t!%(A-;%&`oaWbG3Nr9FBX7w4W@Vi#Rn{p
zAW~Em)F%Tu?*~dHAlHE~%!TN=7*^+^w$7rVt~W#PPkMiW#lshR#set6K`8~~T2L5+
zFsv>^Z{N@=|AFj;q|z8@=nR(DA1R>&Dz!l&2ckhf0%3@+Vqw0*C<AG?3J&B7Q0fG^
z2o$%-DV9`Up_>D0<ABUZ?(;!>8VB|1V5wno`4r@OkgpJN6hA~<4staJBV3*Ub2&!p
z8kiauSD1rb4Z@(52r4rXK1hW5V35?X#P|SIMuW;tgfEhyz8EYuEHS<S#Rn)ZKxz>_
zNrw7lu+*?1*MY(sRE~jM3Bt(b8fp!j0(Ct-=IUTQJdn#^E&{m@gh71FawZk(I#Y}>
zCF-rE1BDlITN>s%WY@r42ujEFujkXCp+L*kXq1KmEVMxJ2ns<^xrMvtONaW%7=3ix
zn|dpEarp@3I#7IqFvzu_Tn)k?7lYy#ly^XKkUBa8=2DCnDC$miblX5C5tCA2t^v7(
zR6Ph+W<p(QiZM6$0M<@^05Y9guB4VZpcDcsQ9=0+RyyFCc>=`|2!rAYqzB}CWDJR~
zELccjjORXqg~XHL6cQl6z)~z`Xk<e}!w4hwqMmm|jrfF>03cUD;vxrXJGI-lpmVNp
z<rk1EKyn}&<Ps1D@j>B)o*zMKKyn}&qz;53F3*L!oZ7V_IW7me4wUW@uFHeE&X~$=
zSVFD?xdIek#D*ov#US;VKFEjqfZDA;N__zHA+n1>z5rqDJ}H3ugxWiq$ngoR1SKYZ
zK)xYH9mwY}4AO%fGLXDd2nzv>)c68c;=TacL~0pBix9vTzaU?M+=s1nz@{D+8Xy{^
z7c)ePpdm6C@+T}!fZ7=#*Mew}%V8KKhUtUiLEr=Q^a64@$h9CE=6Vnf62tUC3Csr=
zH6LZ|HeypHvWsxLvUGr52}<7}400g|V-L$Rs4J<xdyUwzM2>Ng3$f)^<k41g%mw)#
zQl6DVeM{{=CpoU7wr@dgXHa~DFsKa;%45hQ@rY2UfQACKM>VJ!3ZOg&3Ly{;aybY?
z;;<6xdTRHWsqK1DJb-)vO5vcm0riJKzQG(Tu7diA7CWFQ@f9cqfqHNF=CLWwo1hQ^
z<#|x(fKn~UK3KhmtG5UWMNsI1(gZSwq>pN7Xi|HXKe-`^>{^HmYX+GML178Pps<9v
zuomV*^j&1$sD~TTGKJvEGoTcM8K!kG7h{Z)pq`*dYZrsU5rjc$8Dtl735?z4^>lPO
zEN6krB#>)BG$__V7{mwVL|9uHpIS&NX@I(%9zAJ%t^v6mGetDQT!+zjDS-7|3PAn^
z#S*C0A+?l)g&oLtZ03OEL2@7(6uuw~3AZMw%goWov3yP7>zPbQSkDBq9TsaaUm%yu
zARmHUf=wMP+(7ckE(P(isi)LjP$<ySY)A+<LqnKatNpQs1V{|I)`EpRdaVT$2Zc1q
z_aF=k0T>^|2APE(gCKd39Eb+#0bvjyn_7?<HQWNS0}_HQ&=91?c{L!{q5BRbMhzF^
z3K3AKf#g6m$X6f?<Acg)5Fg^xR+vvQdXuE@<N^7LmOcgf2IhL~zG;K{#u9y2&9??#
zw$xBqwqWxm%w-@~!O9U3n^?7=_`s!}STk_x$L2PWeISfW4=wG4q`P)#OqrqAPrd^1
zm=d64Oc9DTQbGh~7PUec**r+-bwERp8vE~X`3#gcKxqe`FF^9h>PS@!@l7YpHyCrK
zq@PJbPv3ybB9QMf<EIPiBQx|h+una*t*<{Imx9_Rgj!!9b?BukdbopJL5w=&_(8WD
z<O`TN$aNE}?TKEq!^DZT7i2dKL*l&~8cN0(p+x#lU}9VW^Btkk1i2EV78J@LHpI0(
zFxOI{S4b<@f>I>H#l0{W57O`jr8tCZ`=G9+)>#;ohA}8CVW|Mb2Duc3L1G}Y2R{Gw
zLqp6Iy$<yzeSQk$BSQHC=5uW7Kp_N@2jv+M8x&F?3=#vGO|8%ZnGM1qvym|*k4}Jw
z$Uv35urd*pvOzuv(J)tn_|yyukXhJb0%Q*;7GU-w(~wY_I6$F<D-8_JP?`h{B|--a
zqx%Y5ykHXp#SwB`;bKGLY%<I>=%Xgys0RyE+cmJf26G*Z4{_BLsH><kqeN^N!BPvz
zMIb%sJ_gCd<ml6mnF<XBLbnj1yA<Sl^c;*$3|mS8`4W_~K|Te^qsGfLn2XT+!QQC%
zvC+y!FxNp+$#ke|Ofk|f=_}Vr2_MiHI&4iaD4alg(R~bZDNK$&sbvN<G)yq!W(%xu
zumzh-L1M(F7Eo#c`3gB@fcV(VfT;n=!`Psd1;QXPTzWuqAoU=1APkA?nK0jB#PuGS
z@AiO9M2`oM*kJM<wh%=4au&>&bcjPx3I*8=3U6$o1d0P(t|dkf%zTg?kT{$T^&K_#
zrV(;6%yv>;ic3AM>_ddk9BAlJV{aO*T#GAoKxqXe2ckhf0%1s;&xQF4qj$RjmTxzJ
zTuY67OYJxZ`39B)U}htHHV@`Aj9wDyyY4_CfL=;rvmF%1#Ht0klvwqk8VyuSf$YRw
zNiiSlQ&Wt+y(eICcmm|Ap%8~4pMv5PghB2Ftpvx6+XYY`Q)3-AC}e0Ax1jg}`2-Z#
zko>j~=5mZ0g!D6IXz6lTm_uB>2<qyAEc=M*V-pjPh`3k`^_3Aun}vFPD^To%aycwK
zL2Tqy0plZFwFKrWjB<<g8GKSw2sT%NT#s<sQmD)5oeGe{4x5j#iNSmW3SVUXFgC(h
z%b>oZ<%|>|*MM9J@&%~GBX?c^6jvZ~U_L}=gUkojA)qxnAUk0B3D@{O$UG1RmB=7H
zpb$gGkdk3JH1ufM2L`zU-8a~L331U1sEeqvRvi>(gvu09_<+I{6vwc-8QrzG%mJwb
z>4SwU#0M*(KA=}Vqr?Zuaf8hqkk25#S_ShJM$Je4{x2wXU~>&9yg_mxSAp2L;sxDT
zAU+6#@&PE1BCl_Q_--}Schoq8msnR3N_C)=1#&gWw-8sZ8HBC`xe$aAu3QUsr3LzV
z0KO^kH3BJQt`UHhdbq+Elqx{s267FEO^iBhYH4K;z3qaen|06_p~d<dLLmi88=#mV
zHQ$0l3WPy=kz<k8_6?MskeFHzjVWrJTa7K$K(PP{Q4o#KryzM`b=cIx)(9fo0}}(~
zK}usDWG5t4Hb6s#8oMWuT?<P~=)MK{oEUW=7ow}h7Xl#l5Fc%X`H0Hv_h{iGTxKGC
zvkB@OYOe^vWixu1<5CB5AxJGKHG<fXu-^=GEykD`>H7qTNhKf`5Tg#{LXcXJYe9Jj
z#D}<e3(Un-UJ-&T+(51%#l@ifgK+Ium}@awij<9!P}8-b)+KC?2@=NJz^-NqHN@Cp
z=BvU49fxyLVIuxYCr|)_8k?Zb8mv$NiJ^B$kj1FwMPkeZMHnoi2fI6mnp+`BYC9}R
zVO(<W`vBe&en4VNn3$*nl^~#~1<}Ys2kLQv`asA@5`FvtUs45`2P&CBX2bG4$b1+E
ziGz|JC2mB|9{B8}hJCaS6HwSdlEV&ga$pX%L@&jCOW>)YWRRu?P%MCa0>hMsJhFL|
zngvR6q{Je!-7x<^VrVBUhS2xp`8L2~s9_Mt5GZfLd<3E?4Ovh~A)865dE~?tvi%@G
zfiNVtb`9a!0{IGrLB4@uN<$Z99tcy*JW^u|WH$)I?1#~i*xC(^EozPmQM;Z2#Q-SH
z!(0yHgF=DU_JHi8r(KX3*#nIcOZ5F4z6J2ou7HlE9kDe(twIWvc0p#Nw`@RSv`7WG
z?1RM4UTEx4b6q{T=@^zLC=CgiIn+q+$Yw!8Y#%Jd&}UvL*qcU4h=JmlT0VuDMGNyF
zp|^h!g&xSaAWRM4g3JM7kXay1X&MHZ1;QY+Ko}Bw2cV%xt$R}_DQ`gK7xMT9r9MVC
z59DiFm<b8VgW!;42{kmr=q9~^S4(ebR4sub7UVUMmyx|mO7{X3b;MR>AUi-9WG6@;
zvBeE8camZ+wd|yI*nq+Zk~9v1lLm9BCC0>J4?Jb`4C0gls<J>a2l5dNQzI78&83E!
zpwvi8%%Zyi=08Zx9UkH_2Z}WihWQFaQzMi?W`Z!tJP@XanWV-X$bJw8*^i7NF?R$U
zb1b2T1{l4B8SqRuV-ROLP*ILt8^XLujiAP6HZ{y8H43r01LjXiG#&*<BXcM<R{+qf
zvZ7Xq4n*Zhik*;{ItGoYfoMsBiUCk)gVGZy{ealW1v`ijN}H6#JhFK(JE(0YBqWay
zvXBIYE(lZ0*C4Y%7-Sv@QzLdj=7BKCJP?M2<Oy&{vV<BMVzkMRz$<|xRICJG^Pr%F
zfh`Mxyb4N4ASZ&@#HfR*Mc0QfG1J2xFn59cFc{ndiSm=+C}$3())rx6d<H6ZK(Ps;
zkwb;lb|^?M41>%Ag)E^|1X2%D3$q&%3a6l<K(8q~Sb77w0@*bnHpmy$2nljy0+w%J
zAq8>|xn_a%QsNho8z3=s8WuxznzEyI3?Ub`AYZ{^n%s~D#S*fal$u9QOd;D3@)HO{
zV(Sbvw&*npL@2gkr2;N>AeU1k1c|YS-gZG^<SaBs=rzrNE2Kf83`)Dy2nCoq)HVwe
zV&`BXMyF{8LLmmrZJ-bY<<`Mca)4q1<~C4FKtk_4H1z1zy}=iHpjsArXn@iXL^luQ
zYg(8I3CRoKkfcMS9AAQhtri1CHpr_W8rj>hf)AHEm|Apw*zCk6M#y}Sn?RX@kUDgC
z!1y4&*zCn7M#v6I+yaU6i?Ari*edQD0bipUL4`G{#Ci{uzd_ytrF`Td!xj|iYLV5G
z+7<+v1=0`0AU9ANRUk7#=1}4nP%;LYGuYe(3L8qp8005NGQ0#%hRmUs7&TxAd^Dhg
z3Zns(q(bDF0L3aORzMi!3lJX^BG}BwCPqwZ2APS^&VjTS5^I;iu?DIyFs9G$z@zOB
zjiZg6>KGRMxb#vIw7AR!MJK2rz}7q<#yx}8O^~E;1)daWJf4RwH=&Dxf*BUv_<{{x
z9ZW4K*TLAN<Z4>EW3age66IInQ9b~<pO~NqMK8A8Mo2HMf}0Ze3|2QmlEO7`QeY0H
zbr%dh=0F)0R0V@FJB$qrDG(pVCN>j;^bj(SR%Sv%@;WpmX+5io9+IH62?|9}NWp3&
z<Q^6*R6y!LAppW4^{^0w@j?30&Bi7MG6$p|BuB1&17#;9rf$Gu3ZwL+?$ROj(1e8?
zwvZ>KK@N&h5C(-J48vjs-A+&}p^M>i2gpnirly%7`ziGs$UTsly9tXq8V~T}3NcW~
zfm{O%ZBl%O9P1!6Kp18Yx_zKfLl*<t4N?!HVS3Tc1j%F5i%kq<4oE$ShM5OaN2#Ac
zW<g@>7BseKJ)BHzNP%1i@-fJl=z6e;fkFnP9z?_Rf<hd`24RpKHhtK{KxTl{gJ_su
zWV0Y4b{iIA7$uVkyjLbdVXq8iE661vSCSgnAeX~1%nbC9gqaVbiB(IfTR`?eLhueW
z1gUka3L)Q7GZc_R4C0%+P~TAN{BmM^19Jt)XD~LzHTR&dq1G**pjgM|B2XO!vK=H(
zs}u^e1Li|e3WC{5OE-b+frRFLSZJc3jNq#S&+9r=$m=klk{X&IeIN{SF$@zELLfaL
zOo|?AxD8|<BxE1JLKb6{sscP@6{r}purve<9rRW)$am;z3KT*h4C>{A<Uwbng7kyT
z0*Qls1LA|^(Dj1EV0PdevjUj~G6#lnnFmq_vJ)f^V#Cx!Lh~UsG-+9jgKR~20mwDT
zK85vZKw{`20@4e@<b)<RyI~;!a|614xb#3m>=7))(9a_E<$<R^o*|t6Kp_pnpwI@L
zAOQ*`P>8@n4Mc;)u*DQe9Y`-o9>fNz1z||YJ%)yy3C3wW<egwmOuT_yLW)aCRS)sm
z6PVA?(}Xv9XX+F387PcF;YzG)Vd}}%1M%fkI{FgiVh|?Q#UQmH3{nrm#Dok;JqUx;
zgD}LG&tSeppYrl1?=*7sxCGe<!Y~(7qYVVBFJPr8DDQ#H2hkv(fiQ><>z_b;`W)(0
zTDJH=J_TWr?Vx-}4d3DmQFI@J?8K)YBu|N(Kp_hW%@@$nG{HFAeF5}LDdz<s+t6JD
z5<`zakQll;P$+^hwL%h>7C`2~*btw*g!zO<<uu3_APlk@<Wow*A7lo|^`P3SoPmKM
zgMonocG?$A4iQJMpgyD4eQfyR2$T{*z69A0atX+%=<y6v1Cj&LAax)N;)B$pn+@_M
zOb$83U~NS7)16@AAUA+8$ZTW`iR;(U5TVxj{OGO**$l!UmxFu>VuM@(;)8riY=|K1
zg}D3;%;o4kEN}A8o}-n^LGcFiF~rqxp{_Q=C?!6?N{J7ka6xwsNDMu!Kw{|XK;a9*
zpzuWwGY}u79^DL>8jw7U4RS3AgTz33(Cq;E2qs6pl=u!BGSoT^9NpERPyu0(%aMHw
z;)8riYRDj)0g2o9FdtyF0;j;@b_&QQbeDm|&}|2ap{oPA9)v+IMz$Tq2dPIl1EvNf
z4`YMk6@)=zpm+qaA+G-bbv?DtGAGyduyPCJI%HRZTngeN`xKvAh_61ve1#Dw)M=T6
z;sb<1E<+Au5Fg}HVnYGhcckcn`0^9XmsC1qmRJ{ne2LB1_);P$UO}}rsN?|Yg`LC?
z3IT|3KEr%NhxP}^1)%r?*$(mz%yrn>5}^2`%x7PqJ~PGe*$r3;aRX!vdiaCH&}{~Z
zp{oP=1cX68A=PK-W`RNiWDdw4P+Wn=0zjup!txw>?TxR{5TMrS`{=F)`5uHpE+-`f
zKx#p00FnZ~!CW~2Z3|GigYpNq6bSMe2!rH7@d6rGM7a7p%+(mBKn*N))qr%NhZ#tW
zmZ=NmBZMn|KwW7<hY}eSRv-+r9h4$KK1KIENFIbi@*o<7LFz#mBu1(pNDBH1^(D1#
z=p)9}=&1tcdi1yjrFD=Thz9u*gh704J_W@cOpbb`z%OX%(4(yficb)xG;}~A0SYNd
z{Qidez><b*6+kv(3wzio4|)j<69>g3wm1ic0?0g&xgeh)V~{vVA9^|isfEdrVkS%v
zKJ$>%6U^<@^b^Q`kl6YIjV)S^bRmZnDLw<`LQu?rTnod5LJwUJA@!h`rPNIzw?RVl
zFEli1H49JekVFnGP)LFJ2;cmJ`o@fgt1`&-5lA1%r7#R~Jt!`5<tk+Lgw%q}Atg;9
zn+<ahBt86xhM+lmYl(_;-5}dxJ|flCppqXpCJ1vm%q)B%iLQ^39&$ny-42-hz#;3)
zzz7an&`u8Y@gm<H@Y%*46wWr{3R^-geUN`isS`kcBNT?{<v+{~AiI#`9a)@IHxT0v
zT=wB}A35%T#4{s2o-szIeJkMG{VFKj?ng;HqvsP)_z>e?SbSqs3zCDC#h^Gr#vnP6
z9q9JL{D3bWaHSJ;vq0u!a~G&K2H6j?3*;sc8<$#;94<c~>qWO4WHv}I$Xsmh#pgeC
zvp{Ch+8>Z~$^=iR82uXG7I->s8D{AeJw1Tp6&b_g5j|dDaf~l6vBf<oKGDqriBTHI
zg!}>W6CuBV+=0!_`209feu1P}W_X&V<#j)>xTbgbfXW_Fyn^Bm854>}eDQ@XenI}C
zBpz|O6Xa$>ZUxzm%}x0HKr6RG(iF?EPE(-t1j3-WLB@pQ3156*i$74jQW8%fH-a#-
z8)0lhZUxzm%}x0HKr6RG(iAH^O&Jc-wm!Q5VPzh!yiF($@P#$DxB<l(scrzJ2SV-u
znTyRneC{L19guit8?x~X%4;AD3R7fED17mSBerk{#RsnN1=$b6$o9k7gxmo#7n^<f
z+((W(Ao0u&iD%|eYF<~5%Q+yg5z{;b1r;&DM66k~Fb@)Z9FX7x4Siq?v0s1>eO#d8
z&<DP-B_-K_k|zj*hIT-crl2rF#>9j_z1#wF*I;uGB>p)e@y{HJaR8F{3D|7r2~e^}
z4=Rutdh!K{p{v7Idl3o>^qLN)9^F2eI*>fb9#B#MB?%A)xdSwB3lawfKL|s-%mw!{
z`cSm@8*i4W!3>NHAmGhWG9}6R4K_c4#L(RW5(C)@n!AV1vViP{xea7Lh)s+-d}{IO
z2g#wk3q6cLW`i)uEigF{AA}*{!wm}`T2DNHf(K+R$SEMF6AC_3)k3_(1M?2G?ouY?
zOj5lA3kHaDd11~q!kD<vfn~@XY@r1bLyvrr7%|R8P6#09Bl987;)6R2efGgS1?H?2
zY}SFq(47S`9%KzZIglDqgo7|hEsPIw4nM7(0~`1TMGnY%5Djt)2!r^j&Jm!Ma|lHq
zcIODv!Z|P>gM3eH#6coU2<{y8CWdzgEbU}~j70Y>NDNonA;vkNdI02H<j8|KNf_oN
zWAsj-cLmHz6(H+r=_F8f1#0ntyn?Yj&sPNQL@KN^0$BkH2@nlSGa&C_3ti+;133}I
z2Q{!jp$Nht{U8hpNl~~n(Km{EH^4%&0b~`eLlWdn5C%CFWLF6T0|UsJAPjM;*kE%i
z$eGyNDB>`uQe)FPDAI|Ed{E?q)PZPFsDUsf<RoCuF`>e4EnLn4g$8o#36yBCrB9F(
zL1uuQ2V#TNfwB=O;z9ahMK`YgCP*(Rp@8%wV~AHJVP2(1gBWBL$cZ2tWEF@FaxT5R
z3i2ihgR(IwctCDI#t<({!M%)rc%AnZSQfrQsk4doGRV20U;-sVkh4KC3kp710SS@=
zxdkK+@)G4SE)DZOHMW$2oC<Oxhz5l;hz)Wsy7xiegUM0PD>6gLE1;MLVNft3V@RTu
zg?j~~UReQ4lq;}>EJ%#lVuM<-0`dk3gAyz#Q6hT>#D;iD4(=t4e6|MWr8OYuqdN;E
zHjrKd*$=9xKwd${5O2xDyhY2p3Oz_*`3FS9oCI->0?aukbWS*+$Oo-|A}3pe5)TN&
zyaJ;kUQ&d4$q=JY=mJYPE}%$44<(QoF$o7&T0yK+f?J1C+k3#Q^8gu5iFGjFLabD#
zwUr?2Ko}IT5Gz$+RvKY+0RmvY44}l9AnQOFlrCU}0>o-nxYZa<`w*DbA=I)OWDYFz
zLabMVTaS@LB4E}>fV7btzaU?OFv#a18svMBS%|<_hdF^(DH>KTKzytLw+>^-&j;pX
zA4+@-vJT{1Q2s~wQWIt^HIo=9KtUK}9VlQy)`5x~)I6*OvznR#4Duz+Iv5)gz}hhD
zsMTr)`4HqQP^6%jVA#Y!z6YrX(J&_xqYk7Ngh6_VRS)s94&2KawK;jMW^%j?(*uhQ
zeBMA;3-UGyld1>eeO<WsF^W&pCk{Zd26G;^SU?tob?-o4200x>WAg?sbs)7M4AM)C
zxgfO=FYCd)jIk=u`wOfT`2sQ$RMvs|C9rt_kQjO*CdSL?=778kayCpa$Y~%xNF2ll
zVGy65_JQn#1dl!}c&If<Kqy7fDtJJ_2`XJd{W4Hc!@8abuNuI-O0VLM)?Nh#H3)<3
z0W}stK>+eD2t#7maIkn8<}_GB2RRuzvw`G6av&Pyd>96afiea$>LFe>f_d2(V;^w`
zEWdYPGZG|5YJLYLG>{WPG`f?qnT4(v;$&l(lc{mM0LUqzL;<o2mzO|tAa%qznN+ht
zW<tDW0`nF%CdEL`CDkb~Z-Vq8oMk!~oCR_c2qT<j26Gm*CO1IFgF=9mL;%tQ!k~f_
zqz_c8Ae?9pcOpiQmh?kBh>1{;Q$SvTsRcO&gh65;=V8lYFg?g62_!vQz?^S{(Vr-R
zl?){yW6=v6kQhFz2+3n}KC&3Zd6t98d7w}PVNhs8oM%N(=YgySVNgVaXb=WD1%%PX
zAkMRfIggfQD6DmgE5boe14S&%nZ&3AsRdz}UYJ)Q-m-yv3u6{`2Q1C*09ivxnx(b3
zKwbi2?4^n=%!{<Fuwh;Vl?tGc2GO8U2VoE&6olBaI7ki_qad}ASh0gSnI4G^<W!I|
zKs3k+APnN;axy5S(dF@ZA7mcHTlO$-(XztE=M<2$L7@%u0>W7iFlW)SW+db+kdr_d
zdm46xInf9so07L!0%R2^G>{{ikP~4E1QMA}FelL>jiQ$}Age*n1<|0017Q#!+4-=3
zH7tT*^2C@0G7sV{XPCEWkr;?^3dq@@*a2beiNOWtL~3UvSjd4w1LO=`p$n1&sRgM6
zVMxfi!ktCs<`2j!P-uXx0$GjASs*nad1Un<JrL))!JUUOe^CId=?Xwbqel`*3|rBS
z%{q|vgw!LaVTjY*VNNr~7<wmfwG1Ur1Eo3KV;mkZr<!7{_`3rO#XF!7phhT?Y6d7|
zu{j?WD<E%y2FF1I1F*u5y!9ZSuwbCpbRxbOLk|X!H>vL>FSwU5#x+PkAsm(kLC%D+
zL0J*R$Cf(C^%5x9Ko}J3pqK^Ofs7&f%Ny=Bj6}NymT0$Na~4R9)T~df*aCSAgh5^d
zxdYZ!#`K;K%zN~x4{&)86oW9QK|<XZ<}5>u`hdLYHBe}P5)gWV0`WmsL#*|KSxc?y
zHGI~B<PcW-)5~g*wIGbJIsk4pMioNd>@CQ6Y=I3D2VqdMgylF$zz4#efHCzI1IuMG
zpp=ZBgg|1X<T8-;APjN>NG~jBK%5Z-cLqi|mH=}`0!SM*odI$VC^SH3A|fRi<`i1z
z09fXOgg^+)S|chgBmnsiTUr2F3yK8P`R!1+^;Av=AZLIu$VyNI!r~p>X^>C|gFA!D
zT_%vPL2(SS7UT?UNf#ssE0#cNAx;X1I|*Zalk{C2Amc%y0kVo1C!woFI4c6?ENYw!
zN{O>T5esUgfy}_1UW<e~6QhWutd&e^=z*LG3QbUk2k8Z2P-ucMB%-6>PQ}OqEwD1E
z1(bF`<proU3hTRq#IU6&kagrlG|Z_W{UGOp?896F9}ROZwI?oNAp>$c$ZH_$De*2S
zae%x5GJ~=Pd<@*n7>VfuEQT&n!^!mYGANco7!>QEU;-sbWDH51v2d?rBu>hfs0?Im
zgS-vGAg_a>4OTZ`dOr^4eQMmg04k|K)`Ai^HN6ir7v_9O%*Df;PVG@ZLQV%c2joQ%
zjq0QXn3Jg8#6flv$XZY+gRF*xL?S(`1z8Eg2y2tz)?$nyk+%jGWIQNfk$nqeLwuSH
zvy$4~8JION>p?jOwIP=Rw;H32q^wVY&1&=$w;{eyg*k!Rqp=_-fP4(9Q9#KH*4zMv
z2B`eNRtbaTKx#qh0fZq=OoKZSV{n1=yHG&JBc};cok&P8!rAF?XJa&vC%`Je380W5
zrvjtY*&uI#DlkwifiTF~APfod47k%Vn$eUk698F74X4AL4+=WW)>bCW`9>J^FJ=8?
ze9ng@EJP?~4TLj5Ap^n)XJo^jLFMi`zR-Zlp_X?!aHn8o3CbqfVAf$PY+!QO%fDQh
zlW5U310{7(WT59XP&o(6`NWrjd2nZ9gdBAW6H=WCQV;S5EOgOB9+Cd?VNN%qQThWp
z1zQMX=E#D9atg?aAdCo=Lby{fB9uB=0c0JvPyva9Feq(;<PlCPf;$PL)Tey#1Y{k^
zT9B2XGy}pQt3hc9Bo4wLHK1$(>sml6qhh$TFq(vvO)G+&0J0KfH6drg)PO1+kh399
zD}g%=qh&(g=^P+yKp{fNX)t-z;<Xg!9BR*U!mI;14}?LX1qumR)eCYmDB_4O&&uFV
z#Yk(^X<>kz3Bsf}6{H^I6_|H%mucm2=VPSF8L+n93{Z%GLJ2gMLdx(HEKP!(Ly7ZY
z`a#|SVNkjSVMyAofO&yh+ebmxA%`A_55m~IfLSzG4y1EH&IDmZ=v2X-gONU|R~CY-
z#1=XraS#S21du$!S=Df7VWd0im4zUyLDquOE^-bBiGwi6S)jCyT1?l#orW<PN!i+5
zP|^ii39=f52{{c}4I<=f;m)IBeT!^0a>&8FfLdJF!JRY!kqL7W_EvE{%xSc)NkC}~
zl#W2o0Yw(500E^f;%kxyxKlB5Ds^iTQk)7(2q3S3yah_TxNDL|xbra*0d;E<N}Lb!
z4k+7!yaU3Zm;zx)P0|GS0!GfC1FH<@fFc#-1WGDHkbYtl3&_dHF#zI&FvuGqb70vM
z;f-dPH)vfUf~-gP24;oWGLX&zg%k)QLZ=n(9E@y6-3k#~=zyF8!l0}Kia><3+ThN@
z$UfAq5D7U8l)gccif~#x+-Vrug1QwVA*X>t4isu2bqMEmz@10q3K3*2C}dy-2Xd}N
ztq?n5PBKRiF<%?_nYcD2oQVtaIWYwe$Vs3a2nrn#8>A0}k@@7914<O2q8}9c$okQj
zM8bLpAUi=AWG^y?xe3_}klBQ4NtixlGa+Sm7c8hP&}SZeJ>Ws@L4%+MIR)e_<kSEP
zdXPBC%OE)rn;5Tx)Pd9^n*|CMQi2@WZjk*j3^EsFKDq8dHy0A@-LPP%M+%3{--Ch&
z<Q)(V@*aqd>^u-3pI7mjL$Bb*=LTfIfcysu>Yl+8)S#dRVUW}48Pp&<K^SB&GN!~0
zAoD;NW*#J{dubiiuown89hBk-m3tuPfz*PWikyMb%^{_rAl6=x-7rjy8;J1>$UI0;
z_rZdiR^>K&5Mgs3C|E&WgB2(s=YjYjaaspA$PSQQ$aaJ5hhdN$NDV%B5bGz9nUG-b
zhXuPC`arKQ3%t5yp`g0N73{dmJ&@B#@hU!ZsBIP`$R@yo4CBr}?-#IM;R{fv0<{m3
zV+$mP9z*zI6c%*EdV`cqk8C!`d>AGa^yvB^K{AnEK>`Xc5C%DykW)dyO-hh}%mrbP
z*~plXJs>?G4ATP%l1Z>2vBbz&HSmm8LqWy@S&5vwKztAeITtyWKzvY8z)~K*3<FXN
zG6UUQbUsK8NG-@*Qi2%SU9jMSnL~{GKzcyxLH6Tw6R~DNqG~cEsz6tkpf9WNJpzxb
zBNWCJ$ju<PgTf5meaK?4E)yuMK<Ys>%-_VQ1E~dJkY13Tq=XYO=7Y>7MjglvFbvW&
zSlj}M^C^%xXAY%Ry$A9x2!p(i99-mNKT^#B1v|)IkUmnPAJp^!=>yRqGhi4JWK&^5
zMvHn6<Wvv_Sr1C_AU4Q)5Ff<G=M73rAY$x6b`wZFBuJ*gg2V)UT*vzetSN8=6#K+B
z1wdW}c^RY!R!HC~5J1j=sYN#*6oW81<lup&KJ;-Km^jFO5C)lzj3M5g4)ZQGb_;^6
z0b!UkiS;gWOhBA91MVy;4ETYZ1j3{`3l!p@5Q8{rCd^5+Oc=0816d7nE{Fy>0fa$(
zP$HsF!k7i~E;UvfgF+vKL0$mSAPn;oX5`PNm(?I=fH24@kVu{bvzi(U*g-~vFg9Pq
zA{JupT$r`g*rNxs5`=MC3!0cf9m}2vvz{7<cYv%0VUYD8Um{xv5~E)do)7aD#%;si
zq|dg3yad7^?|{4jnk9gFhuCrsq>mW2kVLTn=6q@#AO><e2vg#GP)<c;m4z_pQDZSa
z$Y~%<sq;W)Ae^`e?nI1vK-we#Vv28A0>Gyhk^mONoKLR=fX_*ww2qnpmcX1xuLMAh
z^H3AOQkWBI9fG*(WKaPJN{=9n8G_4T&ZBh*lHxqn5L^y(A~m)rfQ$xV^jr=~S0HiZ
zP=fIxSz!e|oB^^DgkjDAu|XKbhd5&;+!<6@9u6`ZgwaC-BnHADanNKe$a>7QvkLAM
z^uZ`HF53fH2f`pLL6sD&as!EhFi0HTDIjr>(;y+Unwri5<r<KWK{Ut;5F6wySkZ~E
z`vh^$8n|<)(0>3q1LOn{O~^T*P(fA;QV(&`TDX&_5MdyvAX@`sV+$d4bqMFIgE@y9
zhySBn19A>1QzNWf54R3u?F;F9C2?5?3Vei>8)$7M$T|>4Sh*2yB}RjT^h;Q<`4S|C
zux=B~I!bO~h1IIqk`E{jk>e7?CPp2|iKM87<dDs9?_gwF;!htP2=9Or0m!?c*Z~!l
zpy3%%(80Emf`S9$#Vv3zVkD3~u+gPGpb$o{u|Z<6;%_i`5#(i1jR5j82!p%`!Vqt6
zg?ketw;zCc^8m<8L&2LMuY%kOsuVH3x(()4Q;ZmW0`uyV!Qxd~r5r@eZ-)g3B{wP4
z%9)@%3d)xV=k9<z7o$N({PF#?b}lFZBAmQ)usa##To6V$c^BNt7zI1=8$xLr${^>0
zFsSebm1+p5?}j@aBYP7+gg`H+gNhkYDTQ$U9=P+-r$N1mKYECk&W9DKka}Y;+_@NS
zWa3XxptW;Bi3D~0bRW#wl$@AE$l0Lu2`j)rY)~YF_@GFKwG5CuMj$mHc~F4|VuRF!
zFi4D$9#E`+yotLZvmfSVO2$^{>19wbfV>Y$Xb0fV$7rAszmtXD&WFV!#0v*uUNFI^
zx`;oal~^x;5(Fsph)p~owIEEYdWhE!!M%nNJJf4Ff<hDIJX(7Ve?B@4^C~@x4tjVM
z6s#bOyTyD2=4DE*NTYYcrL}#a%mPXLM`6KZOv5@56ho9Ie$*267~I)3Y~xYO*&uTe
zA$}b0bd2JL`jr7Cc@X4m5C#<~sLj+9FfULtTMtX)poj%o14<w;HYl{v`H=K_5@t0u
z(km!*K-Li3Kmj=ugh8PJDshl^N`lM*$$>Cth@FBto01)run>be8$`oG3YQl^&H{xh
zNF4}6Lh&@*IT+c8_#;-xaSkYSKu&@=9pR)iFegzmJ4LCJKoN(#6?zu#MD(3u-ozgQ
zOARN2LJ@?qr9(&rpBrpW1!WgdXAv{v&%>NbkF0?!^bzUs0?awoXd;1pi=5m+Y*0=G
zg$DA1M35M)SB1-oFukDk35s}-ewbQ9tMNeT8l(s0O=Jv7EEi#3rezGEdl^(HfxHBw
zVNQZL=Mv00l*|>uLIzt5z^sH=cNuOSMtw~DToEDbK)Ds<ONf<MXl*6PIuJ%!c@<_Q
zHL7fEfeNw?gprdP#JAVrR#R{!JiY)1Sqs7-t3f#lRL&rLejRQ-MyXBwNutDB4{`=5
zqk&R8!U;EEPM}qa21N+Mx|?w8FcuXPfB6MIAA_s|1vcuq)Ge5`<jhTg0t{55!N!z8
z@dzssVPeS)3=AMKkkugdpyC6YI#B)wMFdD5<Wx||fG~)UO+BUN!psKQ3t~f({cU(~
zU?lr9u&J6eAm@O*g6?%}Vj!<0tE1G}xPk-Z4dmbh1vjYD0f{5eF@p4i#6W5h-o69#
zwlPNHBYfluy}S)_G6;j5jolk}Vcsys@WvZh?7bQ2*aIbQQ0&0+A;>ErH&HvKfPx8<
zsPDmo#l&DrXGi;#peY*O%o$vMetzByV8OBg<OFOniA{{!P6Y)4wqU_bsQ2Ms!I-RA
z1M|w7f$$0_MsejgP@)B4P>g`Qis`KfaBq<@P)o}g!sjjGV(cNzYi8(P^JReVvSYxr
z%Z}FG1H}vo!&(67;{%}7gr4AGF$juHSTNv<eO&rMdO_xZQUwS@^6Dd4a1FNl7?ct~
z!34sz3Mx==fXqh($75J<m|)DU5?%~5&?yJx6%Ymm9Vo~^-h#zCNF2n6#QGDsH!-?1
zgl97blQ%($9oCY^?$xI-uaYygKbX7<ia}7Ug4_g(NsxCz-Y2)LfTV<Hu;4Ml7~~_o
zzzyUS<i;j8=aAa~1tnsTmq8fjZS<lNRum#PZ7}on^P%8HP*8#rGRRAy*acxw41@Tf
zpuq3N7cei9bLlC_YI0%`<TRKw=^67dJ27MWB`g@ox%8A)!2ok6a%|$NyJ2cUUWDY&
zS1>P;GvZARFCjY#<OO6t#EGv5+KHed0)#=KhTVy8;7-J7#uHw1M2%1cIS+(ESs7HC
z!JG=pLm)9&_J)M$TbOgn*#l3ib1NAb7(k(hEepdU8zc{MD#+O&4B```9^^G#X&sk-
zkl7%6LH2+!BuL)DgM<zPB?A>CAa8?G3Bud&Vcw=kSDIXJgHi@4reTQ~7K5l>`2h0@
zJxVf|S3ph!g(kX_(D{%=_7UbpGxYHsUlDlUQG~+2Bd&B03N4Vckey7dT9_BHsmCS;
zOK6nX2eT7o7KjZA!cVXuq*Vus81H}*BYLre?p085gD}YHAURy-g5+Rc1gV91^E2F=
zG#cW<<s6WcK~4gt5^}r=@(Rcdgjc@6ykd$m^+EWeNn$D+kXO*vft&!sAg_SrL9qp@
z7m?ec=<}7Z5q^*tK+cC@kQuPxKtGNhCI?arV#D-8V(}}?>t+~U{{b7_`vGzuy7O^~
zq4&kGsRMZ%gkjDBv0=``=M8LTfz*S{0m&f;A#7d-JqTgqAUi=AWG^xXnS~le-(W#R
ztC4z;)6s($<TMb5IS0fBITOSOc^Q=ONsH<4FmI5vEe+jyASa{8G$_<S&I8dfrxK%%
zRJD+V`vdMB8ui*i-k^ndKrsn27Z!`4L<_PDdCCal#h)-Q(rRE5WG%=kAez)z0O<q8
zF0E4r$X$@&_yr3NayBv1+L@r>fCVGOxxeAgMIVLtCcKG>Ud{z2Mnnkz8SG95ITwTx
zPW}sbGR9~%;hjMA2xX9SK^RnEgYpY#gbU&Pe{knxj6xG$<VtVngS-K1p@8f~c;P?X
z3p5&70A(xW>;+<jLXuGZ19BorEvTRcu_4)#fhmJIgRO@-lvWegq&OE8qR37JITOSO
zc@GrI$ZbK87$Gmi^kP$o%`T9AkYHhi1q(Urh3Vl`kk>##3-SiS*-S8J(`xJ<WF^dM
zkhLH-u9yHh4I~draG>mtt!4+w<I)e31K9_%2ZSLp!wmBxt(u=8E0L`Rv2i((STBOi
z1gS-Mg9YY|!O;8!ISu4ZLNNtW2g0D(1eJUU@36wXgHh^HXH^6!1VK&&(I9I<Y)WGV
z<P{JGc?U#;yadX-AoC%KjSc24dL}lI7eUSeIfGJfLcGBa^M(;d=Ya6KeNc#yQ}CjD
z1H=b;3F0IUxRWp%Dui!8qL!0Dp$x(ZXK})vg+AWrP56o-YC8**E<tG$;XE#w^T@ed
znHJ6ig&xR>AhQw9<c2$wj%|IAQ$QHx41C!JBnL~2AhnS6$OCgSIY-Qbd{0gm0XY|h
zLC%0V3&w{yix=iBdPX?N2_Ot}7KjbPp!5i$F(aH0=1g*qu%k5eK+Xa=3Fcf-<bs?D
z;zL4_ALcxIc9lU+0AY0Jfy6)<Bo3lMP6q`6!np!)=VFwNwCluyoC3liXMrkKSjP;M
zE<t>3ZD^1jD6~ON2VqEP3&Nd`F%n(@8$+!C842<=dbYtPM#yTAGeAxUVQkI^IT0j>
zaH0_0iFE8|f}8-tASZ%o<cJ4(0Td*Jyo&IGFx(3m)1Ni4P_6+Pi7iC2i4n3I<V29O
zK^T`8Fhg2oFgg?D1rSDrrYPK*80}Z;EC&FE3@8LaG{{;I8{`BK#ub_%IS>ZPgAxl!
zA0&&3!JSLS_G@w@AJ#iSp40^;FhU6j;cRi3vyCva9pSqrsp)J`EPz55;WP=D(`Ypp
z4YCpx=^z?pEr^XRL_y91Va!OEggcRrIi8RcK?wj>RY20F6wI0AYzrhMl0i;G4oMgr
z7QztcNW+{%s}*1%>p&3(icnB!pnC!2OpqLi200OgL41%N5Jrw@ba9Y8DCdLB1nGl#
zNe1R6a*nkD8HHZyV#~oGt3l3zc@Jbgh!652vFc%ZKxTm05a-JdX6K{F63FSGpg=fZ
z4(@y!Ruv#CL7|OoJ%|qy2Vqd60nx~b10;{E9unH}aOcvnssK3!-FgroBo4y3oC}fz
zVURq?D~M25fIFLpZD5eqAZH?n8i)@{1Ry7oU(+eVoll2Q2RRoMav-PU%AO!OkT*f<
zKp4gc1p$Z;Ni0e*XPaPD(A3!l0CEDzN|5!$I2&Cp$f+Q`xV(w19u$Mv#+)HuRfc($
zoDKV=7NYd<D#+^~400Q&Muyo53Nnb7RbXDG)y4;E#xBSkq|{R|?-H9jK*0jTkl;~;
z1rIIfd_XY-aw56T1?dF^Gsx)>XRE=TjWM`O_}of*g)+?9Anzlbt`2iLt(FFYLJDLh
z$a+#k99QuHG7DGn1Cj&T3GyNcLt;n+?#)41s*>YP^dLoeRTJ)2jIkf;Z1e?%F1q!g
zP{$P`$ZGL<6=p8ND_TR<D<Cg}Fv2U^aIavjYFq&uD_Q{xL2ShVHZgo10BYtlkT*aW
z6mzKULmjx6Fk0c%n{WhK4N6?J@)9V>K~*&(rgY)n8l*7=3K9?o#Re#4pl`s#%tv}K
zFVVYNMRp1(7D1tnE#ZRXKx#p;0K$+$TOa0ZTJ5s{SqTbBkoCw33?>e83P=u@vyt_J
z@(1pwt^v#|v>bLrb`H$x=w3nQLqgebus99md=LgX9pW@2nA7N;Zb8lfVUSbsB><2d
zC`3W(Kp5g|W0<q)oo+!+24Rp>Ku)7P-I~C>Lhp17@&*XQyaEbk7#|YKrZA_`a<~{|
zB`743O9zk`2*aEPVuLWK1OS!EAaRHj&EQU?;czj?YEqmCN))Iyo;l2!w46F7#+jf{
z28AMQ&<7HV7I5cb^scD4bOxLAK+XY0CUWRPoMZ`ek}-ypXxrffISDz0Ku!ZiDDjo5
z72N4`7}Wzg7dZrpbvk-%fXsk|yfw`Ev|O}@ZapZJL0%!p`5-;G+XXf-?+mtt1lyDZ
z@;1!-pjZH5NKDwmoKEjj1LP!7sKQDKkn=FB2fM-KJWzrHVMOTJ!<<LU^)MhSL7@W*
zF_6_THYoJ4ZPfy)2VsyrYE9+<cP_@FJnEbc2a7;tC!%K?kQ}jv9Y`(AOpw<hY1R?u
z4O5H?nK~!?!K?&X4+?cqNRr|WQ0U{+3-Ss`p4ec3c-sl)ZE~)F0Ch1*9R;S9w_(8x
z3I>SRoQI3oKrs)(2(P)oy@t`qpw1~ku+Ri0C|V{~khee>lqyh*6<4?yF&ZP(IoWF<
zya>vhs10m4m^W#;h8+~bgklmF9H2x7QVVh_@l8i}m^TKx7$Mdhgn|SV43J{P1Liej
zj6pxb2gFmeghclmG9Thp&%x?cP-ufNB2>NLP95M@1ISq*400Oj>E0XWje$=0ATNM0
zA#Z@xfV_#TPzULUBqkr2*9JP>gPaV)Ag6(xj+yR#2dh&-&IVzS(;=bi2X`u+mY#rW
zR&32okh4G-<TOyIg4iGoQV%NoQL8C`xU(@Pbf~ui7i2xiDY()mvK;;%Ucg{>F36i8
z3<`Njq6maLcYqTGEadU^01?g(f;pR>g&!#7KrKa(Q$ZNy98kglrCDOyD<HKXGe8)W
z)<HDLJWx#yG7AzT!Emq9vGoaa3dma^3~~<0xwyOrQV)^`sReljJ)6VygVcdANDauV
zAaRH{L*U-TsD+Qfrk;<0tOtb-dQT6V7`7MyS&i&W5Fg|Wkh4J;<Q$MZ$R1+!6XQ*o
z9UwP?%!hb46y{xeEX@UZ4HV)qr-1k%3~~<0yU5iTh!0W+!pM41ogW5wJ{^k*kdv^*
z7Dya~Vcr3;K^WwGkb00fB-F#<&c?_~)Y~uxvKkbMpb!Q*0mKJ61H=bmkaIxtAaM{z
z=OgPu4fP1P^9NC=gPaTU9?Toai3t+wk#J{YWNq3F%Yi}`<SbB#gD}i_AaM{zcQ!~2
zghApU8l)FB^rK+jpx1;cvQt1_0AXz20F_p-HWG5eLe`J!m1wwEF!tRNzHJp`6nX(n
zOkoKNRggD87@JpMYCv|t<U!uUW<JQPpu`4J2f~oV76bFzU?^BY-T`53F#&TvNFL$*
z*rDWnkkdgJ;ruwb^9Na02RRctOM^lmBnHBu*a68Qyb%xc#$e0pAm@Udikv?{Vjv9i
z1}Lw<iULS}NPv3<V=`e0tX;JPqzz;wdbY+UhAk_CtOtcCC|iS^0pf!&%&D-&Um!Ig
zyRex7l0(kUFh0l}M9d_@ykm;d`?~`3&J~aq*qn|{44YFxR)Vaj#5?F_;A$#@+=|T&
zAp4Pn0}`7_Fz?gy+#YnNgZLm!N^FAEgS-F>LR{Vlc@d-*5yQ#D&5I!KfiS|0DR3{+
zX`MH@c@pGoVqy`b4unDWfKmzEt5c08Gk}3_D%{T)H3Mz?cOZ9x+yu(}#FRrI^`!b4
zl!;M`-88s&2T8GuoY6qu138)aawi?`9XihPfnoxLvBe&--T~<cc^6+f1SwiGU|uxF
zXcJTK+(~rjfcPLxj2Dr;3~_SiV0SXeYak3S4}7!WPR2+`v{{n|Dw04>0AXT68Ke$`
zLHa=I5l+vBJDtvh2O#HwoPu70Atwk>8V9KZVUW{7*&DS)%7Hr{qtc<?k(nUtK~8~X
zHxM6$Np(K59z+b}!o4s^vM9)zAPjOM$lJ&<0?DFza3>FvL<4dz2*aEVVuLXDM3WD5
zz6nNin>M>=K+XU;2ZXUD8jv^$gVcfKVcr0xcjBwF0=Ty@@&awPFN2&5ayAHqoQ<B9
zLE<0`QU{VJ<Smd|P=bTyGi=RkNCGT`d6iykH$h$iIS+(EULn@2n0coN?sSYMBW=eG
zNp(6%JqUxm4oZlq1#dCj8-uI}KzA-EL1D&13C#H>7=zARU_I_FAYX$bnjQlPppXV(
zkn=$_ERhk)GcfOg)PXQ0f0V+!XN*x43{15M@)8K+@*ZXkmBGD$5koz&80w*C48ckz
z{4?X_aAyyae1Xl`pqKz*P;7t_8Y1CTz@0xx5)L8fgHix$$y5pV!XPPNVP3#D>5mx`
zRWNVRYqtf+37|v)!k|(Q6sjPngAxa*lmUe_F^v(BTFmTR4fhU47m0Q&YC&EBVN$#U
zQV;S1$m^gW!PW=?$zyu42JS_SyhHo;7cIO93JMSg1p{g^Q49AfMjoU6xEHOx3QFjp
z#1BgNkm|G!?%g34v!K`mVNmRWf(<rQ2=W^CL|hN|>HrM!ft&~mc~IWN=0uQ_L9s(j
zF$9UZ2Do<yNwI@mY~b?_DAqvopr8QdKWxPi#EXqEFAl^IA2Bh8>}80Pn+Cg+L7@-A
zphO69ax>h?7zv5?Lwv-9GRV0g3`#7Z?2d4H3*6}g&?F_*>7eWlDgqGBZ-qO50LI6#
zIUf|tAPjOMD2rmI?>3l|2V#H;pOcXj2gvE5&?dexZHIenkW_=9SO8&Sy#-PW@;2c@
zwFBnWff!&SB?d7QUnkt@gCx6yLLG!bu>?wRu<Q+r2~6jA!JJR+lL=wlszFX5l;%NB
zBSsyp2p~l*rkA^6UN%D?>hNWOU$e_X<~6(Y@;0sPfdw%r*g$MZb=(6BGHRb|Oe=4I
zQWVGwpo{?W28<74L%h@r_Y##`-Y_qLN<EOZAR1&Xh>e_ZajC<m7WwX3kU1c)fz*L8
z#5;X(?_e~TDC_X!awf<b)bI`{hCpV4f(cY);cj{N!@NlCD><p@MNnmeK7b7?{-|LO
zBE~1cf`i)UDp13jpcsWEQb+=w2zM@(+m+OGE+{b~LU<C~$yBb|spVvlb3quCZ$Nnw
zR@Fm7d@|hm81t@_ZE~ZQ^Fdi25!zGW&PA`5y(yc4pr&&{Aq|_6fQ0f?n6s&U-5EY-
zgR&MV;z6N@nboGjorIBYDZB2B6eoe42f`pHLPBaf+*uf-n3P?2hR<0bCx9?0;$Z!B
zScrkd5zd<da~`#i;KS!UP!<3=4VI=rP6Op+LRALB*)w6zHbW0pUlDj~R)oaXEWTO{
zxh(<;L6DO{-T*lX#71^LKDEf|>17|xPEh^<u^|a_7Ay!&G1^WKU~``jK;FXUOk%}g
zF^Vrphz&kaiow=S!lxf32lF<_>o7hj3PF5GP|Su01;&`e3s_LR7%D*l@)`((f(tqC
zgWQiTI$%Kp3Jws41j!tDkYIEa$U8|1TZ+Nv9FWsNP9imPfxL?=b%4wSVUQZ+AOWQe
zkT}BIbK%~`sE+TzQo<cjNMiF2v0}t}8{{+)200HFBp@%K7n|7ZCC3hsJ&@R+2MY#j
zE$yJhiJ+8#?rl)8fH26(prFBw$@wrZP^;fWju$}AMo*L==YmoYp%}uZ7v^1%*FhMh
z21J7rHOPH1dCcHg01FOkT{{A@4p)~CS7{G&F1FwRc>#oB&c_uzpr8YJ4Wtf)VSJcg
zNa9@x_bNuwMc%$GT6z`a9hmb$-T|>e800;OcNW3DgVBW~Z^tPuy#sPO2!mo86pOIF
z8OUqMUWIsVG0bab=w9<>fVbip=+KG-ITz$?5QgO?*fciC=^(FAE5Cxw2gNQvb3t;T
z-~y=wVMuT-fd$t<7ipki0%4F>K&c5is6fsJ$$^{=@&<?v!k{1qsfUSE5~PseSPBac
zYAvq>d4-%J4dgu#26+!;26AwK#6cM5Z4euT(Srjy*dX3pHk7=F?j4X9K^W$3Oz$m+
zd2g^JW>Cz5iYria1LkZ>5-GmKj~N^*V8KD_4lT%=$cY@}U650W@hT`-Kp3PR*$W^(
zNI$yS*u<z|AF>-jY)FP#2@67M4Gp7v2jpdtS3xw$S`ZtX7eVqMIS>s}2g1mFh;vuL
zor^JEMc#=j*qjT>gCMIxp$^KU$Xy4J^FZ=2=YhNetK@Kv-hk9XoWC08d|EXqK~4ra
z9obp98kQhufiOrtK5yaD4+<LehAGHAh*#IZyh^RvV)U4zw^u>24#FUJAQyQEFRz7p
znOgVQfvg%1#SSDztb+v)wN`G>DtM4H1H|d;VNR#kg>aNO9ap{s<zrm=9wZ0y7AUcT
zFpLjM&>%j<n;T%>q}IhIxV#B+3e0MdQ$TDG1~~=9N0x^;Z6nNS#u)um@~)L9)oGwm
z2Zb&u>*IFjCYUp69fF|H0%2@U!5)H}VNRpgnT5DQ5acWn289@?bq{kQDD+@eC8>1^
z$V(tIK^WOwNaEN6^BS!hF|fJ@L?e3#R}jF|!PJ7%Gl&gw?pC;S(R=XT<Zb;Y<P1>g
zfkK~<I*=1VYC+Bgg*1o{aq>2}lWElM0)-gJY7h;w7Q`mS$sp%~Feq_=^ue+z#QED{
z&ZpMQE1{4CIUhL`VSJF+AWqr=cM?X3k~i~8j*~zk2Eqtu?Swgt*5xd6M1!0F!l1|l
z(V$WT<Q<T+U?B$*2k{}Hw+rq}j4225Rsevk!`9S7b|#1oaw2kn0F>4U=|wnuH{976
z(}(1(RmW#FHm71Me_<gEawZ7F_^2Vi2j+BY?Rukz(?L!IITsc)*t#PidE~4JNkDsH
z-l5h-J*0RC<ZKYe9oqZg&Z9%=fSyo5p^5A?5E~L|`{B;Q$aX2PnZ6WkSpu6FC{kdN
z1F{0dCPp1dEyOtoV9qhb2qhPob6l_)iA@Zfb3kGU>kh)L!<g~*fLZ5(&1h_5AfrJ(
z2GJlZU~E{rfcWqb+)DH%8{R%JD}AsTk4+3@704P;{G!J>NDkx-h_#1d)*4}4mmC1I
zHUOIy*u+3a<FXdyTM$NAeFSbbMx2sfFM_NF84t1^#0FUdV&ifqx>}HWNQfMTIm?hr
z5rJ+M$XOsKg2WNl9)np+jjIS?#-m#W;)6m2Tjc<94oDuBX0Vm8kPtc!cPbTHwbXDb
z$mt*qDw#peL68?fAq&C~C!c^h*@y;d4P-UQDp2Slhc4k!!jo`kVWc(ESHBSoNl@q@
zH+*oZ2YCe&YNue%qeWIg4mFTd5Z0cCSxc=l31k&0azNIA*dVLW`4AtUp_kPlXMixs
zDG;mA!mY+wnHvKuiDN*?3q2mOiNSn|?n@9KWGx7TDh7~VP>w-3;~dNxw2pIFCWZL^
zJj_~Z$2rIvkWZ1D@}ST``1k_cYK#G=5Lm#6fC7rtfX8JuDDV;1U!<q?Age(bVf`hz
z^%#YD0?hXb)bc&ZY7hoR2*^y-q<0zS6l(3k0$GJz;(@HhR`r2$1uRW~*dTSF>JQXA
zMb@9hz`&5tz`#(!z`%ek4$=#<3lyp_3^E5>0syH))(f(W7&oEY0m(L3U_nNWdJW_>
z5C%CNM1wHMJ0L!?GYQvgSK-d0LcIoZ1_*<k0ir<|*;(k>5+o18=;}dwAfa{*<~(YY
zaUiQe7~MJ$9~5Z_Yp=tsrA8SCvJ!+r)_`ab23ZB-gRF(|AwIqVx0(uN9LQ)823dtG
ziG$=p&Vg8elNQ#43K38a1vvxR$1py^`de`8F$QBuzd{@2V~`cdIRbf*8zcvE4#;{C
zhVfBD<TlJn)ELMn<Rp;QAZLKofkF*bwt<|O#=yV;av}(W<dYc~7(itaNDU}gfy5yZ
zcn9WOYAhxKIRj)Jt`H>Fxy0%RnFqq4!~*gbYylF&dv{^pqsCY$wY&$4A&|2X&b$YA
zCPovH^vzq;awf=$APh=0pb$nl_deXYRJc`=ROf=61ab~Y9|(h-1;U_&hZ(XD;7%QY
zkOesvgh9?lPoLPG`w-?_YRqzgLI-3Ga(2R3#e>vhhTtQZlc+IjjO-*(h#)tZkmW!H
z87S01@-Qc(H_~9@pb!ROP$EFa*uC=@?j4M@y8_lMUI7X*bmw6cLv}K{dSp3ryn~$W
zuz3q41}Vdzz`cZ##z{YN7unhL_7W@>A>MjAl)ME>9k5OhC}v<7mIy(7h_{}>y@k;)
zBYj;vtz(Ojw@_p3Im~O+EWMCRBv5FAoCIpifQklu%`-@7zkoT3nhgMAodgOUkdr~N
z1j^2!VgRHUxp@Y0=1aIUF-j%U?`Z~E2eKAq703yov<Kqj3OSG(kUU5q$eAEMsPF~x
zK^UYTgdt9S1$Qb&X+wTng%YR2oC_+gFk67HVa}ySmk<;hppZg$E;1hyN^j_4Ey(8}
zjEJ1KaBHbBzlF=UAaR71?_gFMQK1J4G8z=XAYXwn$QlqIcL(M@%zA3ZFUV>ThFOmp
zzaL=M8c|^y73OOY4YLwgW(9=?$T=W&APh-vAK}iSVn>)5=YT>9<Rq9{gp)qOoMenK
z0a*cSXjOoG3Cg#~g*i4cknzaYfY=~)*eU{KCxMDH+@<bkxD%;Z>VljK3I$S}2y!wA
zgMtFoih!8`G6xdDUtmt9WqJcyj~o*qXMnr`qCrjpVTg0S!kj~mX?u|I$ku?^up|z#
z?i<`XDi&)XYe7DOSqF*$5FcXYcUoHsvJQk1R{nrlNsVDdkncb~1JNKWKx~k8APmYC
zpb8!&4)N_zxYblF#Bo^-%DA9Zj<Eh0+<J_R7y&CQB0$>EQz$kukhLJ+f@qKxAU4R?
zAPn+3hz9u{WEMys5(2+rPM}qahB*gf-5<Dh7z62~uL=Pf4e}+32Kfla2Kg2?JqEG%
zFU(qMCoyEJK-PouFDRfv&Vg9{4{kM;;}~Q$w&UM1o$w##1Zv#$3JPqH)gWJjFeo{J
zQZ+~%6at{6NpAH7(vRtU2IdUr47MKTP-@kvAm<~89LQQ?oDGUp5C*9SITOT&L^31X
zi5Q(*($__ULXsLz1jP;rgUkYX4-`uXCo>HeCxe`at*wF?%FHk)Q)3?hzEB3GX>1Jw
zP$+_&3MxiH^01*eTtjrAL;{KdP^cqgOs}!Py@pZKlYY+sKzj}3We^6%EGWoO6CNwf
ztJK(DM-Q)pyaK`??|@<+IVM4DP;7$aK^UY4)5~lyFOyoyf)WtOc#x6U3RRFCNDast
zAPgyu*kRTgkyzz`i~<>n%Q}z`LGrLr2jvBj9?XFq4u}(&LNTg&6<EVY1!N0)P6L?+
zvH?URJj4kxjXBg1!$Srz;|%Z_2XZ(FgQ5o(g^;k~f*Xx7tYZQ*+Jso6LF!NwJ2%{T
zj0UC-%y=D;He3M#3u@H%3=iB`jN$_SdR|btf&u~*#vp5nPgJ~cD>0%Jf6I*+D`8Fo
z`4$xIAPk8gKDf0M`V?0HgPaGl4um0A^24pf=)r5i;!Fb+c-V}D`2=Px#7F_Skz^zT
zWFtXrgi8gfVJyf<5Jng)1UHt9JOMHal)6AP2xDeBVYu-ajU@|M_*xJOUy#ut3^E?1
z7B$<6z^$Ml3So&65{060BgrU(KrRIZ3dl&1i&1m47~E(Q(>Taj5C$0y3IyVFvpC#J
z5_2=iIuOQZB}^XVTTrS3VMriKz^o;8l?cd45QbTcp7lUh!Q?=-5~z?TriBJF6NEud
zM8**3O2V9LNMbSq841E5C!v-EQg9<lED1oyfiTQS5F6xTh)boZVJyf<5Jng)12>k$
zk^p232!mV;qCpsx@Imo{FkTjJJc%U%$T$#2Hy)G;P)h<ixD_O3TafV}46*{0x<IZ+
z&9?F|qe&f902vFyAR|Em0<r?+a^kbE0^Dj6hv7lif-o^wgVceX2J<~AUx5-cW@}Fo
z<{VPnXCNy;7}+@>D-cOU32ro*DG=RgkaY;-mEp!?OdvSG>LLeF+@KdYAY(un*?15i
zgh65;J*X*B1!fhg6Gk8-K^WOsP`rSwAU?^c!mKx>ARs|jf_w$C9yOlS;Kq}gmC=0%
zG9Kh>L|my;+X|5JAdIj=gBn(Vj09zCP?Et6C{4H(7+Kc_mf~zc;SACas={EC{UBpO
z7!*(-8r=$z7{Vu7aBD~`_dvNF<Z9egkJ@mfG4>we-@6Di8st0T6Rr--I#PSHgscMv
z7|5p}b)Z1SUB2nUoq&<J@$V@B84a=mmygk%0OEr%EN(I9_4Qy*A$40FvQyBL7P1^D
z6hPL4k{-F^TaY%1KHRAop@x4@6xlh{aw;fPK^PR#An$>43NnT`*#PcjjLeCDb3Uz|
z3`#(tf*0AzAU4F=h7f0yJlX<tHK?pbm}UeqjX9LmRxh$~ps+?5X-uh+Amcz7VWbJ%
zNHPmJkZ~Z4oFX8>X$m)*#Htb&03c&Q7-TeZ%t4GdgBVYCY=LY+go8Q6H0Dr45<3(i
zV?bpShz2<nwQ*|!H<rvWfEkNC(SmTbCER!t`=2mlL9PY49u&O9*CkdkYe{Xg6KgFf
z)<Mn!sR#KMgdwqR4YQim1v?<4VDSSo8rjFN0ve<SBnP5FR)R3ZS{t~vWF!w{Yf;Mx
zTez`g<UVv`L4k&Fu^qLH1{n*&2&3)cMpGCVuyBTiu>;Ila#|>$z(CF;plC#nVp#JJ
z6#gLJfaGD;BYfrvcLGK~2LFyAkWu710i+MqTmU%@vs35<a}udrZ16b=y*WT^NP$8P
zq!#2{Q1wrF>&6-GJ&Xbp|LSv6ya!TCj?+P2C8ksaB_J4vq-7VFw@5we5ad))g20xR
z(PIsymK<+^5*jEEz?=-DAx?K4%uWY68-!s_hS3nGyTP1JYUdmjlGs8W6jC57AjZ4H
zjmKz-;~#OyXFSX|5Gy?3R*=yt#O4E#@gNM610`oz>lb33C)_%W+=qYK4wsKW;)nqA
zf*8*fieWkrtdGtE%3mNyf%<3=M|s0^Vbm@xFkLL5h=Hi~fvLu*K{;TmIY6pGfdJ9r
zON0)PY7mC#@Pp~Vh<5>)9ReWLAPfp<P|*m{=?~M1(Ge1X=@bF!0%4F&SQJ5Y2f%dW
b2?<b$LsSRCRO7J=q#9%oa`6JzU8)BFh8of$
literal 0
HcmV?d00001
diff --git a/SuperTagger/Decoder/RNNDecoderLayer.py b/SuperTagger/Decoder/RNNDecoderLayer.py
deleted file mode 100644
index 69ca8fc..0000000
--- a/SuperTagger/Decoder/RNNDecoderLayer.py
+++ /dev/null
@@ -1,177 +0,0 @@
-import random
-
-import torch
-import torch.nn.functional as F
-from torch.nn import (Dropout, Module, ModuleList, Linear, LSTM, GRU)
-
-from Configuration import Configuration
-
-
-class RNNDecoderLayer(Module):
- def __init__(self, symbols_map):
- super(RNNDecoderLayer, self).__init__()
-
- # init params
- self.dim_encoder = int(Configuration.modelDecoderConfig['dim_encoder'])
- self.dim_decoder = int(Configuration.modelDecoderConfig['dim_decoder'])
- self.max_symbols_per_word = int(Configuration.modelDecoderConfig['max_symbols_per_word'])
- self.max_len_sentence = int(Configuration.modelDecoderConfig['max_len_sentence'])
- self.symbols_vocab_size = int(Configuration.modelDecoderConfig['symbols_vocab_size'])
- dropout = float(Configuration.modelDecoderConfig['dropout'])
- self.num_rnn_layers = int(Configuration.modelDecoderConfig['num_rnn_layers'])
- self.teacher_forcing = float(Configuration.modelDecoderConfig['teacher_forcing'])
-
- self.bidirectional = False
- self.use_attention = True
- self.symbols_map = symbols_map
- self.symbols_padding_id = self.symbols_map["[PAD]"]
- self.symbols_sep_id = self.symbols_map["[SEP]"]
- self.symbols_start_id = self.symbols_map["[START]"]
- self.symbols_sos_id = self.symbols_map["[SOS]"]
-
- # Different layers
- # Symbols Embedding
-
- # For hidden_state
- self.dropout = Dropout(dropout)
- # rnn Layer
- if self.use_attention:
- self.rnn = LSTM(input_size=self.dim_encoder, hidden_size=self.dim_encoder, num_layers=self.num_rnn_layers,
- dropout=dropout,
- bidirectional=self.bidirectional, batch_first=True)
- else :
- self.rnn = LSTM(input_size=self.dim_decoder, hidden_size=self.dim_encoder, num_layers=self.num_rnn_layers,
- dropout=dropout,
- bidirectional=self.bidirectional, batch_first=True)
-
- # Projection on vocab_size
- if self.bidirectional:
- self.proj = Linear(self.dim_encoder * 2, self.symbols_vocab_size)
- else:
- self.proj = Linear(self.dim_encoder, self.symbols_vocab_size)
-
- self.attn_combine = Linear(self.dim_decoder + self.dim_encoder, self.dim_encoder)
-
- def sos_mask(self, y):
- return torch.eq(y, self.symbols_sos_id)
-
- def forward(self, symbols_tokenized_batch, last_hidden_state, pooler_output):
- r"""Training the translation from encoded sentences to symbols
-
- Args:
- symbols_tokenized_batch: [batch_size, max_symbols_in_sentence] the true symbols for each sentence.
- last_hidden_state: [batch_size, max_len_sentence, dim_encoder] Sequence of hidden-states at the output of the last layer of the model.
- pooler_output: [batch_size, dim_encoder] Last layer hidden-state of the first token of the sequence (classification token) after further processing through the layers used for the auxiliary pretraining task
- """
- batch_size, sequence_length, hidden_size = last_hidden_state.shape
-
- # y_hat[batch_size, max_len_sentence, vocab_size] init with probability pad =1
- y_hat = torch.zeros(batch_size, self.max_len_sentence, self.max_symbols_per_word, self.symbols_vocab_size,
- dtype=torch.float, device="cuda" if torch.cuda.is_available() else "cpu")
- y_hat[:, :, self.symbols_padding_id] = 1
-
- decoded_ij = torch.ones(batch_size, 1, dtype=torch.long,
- device="cuda" if torch.cuda.is_available() else "cpu") * self.symbols_start_id
-
- sos_mask = torch.zeros(batch_size, dtype=torch.bool, device="cuda" if torch.cuda.is_available() else "cpu")
-
- # hidden_state goes through multiple linear layers
- hidden_state = pooler_output.unsqueeze(0).repeat(self.num_rnn_layers * (1 + self.bidirectional), 1, 1)
-
- c_state = torch.zeros(self.num_rnn_layers * (1 + self.bidirectional), batch_size, hidden_size,
- dtype=torch.float, device="cuda" if torch.cuda.is_available() else "cpu")
-
- use_teacher_forcing = False
-
- # for each symbol
- for i in range(self.max_len_sentence):
- for j in range(self.max_symbols_per_word) :
- symbols_embedding = self.symbols_embedder(decoded_ij)
- symbols_embedding = self.dropout(symbols_embedding)
-
- output = symbols_embedding
- if self.use_attention:
- output = torch.cat((symbols_embedding, last_hidden_state[:, i, :].unsqueeze(1)), 2)
- output = self.attn_combine(output)
- output = F.relu(output)
-
- # rnn layer
- output, (hidden_state, c_state) = self.rnn(output, (hidden_state, c_state))
-
- # Projection of the output of the rnn omitting the last probability (which is pad) so we dont predict PAD
- proj = self.proj(output)[:, :, :-2]
-
- if use_teacher_forcing:
- decoded_ij = symbols_tokenized_batch[:, i, j].unsqueeze(1)
- else:
- decoded_ij = torch.argmax(F.softmax(proj, dim=2), dim=2)
-
- # Calculate sos and pad
- sos_mask_ij = self.sos_mask(torch.argmax(F.softmax(proj, dim=2), dim=2)[:, -1])
- y_hat[~sos_mask, i, j, self.symbols_padding_id] = 0
- y_hat[~sos_mask, i, j, :-2] = proj[~sos_mask, -1, :]
- sos_mask = sos_mask_ij | sos_mask
-
- # Stop if every sentence says padding or if we are full
- if not torch.any(~sos_mask):
- break
- return y_hat
-
- def predict_rnn(self, last_hidden_state, pooler_output):
- r"""Predicts the symbols from the output of the encoder.
-
- Args:
- last_hidden_state: [batch_size, max_len_sentence, dim_encoder] the output of the encoder
- pooler_output: [batch_size, dim_encoder] Last layer hidden-state of the first token of the sequence (classification token) after further processing through the layers used for the auxiliary pretraining task
- """
- batch_size, sequence_length, hidden_size = last_hidden_state.shape
-
- # y_hat[batch_size, max_len_sentence, vocab_size] init with probability pad =1
- y_hat = torch.zeros(batch_size, self.max_len_sentence, self.max_symbols_per_word, self.symbols_vocab_size,
- dtype=torch.float, device="cuda" if torch.cuda.is_available() else "cpu")
- y_hat[:, :, self.symbols_padding_id] = 1
-
- decoded_ij = torch.ones(batch_size, 1, dtype=torch.long,
- device="cuda" if torch.cuda.is_available() else "cpu") * self.symbols_start_id
-
- sos_mask = torch.zeros(batch_size, dtype=torch.bool, device="cuda" if torch.cuda.is_available() else "cpu")
-
- # hidden_state goes through multiple linear layers
- hidden_state = pooler_output.unsqueeze(0).repeat(self.num_rnn_layers * (1 + self.bidirectional), 1, 1)
-
- c_state = torch.zeros(self.num_rnn_layers * (1 + self.bidirectional), batch_size, hidden_size,
- dtype=torch.float, device="cuda" if torch.cuda.is_available() else "cpu")
- symbols_embedding = self.symbols_embedder(decoded_ij)
-
- symbols_embedding = self.dropout(symbols_embedding)
-
- # for each symbol
- for i in range(self.max_len_sentence):
- output = symbols_embedding
- if self.use_attention:
- output = torch.cat((symbols_embedding, last_hidden_state[:, i, :].unsqueeze(1)), 2)
- output = self.attn_combine(output)
- output = F.relu(output)
- for j in range(self.max_symbols_per_word) :
- symbols_embedding = self.symbols_embedder(decoded_ij)
-
- symbols_embedding = self.dropout(symbols_embedding)
-
- # rnn layer
- output, (hidden_state, c_state) = self.rnn(output, (hidden_state, c_state))
-
- # Projection of the output of the rnn omitting the last probability (which is pad) so we dont predict PAD
- proj_softmax = F.softmax(self.proj(output)[:, :, :-2], dim=2)
- decoded_ij = torch.argmax(proj_softmax, dim=2)
-
- # Set sos and pad
- sos_mask_ij = self.sos_mask(decoded_ij[:, -1])
- y_hat[~sos_mask, i, j, self.symbols_padding_id] = 0
- y_hat[~sos_mask, i, j, :-2] = proj_softmax[~sos_mask, -1, :]
- sos_mask = sos_mask_ij | sos_mask
-
- # Stop if every sentence says padding or if we are full
- if not torch.any(~sos_mask):
- break
-
- return y_hat
diff --git a/SuperTagger/Encoder/EncoderLayer.py b/SuperTagger/Encoder/EncoderLayer.py
deleted file mode 100644
index c954584..0000000
--- a/SuperTagger/Encoder/EncoderLayer.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import sys
-
-import torch
-from torch import nn
-
-from Configuration import Configuration
-
-
-class EncoderLayer(nn.Module):
- """Encoder class, imput of supertagger"""
-
- def __init__(self, model):
- super(EncoderLayer, self).__init__()
- self.name = "Encoder"
-
- self.bert = model
-
- self.hidden_size = self.bert.config.hidden_size
-
- def forward(self, batch):
- r"""
- Args :
- batch: list[str,mask], list of sentences (NOTE: untokenized, continuous sentences)
- Returns :
- last_hidden_state: [batch_size, max_len_sentence, dim_encoder] Sequence of hidden-states at the output of the last layer of the model.
- pooler_output: [batch_size, dim_encoder] Last layer hidden-state of the first token of the sequence (classification token) after further processing through the layers used for the auxiliary pretraining task
- """
- b_input_ids = batch[0]
- b_input_mask = batch[1]
-
- outputs = self.bert(
- input_ids=b_input_ids, attention_mask=b_input_mask)
-
- return outputs[0], outputs[1]
-
- @staticmethod
- def load(model_path: str):
- r""" Load the model from a file.
- Args :
- model_path (str): path to model
- Returns :
- model (nn.Module): model with saved parameters
- """
- params = torch.load(
- model_path, map_location=lambda storage, loc: storage)
- args = params['args']
- model = EncoderLayer(**args)
- model.load_state_dict(params['state_dict'])
-
- return model
-
- def save(self, path: str):
- r""" Save the model to a file.
- Args :
- path (str): path to the model
- """
- print('save model parameters to [%s]' % path, file=sys.stderr)
-
- params = {
- 'args': dict(bert_config=self.bert.config, dropout_rate=self.dropout_rate),
- 'state_dict': self.state_dict()
- }
-
- torch.save(params, path)
-
- def to_dict(self):
- return {}
diff --git a/SuperTagger/EncoderDecoder.py b/SuperTagger/EncoderDecoder.py
index 273b3e8..1c8e510 100644
--- a/SuperTagger/EncoderDecoder.py
+++ b/SuperTagger/EncoderDecoder.py
@@ -4,9 +4,7 @@ from torch.nn import Dropout, LSTM
from torch.nn import Module
from Configuration import Configuration
-from SuperTagger.Decoder.RNNDecoderLayer import RNNDecoderLayer
from torch.nn.utils.rnn import pack_padded_sequence
-from SuperTagger.Encoder.EncoderLayer import EncoderLayer
from SuperTagger.eval import measure_supertagging_accuracy
@@ -21,6 +19,8 @@ class EncoderDecoder(Module):
def __init__(self, BASE_MODEL, numPos1Classes, numPos2Classes, numSuperClasses):
super(EncoderDecoder, self).__init__()
+ self.bert = BASE_MODEL
+
self.dim_encoder = int(Configuration.modelDecoderConfig['dim_encoder'])
self.dim_decoder = int(Configuration.modelDecoderConfig['dim_decoder'])
self.num_rnn_layers = int(Configuration.modelDecoderConfig['num_rnn_layers'])
@@ -28,18 +28,18 @@ class EncoderDecoder(Module):
dropout = float(Configuration.modelDecoderConfig['dropout'])
self.dropout = Dropout(dropout)
- self.encoder = EncoderLayer(BASE_MODEL)
+ self.bert = BASE_MODEL
self.lstm_shared = LSTM(input_size=self.dim_encoder, hidden_size=self.dim_encoder, num_layers=self.num_rnn_layers,
dropout=dropout,
bidirectional=self.bidirectional, batch_first=True, )
#Pos1
- self.pos1_1 = nn.Linear(self.dim_encoder*2,self.dim_decoder)
+ self.pos1_1 = nn.Linear(self.dim_encoder,self.dim_decoder)
self.pos1_2 = nn.Linear(self.dim_decoder, numPos1Classes)
#Pos2
- self.pos2_1 = nn.Linear(self.dim_encoder*2, self.dim_decoder)
+ self.pos2_1 = nn.Linear(self.dim_encoder, self.dim_decoder)
self.pos2_2 = nn.Linear(self.dim_decoder, numPos2Classes)
#super
@@ -47,24 +47,27 @@ class EncoderDecoder(Module):
num_layers=self.num_rnn_layers,
dropout=dropout,
bidirectional=self.bidirectional, batch_first=True, )
- self.pos_super_1 = nn.Linear(self.dim_encoder*2,self.dim_decoder)
+ self.pos_super_1 = nn.Linear(self.dim_encoder,self.dim_decoder)
self.pos_super_2 = nn.Linear(self.dim_decoder, numSuperClasses)
def forward(self, batch):
- encoded_layers, pooled_output = self.encoder(batch)
- encoded_layers = self.dropout(encoded_layers)
- # encoded_layers = encoded_layers.permute(1, 0, 2)
- print("encoded_layers : ", encoded_layers.size())
+ b_input_ids = batch[0]
+ b_input_mask = batch[1]
+ encoded_layers, _ = self.bert(
+ input_ids=b_input_ids, attention_mask=b_input_mask, return_dict=False)
- lstm_output, (h, c) = self.lstm_shared(encoded_layers) ## extract the 1st token's embeddings
- print("last_hidden : ", lstm_output.size())
- # output_shared = torch.cat((lstm_output[:, :, :self.dim_encoder], lstm_output[:, :, self.dim_encoder:]), dim=2)
+ lstm_output = self.dropout(encoded_layers)
+
+ print("encoded_layers : ", encoded_layers.size())
- print("output_shared : ", lstm_output.size())
+ # lstm_output, _ = self.lstm_shared(encoded_layers) ## extract the 1st token's embeddings
+ # print("last_hidden : ", lstm_output.size())
+ #
+ # print("output_shared : ", lstm_output.size())
# Pos1
pos_1_output= self.pos1_1(lstm_output)
@@ -77,9 +80,8 @@ class EncoderDecoder(Module):
pos_2_output = self.pos2_2(pos_2_output)
# super
- enc_hiddens, (last_hidden_super, last_cell_super) = self.lstm_super(lstm_output)
- print(enc_hiddens.size())
- super_output = self.pos_super_1(enc_hiddens)
+ # enc_hiddens, _ = self.lstm_super(lstm_output)
+ super_output = self.pos_super_1(lstm_output)
super_output = self.dropout(super_output)
super_output = self.pos_super_2(super_output)
diff --git a/SuperTagger/Encoder/EncoderInput.py b/SuperTagger/EncoderTokenizer.py
similarity index 95%
rename from SuperTagger/Encoder/EncoderInput.py
rename to SuperTagger/EncoderTokenizer.py
index e19da7d..865f5a7 100644
--- a/SuperTagger/Encoder/EncoderInput.py
+++ b/SuperTagger/EncoderTokenizer.py
@@ -1,7 +1,7 @@
import torch
-class EncoderInput():
+class EncoderTokenizer():
def __init__(self, tokenizer):
"""@params tokenizer (PretrainedTokenizer): Tokenizer that tokenizes text """
diff --git a/SuperTagger/SymbolTokenizer.py b/SuperTagger/SymbolTokenizer.py
new file mode 100644
index 0000000..8a4948c
--- /dev/null
+++ b/SuperTagger/SymbolTokenizer.py
@@ -0,0 +1,56 @@
+import pickle
+
+import numpy as np
+import torch
+
+
+def load_obj(name):
+ with open(name + '.pkl', 'rb') as f:
+ return pickle.load(f)
+
+
+class SymbolTokenizer():
+
+ def __init__(self):
+ """@params tokenizer (PretrainedTokenizer): Tokenizer that tokenizes text """
+ self.index_to_super = load_obj('Datasets/index_to_super')
+ self.index_to_pos1 = load_obj('Datasets/index_to_pos1')
+ self.index_to_pos2 = load_obj('Datasets/index_to_pos2')
+ self.super_to_index = {v: int(k) for k, v in self.index_to_super.items()}
+ self.pos1_to_index = {v: int(k) for k, v in self.index_to_pos1.items()}
+ self.pos2_to_index = {v: int(k) for k, v in self.index_to_pos2.items()}
+
+ def lenPOS1(self):
+ print(self.pos1_to_index)
+ return len(self.index_to_pos1) + 1
+
+ def lenPOS2(self):
+ return len(self.index_to_pos2) + 1
+
+ def lenSuper(self):
+ return len(self.index_to_super) + 1
+
+ def convert_batchs_to_ids(self, Y1, Y2, Super):
+ max_len_Y1 = max(len(elem) for elem in Y1)
+ max_len_Y2 = max(len(elem) for elem in Y2)
+ max_len_S = max(len(elem) for elem in Super)
+ Y1_tok = torch.as_tensor(pad_sequence([[self.pos1_to_index[str(symbol)] for symbol in sents] for sents in Y1]))
+ Y2_tok = torch.as_tensor(pad_sequence(
+ [[self.pos2_to_index[str(symbol)] for symbol in sents] for sents in Y2]))
+ super_tok = torch.as_tensor(pad_sequence(
+ [[self.super_to_index[str(symbol)] for symbol in sents] for sents in Super]))
+
+ return Y1_tok, Y2_tok, super_tok
+
+ # def convert_ids_to_symbols(self, ids):
+ # return [self.inverse_symbol_map[int(i)] for i in ids]
+
+def pad_sequence(sequences, max_len=400):
+ sequences_pad = []
+ for s in sequences:
+ padded = [0] * max_len
+ padded[:len(s)] = s
+ sequences_pad.append(padded)
+ return sequences_pad
+
+
diff --git a/SuperTagger/eval.py b/SuperTagger/eval.py
index 3017c7f..dd99757 100644
--- a/SuperTagger/eval.py
+++ b/SuperTagger/eval.py
@@ -33,12 +33,14 @@ class NormCrossEntropy(Module):
r"""Loss based on the cross entropy, it considers the number of words and ignore the padding.
"""
- def __init__(self, ignore_index, sep_id, weights=None):
+ def __init__(self, ignore_index, weights=None):
super(NormCrossEntropy, self).__init__()
self.ignore_index = ignore_index
- self.sep_id = sep_id
self.weights = weights
def forward(self, predictions, truths):
- return cross_entropy(predictions.flatten(0, -2), truths.flatten(), weight=self.weights,
- reduction='sum', ignore_index=self.ignore_index) / count_sep(truths.flatten(), self.sep_id)
+ print()
+ print("predictions : ", predictions.size())
+ print("truths : ", truths.size())
+ return cross_entropy(predictions.flatten(0, -2), truths.flatten(), weight=torch.tensor(self.weights,device="cuda" if torch.cuda.is_available() else "cpu"),
+ reduction='sum', ignore_index=self.ignore_index)
diff --git a/SuperTagger/utils.py b/SuperTagger/utils.py
index a0438f5..e257eb7 100644
--- a/SuperTagger/utils.py
+++ b/SuperTagger/utils.py
@@ -5,7 +5,7 @@ import torch
from tqdm import tqdm
-def read_csv_pgbar(csv_path, nrows=float('inf'), chunksize=500):
+def read_csv_pgbar(csv_path, nrows=float('inf'), chunksize=100):
print("\n" + "#" * 20)
print("Loading csv...")
diff --git a/Utils/PostpreprocesTXT.py b/Utils/PostpreprocesTXT.py
index a4168c8..fbfcb8d 100644
--- a/Utils/PostpreprocesTXT.py
+++ b/Utils/PostpreprocesTXT.py
@@ -1,4 +1,5 @@
import itertools
+import pickle
import re
import numpy as np
@@ -115,17 +116,15 @@ def read_maxentdata(file):
Z = np.asarray(allsuper)
return X, Y1, Y2, Z, vocabulary, vnorm, partsofspeech1, partsofspeech2, superset, maxlen
+def save_obj(obj, name):
+ with open(name + '.pkl', 'wb+') as f:
+ pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
-# Txt_to_csv("m2.txt")
-
-X, Y1, Y2, Z, vocabulary, vnorm, partsofspeech1, partsofspeech2, superset, maxlen = read_maxentdata("m2.txt")
+def load_obj(name):
+ with open(name + '.pkl', 'rb') as f:
+ return pickle.load(f)
-print("X[17] (", np.array(X[17]).shape ,") : ")
-print(X[17])
-print("Y1[17] (", np.array(Y1[17]).shape ,") : ")
-print(Y1[17])
-print("Y2[17] (", np.array(Y2[17]).shape ,") : ")
-print(Y2[17])
+# Txt_to_csv("m2.txt")
X, Y1, Y2, Z, vocabulary, vnorm, partsofspeech1, partsofspeech2, superset, maxlen = read_maxentdata("m2.txt")
@@ -136,6 +135,13 @@ df['Y1'] = Y1
df['Y2'] = Y2
df['Z'] = Z
+df.to_csv("../Datasets/m2_dataset_V2.csv", index=False)
+
+
+t = np.unique(np.array(list(itertools.chain(*Z))))
+
+print(t.size)
+dict = { i : t[i] for i in range(0, len(t) ) }
-df.to_csv("../Datasets/m2_dataset_V2.csv", index=False)
\ No newline at end of file
+save_obj(dict,"../Datasets/index_to_super")
\ No newline at end of file
diff --git a/train.py b/train.py
index 187b93a..a46b4a5 100644
--- a/train.py
+++ b/train.py
@@ -6,16 +6,18 @@ import numpy as np
import torch
import torch.nn.functional as F
import transformers
-from torch.optim import SGD, Adam
+from torch.optim import Adam, RMSprop
from torch.utils.data import Dataset, TensorDataset, random_split
from transformers import (AutoTokenizer, get_cosine_schedule_with_warmup)
from transformers import (CamembertModel)
from Configuration import Configuration
-from SuperTagger.Encoder.EncoderInput import EncoderInput
+
from SuperTagger.EncoderDecoder import EncoderDecoder
+from SuperTagger.EncoderTokenizer import EncoderTokenizer
+from SuperTagger.SymbolTokenizer import SymbolTokenizer
from SuperTagger.eval import NormCrossEntropy
-from SuperTagger.utils import format_time, read_csv_pgbar, checkpoint_save, checkpoint_load
+from SuperTagger.utils import format_time, read_csv_pgbar
from torch.utils.tensorboard import SummaryWriter
@@ -24,9 +26,7 @@ torch.cuda.empty_cache()
# region ParamsModel
-# max_symbols_per_word = int(Configuration.modelDecoderConfig['max_symbols_per_word'])
-# max_len_sentence = int(Configuration.modelDecoderConfig['max_len_sentence'])
-# symbol_vocab_size = int(Configuration.modelDecoderConfig['symbols_vocab_size'])
+
num_gru_layers = int(Configuration.modelDecoderConfig['num_rnn_layers'])
# endregion ParamsModel
@@ -35,7 +35,7 @@ num_gru_layers = int(Configuration.modelDecoderConfig['num_rnn_layers'])
file_path = 'Datasets/m2_dataset_V2.csv'
batch_size = int(Configuration.modelTrainingConfig['batch_size'])
-nb_sentences = batch_size * 300
+nb_sentences = batch_size * 50
epochs = int(Configuration.modelTrainingConfig['epoch'])
seed_val = int(Configuration.modelTrainingConfig['seed_val'])
learning_rate = float(Configuration.modelTrainingConfig['learning_rate'])
@@ -115,9 +115,8 @@ BASE_TOKENIZER = AutoTokenizer.from_pretrained(
'camembert-base',
do_lower_case=True)
BASE_MODEL = CamembertModel.from_pretrained("camembert-base")
-sents_tokenizer = EncoderInput(BASE_TOKENIZER)
-model = EncoderDecoder(BASE_MODEL, 20,20,20)
-model = model.to("cuda" if torch.cuda.is_available() else "cpu")
+sents_tokenizer = EncoderTokenizer(BASE_TOKENIZER)
+symbol_tokenizer = SymbolTokenizer()
# endregion Model
@@ -126,7 +125,9 @@ df = read_csv_pgbar(file_path, nb_sentences)
sents_tokenized, sents_mask = sents_tokenizer.fit_transform_tensors(df['X'].tolist())
-dataset = TensorDataset(sents_tokenized, sents_mask)
+y1, y2, super = symbol_tokenizer.convert_batchs_to_ids(df['Y1'].tolist(),df['Y2'].tolist(),df['Z'].tolist())
+
+dataset = TensorDataset(sents_tokenized, sents_mask, y1, y2, super)
# , torch.tensor(df['Y1'].tolist()), torch.tensor(df['Y2'].tolist()), torch.tensor(df['Z'].tolist())
# Calculate the number of samples to include in each set.
@@ -144,13 +145,14 @@ validation_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batc
# endregion Data loader
+
+model = EncoderDecoder(BASE_MODEL, symbol_tokenizer.lenPOS1(),symbol_tokenizer.lenPOS2(),symbol_tokenizer.lenSuper())
+model = model.to("cuda" if torch.cuda.is_available() else "cpu")
+
# region Fit tunning
# Optimizer
-# optimizer_encoder = Adam(model.encoder.parameters(),
-# lr=5e-5)
-# optimizer_decoder = Adam(model.decoder.parameters(),
-# lr=learning_rate)
+optimizer = RMSprop(model.parameters())
# Total number of training steps is [number of batches] x [number of epochs].
# (Note that this is not the same as the number of training samples).
@@ -165,15 +167,9 @@ total_steps = len(training_dataloader) * epochs
# num_training_steps=total_steps)
# # Loss
-# if loss_scaled_by_freq:
-# weights = torch.as_tensor(
-# [6.9952, 1.0763, 1.0317, 43.274, 16.5276, 11.8821, 28.2416, 2.7548, 1.0728, 3.1847, 8.4521, 6.77, 11.1887,
-# 6.6692, 23.1277, 11.8821, 4.4338, 1.2303, 5.0238, 8.4376, 1.0656, 4.6886, 1.028, 4.273, 4.273, 0],
-# device=torch.device("cuda" if torch.cuda.is_available() else "cpu"))
-# cross_entropy_loss = NormCrossEntropy(symbols_tokenizer.pad_token_id, symbols_tokenizer.sep_token_id,
-# weights=weights)
-# else:
-# cross_entropy_loss = NormCrossEntropy(symbols_tokenizer.pad_token_id, symbols_tokenizer.sep_token_id)
+cross_entropy_loss_Y1 = NormCrossEntropy(0,0.15)
+cross_entropy_loss_Y2 = NormCrossEntropy(0,.35)
+cross_entropy_loss_S = NormCrossEntropy(0,.5)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
@@ -211,7 +207,9 @@ def run_epochs(epochs):
t0 = time.time()
# Reset the total loss for this epoch.
- total_train_loss = 0
+ total_train_loss_Y1 =0
+ total_train_loss_Y2 =0
+ total_train_loss_S =0
model.train()
@@ -221,8 +219,8 @@ def run_epochs(epochs):
# if epoch_i == 0 and step == 0:
# writer.add_graph(model, input_to_model=batch[0], verbose=False)
- # Progress update every 40 batches.
- if step % 40 == 0 and not step == 0:
+ # Progress update every 10 batches.
+ if step % 10 == 0 and not step == 0:
# Calculate elapsed time in minutes.
elapsed = format_time(time.time() - t0)
# Report progress.
@@ -231,23 +229,36 @@ def run_epochs(epochs):
# Unpack this training batch from our dataloader.
b_sents_tokenized = batch[0].to("cuda" if torch.cuda.is_available() else "cpu")
b_sents_mask = batch[1].to("cuda" if torch.cuda.is_available() else "cpu")
- b_symbols_tokenized = batch[2].to("cuda" if torch.cuda.is_available() else "cpu")
- # optimizer_encoder.zero_grad()
- # optimizer_decoder.zero_grad()
+ optimizer.zero_grad()
+
+ logits_predictions = model((b_sents_tokenized, b_sents_mask))
+
+ output_dim_Y1 = logits_predictions[0].shape[1]
+ print(output_dim_Y1)
+ # output_Y1 = logits_predictions[0][1:].view(-1, output_dim_Y1)
+ output_dim_Y2 = logits_predictions[1].shape[1]
+ # output_Y2 = logits_predictions[1][1:].view(-1, output_dim_Y2)
+ output_dim_S = logits_predictions[2].shape[1]
+ # output_S = logits_predictions[2][1:].view(-1, output_dim_S)
- logits_predictions = model(b_sents_tokenized, b_sents_mask, b_symbols_tokenized)
+ loss_Y1 = cross_entropy_loss_Y1(logits_predictions[0], batch[2][:output_dim_Y1])
+ loss_Y2 = cross_entropy_loss_Y2(logits_predictions[1], batch[3][:output_dim_Y2])
+ loss_S = cross_entropy_loss_S(logits_predictions[2], batch[4][:output_dim_S])
- # loss = cross_entropy_loss(logits_predictions, b_symbols_tokenized)
- # total_train_loss += float(loss)
- # loss.backward()
+ total_train_loss_Y1 += float(loss_Y1)
+ total_train_loss_Y2 += float(loss_Y2)
+ total_train_loss_S += float(loss_S)
+
+ loss_Y1.backward()
+ loss_Y2.backward()
+ loss_S.backward()
# This is to help prevent the "exploding gradients" problem.
#torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)
# Update parameters and take a step using the computed gradient.
- # optimizer_encoder.step()
- # optimizer_decoder.step()
+ optimizer.step()
#
# scheduler_encoder.step()
# scheduler_decoder.step()
@@ -257,7 +268,9 @@ def run_epochs(epochs):
# if use_checkpoint_SAVE:
# checkpoint_save(model, optimizer_decoder, epoch_i, checkpoint_dir, loss)
- avg_train_loss = total_train_loss / len(training_dataloader)
+ avg_train_loss_Y1 = total_train_loss_Y1 / len(training_dataloader)
+ avg_train_loss_Y2 = total_train_loss_Y2 / len(training_dataloader)
+ avg_train_loss_S = total_train_loss_S / len(training_dataloader)
# Measure how long this epoch took.
training_time = format_time(time.time() - t0)
@@ -274,7 +287,9 @@ def run_epochs(epochs):
# writer.add_scalar('Accuracy/symbol', accuracy_symbol, epoch_i + 1)
print("")
- print(" Average training loss: {0:.2f}".format(avg_train_loss))
+ print(" Average training loss: {0:.2f}".format(avg_train_loss_Y1))
+ print(" Average training loss: {0:.2f}".format(avg_train_loss_Y2))
+ print(" Average training loss: {0:.2f}".format(avg_train_loss_S))
print(" Training epcoh took: {:}".format(training_time))
# writer.add_scalar('Loss/train', total_train_loss, epoch_i+1)
@@ -287,10 +302,13 @@ def run_epochs(epochs):
# run_epochs(epochs)
# endregion Train
-b = next(iter(training_dataloader))
-# , y1,y2,y3
-a = model(b)
-print(len(b))
-print(a[0].size(),a[1].size(),a[2].size())
+# b1, b2 , y1,y2,y3 = next(iter(training_dataloader))
+# b =(b1, b2)
+# # , y1,y2,y3
+# a = model(b)
+# print(len(b))
+# print(a[0].size(),a[1].size(),a[2].size())
+print(symbol_tokenizer.lenPOS1())
+
--
GitLab