pax_global_header00006660000000000000000000000064143257124460014522gustar00rootroot0000000000000052 comment=3918e56f08aba0bdb65f095e78e622df9e174934 pangu_2_6b/000077500000000000000000000000001432571244600130105ustar00rootroot00000000000000pangu_2_6b/.gitattributes000066400000000000000000000022271432571244600157060ustar00rootroot00000000000000*.7z filter=lfs diff=lfs merge=lfs -text *.arrow filter=lfs diff=lfs merge=lfs -text *.bin filter=lfs diff=lfs merge=lfs -text *.bin.* filter=lfs diff=lfs merge=lfs -text *.bz2 filter=lfs diff=lfs merge=lfs -text *.ftz filter=lfs diff=lfs merge=lfs -text *.gz filter=lfs diff=lfs merge=lfs -text *.h5 filter=lfs diff=lfs merge=lfs -text *.joblib filter=lfs diff=lfs merge=lfs -text *.lfs.* filter=lfs diff=lfs merge=lfs -text *.model filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.ot filter=lfs diff=lfs merge=lfs -text *.parquet filter=lfs diff=lfs merge=lfs -text *.pb filter=lfs diff=lfs merge=lfs -text *.pt filter=lfs diff=lfs merge=lfs -text *.pth filter=lfs diff=lfs merge=lfs -text *.rar filter=lfs diff=lfs merge=lfs -text saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.tar.* filter=lfs diff=lfs merge=lfs -text *.tflite filter=lfs diff=lfs merge=lfs -text *.tgz filter=lfs diff=lfs merge=lfs -text *.xz filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zstandard filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text pangu_2_6b/.gitignore000066400000000000000000000000441432571244600147760ustar00rootroot00000000000000# backup config backup_config.json pangu_2_6b/Docs/000077500000000000000000000000001432571244600137005ustar00rootroot00000000000000pangu_2_6b/Docs/compare1.png000066400000000000000000000644511432571244600161270ustar00rootroot00000000000000PNG  IHDRnyJsRGBgAMA a PLTE  "!*+00022$:;AAADD1$DrGIPOzPPTS<-UX`[`aecfpdI6efopuor~oqsuU?{{}~ׂ׃_F؈ܑjNݘߙtV᣺⥧䩮~]沴ɳ˵织dǐkәryġˬ̭дӸҷӸջ tRNS P{ pHYs&?fIDATx^cOr.d:-'bLl )Dza`KPv^x-"=ܚ[UO=]}Ε+ʯ'JT)Ou\̟pq%>b@iDn_T94xUN0g!\X؏4 [XqShAF >CZI9: ETO4>)h2> 1MNG9Z}Q~4sxZ|P}$\7e6 OSvX/؁;/wFA&7^w{)+Ң8QJ'TU(܁iNx!)N!,)뼊'pNštnNJS +N-Z-?J}?\j[iyg!_VKח8O_0Ӡ89=n'\Ž֠g󥥱Uwjc5ŭ.R;nGvTQ [ u(J]g $AI i#ND$gʭ>kXg5(6QM71Lq=hA-rFqH4\51WvУig;y6򼆔2vVMz8=oz8ndoۭH鰄 cxVF ۮLs.!OK~9˔ үCP9h!*&J?OJnp9oo #K!Gs`'JEKn/[TB=.nHJCX"fbU]r7 ɨvz+N-8ŠS\1FRop^iqڏj_[|8(K g/V*{|(6@ro '~"3ݻ7_H'(i)I\xS*!N,r[y\ɗduS|98VP$g~gAeQZ)Y+NN:S}wM>)i%nڜA<˓t{6KfTE*'x_d?h\5SY;k&VCt8tqdͷIhnͷIZYhv9hMrQS$VZ\q>T#{:CoL5ɕ+z@yf6 ]]]r>Zޅqt1 VWR3  .Ғ[cYZ;uǏGQR[HFSRk@] .P8ȷvqC t\彟CwAL*͊\gsy 䎉ځHթU4_*Gk\ʭtqi!G\1L׋7\ `qƷo=1{Qa}ʇr]rYnwU߯ ѻJ{K[T͑%΍"ǷgǢĖ6$RIHe,)̝EˍpVt3 EnKsZD.[,*ۥx(:i4 &ZXqjaũ=WQ"L 7-GOr Ed[EYⶼF#~ KTpP4yxkb\9t?ѭ;'7_h4;s?Ќq0:{irR3)".4UcȁfYpa"r &Y}3 BsȦ¤@MĕC[tf~knF{R4Yf6POf[bUO7ӕ$>G3)?[yVd7FCn êTqtp 8:7E!rm})ϹVNôҬjrH:h#r^8[=gWN>[9ҧG[\ʡWũGtwhꁨ*.r*V95ʩUN rjpΕ]2#(GvZkp EXrN^wV_QhuRc_zÕ3ŋ,eV?QsBr07=EOU ?_Ms4K CE&hzMa\]Cdy.Vzs[Urq[2M q< 5^SVN(,[;nyp6۬@Wεթ)u1Lt8֞T TР#zNgsq0,O_9pÆyTkl;ꄜB4~aT=I3#EpU'UƁ3\laXϑůM rP tܦ;/{& :]kcJ>ŮʩUN r*#orƷc3uS’ԯ>(qn{$p PPvs+!oP>H 84'.npc,迣8zRyRqT%C8SI!fɇ\|ImʄϽBPFn>:T6K,oJcJ8ΌeWh^IJGOlsr0^r1CeT-o,R2E$6Zks!_ Gx=K.sU bϐ.Yq(4߶ x̂rPV\fʦ)^VzP{{r 7h>L2H^yi|j6xe@Ǹ{u"}$F m9fMAz?Fn.@!!U,ʹX*V95ʩUN rj`Srtuo:W~=] 0zea[wn )&z~L+KJoKQ\)-z4%ÕQDÅ*G 1C}^q)ʷJ~ |1Fɛ&5j.L ¦VnR,HBuNc^{q _:o3bpHT>Q ' ު0É\Bݰ>}1 m oiy/Q7R "jfj*#˞g,F=:Z9>8/%WC}t ,⺡v/"TpT7o$H:ZBk@eD}T~EzHDHCuem!M"+/fW&xk|@]PUO;au_gSX wxjЬPOiӨhS:FNR! 泯)&bji^LuXHvS?9%[zV&+{IVRFn;1?n׍)\\j,B0ٺi~D^(hZ7.gxE u,K Ln^?W>e|!媁΋fG;opooakm O¾ E)N"&=Ϲ 5:t4kntਇPNnEϵM65/ g/{y"'} mJ^|8rI.KQn.Nh`Sgan)8zrb6L Ran#< ~#an0nppw7zU F0W7]>ZߡaM ӧuZ̆)bn)bn)ꦘͿ^V/q菣5VJ_՗H%q 7ʻaӦnкYZ}wmuisn󥥱/*qL9fq]p\{ACDDž(wxn/a,VwNGq߭*A}%7E%Bxlij>S1Uh_A^QP}^Uf+[\ztTO҄>) -W}fu@uCѡNXT7 ({7{RI' (>|ȇR>RFuPy6Z74OTFWy-X7<к!~&A3t~.^nSiH_ *S݋TFާü}unSOM2-5{/PݐHd#usRWay=O7"u0Z̆u ) }F#WuLUc1Ȼ)F裺{6P|2G;roznb]52Qg͔t殺@p!K!rMuC} $@ꆏGWnn9 utJ nIA=9^T4XEu6f#G?sߢ>UEzRyղ{\ץo5}ŵ +@usnnѥ GV#54C\a'NMB.u$SqqT%7WK\tnnuCuh]кyM}abuSM1V7XcuS^CRD:XcuSMn )N53($fI}k8I\^wo%oʜ˔3}ӧ %5}8Yf?g̷P7gwfɍӇKCX6w#T7+o0FYh̛Ã,6@+S3mL~W 將\9#t?3-Ŧ/Yv7OURe u}7!Oe#ʔnӇ|i{gihy%r ;reŅv,Pvcq{P-zzB7qZ"M.\7%Q GnW) ߎs"vo)?I,*%5qIِɏs+ˈR >T$hF3qG6%cp:#]73?-_(mߦ:KٵZM%bn)bn)bn)bn)ꦘ߽b/Fʕ_7!Fi'M~/9r?6A?ntV VkcPN5DOϼޕTcԡӏa+YxW9‘3Pa{soI7LJ]kJyt-hr:9:y)x1?_1 [w]^;VOr$s5L}ΎB~-|~~N& 'G7 T f"} $~MndaFSLfr-FW֔tBP#4(qw?NTWH*C|QB7$RnH#n|I ],Hvc]5X6>K]/+{#hɵQEbFdhW~?=/طaZ)FH7 -9&7uj"t8ݝhiFC7mg;Tnƥͺto0I3an/w&wZB fa;w)b|[UtQXKXN`{_u!'4Q0S\N*Oѥ!?Bin:mn:nZteO>]b\bt窛W7 BHOӋM$nf(NDa&0ݴƹv;t~CT1"dS_B[#<.nPtO/P$REn?-`i^ͦjc{&h4$tSsâ+PM`ibi M1ݴ M0tMk\fQ?vgtl\~C)Ӎ 篛q l\GY KvQXpy2CyG<~Pylܤs\rTY@J7!<݌k~%fhHR69];BK͞AYpD7Bѝ@-po% ܖya4/ )6܊3YP^0 ̌l/J.dJ(6$T !?]ڽd V`8!э0rL.?v"(,]GdtCɀgiw޺Nt.tgqN4z-H" {MAe\vͿG*q$݌p<4KA5)qW^M(v=Q¯#:t0R]%YM1zӍ L7F3nf0`1tc4hӍ L7F3nf0`1tc4hҍk+W~]$.d镭+7Fn&;  ޚ`ES%L7t>L7t L7-aIO`-pnh;:bAЩ8[7[Xlemuv´PCFN>DLKO+$ a QS7W-M~P$t>9f\ǟCַgy +ɍ/5=\XBkn({l"݆I^)#RGbH9>|TymHq}ix|z|xX?y9sr3U)Rt%fXwb+_RWĬ oW5O=ߊlue]ٛhaZNhڡy "|vXY 9Ω#|tm~N'D E4_n>42 |Ih%ҍT56LFHٕ"fhcb~xQR'膕(LkoamA7{)@{O8 h:ș@F`D @y+ 3gukYD?'~.Hȱ!u#5r-}Jho_$ !ϝn7> i 5]Z#o7NSd :9I&7׎~PO9NIӤnW!tiݴoBi-ޅr&֍ tCW5ء<|8Q+KLJaU8r t#*{ew`;цN"Ģ~L3B {98D%͏I3`b, GL5FnxÆчlX7kQ4t7zI0? ,GxΧ& ' v%!~}Ea(;LL! LHjјX7>+1GGA u:&A/^7S4A7DEGjyZ= Kb\ @J7/ԿyӞh_TxƱ@z$G8L|ǡitHB,!Jne8VNdnף7ೞ8j`8uܤ[>YJ7beŲ-?ѽBX7?p'Јu|OB7r~wiL`]+%%:l(t9ZX7#/i&us7Sc<[HDQkuS9=n0R9ͧ'9|}?ݫuhO9^UQ+_s'v!O͉~CBWm*6M$n /~I@G9[k'{M]L:f@("7HtT)ǨpL{6CNTH47Jr6DovT)D_Ck3umݍʙ%>)IVsnO>=*c[=~S@ ?HRg._'YP 57~N[qҮ*Y݄[hmsQnGnڇSn@{0+ur)u#+n6=v4𙽔7MWѿߏ/>X7tNUʱ8SpF~{8}#nktV|}:٥'ؓ;Oq*qq-u2݅uh8? m BQ>:G@'_hx:3FWuɠK(t_eAyuoY Ǿ!MzCnhFwrhOGCF~Fl1X^ns)u{$F"t>B^rP$<]$u#*&F|o}/A(&:U!OS p9u7B[ܙOD|qQ!/l~񔽍6dDq ЀtzGr:uG7Ln'toC|xϞJoɓ[`~źARQD>5JwutC;HC E^::ѽJ=ɑH1{tDqU~C0Ge|ؗA7h/O!TF>"{Jw -&#gUCA WZD"N'u#o4 k=,=e99%tX7g.OtHOCrYuy:5)n~~DGp_4M8hϠM7m2L7nڇSn% ET.ͷ tAK7SFOKbqo ¼z}4D M9G, "vl馃n E۳ѦrYw^\074hFn:ȹE6chɭ1t7Ӳ fʗr>ϖs7+.F" 7tK1Y|hiKӸ[L7t)L7t>L7t L7-a an`i M1ݴM0tMKn:}n:nZtAL7t)L7tbiNa0tM0t M70ݴ馃nGO/u@W֤7OD:>oX7(H7,I]"7wB&_bNaiNaM@=nGojxwuսh 9fu.RyLbݼz/s7ܵW)YCZAswy2<%>nhݧ}ʈv֌bc*^i[u6+%uGuZFxT$X4sk_}?@%)T.c[ f!Y>0n^#!Ùeyn8E4PD2qTRTiBݠO@mH7ywuqWtV}?p/ѡ!l:E sPRnky>TfLԮ §5Ǻp&qJ'ۇ>Jqנf/ҩwo& }O;Zw :9REEKFGdU+ c~qD~[3ݠwBtBΥ^P7NQߊlwtCa Nq";i8Nhn.Bq!Y|u+}R93htd3iJ5>畐 4 )ꆪá[r-KAtC]Ѫp {GҤ&@ӣɃ\5պ~c~KrwnFw{ #Nkх& tݏzQ]0 TڀQ܋,}Ҭsf8Y"TnEBI;>y!6&.u.I<%}ʡb 1&{u@G3An ĺA7Lo:}n:nZtAL77utԩN8M|FmҺjR9c9|j^Ѥnؙt3XWg&H0ݴ c2ISVw1ܫtWUXtmǗ޻#|3nc赵'A{)YDbV}bnx}%uth0w <+*]˃wSgaiA7gל<97̓  xI*9?dוduw+i?9pA7I }x{J|?X0/f/n~ƎSy M 'PHƄY0>&yiR7ܹ.Pw|fj nniz0ݴ#Ұ>?<@?5i dьnНYrnj)=g%nk?VnS@uMrQy_ves˜Ϻ?Fѩܨz%YnfytP"`CubinZF=E:6M>Y}tAL7t)L7tbiA7zkǗ(~2{ Mai >W+Qn1gBFto˃ry4>B++:OQr͔:d ]Z6##!s4ӊ̉--`fDz1xH{jWC .(ߢTm+vE[P\n8+g+xo,g$ػvnB7f^ a^>";1F:GbP`g7JnpKI%xѐMQ}dt3>bm 8R n"gZe!*K !%f#h,.35.x0BZ]#ҍA tQ1|͌H7g{n*5HF|:&P&6WGh\١D3Uo>gh9^FC.>tCFN.sނ xLX3ثoGyݨX7me(=8P8d]xYjnU]R7jfTRYLxym# i_r4m!=&>.<ޱBZf0(ɘ *T+P{߀n0h}7 nope7B0h0aqJ8PϤo0bwHF u C+)=jޖ4A }X74EP O1`^<3C'[%V`;%j&<@ߎ֍Ssp0%.#!b9ykyNR!,>t=g KB77)sjRѾoc g.|5VYx}VCwF?mHbcݠ lSfyK2!MF fWJGq)1 b'He?J>ZJlj'(>,jȌFH6"KB^2oO1) (dyKhQtP|Nev&,QƘzxLϚVry}?F1+{їNyĵgs,DUPVHb(4Șh}/&N/Mȳ~hv7QRhZӏ/iD  0yupvYޏɊ4yvwiM|"L͗җ_iDK킇h_&2'Šinpsj n>FC'v#៼vs}Pv3%i6'）XnT#o_7ջV@;TO(Aw.T~@d[,ъ7 /,윯lE4$KntӢҭIVEXR@$D;=vǜNvჃ-S"܆[]Pt&i'Lh{ ɷyvӴУã|G#v o ̒/tϯharidcS|w_ /ѯA-a7H5%%ɽs'Sɖ/$HրY0y`9:9 b؃8G"TbxtPL4# kYw8ntj^Id톯ڍYm7z.HJ?'P![P{фd:Z(>_>ߚ.9(ƷV J,Uvs}MK$7W}I)](t2RO{dA:z6g7I()GA˱KݠhdDەM$n.8r]n3ץĖ3In'$KŠks.'mHJTMp@r@pR1ȴѱލ"d.ff'dI'˔jA>yvXWc)Mv|Z Xw@vs q7su{T>df?}=K-U ~UJmF$tFȱS9M[S:>MH{fJDpZN>Gf6NYgKU@P'z}E2QB.'"zw3tY#[O<]41Jn(9v~GFn @#ԥ7ikR^7|-jgk93;o'~ MRFY쬿vC7~Ni?|Oirk7\*>.&5gď$gJ;wOЖu Bz ~QWvROXCmt&zI]*c諷]40 ~jT{79],ηFX^8ѭIF;N4~%5 2s|i|7HS 鱶)+1?J3wǞCv$ȜxvujL1$KnE4*n>"uYZvӴn7vs]r }zAݼ]YcLW0Gn.gn~avݤI\yn\TN2^dN qj]DEoRe7[WE^kj/ nms ]L\tp5N0y)S$g'Qtr4Qs'}!"ٗ0Isa{7G]I>ȋފx 3"nEL j.].Qn-(=H-9G4}G {7j{e+㔙RB_ӝ\Dݤ'S6&wyd?KOJsfTLJ}_l|_jC{v8)tx(ܩf_GWS|!d_to1n\k7;&)vNRz Gȹvt=^#+mdzC{\w#qDſ5ذ)aM^5"d~Rݤv@>8mcl2&.P/1(K OzC{ S-OM^>Op.usx.y>;4M l7z5$/݈b7]DJ71:2$B/tɆƟ⹰Bdcv" kMRijPOog7bML"%`ڋLSs?>ljx|If%[D-}i2=婲'`/#Q4n=!*>@_3]:ȵ7DQ>t^wM[ӑ/FQSpF鯎|S`7~k7 Mm7dTы:o(z|[ov`G7ݨ}hWs !~RzQG?YRhnP~L&Ŷ7&Η鷗;e?_sTwjְ4' hz7I2OǨIL<ؗⓩ,O v9(>p`ݤvs(@ȽCN]\i̜M2yavnzcvnz&M^{岸%ɵBN=ǿ>gk!‰KMSnpxW<Nꉇn.f7=Ovwazb39h9bvs0yj؍Mݬsdݯzo`vs0y߼̱t GN?f7 ^~!In,g{KMS`7Iw ҫf7 no%;3NvzΘ\2nz< Ǥ6@]D/zΘ\2nzLǟ|E;Хٟߑ/H Xf/3f7 'd*N ?OG=:gn.f7=Ovw\2nz&Mcvӿݤ1yn4f7=Mbv1_nҘB%Wn]{\[%ruT;; _Mzvp'ȳYwks{n6F75ryrƹsnõ'n6g̹Jd'x9`fZ ?š޿7LռռvaQ`7c`kPc,5v=ջYuRimn<^rWy ޸ %cw5T{jp]u[96vyb;x{p/x<w[ 4=8 F'1$u$npz 5a5$W8bRX& AF MH;wc`jHZ w#zjh1^8D!e7Opla߇aYr;pn] (KeL5&ݻawykItvnTd! UOsh]g*^єQe7qKs&_܀ޤڍJIy[} {T-)">zpWRd çM[kܫ>C0O@b:nPm|WwމjFuY9#=િT 0_}uCwUO\/Pܜs Աy?YC GXp͙qfqK.伾777lpΝ) vY`E &G*G䓵xAkBdl>zܧ갛L\*&gNiwxu߿MI$n. 9vsQȱE\j檻>E֏xNlr_1df7%1_nҘCgNK7:w<+'vN{@grdsmJ5ӯ~Rag6FP&k*1z/~ѹ9YZ FBbC N1*+L"+#>Z,sڍ> K굛Ğq612)iӻ74_1G.l4IzrOr#Aou'R!qY8ӻAާFЖqe&3o|uwڣMAD.'N6:Ef4v`vӏ\B V苨I{Lj߻63 g&*U*Vz2 R+K59#Uvp̘Wy )\@ş0G.$@M?aL"&r9=RB@fatm@ΏY'pjxvkݔb_&яugΖvpB0Gn1%' _O8PBKғLݱYK6m5}ӍlhYt3Lr[%*l/ڄhPC̸u3Rĥ'{CZa@`l󪾋԰nP,J f$mg!ҍn@c=[Jq@<o9<+u9QJ" plz?p;TSmwcDLCpڌym a]P$f{2ɭZ)# ZW3837 `Y BPpsa&ȩM748RJC2*b5! rrL Q̠|5uMYS<d /r͵uRoGʙIF(1xr٠ ^MG^D ((fX2vjِly( ిOн{2h+5SHH.\uP|f4HU6l Cn\|h&xciצּ@"5w*;tX% idr85`D!c!<aqX*o}863=^YrβF+.Ed! N<7C 9:ik DeReJLT9OIMlg.X&=578;%ݟt=XȜIF"fP4X3zKЅCyPѐhr zPnt& H.37Qh7]ŚKe}"\J,Y}$fBA !ESiqjyC|)ވUeRgۺ٭k QdJ*%HMOe58&Dx\ XVYa=D!0٘ǢXi C j8hpnDh%j6`$4JTDin :@PViF|wfOl!HmoO 4 %cqÑ9̔lwefq}@ 8̬eq^:,nw5eΆĄ~}(8ӁN8[R^j>6UɉDJn6ki(n+0 VT3J*%j>"2]txq0%~ r2Mdm[ɳb`S8=W8@8ty8@P+( 5+#4TKD.fJHnm,I):3}$UM8`{t:Rmgpԗ7eZYz2`7dv=b%4[9Rgl0GbY1ymh2T9|C*D.C).`פkS9qKtZ㭈8ZHaC1XbrP&XnKjheī8EP hn79^NA/f!cd{wirO N34Syۛ8 q$(\[aAiYu,/GuݐHn^AaPXd,2&5ތTZ6MI4coE'!H+O%fPcaJߛEI$-cU71!uѽ)5-A:Ra A+m#aLj~6%@g'+uvB3 /&~b.fvn `AˠliNjJzYSs^7xVqw|ts C(sNHK`IΕV֐ Jy>_U<+\dʜSo+:5 82 G*Y W@i0a L+F0YF1vm7%HC ÕR&6XB靛&CXt h~]-SopdfEHVHZxT4 EjCU ꯢRU1m8|΃ yd\ sJeixTOSŷ(8Rd`6\!d q~rvqYfc^0 h.Wҳ4q*>sPH&$Sղh,ݩ :ZxE۠2 Fb\RfK5R*o`=P BO*aIχVs>Ysfر]WY:'D.lj8r['lUX.|\NGy]Lnrkl]f _~84l< <2:a7lQWU`K CGgEQ 9AqbV`8?L4 ey{h؈1M j5`$(2+~),hf``U^Rh-gpf):k= !.΃-:=L{$HtP2E{ 'x;hg쩓l6+44:iɟ\鳟eV nY6#&f=,y1Eno?,I0d8cڅ5ba3 &f^5l&V4bN^fR>hlV$:VseE6;VfRNCFsQ ~}Ӣyaq̣{4g. & qad|v,0+Wo1< FčX褡ICƯ2RacIE!Kw6:]ٵ)7?nѹs^Y[3+=jth56aNh8Ӓ arV}6:kq0<^r]YG[^ +sdL bԂrѪK):x/.}h>b ebdm{oUJ;1Ǚᎌ{/X,@*(㹏I6 e_d+{>%:&eJu b8u.+(62P ٢EFijr%+=*sQ`MsJYW5ύNkȢ>|Him囧5ZI]N7EQ^aCc/%~V0E+ #JTI*ynjN66׸S-jkKP7Lŗ2eр5I֒r1xQ$b>8:8@KJKK{8"dyn * J卓:̷ SO q,$',YOE` Lai⤠&N 8)8xrUbݑz9+; `988^Nz{![&ҠY[Qh*Ԯ*vG00kh[Jxr,.q!9>"x޷t*L A,UCl7 ᯡ)5_S϶)9[6'g{g'8Q.b%黥.I~?&I1nyCk7$r=gY>{1}l6)LrD^NH#6\7>qtsRIA'M4qRɉof2? W5r8%ʯ%q]F>Ы3ll¿s8s ^\$hykvwGs7!q#<{|zp~ĹVI5HJBlⰪn&6q2RjBL>8)h⤠&N 8)h({ӥ rc8ҭr]D&MdGI;N9xUKX[|8m6,=Y \֣p>.j<Sjh7W_``%m =\" ?"剴Rf7dmZ?!1Ң mZ~S8H%~#vI|nW§hKxF1HM\d,]TPjCN:R,Yu$m[yJ)'ɀicװiҨ6!i#`(A` m ?i3e7 /m@X [;F GHɦ WHE~ 2EbWxN7iC۴iÏYBژ;(^)PSS>6MWú'Щx7C`,|V;P ?Mm7f?ǔ&kb1mٲTX7ke ߪJe|=-, P :IBN:Ϗ6KNR LhwD[3%)BPwib4H}Kn}H?N7C6)Ĥ͠H Md C8Qٷ W|_/e_a(؉<p䳦LQ3>( 6̿iC)VQŸ=t7; D7iaL qP۰e C1z?PP(m`UKӆj4)kQ4;PӶ64Rb : yoLyUuk, 6,-\+:FaaPN r8SǦ 6mzdNá.A 唂@@R (QqSMi V`ZꠗIJɚ6hЀ\݁Q$BimM| 5S_s#wQ{*HiOѣ1AE \֡h t307(Uaism7mO|#q.ƕ6fM~yE*GdӦofg$qk{ AWBq0f<$.H:sV7eo.+ma^-HCxu;q ӫ}~%,Gx]7mDx]h$iM26h$iM26h$iOe\8=_qC?_g'g@}]D&6c`~wv+^Ai3Cozy(mN]0] uJ-1dV֯vEppo)1%E060}fP-g7Fӂ?3:lDqS9 ݸ' $B^L7F^J6@7Vy `i\{}#b]pBi3P,Ȼ1 L\9t{urce0RŰBܧ2MJ<%MnSGt3 25"\C&[6wSwLf|^m|i^ \) qf;,u:_Av mA)Q-mP' XѴ!߄8 v&qMd4mѴIF&Md4mѴIF&MdnO{pz28~>xM"6ش9`ش~b߈=Q\E7>Dy&H6h$iM26h$iM26h$iM26h$<((>/^a1+ + \羙߂nf޽| ?I1?RP7Gi_g@ -K4vʌL7?+T9\tH7~neB@7Kw/5G5)~uoow]'O7Vq &a_x\ӵ7 3Aћ=LyJ@ܧ/HЊN&}7y+zg[ANh~Ղb\/(!6* Gu(r¿ g%~K4_[1i6N7S;93FX7p$F ֢l*(/#w[G( ºْfd;aKH70՝ANGNB!Q'ZSo_ \p 6O )B!Π~C`04 ֍< qnNfnP3-x`_醾8e^q l?9$btC_ 2Rͻ n@;-9):,_ndK MYQ񴊋tX7Pׁ8)!$#@c~}VwHT`?ښY\[[#w[iʊ,^7[C<Me(Aޥ9_,3iQ"k)laf ªG V8VLnFŀĀɗAu3 sߌ&+MyG#5 {j`4['P{#nQO-xVDd2/E7܇ᣩybikPXOꢩ7`L3Dt|鶹zm+"X@qU )IxPjbZC[:fH)R]MWnuTCy `5 oqH(}-v5M:h+_u!w M.DGo0F]uuº^7lnLb?3T7?s|АW'"kkt'*M.xY>￣@ӽo)|ebtCU N_>}cӧ^,}=Ϲ ݔtk:r`$nrx% U?b9#\HHQݨnpyQdEuɂFuՍ& MT7,nT7YP;oG?K`qf?U23'~'q)MT7nq`^q@ 0fxn)%qh'϶ЏH7V:rn(o`Hٺ@t2b3Ζ@7mU,Ѯ}PG؋i{4ty,ڀ.=Y6 \H5M^ ec # 4w7N7Th9MqѸȔF_|4}b q , g^p9cMJ~#u'Fr*iqQe\?CLX /pbP/OfW'#/ BfsLX7#Du#\P݌nBMPSrT79R4݄P5b"A7QHtsua2(;AC9|Nك*!VͲ.܏| )nQ 6uW'g w󤣺ɑ醻HWz-Z砭;LQHt8>{>,bP(hFl*BN7x))nkFRT79R8݌MnrAu3vT7YQݨnQdAuɂFuՍ& MT7,nT7YPGOO#ῐ8#_׹oBËӓ2Zud FՍcYoR JX724n9> Bl$NъX)Ln<{s-W^쾑08>{&S8~C=Ž~\5hXYŮ_3_Q@0?&`p94AuCi5\L!U (V|6pd&!P.o^6$_}%ؓXnlHV)c!U7܀9  Us*|2/VY-x>x$H7^"CGnFHX7̦!!$r,B0YH7q )c`97s֠͋W\(7ՍՍՍՍՍՍՍՍՍՍՍՍՍԍ|WQ租)J&DA28I`/#yOׇ&#MT7,nT7YPȸV<Ȁ1K0zƓ2n<T7?]j7912ݜg5>}swgCWͣycjkƘ6f",u@X?O`[Mo~6E6N[pƐlS}`S&̐c`h!UgR \uX=]?'K%P `mV uu۰I lLO]U6nS^.NºZuYKuarVpnςdpx؏SfK})A>0Ze,x1 nr ;ڱݔ#8s׀Ӕ:)h>PZ2Y(ff kӼBf%ͣ5̅XnV 66%ۆf,,Hjne-!U$OݔH6+'IeZ߮CZvN7]&{nQ"nYO@8@C( t:7ю,ĭ -)&PQPii>6+TI^`NQYƐ@Jb<*G,'Xni8;; Zg8z1{|H7S4 b?[d0n (FEzna#f؂PYq7wEݸuyH%/8G$bπV7XIlfpv7Ite(랐9}~r+f[Ə5*;Py Da8d $k@h nnjE̺4~T[CQ`3N78S5z.LTw0 ?n`n t{<-nK`7:-ú3Y&`AT޽iɚO7in =!hjzq:Ē '*O7n^k tB O7)/p#vigiN ]d@7fJ @~f ,C8*@?M+0nQ4QG55!Ɣ<5"l 7`0݌MFT7,nT7YPݨnЧWrWdj*]VΡ"hT{bb+WZtcgt1{5/`GfMGWtCݦ<'* ,,UG&ݘF p?^.W]{Ўn\p|!V4:tC=y(WVjj 0t;4v9GaQ xJuw:cL)ԍLxѾ>& n`p4[-$ݘGnNR^N6-ݰTp'2Αnu')Dg,"e | aנ;rҥ)gCQn?7u6GD7-|1d$oO(o߷JQʄ1{(+h`dsdxEA7|uĬ5n `D?;KDG·sHtc ~#{QK FOtw(Ǧ*mnJUMBUlZh бe"Oi  q@D7L10fm4y=~,>wm tCÝJwuy6G>.%sc#2R&x\nhH;<n O78ୀhD/"[M&quCA$Ryag0D?(Soup dnǛM8<%[p\0LwOeq@F.l0-AM[ ͇JUS|>J3V@Û!8M%'=np4QQ+OƮZSv`fs`TFQnPS Y.mm +R3#Sn McJd2 p{enot uuCc3ף)d#QRf~5Y*YTX.ͺao ZCA_! 1PSm;܂l5t 8]Yvp[TlIe.͓F8 M+$U:35>CsPux~KN!Wc>[n@b-|dtjDנ{rDuoS۬ ¨pvˮAՄLߌMFB3APdDuɂfbu>^:XDG n^}OeBhBW*4e#@uaݸQ|+}Lުe_w6"T7Z8>*ࠕDlZt˂j`Tn!=?x 8`Z4tn}BuQ݈:z)j!1 ?_`TxPCn| ^{QP[]͑ɇxӷ?" Փ?}ՉFĺ8 |އ r U7|)#AunX_DR~&FD7T')ԭɁnpXGݠH+lxU7`?(P Oz=N#zMӴɇ@|}ӷ}uA~3.T7ztFz}C]ʎfbu3RT7,nT7YfnAu3vT7QݨnQdAuɂFuՍ& MT7,nT7YPݨnQdAuɂFuՍ& MT7,nT7YP|o%0 qQ_AQaQ(Y@F8 _?_ \F8 _z&MT7,n<;أp=3xxkɑBH#bچ:-8|::;F 9||Dg*lDɂ7/'t*#zuV:m=@)Mm/old*nst/4:挜 @7guPF1/1ӎ<ݠ@+ Vn9No"ƣ׍5R 98x*@8& =*l2 8ݐ<{6BݐVD$M62C:З!8ƏL;—DfyFn<) keTnxKW L3Cӡ(uÛw؏L;/ aA?9tnD.oRq;N6n'΃rsO7<h##';LjoSXE9͛tu4V7{PC8O. ]ץ7a<7O(=ݸe)̓g'>OC=r$_}\vxf w ddpm6@ oxnhԀrjNp ~sڢ0AfFD7Sܡ)DS6'a,G'J2`Wts ҐSԫ7;)UtCmwPfo[5|}3+ڱD3b٤dݠ~B \f^&@7C9=`)' al'hY6}nhVƑ6|'/)Crz`͠9cEecUyx7CR5<[7TQ2~z۩T(uӭpt"T頭~aن81tY\?D̘RSNM0><݌O7ׅ'P":S b)nF>7;nT7YP;S52MN7!,]~DBKkMN7(D7vTT79R4ݐlC/\r:_o,%pT79R4ݠB]0˰!׉QHuc݃.<,7g—a)^T79R4hOK HBTY'ZNXu3vT!Ͱn͵QpAHUb֟`5T7FAux7?$ڋ7孟t3-8)nw AH3gpzwoN7Vmnrh NB.p97v {B6ٍ&O {~Թ|))nF>7;nT7YPݨnQdAuɂFuՍ& MT7,nT7YPݨnQdAuɂFuՍ& MT7,nT7YPݨn_`w^&tGNd}1W/h9kahx0~P} ~"axf©xaHփ.>71gXeDf|Wf@7={qzм t~[!ρ"wq:٤{q̻{qqi9 w2AR0qP [tcùij/[>G}<^qfDyp9 wi7Ǜ + <tZ'~h||sl$q)rOH}p+ ,^OuC3%T#9f<3t0dYc<58-[O7ÝbƹG!x p9 nΝĕ񐦛 xՄ:ts ͜ !!n<48ZӍlMn3GUUwyblfAčc`}ZĂf -k%nC1ef Q UVЈk0fc#py3T=?`d-n@۾`uJfFvIZ=YvӄHK #l70AF],vNST22׆R5:) q©cvI΢l81v~syGnh Yc )30‚lv?a&`<(H=c|[a0QK_݀$,VuXT*aSؒ>:a=F`Ll DKBȱ$ > -.q+dBnh`% l7"ưp`鴹ÁknH31>3ed`78e C݀c+Lh(NnM 0np$6BNE!i܄2 kGvF@AOnNL>S5tr( ~9 vZw0FɔŇV3 |v f7dm P"õkn"tdԧf l,Maa5`4`hdv*:WV eI&CV]Ty(+_݀&MQ>K ֭Ӹ0n Enf GIv+p Y.mw SjQߍ vp@$pT5Fd]n@0X26HDc.k$!` +k r) Pd/ 0Ҙ ʿ4)F{rN<"*vb6ڞWcMsL'ԍUdN0ߝVL 5iH ȵ>Z6MHbURJT72跛;E Lh(`#0\$vpo7`]ž݄fpL7`msC0n8;%vIP8i7 ta8VG_(H4iS@?AmG*nK6];lc ~d%رpƤry`m"XX9=o%n{W5fL$ pTc0hc͡U;^*nvL[+[qvxH}jJ;6-{ۀ W'gqo/nn818ɵv<sZ.b\WV|D[\LdMHY4Iv g7Qh)&䟣"n&BPdO9 vkIf*r>>lfLE*.Dݜw"]i\Zag7`2Ҡ MvWs*(nd|ĝ K~vrgunMAȷvS`r\n ̈́v GP)j7rmǸ!% ҩw1(w.zwRiV 95nW6m?ƽi] ߟ5fzqw%Z߃ݕhD}u0~i} & x)LjY*ci\#nd7Cn l\gNqDn1Fa#m|ք(%sK,>Z.7lLmf$ D/d+SshDD&V(k7\M|mLWlPĪ8ei7}FY ˸ <"fɔnaN]<Og7(Xd)"DK^ddīR^nKfs K\%pOK=G"g]ajt$<ԻQM/M5шsR^pԥ1ZG\&@ Z:vApSo&No7t RC&azFRn@KoNB;\ n= vm +{nów={Hp)1 v#m\o6>-nS6M#=' ?K`tc6*:L0Pa+c`P":KJ&gR`d.[a7\hB}vJ1,f6 5Fg7Qnoݐ`1[Gb!f vi?v#og=&`{1>BvCӱMP,yH\"`?VhUg7Ph/co'bMiJQKs܃!nd4M9xuCd* Ϗd& w#No7$'1݈EXޝ1v;/a 3XqÀQږ]BD@y ($vӮs.kN(u`vå'lM'2e!qMTBoJmȎ䋝.#nl E;-.9Iݜxp*m^3R{eV* x$v3$~ ݘ5woJ 8}ǚ: ,z=A`pM'T#z!D~3Bn9kfNd!:COUhb|1vjPWVn_-+6FU<>,wi)4hvuK|l9T-H1x/0h~gV܊g7n W o%t8c }~-,ׂMAv=2\hCmd{[iUij6+^Annmd}[zK$vSn bSv3j7AfQn ̈́v!C'^ _zy:/ vө7db2Qn B݈|G_|rBvsؗ2ɎN4RH.ʹ+toNkǻƠvTdׯkZ|!'X582iGn7v!!j78I)xBҿj7!n&/{Os"W)ivc>Tt:S8iyj7a8>H/` tq.SvS5;Sow")j7a(eOzH1&E!n{PޓuPn 0vM15MBF~|B nlEdrv:nj7MAH/aoj;>QGN`9,Mv2O榢vSL<sZ VBڔgG~P!HEUb7ЄuuAj7!n^~|ǀg'P寿 շ8OK4ݭ?X_E/Cb)j7nD b,.`~FӏIç_~5nj?ة#usIc]/Cg v:dꆡvT"l7_d@Spovٵg7fPn vOj0Os_qPŨ!vsQ)j7 vSn&@ L8j7MAPpnv3j7AfQn ̈́v GP)j7 vSn&@ L8j7MAPpnv3j7AfQn ̈́v GP)j7 vSn&@ L8j7MAPpn2(Hڍ(AFQ1Av_>ZoJIh+ws{q]j7׋̈́v GP)j7 b7Fg/İ rFvStnn1̉ə N3)vsh%nN/@decf$n^B]‡x' z<$n+TrmSeӬS:IGHiԒ))sU2ڔvpq\_`n .A3H4jm\Iϓ塙6O؏+qvS)o(q)*Ӹՠn˄J[\v2l4Ɩfi"vISM!nP%jVivJϟB9o7'ƞߙW":صc7. Ir]r_[̟%-H)K[HirjBEn5MPkQEl&!e/ew3_\ۊv"miPHCDB͓ R=Y@1跛 _@ΔBo7-=h˹8 Ȏ>XvSP MB &%vļN6!3mnGAY B9o7^:gbKh pkl} 'Bo7r&GlM/\z:tv HHjF7n)(g7ͽnM\T9]{65j7gay@1:.8vM* cNo7KOnʽY7Ty2b_&,n\ap- n73*37lf7i)K-Vz3&ngYWY톉,,Ϙ~cw>drj[LJFvStnn1j7AfQn ̈́v G\YvsIäg!@ҽ3O j>yUP):j7D3lA$>/h{PV&q/Y3;םO=OלR):j7Dy.`c`yzt5݌< o]w3E(ώMQ$MPx5)]nmf2^ n|n& nw/guX޻|EMu|*\pm)Mj7 piߍMF| }bnM0모[L+SusCe"Cy;'v+WyIk]vsI=f;_Dj<3f}bQzxIog7}lVbζ}r&MQŤ T>Bwߝ$3S̈́vMDbn ̈́v GP)j7 vSn&@ L8j7MAPpnv3j7AfQn ̈́v GP)j7 vSn&@ L8j7MAPpnv3j7AfQn ̈́v GP)j7 vSn&@ L8j7MOQek7x1o$(JmL(2ZnEj7 EQmMs,fSF<[}qݻ2&0*yfsd(Òl7w%0 sfUB{^͛]ܕ .f)-%nMlVv`7vC9|Θϥ>4fs*Q"`]`vqSޔCdw-RQvcfsp|UoLc&Ӌg2~VȓnŌ 1ls&, uoO° $en^9 \y)vŦ/Bv"f"U͹\fw~Nl50X_ş4d2A n7t #KF݄z,ݮQn$nvWbvD=/MsSsdVcfƈKLT x凱`7<1SB3j+2nhD1J vwLfxT82d7&ڹ0|4oyj$x9YtiLш`: a7(CVx-Ǯlۍ v]*d~qe?!j7>MD7el7Pz7ߞ_ݜ 6qU٪4zz 8CS}nf}ߊE;lʹdy`R"nn&vsq|ZNA)9[;nR~է椛;8ͭoМW͋d?xժ ޔxկ"9an`3σ>9 M)v}@/FTk7x 3Tdx/DP >ʾ И(L쮾H-V"BW+DsIR%FQkDFQ1v(ʘPQeL(2&nEj7 EQƄڍ(cBFQ1v(ʘPQeL(2&nEj7 EQƄڍ(cBFQ1v(ʘPQeL(2&nEj7 EQƄڍ(cBFQ1v(ʘPQeL[?D𧽿/!EQ n'g$(J>6j}IENDB`pangu_2_6b/Docs/result.jpg000066400000000000000000000543571432571244600157360ustar00rootroot00000000000000JFIFxxZExifMM*JQQtQtC    $.' ",#(7),01444'9=82<.342C  2!!22222222222222222222222222222222222222222222222222" }!1AQa"q2#BR$3br %&'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz w!1AQaq"2B #3Rbr $4%&'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz ?%ƱqxG\q<G,??Uu&/u&/u&/lG3!T?_4ZǤ?K@nQѺyER@nQѺyER@nQѺyER@nQѺyER@nQѺyER@nQѺyER@Bq W>a?~?MKEEyEF?MKEEyEF?MKEEyEUN)ŐoR'8O[isfDtk$3 +޼eD.Ԯ.ŵeX3C'Tmqs\(hq<+RZmC batVA5S\ytDwQO9YQ]Ƿ)-UWq<ˋn~&e.#bN1d}rgO-ͼѣ0KI"$G(cqs4\(hq<=W:ơuKHe"xh%hٻ3q5}ql-=28&w7-ژxFvu&/ğ[եt{{fU.^k y-rz6/E.4-3mgRF:)~c/Cηu&/㵯v6qk+J;Y!2bПݗ7TP_hO~(Bv_R@}?/o KEU 8A6=/ߦ -ʥПݗ7Q¥"Bv_Gߦ hO~*Z(/'eM}?/ohПݗ7Q¥"Bv_Gߦ F$rI— _7soT"n4˄Bqj9lPv8 hO~(Bv_\H9jv֯3M䴥G fV00AΎ}iZiZSu+k\8H !Z9?;aIFp8]f}[Y{75xRF8X;K?fI`ŤbՊɫٝ.ͱo[yg?j׮jBv_YwڵŮazkX08GElsֶjkrUE'eM-BnPI9$K‹j7eF!kdu Y;nP'eM}?/ofz~5k;kWZRwX+c GW?4)b)'yiR8ШI%>` P5~Пݗ7Q¹[P{m2luBgLIYH9;:6-]Ʊ\,N;w#g0t~Пݗ7Q¹]Gz͔zqyo F 3{5=c^Zm1lF< 靍zJ>Пݗ7Q²]f]SQ"dE١%sg-ڬhZ*+WAŒbSw hO~*Z(ߦ >Пݗ7TP_hO~(Bv_R@}?/o KEUiQI7oU~Пݗ7Ps-R? a^"6kj>g:tIxe<08=m*F#A{kk{i "+u =Z`z+_ևS]#9'z?'mwcGm۳8wTڑ?/??}OjM?jze*+20 ~i0ɫi_΂f m*9g4fA\`bXc'֮J͒%QE#3`Kug6R9F=:eοik%4sˏvpHE8(,?Y"O!$l>XTזO5M^ɵU e U52v}yy%M9Ame%P[FλHPqr@ a`+x.~⻂}t5Xjko:ZqHfet&ٓ fPsvN,kIe?BtW̞n7]8'8洒JM#8vV h$򧷜/6Tr #uẹJԯ%nn-#B0#p@< AR;6Y2$3)8HSKW?Ar7X;ˉcCGVuars֧ȮۏA/Q%DѢ2iHS ֵm^^@"6 0Grckο z}@Kh^d+PlxWZz܈$kز*p <9<˚.{l ,w+vOT;X+,nݶk*|~GRx.un53[~ oFTӊⷳS[$ZjMvk"qON{[hKYM#Q[Aem+#) ׊eksX_K?KK.e08Qc{<>TmcfOH}(٘ :3۟l% -~gMGIcY#et`YNAHoʱ|QgKPvERAId|w5u6UNW[ ONN6#gerG\~f7nm<ߗ4~Ts#_lԷO}^CwX%O2'P̥X38)Foh^!ͼ,J}Y'x~J,Dmep" pS=(ۧɂW.m̟i6VYk;*#9$g;:d4ig4m!s܅ dgN֞lJčn?}, w`L~P \K[[_hQ>쥟n{Ys&|l7~&>!4}-%-4bw mHrxN_^_j04PF";r=j񛾷qj7m],vAN8&u95]gFmNDn$.#X` OW.7gDžkBI-իo+@cۧ-? ;'Zizݤz=UPOk''ud.5{Xm4 \Jc$Ƨ(v *=~}ny6I%{Sh[iK.(pq^E"&5rE_HdDS5s;»z~= tmq7kT]5%("IfX2 Iopc[Sx.WúYo%mH>Eϴ346ܪ;92lgֆ=$k>6q}V ( ( ( ("oZoZ)ihWk;IYq$`}<n?_T-ۏ?Un?_T-ۏ?Un?_T-ۏ?Un?_TZǤ?KUmÉ#`1OKz~@Qm߳Qz~@Qm߳Qz~@Qm߳Qz~@ hZ c ˞(y#EisP p7m*/m*/m*/.oʹnoq׶׍qqn P:%zVY$dl93vX5u`<:o}!k4I`1Ss]tK 4ek-HH9ȫϠ JvX⩒$S6ׇ# AMp=iocT ]crȨ_*e7$]'k-mp p0|ltojm`+sĖP㜜y>LVq)Ųp)kk9gZ"2١kyiL `zW^C)ʰ#T]BisLS[RA8' Oz~@u%q=bgm*%q=bgm*%q=bgm*4i4!+Gydw<>%֣ D/ ,?GUIc^Aw!nD n:h:h^ ,$U%Ȅݎ+1t[V_>HFqcs}t91n?_Tm߳S_$7{-n1eq46إėP۰Ğ'KO{^D,Ifܣ?21;ݷ/=뱋a1q=z;^w8EmO̻T/,|FGNUңT5WW œ,! 8b3m*/:KEc~],7o'kv8b !I=z=Ez~Fۏ?U? rZ*-/6X]ǤͿp(]ްwZcr6KkY6d F}>.ۏ?U"4$Ez}zrio䯔yn0jU I>ͯhq[K6"]~YBo8#5߳Qz~IRxFˊ++f#x&1,а=\5ś[X]]-ĺʴ\@ $`98%8+ҊNAH=AUZL sc*pFT#SNZmocm֘< WڌF-g5ɦurvd ץsJmulcu c*1cm߳Qz~OP]/u,qsH鷷S]Ez~Fۏ?U %q=bgm*%q=bgm*%q=bgm*%q=bgm*5j,jc`g}]/E]bM/<=kh.,TE+\ǃG #osΥ1ޗSm˼qA-U?i^\:5jMՊ NEP0((宴xV ;H S(eFTn14\l\O-.$mi<ۈi#iU?qRv~կ!-"K8; u*/ 籣~6:ޓJj..2GY7c4m,a,G8k Nlb7˘,?1 =X8}?k'c C)H#p%9({;t`H%(lAUWT(sO<4,.๸յ FKhV p4@N98ϩ}/7\(hZ*-/7\(hZ*-/7\(hH*4eF}>.?MKEEyEF?MKEEyEF?MKEEyEF?MKEEyEF?MKY໇Q ~LY,y$+oVIa3Z[?_4nQ1x H ϒF#`s<'sjRLXdhB㜜V?M?_4k._ƹ)b@EUd9ت-?fqOKyE@QnQѺyE@QnQѺyE@QnQѺyE@QnQѺyE@ mo-[k;9YhBkg8bb*QP㸒cou&/}RKX(HNXL㞵QnQѺyEGK-?M?_4-?M?_4-?M?_4-?M?_4-?M?_47}\KUYT;>RQ}#@*gi3ѬAqoHhO~(Z*/'eM}?/o hПݗ7Q€%Bv_Gߦ -ʥӠ':FǷҥBv_@Q}?/o KEE'eM-ߦ >Пݗ7PT_hO~(Bv_@Q}?/o KEE'eM]ǤͿKUngCk0NPoK€%Bv_Gߦ hO~(Z*/'eM}?/o hПݗ7Q€%Bv_Gߦ hO~(oZ$YThO~(Z*/'eM}?/o hПݗ7Q€%Bv_Gߦ hO~(Z*/'eM}?/o=&mZs:Yrl;}*_'eM-ߦ >Пݗ7PT_hO~(Bv_@Q}?/o KEE'eM-ߦ >Пݗ7PT_hO~(Bv_@Q}?/o KEE'eM7}\KUZtTgp,}_'eM_H%{GTX״QE5"32qCvQU=2u4d4As6vus3;m4.[)zC\TsEh-:Fg8Td9iƋH%#ѓ9/QY>ux "daq!(PNX*M{eycqjuFٕ0*=x8 +3JWR{yl/ []lAà Pk M2RUKA74 lsP7HO=@o,Hb]HeD;$X"[K8.+fubefp%da#;wPä3VXQ_P!0NHEƕqoMM)]]ǤͿdxV,J;v[6v.?uXGҵoʹCDԵo}jzem%Vs #:׵KHҗTzTq³E ;98^u\BmRՖ="3E122Hu;{VhѼ $8 eQgmh+þuLlQBőv9.|O6,6#dklL *b2^vîk[E, #Ue I+ԌySa%.&<۶yb{{· M_ZI.S:P+ǂF4_xB:Mi!.w{BUp"v8T5 uks鐤Z\#,W[k@7642\K=sĈc+(f f#*#⣋kkZ $WrcDk K[i_wa}uuMGTf;xqjujIugokKvSB،l x=*wiO[7F%hUC F'zTkOm^Kx [K u# > mfcK[LaHo+{[%1+oq].K & ߕV`e+vޭ&%Bu+*IijcZJe[E Tܞh9;}:m\>-oM?6 6~@$$.vc]ͿepcmWW}xwUY-tfs=ԑƬHc+qBW>oְsNmVaj ۰ASm;.2!Xp2x޸n-;[To7y[I+-otڝ%et#1.F6NsM gY3jY-$bfbFLuˍPLnᲕXD˅29j;mfMjO5f2CuƱ3F  6q2V? CuZ7Jٶ)c-F`UsuuqaB/ e+)[s"׊,n:JOy$ 6s L); |^"xœl |1ZTA ?YO Feka? BIߴݘnt1h=&m#e zT0ϩ˲w9R>u6Uj&xP-i-ҳHA;R~=OJ9VhڡXgq'ӎkhZڭZǵFh&##PFI^Uot߱j9-7d'lj62c01UasE{WXwξI2S7g5Z;_%Oņ׵do삶-v-)LF_7ݻC.xukhד垢!7ry BʬI%zO*y7_5wdb[v3,[czi Si%ܪcgVxT %wH&EoC=ΐlpNE#ܘث#(_{⋛]jX>`nٮ9b6nIP c5JQNIs%ŵƝ V]\̍wdHƵ-gbx5 w*_ZJ]7m/%Z (c!Ib$ge`* oo5hdT gv 9+F.J̻u5 3H{Qd,H BFU+85wp{w}4Cj#v''i=??آ((((&5j&5hN5 [^³۝:r$l<2sVUcoZ۠׍xb򕺄ޞhWhZ[UcumjOۏs۷f6q<#}_M4~gw_;RjwY)hvؖe:呞& a2:- Mv׎5CљqU`qZ(ro)ioOQEHC_^\zNo̪VxhSˏ2!Rv_j/gM[Yⰺ- 3:ItWSk5U- E]=;Uӵ_[^TWQJD/mfկ5u}.TMsl|- ˝n g5+ioi:k2xH~ܻ1Ua*:n^-cO濬Ŭ` sm=́e1" x gqmo(ϩk*K-=e}b9O5B2J#]7[m,8r=&mnOiM{mxF1U 㪂]@'EKvFnEnImd\rT'?RVyg%k,ˆ`88cz\o8u? \A BiA$.c`?ږip1^[s#W@MZ޿_>,٭X論_i+uku<b+.~חK{Dp]J;EDRS-)'ԚQ=Ynn(oe/O3w-R! ˵Q?0_\}jl˛f.VEy2Xw GɁ]zu * QE ( ("oZGq,6A K6&Y'|j4i4!+Gydw<>%֣ D/ y_æٽ#*$,{u8RMz4;]Q n6AB*gǩ$f3R4 Y[= XI$K $ `3Vcڷ<)u|B ;! s b 19ioC;6 vK)<7J*YXsI5;/uTKfR3. : Wij8' JC~-/gWLoM[lIu I2q\.Dʰ8DH@RĖm3)Q:a?v3wC;R[Ʃn#M727!FO!E{7m~qj7P;]GSⰼsm?2oUP"#y;UWJmPZj4\Z^r$2GLrL0 ;w_+cSķ/.͝6pb<޺K15ݣ(uBHc#ROA^E_QaER+1h.T0ē9Fr1<.oʸ.oXM̻t4mwwW2eqV~-ೃN.14d%[rNxD,Qɧ^5RILMm2zV4(5$6miOm.lwIf^ʘazl+nή]pVn2(5+i < ӽD1Cu,Q"v9{e^3QqҦ:TqJ ±e[wlIXw: X/>B|Aq<?`n?)9u/5mgyjiZnf춷SM'e̞OѵY.Gu/*bVI%[)4 vWj;q|Wzյ֕; uK*T@NOט6"U= Xm2G,0v!X۬2}܇P`6z}haj_ok>7)4Yo"u&mݸzW4 ]G[+=Ԁ3@QՃ,7R{{I!単H'Nσn<]B<: +OC2<.(t)o?a-T^\[gNȆ6mͳn;Wp|Hڞ/"ӯ&@-P4B@Ii+)w\Eqmmceuww*Irl<,8 Q(Ŵ,F4ΣN֌z7֥,mmn76hN-*PAo֥>o֥ %gƱđx gޮm߳U}#@*Ez~Fۏ?UKEEz~Fۏ?UKEEz~Fۏ?UKEUY$ }>.ۏ?Us_R[n?_Tm߳TP[n?_Tm߳TP[n?_Tm߳TP[n?_Tm߳TP[n?_Tm߳TP[n?_Tm߳TP[fHrgR߳Qu6U-Ez~L 'M$]9 P(t6EibK}s׀qNhOH<&+8op8vw` ޤpN8 m*"q=bgm*"q=bgm*"q=bgm**yv6}Wޙ%y܆#=alo67qV>o֥Xq=bghX⪯\[.OنfP[n?_Tm߳TP[n?_Tm߳TP[n?_Tm߳TP[fHrgR߳Qu6U-Ez~Fۏ?UKETkM4[1%=*ׇn.2[JPp:z ֢+=,rH幇%r0psE)C2LTRs~BQ@Y:;IQzTV>ji֋anШyl34iF3'#_EQEEk5U-Ek5U-QEQEQEQEQEs ӬIt謭㕞v/bwukHìQnHwXc9 N:*֡c2Z2XP$w+bU} gcḴa,EHn9{>@8RʧG?H Zj1m^j&4g ]N8;;On,#=7Zk+J.ʤ[8|6$QE!Q@Q@Q@ AO#}ROM-W^UKdcGl$B吡h;$ҺEeX`j4Zve01K܅BB&2 !%GA }|ImbKT-f$*D|;Tg{18'ry 厣,ZՍ퉶{h/o6xKԗwgkn8?}p2698R ioF]m"O.o^Khx#c#-;5]GsXȑH]URGҭS{ (Š((vSu.rnҼV€N]rO^x9lw@=p3S:.%uAΟޗT_gOK[h:z_@Q}?/o_7soR[ZA=dcR?/o hΟޗtƀ%:z_GgOK[hZ*/4}?/o hΟޗtƀ%:z_G>o֥}19F/gOK[hZ*/4}?/o hΟޗtƀ%:z_GgOK[hI*b !#ZKEEtƏ4->ΟޗT_gOK[h:z_@Q}?/oKEEtƏ4->ΟޗT_gOK[h:z_@Q}?/o q7kRV>̜F_zpangu_2_6b/README.md000066400000000000000000000073261432571244600142770ustar00rootroot00000000000000# 快速使用transformers构建鹏程·盘古α 2.6B模型 非常感谢一名开发者将鹏程·盘古α 2.6B模型迁移至Huggingface Transformers库 [huggingface.co/imone/pangu_2_6B](https://huggingface.co/imone/pangu_2_6B) ，同步我们也将此项目迁移到启智社区盘古项目下，方便用户使用。 ## Model Description PanGu-α is proposed by a joint technical team headed by PCNL. It is the first large-scale Chinese pre-trained language model with 200 billion parameters trained on 2048 Ascend processors using an automatic hybrid parallel training strategy. The whole training process is done on the “Peng Cheng Cloud Brain II” computing platform with the domestic deep learning framework called MindSpore. The PengCheng·PanGu-α pre-training model can support rich applications, has strong few-shot learning capabilities, and has outstanding performance in text generation tasks such as knowledge question and answer, knowledge retrieval, knowledge reasoning, and reading comprehension. This repository contains PyTorch implementation of PanGu model, with 2.6 billion parameters pretrained weights (FP32 precision), converted from original MindSpore checkpoint. ## 快速使用 ## finetune ### 环境 (说明：因为transformers库暂不支持模型并行训练，故使用V100会显存溢出，您可以使用A100单卡进行微调) 可以直接使用现成的docker镜像 ``` docker pull nvcr.io/nvidia/pytorch:21.03-py3 ``` ### 依赖 ``` pip install transformers=='4.13.0' pip install jieba pip install sentencepiece==0.1.94 ``` ### 运行 ``` python ./pangu_2_6B/pangu_train.py ``` ## CPU推理 ### 环境 可以直接使用现成的docker镜像 ``` docker pull nvcr.io/nvidia/pytorch:21.03-py3 ``` ### 依赖 ``` pip install transformers=='4.22.0' pip install jieba pip install sentencepiece==0.1.94 ``` ### 运行 ``` python ./pangu_2_6B/pangu_inference_cpu.py或 使用jit模式 python ./pangu_2_6B/pangu_inference_cpu.py --no_cuda --use_ipex --jit_mode_eval ``` ## GPU推理（fp16模式下） ### 环境(nvidia V100/A100单卡) 可以直接使用现成的docker镜像 ``` docker pull nvcr.io/nvidia/pytorch:21.03-py3 ``` ### 依赖 ``` pip install transformers=='4.22.0' pip install jieba pip install sentencepiece==0.1.94 ``` ### 运行 ``` python ./pangu_2_6B/pangu_inference.py ``` ## GPU推理（int8模式下） ### 环境(nvidia A100单卡) 可以直接使用现成的docker镜像 ``` docker pull nvcr.io/nvidia/pytorch:22.03-py3 ``` ### 依赖 ``` pip install transformers=='4.22.0' pip install jieba pip install sentencepiece==0.1.94 pip install accelerate>=0.12.0 pip install bitsandbytes>=0.31.5 ``` ### 运行 ``` python ./pangu_2_6B/pangu_inference_int8.py ``` ## 性能对比结果

## Usage (Text Generation) Currently PanGu model is not supported by transformers, so `trust_remote_code=True` is required to load model implementation in this repo. ```python from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("imone/pangu_2_6B", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("imone/pangu_2_6B", trust_remote_code=True) text_generator = TextGenerationPipeline(model, tokenizer) # greedy search print(text_generator("中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？", max_length=50)) ``` Expected output: ```python [{'generated_text': '中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？\n中国北京,美国华盛顿,日本东京,法国巴黎,加拿大多伦多,澳大利亚悉尼,新西兰奥克兰,澳大利亚墨尔本,新西兰奥克兰,'}] ``` pangu_2_6b/config.json000066400000000000000000000017521432571244600151550ustar00rootroot00000000000000{ "activation_function": "gelu", "architectures": [ "GPTPanguForCausalLM" ], "attn_pdrop": 0.1, "bos_token_id": 9, "embd_pdrop": 0.1, "eos_token_id": 9, "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": null, "layer_norm_epsilon": 1e-05, "max_position_embeddings": 1024, "model_type": "gpt_pangu", "num_heads": 32, "num_layers": 32, "resid_pdrop": 0.1, "scale_attn_weights": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "summary_type": "cls_index", "summary_usepip uninstall sen _proj": true, "torch_dtype": "float16", "transformers_version": "4.22.2", "use_cache": true, "fp16": true, "vocab_size": 40000, "tokenizer_class": "GPTPanguTokenizer", "auto_map": { "AutoConfig": "configuration_gptpangu.GPTPanguConfig", "AutoTokenizer": ["tokenization_gptpangu.GPTPanguTokenizer", null], "AutoModelForCausalLM": "modeling_gptpangu.GPTPanguForCausalLM" } } pangu_2_6b/configuration_gptpangu.py000066400000000000000000000035771432571244600201520ustar00rootroot00000000000000from transformers.configuration_utils import PretrainedConfig class GPTPanguConfig(PretrainedConfig): model_type = "gpt_pangu" keys_to_ignore_at_inference = ["past_key_values"] def __init__( self, vocab_size=40000, max_position_embeddings=1024, hidden_size=2560, intermediate_size=None, num_layers=32, num_heads=32, activation_function="gelu", resid_pdrop=0.1, embd_pdrop=0.1, attn_pdrop=0.1, layer_norm_epsilon=1e-5, scale_attn_weights=True, initializer_range=0.02, summary_type="cls_index", summary_use_proj=True, summary_activation=None, summary_proj_to_labels=True, summary_first_dropout=0.1, use_cache=True, bos_token_id=9, eos_token_id=9, **kwargs, ): self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_layers = num_layers self.num_heads = num_heads self.activation_function = activation_function self.resid_pdrop = resid_pdrop self.embd_pdrop = embd_pdrop self.attn_pdrop = attn_pdrop self.layer_norm_epsilon = layer_norm_epsilon self.scale_attn_weights = scale_attn_weights self.initializer_range = initializer_range self.summary_type = summary_type self.summary_use_proj = summary_use_proj self.summary_activation = summary_activation self.summary_first_dropout = summary_first_dropout self.summary_proj_to_labels = summary_proj_to_labels self.use_cache = use_cache self.bos_token_id = bos_token_id self.eos_token_id = eos_token_id super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) pangu_2_6b/modeling_gptpangu.py000066400000000000000000000537351432571244600171020ustar00rootroot00000000000000"""PyTorch PanguAlpha GPT2 Model""" # from .configuration_gptpangu import GPTPanguConfig from typing import Tuple import math import torch from torch import nn from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from transformers.utils import logging logger = logging.get_logger(__name__) class GPTPanguAttention(nn.Module): def __init__(self, config): super().__init__() max_positions = config.max_position_embeddings self.register_buffer( "bias", torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view( 1, 1, max_positions, max_positions ), ) self.register_buffer("masked_bias", torch.tensor(-1e4)) self.embed_dim = config.hidden_size self.num_heads = config.num_heads self.head_dim = self.embed_dim // self.num_heads if self.head_dim * self.num_heads != self.embed_dim: raise ValueError( f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." ) self.scale_attn_weights = config.scale_attn_weights self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True) self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True) self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True) self.c_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True) self.attn_dropout = nn.Dropout(config.attn_pdrop) self.resid_dropout = nn.Dropout(config.resid_pdrop) def _attn(self, query, key, value, attention_mask=None, head_mask=None): attn_weights = torch.matmul(query, key.transpose(-1, -2)) if self.scale_attn_weights: attn_weights = attn_weights / (float(value.size(-1)) ** 0.5) query_length, key_length = query.size(-2), key.size(-2) causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length].bool() attn_weights = torch.where(causal_mask, attn_weights, self.masked_bias.to(attn_weights.dtype)) if attention_mask is not None: # Apply the attention mask attn_weights = attn_weights + attention_mask attn_weights = nn.functional.softmax(attn_weights, dim=-1) # Downcast (if necessary) back to V's dtype (if in mixed-precision) -- No-Op otherwise attn_weights = attn_weights.type(value.dtype) attn_weights = self.attn_dropout(attn_weights) # Mask heads if we want to if head_mask is not None: attn_weights = attn_weights * head_mask attn_output = torch.matmul(attn_weights, value) return attn_output, attn_weights def _split_heads(self, tensor, num_heads, attn_head_size): """ Splits hidden_size dim into attn_head_size and num_heads """ new_shape = tensor.size()[:-1] + (num_heads, attn_head_size) tensor = tensor.view(*new_shape) return tensor.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) def _merge_heads(self, tensor, num_heads, attn_head_size): """ Merges attn_head_size dim and num_attn_heads dim into hidden_size """ tensor = tensor.permute(0, 2, 1, 3).contiguous() new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,) return tensor.view(new_shape) def forward( self, hidden_states, layer_past=None, attention_mask=None, head_mask=None, custom_query=None, use_cache=False, output_attentions=False, ): query = self.q_proj(custom_query) if custom_query is not None else self.q_proj(hidden_states) key = self.k_proj(hidden_states) value = self.v_proj(hidden_states) query = self._split_heads(query, self.num_heads, self.head_dim) key = self._split_heads(key, self.num_heads, self.head_dim) value = self._split_heads(value, self.num_heads, self.head_dim) if layer_past is not None: past_key, past_value = layer_past key = torch.cat((past_key, key), dim=-2) value = torch.cat((past_value, value), dim=-2) if use_cache is True: present = (key, value) else: present = None attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask) attn_output = self._merge_heads(attn_output, self.num_heads, self.head_dim) attn_output = self.c_proj(attn_output) attn_output = self.resid_dropout(attn_output) outputs = (attn_output, present) if output_attentions: outputs += (attn_weights,) return outputs # a, present, (attentions) class GPTPanguMLP(nn.Module): def __init__(self, intermediate_size, config): # in MLP: intermediate_size= 4 * hidden_size super().__init__() embed_dim = config.hidden_size self.c_fc = nn.Linear(embed_dim, intermediate_size) self.c_proj = nn.Linear(intermediate_size, embed_dim) self.act = ACT2FN[config.activation_function] self.dropout = nn.Dropout(config.resid_pdrop) def forward(self, hidden_states): hidden_states = self.c_fc(hidden_states) hidden_states = self.act(hidden_states) hidden_states = self.c_proj(hidden_states) hidden_states = self.dropout(hidden_states) return hidden_states class GPTPanguBlock(nn.Module): def __init__(self, config): super().__init__() hidden_size = config.hidden_size inner_dim = config.intermediate_size if config.intermediate_size is not None else 4 * hidden_size self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) self.attn = GPTPanguAttention(config) self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) self.mlp = GPTPanguMLP(inner_dim, config) def forward( self, hidden_states, layer_past=None, attention_mask=None, head_mask=None, custom_query=None, use_cache=False, output_attentions=False, ): residual = hidden_states hidden_states = self.ln_1(hidden_states) attn_outputs = self.attn( hidden_states, layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask, custom_query=custom_query, use_cache=use_cache, output_attentions=output_attentions, ) attn_output = attn_outputs[0] # output_attn: a, present, (attentions) outputs = attn_outputs[1:] # residual connection hidden_states = attn_output + residual residual = hidden_states hidden_states = self.ln_2(hidden_states) feed_forward_hidden_states = self.mlp(hidden_states) # residual connection hidden_states = residual + feed_forward_hidden_states if use_cache: outputs = (hidden_states,) + outputs else: outputs = (hidden_states,) + outputs[1:] return outputs # hidden_states, present, (attentions, cross_attentions) class GPTPanguPreTrainedModel(PreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. """ # config_class = GPTPanguConfig base_model_prefix = "transformer" supports_gradient_checkpointing = True # config_class = GPT2Config # load_tf_weights = load_tf_weights_in_gpt2 # base_model_prefix = "transformer" # is_parallelizable = True # supports_gradient_checkpointing = True _no_split_modules = ["GPT2Block"] def __init__(self, *inputs, **kwargs): super().__init__(*inputs, **kwargs) def _init_weights(self, module): """Initialize the weights.""" if isinstance(module, (nn.Linear,)): # Slightly different from the TF version which uses truncated_normal for initialization # cf https://github.com/pytorch/pytorch/pull/5617 module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) if module.bias is not None: module.bias.data.zero_() elif isinstance(module, nn.Embedding): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme: # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers. # > -- GPT-2 :: https://openai.com/blog/better-language-models/ # # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py for name, p in module.named_parameters(): if "c_proj" in name and "weight" in name: # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block p.data.normal_(mean=0.0, std=(self.config.initializer_range / math.sqrt(2 * self.config.num_layers))) def _set_gradient_checkpointing(self, module, value=False): if isinstance(module, GPTPanguModel): module.gradient_checkpointing = value class GPTPanguModel(GPTPanguPreTrainedModel): def __init__(self, config): super().__init__(config) self.embed_dim = config.hidden_size # self.wte = self.wte.to(self.first_device) # self.wpe = self.wpe.to(self.first_device) self.wte = nn.Embedding(config.vocab_size, self.embed_dim) self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim) self.wqe = nn.Embedding(config.max_position_embeddings, self.embed_dim) self.drop = nn.Dropout(config.embd_pdrop) self.h = nn.ModuleList([GPTPanguBlock(config) for _ in range(config.num_layers)]) self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) self.gradient_checkpointing = False # Initialize weights and apply final processing self.post_init() def get_input_embeddings(self): return self.wte def set_input_embeddings(self, new_embeddings): self.wte = new_embeddings def forward( self, input_ids=None, past_key_values=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None, ): output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: input_shape = input_ids.size() input_ids = input_ids.view(-1, input_shape[-1]) batch_size = input_ids.shape[0] elif inputs_embeds is not None: input_shape = inputs_embeds.size()[:-1] batch_size = inputs_embeds.shape[0] else: raise ValueError("You have to specify either input_ids or inputs_embeds") device = input_ids.device if input_ids is not None else inputs_embeds.device if token_type_ids is not None: token_type_ids = token_type_ids.view(-1, input_shape[-1]) if position_ids is not None: position_ids = position_ids.view(-1, input_shape[-1]) if past_key_values is None: past_length = 0 past_key_values = tuple([None] * len(self.h)) else: past_length = past_key_values[0][0].size(-2) if position_ids is None: position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) # GPT2Attention mask. if attention_mask is not None: if batch_size <= 0: raise ValueError("batch_size has to be defined and > 0") attention_mask = attention_mask.view(batch_size, -1) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] # this attention mask is more simple than the triangular masking of causal attention # used in OpenAI GPT, we just need to prepare the broadcast dimension here. attention_mask = attention_mask[:, None, None, :] # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for # positions we want to attend and -10000.0 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility attention_mask = (1.0 - attention_mask) * -10000.0 # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head # attention_probs has shape bsz x num_heads x N x N # head_mask has shape n_layer x batch x num_heads x N x N head_mask = self.get_head_mask(head_mask, self.config.num_layers) if inputs_embeds is None: inputs_embeds = self.wte(input_ids) position_embeds = self.wpe(position_ids) hidden_states = inputs_embeds + position_embeds if token_type_ids is not None: token_type_embeds = self.wte(token_type_ids) hidden_states = hidden_states + token_type_embeds hidden_states = self.drop(hidden_states) output_shape = input_shape + (hidden_states.size(-1),) # top attention custom query last_layer_id = len(self.h) - 1 query_embeds = self.wqe(position_ids) presents = () if use_cache else None all_self_attentions = () if output_attentions else None all_hidden_states = () if output_hidden_states else None for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): # Final LayerNorm before last query layer if i == last_layer_id: hidden_states = self.ln_f(hidden_states) if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: if use_cache: logger.warning( "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." ) use_cache = False def create_custom_forward(module): def custom_forward(*inputs): # None for past_key_value return module(*inputs, use_cache, output_attentions) return custom_forward outputs = torch.utils.checkpoint.checkpoint( create_custom_forward(block), hidden_states=hidden_states, layer_past=None, attention_mask=attention_mask, head_mask=head_mask[i], # custom query custom_query=query_embeds if i == last_layer_id else None, ) else: outputs = block( hidden_states, layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask[i], # custom query custom_query=query_embeds if i == last_layer_id else None, use_cache=use_cache, output_attentions=output_attentions, ) hidden_states = outputs[0] if use_cache is True: presents = presents + (outputs[1],) if output_attentions: all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],) hidden_states = hidden_states.view(*output_shape) # Add last hidden state if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) if not return_dict: return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None) return BaseModelOutputWithPast( last_hidden_state=hidden_states, past_key_values=presents, hidden_states=all_hidden_states, attentions=all_self_attentions, ) class GPTPanguForCausalLM(GPTPanguPreTrainedModel): def __init__(self, config): super().__init__(config) self.transformer = GPTPanguModel(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) # Initialize weights and apply final processing self.post_init() def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs): token_type_ids = kwargs.get("token_type_ids", None) # only last token for inputs_ids if past is defined in kwargs if past: input_ids = input_ids[:, -1].unsqueeze(-1) if token_type_ids is not None: token_type_ids = token_type_ids[:, -1].unsqueeze(-1) attention_mask = kwargs.get("attention_mask", None) position_ids = kwargs.get("position_ids", None) if attention_mask is not None and position_ids is None: # create position_ids on the fly for batch generation position_ids = attention_mask.long().cumsum(-1) - 1 position_ids.masked_fill_(attention_mask == 0, 1) if past: position_ids = position_ids[:, -1].unsqueeze(-1) else: position_ids = None return { "input_ids": input_ids, "past_key_values": past, "use_cache": kwargs.get("use_cache"), "position_ids": position_ids, "attention_mask": attention_mask, "token_type_ids": token_type_ids, } def forward( self, input_ids=None, past_key_values=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None, ): r""" labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set ``labels = input_ids`` Indices are selected in ``[-100, 0, ..., config.vocab_size]`` All labels set to ``-100`` are ignored (masked), the loss is only computed for labels in ``[0, ..., config.vocab_size]`` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict transformer_outputs = self.transformer( input_ids, past_key_values=past_key_values, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, ) hidden_states = transformer_outputs[0] lm_logits = self.lm_head(hidden_states) loss = None if labels is not None: # Shift so that tokens < n predict n shift_logits = lm_logits[..., :-1, :].contiguous() shift_labels = labels[..., 1:].contiguous() # Flatten the tokens loss_fct = nn.CrossEntropyLoss() loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) if not return_dict: output = (lm_logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output return CausalLMOutputWithPast( loss=loss, logits=lm_logits, past_key_values=transformer_outputs.past_key_values, hidden_states=transformer_outputs.hidden_states, attentions=transformer_outputs.attentions, ) @staticmethod def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]: """ This function is used to re-order the :obj:`past_key_values` cache if :meth:`~transformers.PreTrainedModel.beam_search` or :meth:`~transformers.PreTrainedModel.beam_sample` is called. This is required to match :obj:`past_key_values` with the correct beam_idx at every generation step. """ return tuple( tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) for layer_past in past ) pangu_2_6b/pangu_inference.py000066400000000000000000000022221432571244600165100ustar00rootroot00000000000000import numpy as np from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM #from transformers.utils import is_accelerate_available, is_bitsandbytes_available import datetime import torch device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") tokenizer = AutoTokenizer.from_pretrained("./pangu_2_6B", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("./pangu_2_6B", from_tf=False,trust_remote_code=True, torch_dtype=torch.bfloat16) model.half().eval().cuda() print(type(tokenizer)) print(model.device) #text_generator = TextGenerationPipeline(model, tokenizer) # greedy search Start_time = datetime.datetime.now() question="中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？" input_ids = tokenizer.encode(question, return_tensors="pt") output = model.generate(input_ids.cuda(), max_length=50) # print(type(output)) # print(type(tokenizer.decode(output[0]))) # print(tokenizer.decode(output[0])) # print(len(tokenizer.tokenize(''.join(tokenizer.decode(output[0]))))) end_time = datetime.datetime.now() cost_time=(end_time-Start_time) print(f"cost time:{cost_time}")pangu_2_6b/pangu_inference_cpu.py000066400000000000000000000021221432571244600173560ustar00rootroot00000000000000from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM #from transformers.utils import is_accelerate_available, is_bitsandbytes_available import datetime tokenizer = AutoTokenizer.from_pretrained("./pangu_2_6B", trust_remote_code=True) model_8bit = AutoModelForCausalLM.from_pretrained("./pangu_2_6B", from_tf=False,trust_remote_code=True) text_generator = TextGenerationPipeline(model_8bit, tokenizer) # greedy search Start_time = datetime.datetime.now() print(text_generator("中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？", max_length=50)) #output = text_generator("中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？", max_length=50) #tokens = tokenizer.tokenize(line) #print(type(output)) #print(type(output[0])) #print(list(output[0].values())[0]) #print(tokenizer.tokenize(''.join(list(output[0].values())[0]))) #print(len(tokenizer.tokenize(''.join(list(output[0].values())[0])))) end_time = datetime.datetime.now() cost_time=(end_time-Start_time).seconds print(f"cost time:{cost_time}")pangu_2_6b/pangu_inference_gpu_fp32.py000066400000000000000000000022131432571244600202150ustar00rootroot00000000000000import numpy as np from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM #from transformers.utils import is_accelerate_available, is_bitsandbytes_available import datetime import torch device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") tokenizer = AutoTokenizer.from_pretrained("./pangu_2_6B", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("./pangu_2_6B", from_tf=False,trust_remote_code=True, torch_dtype=torch.bfloat16) model.eval().cuda() print(type(tokenizer)) print(model.device) #text_generator = TextGenerationPipeline(model, tokenizer) # greedy search Start_time = datetime.datetime.now() question="中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？" input_ids = tokenizer.encode(question, return_tensors="pt") output = model.generate(input_ids.cuda(), max_length=50) # print(type(output)) # print(type(tokenizer.decode(output[0]))) # print(tokenizer.decode(output[0])) # print(len(tokenizer.tokenize(''.join(tokenizer.decode(output[0]))))) end_time = datetime.datetime.now() cost_time=(end_time-Start_time) print(f"cost time:{cost_time}")pangu_2_6b/pangu_inference_int8.py000066400000000000000000000022701432571244600174550ustar00rootroot00000000000000 import numpy as np from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM #from transformers.utils import is_accelerate_available, is_bitsandbytes_available import datetime import torch device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") tokenizer = AutoTokenizer.from_pretrained("./pangu_2_6B", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("./pangu_2_6B",device_map="auto", load_in_8bit=True, from_tf=False,trust_remote_code=True, torch_dtype=torch.bfloat16) model.half().eval().cuda() print(type(tokenizer)) print(model.device) text_generator = TextGenerationPipeline(model, tokenizer) # greedy search Start_time = datetime.datetime.now() question="中国和美国和日本和法国和加拿大和澳大利亚的首都分别是哪里？" input_ids = tokenizer.encode(question, return_tensors="pt") output = model.generate(input_ids.cuda(), max_length=50) print(type(output)) print(type(tokenizer.decode(output[0]))) print(tokenizer.decode(output[0])) print(len(tokenizer.tokenize(''.join(tokenizer.decode(output[0]))))) end_time = datetime.datetime.now() cost_time=(end_time-Start_time).seconds print(f"cost time:{cost_time}") pangu_2_6b/pangu_train.py000066400000000000000000000032121432571244600156670ustar00rootroot00000000000000from transformers import TextGenerationPipeline, AutoTokenizer, AutoModelForCausalLM from transformers import TrainingArguments, Trainer from datasets import load_dataset #from accelerate import Accelerator from pynvml import * import numpy as np from datasets import Dataset # def print_gpu_utilization(): # nvmlInit() # handle = nvmlDeviceGetHandleByIndex(0) # info = nvmlDeviceGetMemoryInfo(handle) # print(f"GPU memory occupied: {info.used//1024**2} MB.") def print_summary(result): print(f"Time: {result.metrics['train_runtime']:.2f}") print(f"Samples/second: {result.metrics['train_samples_per_second']:.2f}") #print_gpu_utilization() default_args = { "output_dir": "tmp", "evaluation_strategy": "steps", "num_train_epochs": 1, "log_level": "error", "report_to": "none", } seq_len, dataset_size = 1024, 1024 # dummy_data = { # "input_ids": np.random.randint(100, 30000, (dataset_size, seq_len)), # "labels": np.random.randint(0, 1, (dataset_size)), # } np.random.seed(42) x = np.random.randint(1,1000,size=(seq_len,)) y = np.random.randint(0,1, size=(seq_len,)) ds = Dataset.from_dict({"input_ids": x, "labels": y}) tokenizer = AutoTokenizer.from_pretrained("./pangu_2_6B", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("./pangu_2_6B", from_tf=False,trust_remote_code=True) #text_generator = TextGenerationPipeline(model, tokenizer) training_args = TrainingArguments(per_device_train_batch_size=1, gradient_accumulation_steps=4, **default_args) trainer = Trainer(model=model, args=training_args, train_dataset=ds) result = trainer.train() print_summary(result) # greedy search pangu_2_6b/pytorch_model.bin000066400000000000000000000002001432571244600163420ustar00rootroot00000000000000version https://git-lfs.github.com/spec/v1 oid sha256:2ceede6a5eabc855ec723f43feca2682dff7ee3e2144882475f264f926e3a419 size 139 pangu_2_6b/src/000077500000000000000000000000001432571244600135775ustar00rootroot00000000000000pangu_2_6b/src/dataset.py000066400000000000000000000132411432571244600155770ustar00rootroot00000000000000# Copyright 2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """ Create dataset for training and evaluating """ import os import numpy as np import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C import mindspore.common.dtype as mstype def get_input_data_batch_slice_map(input_ids, eod_id, rank, dis, eod_reset): """ Generate position_id and attention_mask according to input_ids considering eod reset Inputs: input_ids: the input token ids eod_id: the id for rank: the current rank dis: the slice value for each rank eod_reset: whether to open eod reset or not returns: input_ids: the input token ids position_id: the position ids cosidering eod reset attention_mask: the attention mask considering eod reset """ rank = int(rank) input_ids = input_ids[rank * dis: (rank + 1) * dis] if not eod_reset: return input_ids seq_length = input_ids.shape[1] - 1 # Initialize position_ids and attention_mask batch_input_ids = input_ids batch_position_ids = np.ones((dis, seq_length)) batch_attention_mask = np.ones((dis, seq_length, seq_length)) # Loop through batches for bs_i, _ in enumerate(range(len(input_ids))): # Get normal position_ids and attention_mask local_ids = input_ids[bs_i] batch_attention_mask[bs_i] = np.tril(np.ones(shape=(seq_length, seq_length))) batch_position_ids[bs_i] = np.arange(seq_length) # Find eod_of_document eod_index = batch_position_ids[bs_i, local_ids[:-1] == eod_id].astype(np.int32) prev_index = 0 for i in range(eod_index.size): # Reset position_ids and attention_mask considering EOD index = eod_index[i] batch_attention_mask[bs_i, (index + 1):, :(index + 1)] = 0 batch_position_ids[bs_i, (index + 1):] -= (index + 1 - prev_index) prev_index = index + 1 return batch_input_ids, batch_position_ids, batch_attention_mask def create_dataset(batch_size, data_path, device_num=1, rank=0, drop=True, full_batch=False, data_start_index=0, eod_reset=False, eod_id=9, column_name='input_ids', epoch=1, num_samples=None): """ Create dataset Inputs: batch_size: batch size data_path: path of your MindRecord files device_num: total device number rank: current rank id drop: whether drop remainder eod_reset: whether enable position reset and attention mask reset eod_id: the id for column_name: the column name of the mindrecord file. Default is input_ids epoch: The repeat times of the dataset Returns: dataset_restore: the dataset for training or evaluating """ ds.config.set_seed(1) # Control the size of data queue in the consideration of the memory ds.config.set_prefetch_size(1) # Get path for source data files home_path = os.path.join(os.getcwd(), data_path) files = os.listdir(data_path) data = [ os.path.join(home_path, name) for name in files if not name.endswith(".db") ] # Ensure the order of mindrecords is same in all machines, otherwise it will meet loss converge problem. data.sort() # Load data files and preprocess dataset = ds.MindDataset(data[data_start_index:], columns_list=[column_name], shuffle=False, num_samples=num_samples) type_cast_op = C.TypeCast(mstype.int32) type_cast_op_float = C.TypeCast(mstype.float16) if full_batch: # no need to slice from the inputs rank = 0 dis = batch_size else: # Each card slice a small batch from the full batch dis = int(batch_size / device_num) if batch_size % device_num != 0: raise ValueError( f"batch size {batch_size} should be a multiple of device number {device_num}." " You should change the args: per_batch_size.") map_func = (lambda input_ids: get_input_data_batch_slice_map(input_ids, eod_id, rank, dis, eod_reset)) # If eod_reset enabled, another two inputs will be generated through input_ids if eod_reset: dataset = dataset.batch(batch_size, drop_remainder=drop) dataset = dataset.map(operations=map_func, input_columns=[column_name], output_columns=[column_name, "position_id", "attention_mask"], column_order=[column_name, "position_id", "attention_mask"]) dataset = dataset.map(input_columns="position_id", operations=type_cast_op) dataset = dataset.map(input_columns="attention_mask", operations=type_cast_op_float) else: dataset = dataset.map(input_columns=[column_name], operations=type_cast_op) dataset = dataset.batch(batch_size, drop_remainder=drop) dataset = dataset.map(operations=map_func, input_columns=[column_name], output_columns=[column_name]) dataset = dataset.map(input_columns=column_name, operations=type_cast_op) dataset = dataset.repeat(epoch) return dataset pangu_2_6b/tokenization_gptpangu.py000066400000000000000000000025721432571244600200130ustar00rootroot00000000000000from transformers.tokenization_utils import PreTrainedTokenizer import torch import sentencepiece import jieba class GPTPanguTokenizer(PreTrainedTokenizer): # Ref: https://git.openi.org.cn/PCL-Platform.Intelligence/PanGu-Alpha/src/branch/master/tokenization_jieba.py vocab_files_names = { "model_file": "vocab.model" } def __init__( self, model_file, **kwargs ): super().__init__() self.sp = sentencepiece.SentencePieceProcessor() print(f"model_file:{model_file}") self.sp.Load(model_file=model_file) self.translator = str.maketrans(" \n", "\u2582\u2583") # special token ids self.eos_token_id = self.sp.piece_to_id("") def tokenize(self, text, **kwargs): """ Tokenize a string. """ seg_list = [x.translate(self.translator) for x in jieba.cut(text, cut_all=False)] new_seg = " ".join(seg_list) return self.sp.encode(new_seg) def convert_tokens_to_ids(self, tokens): return tokens def convert_ids_to_tokens(self, ids): return self.decode(ids) def decode(self, tokens, **kwargs): if isinstance(tokens, torch.Tensor): tokens = tokens.tolist() text = self.sp.decode(tokens) text = text.replace(' ', '').replace('\u2582', ' ').replace('\u2583', '\n') return text pangu_2_6b/vocab.model000066400000000000000000000002001432571244600151140ustar00rootroot00000000000000version https://git-lfs.github.com/spec/v1 oid sha256:8902d47b66507db01e44d9d8b7b82e59b8eea3756b712dd1bcd0ca813263e623 size 134