#;7jUddlmZddlmZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZ ddlmZn#e$r dZded<YnwxYw dd lmZdd lmZdd lmZdd lmZd Zn)#e$r!dZdZded<dZded<dZded<dZded<YnwxYw ddlm Z n#e$r dZ ded<YnwxYwddl!Z!ddl"m#Z#ej$e%Z&GddZ'eGddeZ(eGdde(eZ)Gdde(Z*Gdd e)Z+Gd!d"e)Z,Gd#d$e)Z-Gd%d&e.eZ/d,d)Z0Gd*d+e)Z1dS)-) annotations)EnumN)Path)AnyCallableSequenceMappingIterableProtocolClassVarruntime_checkable)SentencePieceProcessorrr)MistralTokenizer) Tekkenizer)_filter_valid_tokenizer_files)SentencePieceTokenizerTFrrrr)get_one_valid_tokenizer_filer) GGUFWriterceZdZUded<ded<ded<ded<d ed <d ed < d+d,dZd-dZd.d/dZd0d Zd1d!Zd2d%Z d3d(Z d1d)Z d1d*Z d S)4 SpecialVocab list[str]mergeszdict[str, bool]add_special_tokendict[str, int]special_token_idsz(str | Sequence[Mapping[str, str]] | None chat_templatez bool | Nonenormalizer_lowercasenormalizer_strip_accentsFNpathstr | os.PathLike[str] load_mergesboolspecial_token_typesIterable[str] | Nonen_vocab int | Noneci|_i|_||_||_g|_d|_d|_d|_|||_nd|_| t|dS)N)boseosunkseppadclsmask) rrr&r"rrrrr$_loadr)selfr r"r$r&s 8/home/rurouni/src/llama.cpp-master/gguf-py/gguf/vocab.py__init__zSpecialVocab.__init__:sy "$!# & !$(!(,%  *':D $ $'YD $ 4::returnstrcpdt|j|jpd|jpdS)NzGunset)formatlenrrrr1s r2__repr__zSpecialVocab.__repr__Ms<X__   d4?AWAb[b   r4gwrquietNonec|jrM|s0tdt|jd||jn!|jrtd|jD]k\}}t|d|dd}|"td|d|d>|s td |d |||l|j D]k\}}t|d |d d}|"td |d|d>|s td|d|||l|j >|s"td|j | |j |j >|s"td|j ||j |j@|s"td|j||jdSdS)NzAdding z merge(s).zJAdding merges requested but no merges found, output may be non-functional.add_ _token_idz"No handler for special token type z with id - skippingzSetting special token type z to add_add__tokenzNo handler for add_z_token with value z Setting add_z _token to zSetting chat_template to z Setting normalizer_lowercase to z$Setting normalizer_strip_accents to )rloggerinfor:add_token_mergesr"warningritemsgetattrrradd_chat_templateradd_normalizer_lowercaseradd_normalizer_strip_accents)r1r=r>typtokid id_handlervalue add_handlers r2 add_to_ggufzSpecialVocab.add_to_ggufRs ; i D Bc$+&6&6BBBCCC    , , , ,   i NNg h h h06688  JC7>rCX#CXCXCXZ^7_7_J!dCddRWdddeee L J#JJ5JJKKK Ju    06688  JC9@E[PSE[E[E[]a9b9bK"^S^^E^^^___ C A3AA%AABBB K       ) N L8JLLMMM  !3 4 4 4  $ 0 \ Zt?XZZ[[[  ' '(A B B B  ( 4 d b4C`bbccc  + +D,I J J J J J 5 4r4rc|||||jr|js||dSdSdSN)_try_load_from_tokenizer_json_try_load_from_config_jsonr"r_try_load_merges_txt)r1r s r2r0zSpecialVocab._loadvsl **4000 ''---   ,DK ,  % %d + + + + + , , , ,r4cr|dz }|sdSt|dd5}t|d}|ds|dd}nd }g}|D]}|d z }|}|s|dd }t|d kr&t |j d |d m| |dd|d  dddn #1swxYwY||_ dS)Nz merges.txtFrutf-8encoding#rrz: Line z: Entry malformed, ignoring T) is_fileopennextstrip startswithseeksplitr:rFrInameappendr) r1r merges_filefp first_lineline_numrlinepartss r2rYz!SpecialVocab._try_load_merges_txt|s\) ""$$ 5 +sw 7 7 7 82b"++--J((--  F 8 8A zz|| 4++u::??NNk&6#d#dx#d#d#deee q66E!H667777 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8$ tsC*D%%D),D)rOtidrc t|tsdS|dkrtd|d||j ||jkr||jvrdS||j|<dSt d|d|d|jddS)Nrz%invalid value for special token type : zSpecial token type z, id z out of range, must be under rC) isinstanceint ValueErrorr&rrFrI)r1rOrss r2_set_special_tokenzSpecialVocab._set_special_tokens#s##  F 77QSQQCQQRR R < 3#5#5d,,,*-D "3 ' FrSrrsrrY]Yerrrsssssr4 normalizerdictc0|d}|dkr d|_dS|dkr d|_dS|dkr&d|vr |d|_d|vr|d|_dSdS|dkr.|d gD]}||dSdS) Ntype LowercaseT StripAccentsBertNormalizer lowercase strip_accentsr normalizers)getrr_parse_normalizer)r1rznormalizer_typenorms r2rzSpecialVocab._parse_normalizers%..00 k ) )(,D % % %  . .,0D ) ) )  0 0 0j((,6{,C)*,,0:?0K----,  * *"}b99 - -&&t,,,,+ * - -r4c&d}|dz }|rt|d5}tj|}dddn #1swxYwY|jr,|did}t |tr|rt |dtr||_ nt |dtrt|ddkrt |ddtretd|Dr:t d ttd d zd |D|_ nt!d |dx}r|||di}ni}d}|dz } | r=t| d5}tj|}dddn #1swxYwY|r|pid} |pid} |pid} |pid} | s | r |r| x|d<} | s | r |r| x|d<} |dx}r|d|gD]}|ddkrkd|jd<d|jd<d|jd<| s$|r"|d| gd} | |d<| s$|r"|d| gd} | |d<|ddkrD|dg}|d g}d}d}t|d!krt|dd"id#x}rS|s|} n|| | fvr| s|x|d<} | s|x|d<} || | fvrdnd$|jd<|| | fvrt d%|d&|d'd"id#x}r|s|} n|| kr~d(|jvr"t)|jd)z|_| |d*<nLd+|jvr"t)|jd,z|_| |d-<n!t d.| d/|d0|x|d<} || krdnd$|jd<|| krt d1|d&|r|r5|dd"id#|krd!nd}|r5|d'd"id#|krd'nd}|r|dks|r|t d2|t+||x}r|dd3id#}|d'd3id#}|d4ks|d5kr!t d6|d7|d8|d4krD|d5kr=|d!d'x}r0d$}|dd"id#x}r/|| | fvr|sd}|| | fvrt d9|d8n$t d:|dd8t|dkr|d!d"id#x}r-|| | fvrd}|| | fvrt d;|d8n$t d<|d!d8||jd<|r | s|r| |d<|sdSd}|d=z }|d>z }|rt|d5}|}dddn #1swxYwYt|d?z d@x}rcdA|dBg}|D]Z}t|d5}||j|dBdddn #1swxYwY[nd|rPt|d5}tj|dC}dddn #1swxYwY|dC|} | t | ttfr| |_nt dD| dE|jD]}!|dF|!dG}"t |"t6r |"|j|!<||!dG}#t |#tr|#&nDt |#t8r.|#dH}$t |$ts|$&nt;&fdI|Dd}%||!|%dS)JNtokenizer.jsonr\r]modelrrrbc3(K|] }|D]}d|vV dS)rcN).0pairss r2 z=SpecialVocab._try_load_from_tokenizer_json..s3IIDDIIqsaxIIIIIIIr4z'Spaces in merges detected, encoding as rccLg|]!}dd|D"S)rccLg|]!}dd|D"S)r_c3dK|]+}|dkrtt|dzn|V,dSrcrNchrordrcs r2rzSSpecialVocab._try_load_from_tokenizer_json....sU,&,&,->?#XXCFFSL(9(9(91,&,&,&,&,&,&r4joinrparts r2 zISpecialVocab._try_load_from_tokenizer_json...sT!"!"!" )- %'GG,&,&15,&,&,&%&%&!"!"!"r4rrrs r2rz>SpecialVocab._try_load_from_tokenizer_json..sV ' ' '!% HH!"!" 15 !"!"!"   ' ' 'r4zUnknown tokenizer merges formatrz added_tokensztokenizer_config.json bos_token cls_token eos_token sep_tokenpost_processor processorsr}RobertaProcessingTr)r*r,r.TemplateProcessingsinglerr SpecialTokenidFzUnknown leading special token z in TemplateProcessingeot)r eot_tokeneom)r eom_tokenzOverriding EOS token z with z without EOT/EOM fallback!zUnknown trailing special token z`TemplateProcessing leading/trailing special tokens do not match TemplateProcessingrABzUnknown sequence z...z in TemplateProcessingzUnknown separator token zUnknown middle sequence zUnknown second separator token zUnknown second middle sequence zchat_template.jsonzchat_template.jinjaadditional_chat_templatesz*.jinjadefault)rktemplaterz$Bad type for chat_template field in z - ignoringrArEcontentc3tK|]2}|dk|dV3dS)rrN)r)ratok tc_contents r2rz=SpecialVocab._try_load_from_tokenizer_json..`sD^^DDHHY > ( ( * * 0+@@@ 0A#'9Q<<  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 X !+1r66{CCK+1r66{CCK+1r66{CCK+1r66{CCK J; J3C J>II -  J; J3C J>II - !*/?!@!@@~O !!/!3!3L>BR!S!SN!N!I }}V,,0CCC8<.u58<.u58<.u5*H/?H*3-- }*M*Ma*PKSU0V0V0Z0Z[_0`0``} E'7!d2?KK%2; :T%T%T+6%dVc(c(8(E +6%dVc(c(8(E HUZegrYsHsHsy~ 6u =#0k8R#R#R$*NN4DTa4D4D4D%E%E%E/:2/B/B>SU/V/V/Z/Z[_/`/``|E'7 !_2>KK%1[%@%@',D4L'L'LCHIaCbCbenCn(@HS(8(E(E).d6N)N)NCHIaCbCbenCn(@HS(8(E(E(.8O{8O8Odp8O8O8O)P)P)PR^$^$4[$AKHTXcHcHcin 6u =#/;#>#>$*NN4DUa4D4D4D%E%E%E$T-:)Fy|?O?OP^`b?c?c?g?ghl?m?mq~?~?~EFI-9(Him>O>OP^`b>c>c>g>ghl>m>mq}>}>}rrDHH -S)q..l.W_Wg &0R!S!S!S,5eIx6P6P,QQyT)21)9)9*b)I)I)M)Md)S)S)22):)::r)J)J)N)Nt)T)T#)S==FcMM$*NN3vv3v3vRX3v3v3v$w$w$w#)S==Vs]]U^_`ac_cUdHd ].3G8A! 8H8HY[8\8\8`8`ae8f8f'f}%@+8[+/<[R]D^/^/^06@Oan@O@O@O1P1P1P,2NN>> -!$%FFHH! - - - - - - - - - - - - - - -'+T4O-O,U,UV_,`,`'a'aa# f.7EV%W%W$X!%9ffMm@@@fB)00-:LZ\ZaZaZcZc1d1deeefffffffffffffff  ' ' ) ) F(W=== F$(IaLL$4$4_$E$E! F F F F F F F F F F F F F F F(,,_>OPP  J}sDk$J$J !.D   NNfBWfff g g g+ 9 9C(,,-?C-?-?-?@@I)T** 8.7&s+$((C88E%%% " E4((  % ) 4 4 !-55* !^^^^L^^^N  # #C 8 8 8 8tsYAAAH!!H%(H%0```0bb !b (dddcn|dz }|sdSt|d5}tj|}dddn #1swxYwY|jD]T}||d}|"d|vr|d|d}|||UdS)Nz config.jsonFr\r]rB text_configT)rdrerrr$rry)r1r config_filerconfigrOtoken_ids r2rXz'SpecialVocab._try_load_from_config_jsonfs]* ""$$ 5 +' 2 2 2 "aYq\\F " " " " " " " " " " " " " " "+ 3 3CzzS"3"3"344HMV$;$;!-0445F5F5FGG  # #C 2 2 2 2tsAAA)FNN)r r!r"r#r$r%r&r'r5r6)F)r=rr>r#r5r?)r rr5r?)r rr5r#)rOr6rsrr5r?)rzr{r5r?) __name__ __module__ __qualname____annotations__r3r<rTr0rYryrrWrXrr4r2rr2s8&&&&%%%%;;;;%%%%))))AF48"&    "K"K"K"K"KH,,,, 2 t t t t----,mmmm^      r4rc$eZdZUded<ded<dS) BaseVocabz ClassVar[str]tokenizer_modelrkN)rrrrrr4r2rrus*""""r4rcHeZdZUded<ded<ded<ded<dd Zdd ZdS)Vocabrw vocab_sizeradded_tokens_dictradded_tokens_listrfname_tokenizer base_pathcdSrVr)r1rs r2r3zVocab.__init__r4r5-Iterable[tuple[bytes, float, gguf.TokenType]]cdSrVrr;s r2 all_tokenszVocab.all_tokensrr4Nrrr5r)rrrrr3rrr4r2rr{sWOOO%%%%    ,,,,NNNNNNr4rceZdZdZdZddZdS)NoVocabno_vocabr5r6cdS)Nz3rr;s r2r<zNoVocab.__repr__sDDr4Nr)rrrrrkr<rr4r2rrs8 O DEEEEEEr4rc>eZdZdZdZddZddZdd Zdd Zdd Z d S)BpeVocabgpt2bperrc i}|dz x}rt|d5}tj|_dddn #1swxYwY t|dz d5}tj|}dddn #1swxYwYn#t $rYnwxYw|dz }t|d5}tj|}dddn #1swxYwY|d}|ddks(|d d s|d dd krt d |d_|dx}fd|D}tj}tt||t|z} t| } | | kr=|t| zdz } tdt| d|d| d| t| d} |_d| D_|_jtjz_|_dS)Nz vocab.jsonr\r]added_tokens.jsonrrr}BPE byte_fallbackFdecoder ByteLevelzCannot find GPT-2 BPE tokenizervocabrcLi|] }|djv|d|d!S)rr)r )ritemr1s r2 z%BpeVocab.__init__..sA F F F$(#' ?$*#D#D!%Yd#D#D#Dr4rz Expected the z1 added token ID(s) to be sequential in the range z - z; got c|dSNrr)text_idxs r2z#BpeVocab.__init__..s (1+r4keycg|]\}}|Srr)rtextidxs r2rz%BpeVocab.__init__..s$C$C$CktST$C$C$Cr4)existsrerrr FileNotFoundErrorrr:rrangesortedvaluesrxrJrrvocab_size_baserr) r1rrrrtokenizer_jsonraddedr expected_ids actual_idsexpected_end_idrJs ` r2r3zBpeVocab.__init__s') (<7 7O ? ? A A Fo888 *A!Yq\\  * * * * * * * * * * * * * * * )&99GLLL0PQ#'9Q<OSS__SS *SS/>SSFPSSTT T|))++1M1MNNN$0$C$CU$C$C$C$.$($83t?U;V;V$V$3sYAAA B!4B B!BB!BB!! B.-B.C((C,/C,r5rc#Kd|jD}t|jD]!\}}||dtjjfV"dS)Nci|]\}}|| Srrr encoded_tokrs r2r z'BpeVocab.bpe_tokens..sSSS_["[SSSr4g)r rJ enumerategguf TokenTypeNORMAL)r1 reverse_vocabi_s r2 bpe_tokenszBpeVocab.bpe_tokenssnSS @P@P@R@RSSS dj)) ? ?DAq"C)>> > > > > ? ?r4c#tK|jD]-}d}|d|tjjfV.dSN@r\)rencoder'r(CONTROLr1rscores r2rzBpeVocab.added_tokenssR* F FDE++g&&t~/EE E E E E F Fr4c#rK|Ed{V|Ed{VdSrV)r-rr;s r2rzBpeVocab.all_tokenssR??$$$$$$$$$$$&&&&&&&&&&&r4r6cBd|jdt|jdS)Nzrr:rr;s r2r<zBpeVocab.__repr__s)s!5ssDLbHcHcssssr4Nrrr) rrrrrkr3r-rrr<rr4r2rrsO D24242424h???? FFFF ''''ttttttr4rc>eZdZdZdZddZddZdd Zdd Zdd Z d S)SentencePieceVocabllamaspmrrc ttdi}|dz x}rR t|dz d5}t j|}dddn #1swxYwYn=#t $rYn1wxYw|jdz x}st dt|_|j t||j fd| Dtttz}t!}||krt%d|d |||_fd |D|_|_|jt|jz|_ ||_dS) Nzsentencepiece is not installedztokenizer.modelrr\r]zCannot find tokenizer.modelc(i|]\}}|k ||Srr)rpiecerrs r2r z/SentencePieceVocab.__init__..s*```)%rU_O_O_BO_O_O_r4zExpected new token IDs z to be sequential; got c g|] }| Srr)rr new_tokenss r2rz/SentencePieceVocab.__init__..s"K"K"Kb:b>"K"K"Kr4)r RuntimeErrorrrerrrparentsentencepiece_tokenizer LoadFromFiler6rrJrrr:rkeysrxrrrr) r1rrrrexpected_new_idsactual_new_idsrBrs @@r2r3zSentencePieceVocab.__init__s1 ! )?@@ @') (+<< > ````|7I7I7K7K``` j*s:2N O OPP!*//"3"344 ~ - -p7Gpp`nppqq q#/"K"K"K"KN"K"K"K","&"6T=S9T9T"T"1s5A6 A* A6*A..A61A.2A66 BBr5rc#>K|j}t|D]}||}|d}||}t jj}| |rt jj }| |rt jj }| |rt jj}||rt jj}|||fVdS)Nr\)rErr IdToPiecer1GetScorer'r(r) IsUnknownUNKNOWN IsControlr2IsUnusedUNUSEDIsByteBYTE)r1rr+r@rr4toktypes r2sentencepiece_tokensz'SentencePieceVocab.sentencepiece_tokenss0 y++--.. ' 'A''**E <<00D$--a00En+G""1%% 1.0""1%% 1.0 !!!$$ 0./"" ..-w& & & & &) ' 'r4c#tK|jD]-}d}|d|tjjfV.dSr/)rr1r'r( USER_DEFINEDr3s r2rzSentencePieceVocab.added_tokenssR* K KDE++g&&t~/JJ J J J J K Kr4c#rK|Ed{V|Ed{VdSrV)rUrr;s r2rzSentencePieceVocab.all_tokenssT,,.........$$&&&&&&&&&&&r4r6cBd|jdt|jdS)Nzj | | j| <j| Tfdj jD_tj j_j j_jt9j z_|_dS)Nrr\r]rr}r ignore_mergesFrTz'Llama 3 must be converted with BpeVocabrrzCannot find Llama BPE tokenizerr) AutoTokenizerzsTo use LlamaHfVocab, please install the `transformers` package. You can install it with `pip install transformers`.) cache_dirlocal_files_onlyc|dSrr)xs r2rz'LlamaHfVocab.__init__..Us AaDr4rcPi|]"}|j|#Sr)r get_vocab)rtokr1s r2r z)LlamaHfVocab.__init__..^s>    ))++C0   r4)rerrr TypeErrorr transformersr_ ImportErrorfrom_pretrainedris_fastrr{rsetadded_tokens_idsrget_added_vocabrJrrladdall_special_tokensspecialsall_special_ids special_idsrr:r) r1rrrrr is_llama3r_erftokidxs ` r2r3zLlamaHfVocab.__init__(s#&66 /G 4 4 4 *!Yq\\N * * * * * * * * * * * * * * *+9*A F #u , ?1D1D_V[1\1\ ?#''>>>   GEFF F G F #u , ,O4G4GY^4_4_ ,i(0J>>#$EFF F  2 2 2 2 2 2 2   F  '66 !7   ~%%%%"$!%!$" N * * , , 2 2 4 4..    2 2KC222&--c222.4&s+%))&111    ~8    t~=>> $~8#3c$:P6Q6QQ.s%9==C## D-C==Dr5rc#TKd|jD}t|jD][}||jvr ||d}|||||||j fV\dS)Nci|]\}}|| Srrr$s r2r z*LlamaHfVocab.hf_tokens..ks+   / RB    r4r\) rrerJrrrmr1get_token_scoreget_token_typers)r1r*r token_texts r2 hf_tokenszLlamaHfVocab.hf_tokensjs  37>3K3K3M3M3S3S3U3U   d233  H4000'x077@@Jd228<Q>Q*d&6??      r4rrwr{bytesrsset[int]gguf.TokenTypectjd|rtjjS||vrtjjntjjS)Ns<0x[0-9A-Fa-f]{2}>)re fullmatchr'r(rSr2r))r1rr{rss r2rzzLlamaHfVocab.get_token_type|sC <. ; ; '>& &*2[)@)@t~%%dnF[[r4floatcdS)Nr0r)r1rs r2ryzLlamaHfVocab.get_token_scores wr4c#K|jD]}||jvrH||j|d|j}||j|}nt jj}d}|d||fVdS)Nr4r0r\) rrqrzrsryr'r(rWr1)r1rrTr4s r2rzLlamaHfVocab.added_tokenss* 7 7Dt}$$--dmD.A3HXYY,,T]4-@AA.5++g&&w6 6 6 6 6 7 7r4c:d|jjvp d|jjvS)Nz<0x0A> )rr r;s r2has_newline_tokenzLlamaHfVocab.has_newline_tokens!4>//O44>;O3OOr4c#rK|Ed{V|Ed{VdSrV)r|rr;s r2rzLlamaHfVocab.all_tokenssR>>#########$$&&&&&&&&&&&r4r6cBd|jdt|jdS)Nz.s ! ! !c!ff ! ! !r4)rrrrlr{zip)bscsrbcs_strs r2bytes_to_unicoders  U3s88SXX\ * *++ uSYYD A .. / / 0 uSYYD A .. / / 0 AAAB A 4[[ B;; IIaLLL IIdQh    FA ! !b ! ! !F B  r4cheZdZUdZdZiZded<gZded<ddZe d d Z e d d Z d!dZ d!dZ d"dZe d#dZe d#dZe d#dZe d#dZe d dZe d dZe d dZe d dZd!dZedZdZdS)$ MistralVocabmistralrrrrrrctstdt Jdt Jdt Jdt d|d|dD}tt|}nt|}t|dkrtd|t|dkr=d |vrd }nt|d }t d |d |n|d}||z }tj |jj|_t!|jtr t"jn t"j|_|jj|_||_d |jjzdz|jjz|_dS)NzwTo use MistralVocab, please install the `mistral-common` package. You can install it with `pip install mistral-common`.mistral_common is not installedzLoading Mistral tokenizer from c^g|]*}||+Sr)rdas_posix)rrs r2rz)MistralVocab.__init__..s-QQQaQYY[[QQZZ\\QQQr4z**/*rz*No tokenizer file found in the directory: rz tekken.jsonrz"Multiple tokenizer files found in z. Using zmistral--)_mistral_common_installedrirrrrFrGrrr:rxrrI from_fileinstruct_tokenizerrrvrrr=tokenizer_typen_wordsrrrRversion_name)r1r all_filestokenizer_file_pathvalid_tokenizer_filesrs r2r3zMistralVocab.__init__s( H -88:[888++-N+++%%'H%%% AiAABBBRQ9>>&+A+AQQQ ' 3">y"I"I  $A)$L$L !())Q.. !Yi!Y!YZZZ())A-- $999%2NN%+,A%B%B2%FN\\\N\\"7q!9"+n"< .8   Y  $.*55 * ' '%)  .02 ,2 2S 84>;Q Q r4r5r6c|jSrV)rr;s r2tokenizer_namezMistralVocab.tokenizer_names zr4c4|jtjkrdndS)Nr<r)rrr=r;s r2gguf_tokenizer_modelz!MistralVocab.gguf_tokenizer_models-1E1IIIwwvUr4rc#TKt Jdt|jtsJdt|jt |jjD].}|jj|}|d}|jj |}tj j }|jj |rtj j}|jj|rtj j}|jj|rtj j}|jj|rtj j}|||fV0dS)Nrz%Expected SentencePieceTokenizer, got r\)rrvrr}r_modelrrKr1rLr'r(r)rMrNrOr2rPrQrRrS)r1r+r@rr4rTs r2_sentencepiece_tokensz"MistralVocab._sentencepiece_tokenss%113T111$.*@AA  JD4H4H J J  At~,7799:: ' 'AN)33A66E<<((D>099!<T^(<(< > >  5()) dn?@@  H**844;;GDD&     ^>  E**5,??FFwOO%      r4rrwctt Jd|jtjkr;t |jtsJ|jj|S|jtj krHt |jtsJ|jj||jj zStd|j)NrUnknown tokenizer type: ) rrrrr=rvr_vocabindexrrrx)r1rs r2 get_token_idzMistralVocab.get_token_id.s%1j6L6LNo6L6LL  "6": : :dn.DEE E EE>(..u55 5  $8$? ? ?dnj99 9 99%++E22T^5VV M8KMMNN Nr4c|jjSrV)rbos_idr;s r2rzMistralVocab.bos_id; ~$$r4c|jjSrV)reos_idr;s r2rzMistralVocab.eos_id?rr4cH|jjdkr|jS|jjS)Nr)rpad_idrr;s r2rzMistralVocab.pad_idCs% > B & &; ~$$r4c|jjSrV)runk_idr;s r2rzMistralVocab.unk_idIrr4cJ|j|jjSrV)rrrr;s r2rzMistralVocab.bos_tokenM~))$.*?@@@r4cJ|j|jjSrV)rrrr;s r2rzMistralVocab.eos_tokenQrr4cJ|j|jjSrV)rrrr;s r2 pad_tokenzMistralVocab.pad_tokenUrr4cJ|j|jjSrV)rrrr;s r2 unk_tokenzMistralVocab.unk_tokenYrr4c#K|jtjkr|Ed{VdS|jtjkr|Ed{VdSt d|j)Nr)rrr=rrrrxr;s r2rzMistralVocab.all_tokens]s  "6": : :1133 3 3 3 3 3 3 3 3 3  $8$? ? ?**,, , , , , , , , , ,M8KMMNN Nr4cldfd|dDS)Nr_c:g|]}t|Sr)r)rcharrs r2rz6MistralVocab.token_bytes_to_string..is$PPPD SYY/PPPr4latin-1)rdecode)rrs `r2rz"MistralVocab.token_bytes_to_stringgs6wwPPPPAHHY|s(td|d| dt| fdd }||t|d d }t! fd |D} d | D} | S)Nrci|]\}}|| Srr)r token_bytesranks r2r z@MistralVocab.extract_vocab_merges_from_model..rs+   "3+tD+   r4rrz-Could not find valid merge for token at rank rurc<|d|dfS)Nrrr)rcmergeable_rankss r2rz>MistralVocab.extract_vocab_merges_from_model..sqt4oad6KLr4F)rreversec|dS)Nrbr)vals r2rz>MistralVocab.extract_vocab_merges_from_model..s #a&r4cg|]:}|d|dg;S)rr)r)rrrr1s r2rz@MistralVocab.extract_vocab_merges_from_model..s[   **3q6<@@**3q6<@@    r4cLg|]!}dd|D"S)rccLg|]!}dd|D"S)r_c3dK|]+}|dkrtt|dzn|V,dSrrrs r2rzUMistralVocab.extract_vocab_merges_from_model....sAOOQcCA ---qOOOOOOr4rrs r2rzKMistralVocab.extract_vocab_merges_from_model...sEGGOO$OOOOOr4rrs r2rz@MistralVocab.extract_vocab_merges_from_model..sV     HH!%      r4)rrvrr}r_mergeable_ranksrJrrrr:rlrxrrextendr) r1token_bytes_map merge_pairsr+ merged_tokenlocaljleftrightdecoded_merge_pairsrrrs ` @@r2extract_vocab_merges_from_modelz,MistralVocab.extract_vocab_merges_from_modelks*%*T^Z*P*P%% >T^(<(< > >&%P./@  7F7L7L7N7N    sDOdn.OOPP & &A*1-LE1c,//00 3 3#BQB'$QRR(O++00/99LL$q!1222  iAiiI\I\]fIgIgiiLLLLE   u % % % %[.@.@%PPP '))      #       ,     r4Nrrr)rr6r5rw)r5rw)rrrrrkrrrr3propertyrrrrrrrrrrrrrr staticmethodrrrr4r2rrs2O D(*****#%%%%%0 0 0 0 dXVVVXV''''0( O O O O%%%X%%%%X%%%%X% %%%X%AAAXAAAAXAAAAXAAAAXAOOOORR\R:::::r4r)r5r)2 __future__renumrrloggingrospathlibrtypingrrrr r r r r sentencepiecerrir(mistral_common.tokens.tokenizers.mistralr'mistral_common.tokens.tokenizers.tekkenr&mistral_common.tokens.tokenizers.utilsr.mistral_common.tokens.tokenizers.sentencepiecerrrr' gguf_writerr getLoggerrrFrrrrrr;r[r6rrrrr4r2rsa"""""""  dddddddddddddddddddd'4444444'''"&&&&&&&'%IIIIIIBBBBBB!%... %     J"&&&&&)-!------ .----(, ,,,,,,-  ######  8 $ $@@@@@@@@F  OOOOOIxOOOEEEEEiEEEHtHtHtHtHtuHtHtHtVH~H~H~H~H~H~H~H~Vxxxxxxxxxx5xxxxxxv3!!!!4ddddd5ddddds5A AAA..#BBB B.-B.