
    #;7jN                    6   d dl mZ d dlZd dlZd dlmZmZmZ d dlZ	d dl
Z
erd dl
mZ ddlmZmZmZmZ  ej        ddd	d
dddddddd           G d de                      Z ej        d           G d de                      Z ej        dd           G d de                      Z ej        d           G d de                      Z ej        d           G d d e                      Z ej        d!           G d" d#e                      ZdS )$    )annotationsN)CallableIterableTYPE_CHECKING)Tensor   )	ModelBase	TextModelggufloggerLLaMAForCausalLMLlamaForCausalLMMistralForCausalLMMixtralForCausalLMVLlama3ForCausalLMLlavaForConditionalGenerationVoxtralForConditionalGenerationLlamaForCausalLMEagle3Eagle3SpeculatorEagle3DraftModelIQuestCoderForCausalLM
LlamaModelc                       e Zd ZU ej        j        ZdZ fdZd Z	 fdZ
ed%d            Zd& fdZdZded<   ed' fd            Zd(d) fdZd* fd"Zd+d#Z fd$Z xZS ),r   Tc                :    t                      j        |i | | j        dk    r#| j                            dd          | j        d<   | j        rd | _        n=t          j        | j	        d          }|                    dd g          d         | _        d| j        v rz| j        d	         d
k    rjd| _
        t          j        j        | _        t          j        d           t          j        | j        | j                  | _        t          j        | j                 | j        _        | j                                         | j        t3          d          t5          | j	        dz  dd          5 }t7          j        |          }d d d            n# 1 swxY w Y   t5          | j        dz  dd          5 }t7          j        |          }d d d            n# 1 swxY w Y   d|v ri ||d         }|d         | _        |d	         }d|dz  |dz
  g}t          j        d| d| d           | j                            | j        j         d|           |                    d          |d         }	d}
n
|d         }	d}
t          j        d|	 d|
 d            | j                            | j        j         d!|	           |                    d"d          }t          j        d#|            | j                             | j        j         d$|           d S d S d S )%Nr   num_attention_heads    F)is_mistral_formatarchitecturesr   draft_vocab_sizenum_hidden_layersr   Tz>Detected EAGLE-3 draft model, switching to EAGLE3 architecturezEAGLE-3 model requires --target-model-dir to be specified. Please provide the path to the target model directory to read config.jsonzconfig.jsonrutf-8encodingtext_config
vocab_size      zEAGLE-3: target_layers = z (target model has z layers)z.target_layerstarget_hidden_sizezEAGLE-3 confighidden_sizeztarget model configzEAGLE-3: target_hidden_size = z (from )z.target_hidden_sizenorm_before_residualz EAGLE-3: norm_before_residual = z.norm_before_residual)!super__init__hf_archhparamsgetr   origin_hf_archr	   load_hparams	dir_model	is_eagle3r   
MODEL_ARCHEAGLE3
model_archr   infoget_tensor_name_mapblock_count
tensor_mapMODEL_ARCH_NAMESgguf_writerarchadd_architecturetarget_model_dir
ValueErroropenjsonloadtarget_vocab_size	add_array
add_uint32add_bool)selfargskwargsr0   feagle3_raw_configtarget_configtarget_num_layerstarget_layersr)   srcr,   	__class__s               6/home/rurouni/src/llama.cpp-master/conversion/llama.pyr.   zLlamaModel.__init__"   s   $)&)))<///26,2B2BCXZ\2]2]DL./! 	J"&D,T^uUUUG")++ov"F"Fq"ID --$,?R2SWX2X2X!DN"o4DOKXYYY"6tHXYYDO$($9$/$JD!--///$, `  
 dn}4cGLLL 1PQ$(IaLL!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1d+m;S7SSS -WX $	!- - - - - - - - - - - - - - - -- Q= QM-4P Q%2<%@D" !..A B 1Q 68IA8MNMKqMqqVgqqqrrr&&$*:*?'O'O'OQ^___ !$$%9::F%67K%L"&%2=%A"+KZ9KZZTWZZZ[[[''4+;+@(U(U(UWijjj $5#8#89OQV#W#W KQ;OQQRRR%%)9)>&U&U&UWklllllW .-2X2Xs$   0FFF6GGGc                   d }t          | dd          r8| j        J t          j        d| j                    | j        }| j        | _        | j        dk    r|                                 S | j        r|                                 S | j        dz  }| j        dz  }|	                                r(|	                                s|                                  | j        dz  }|	                                rt          |dd	
          5 }t          j        |          }|                    d          x}| j                            |           |                    d          dk    r |                                 cd d d            S 	 d d d            n# 1 swxY w Y   	 |                                  nQ# t$          $ rD 	 |                                  n+# t$          t(          f$ r |                                  Y nw xY wY nw xY w| j                            dd          dk    rt/          j        | j        dg d          }|                    dd           |                    dd           |                    dd           |                    dd           |                    | j                   | j                            dd          dk    r| j                            d           |	|| _        d S d S )Nr5   Fz,EAGLE-3: Using tokenizer from target model: GlmasrModelztekken.jsonztokenizer.jsonztokenizer_config.jsonr!   r"   r#   add_prefix_spacetokenizer_classHybridDNATokenizerr&   i }  i}  )prefixsuffixmiddleeot)load_mergesspecial_token_typesrZ   i}  r[   i}  r\   i	}  r]   i
}  i   )getattrrA   r   r9   r4   r2   _set_vocab_glmedger   _set_vocab_mistralis_filerC   rD   rE   r1   r>   add_add_space_prefix_set_vocab_hybriddna_set_vocab_sentencepieceFileNotFoundError_set_vocab_llama_hf	TypeError_set_vocab_gpt2r0   r   SpecialVocab_set_special_tokenadd_to_ggufadd_add_bos_token)	rJ   original_dir_modelpath_tekken_jsonpath_tokenizer_jsontokenizer_config_filerM   tokenizer_config_jsonrW   special_vocabs	            rT   	set_vocabzLlamaModel.set_vocab\   s   !4e,, 	3(444K^tG\^^___!%!2DN-//**,,,! 	-**,,,>M9"n/??##%% 	&.A.I.I.K.K 	&##%%% $1H H ((** 	7+S7CCC 7q(,	!%(=(A(ABT(U(UU$b$99:JKKK(,,->??CWWW44667 7 7 7 7 7 7 7 X	7 7 7 7 7 7 7 7 7 7 7 7 7 7 7	'))++++  	' 	' 	''((****%y1 ' ' '$$&&&&&'	' <L%00E99 -E&K&K&K  M ,,Xu===,,Xu===,,Xu===,,Uu===%%d&6777 <L%00E99..u555 )/DNNN *)sI   A2FFFF3 3
H>GH%G;8H:G;;H Hc                (   t                                                       | j        }| j        s | j                            |d                    |                    d          x}|d         |d         z  }| j                            |           d S )Nr&   head_dimr*   r   )r-   set_gguf_parametersr0   r   r>   add_vocab_sizer1   add_rope_dimension_count)rJ   r0   rope_dimrS   s      rT   rx   zLlamaModel.set_gguf_parameters   s    ##%%%,% 	C++GL,ABBBJ///H8}-9N1OOH11(;;;;;    weightsr   n_headint	n_head_kv
int | Nonec                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S )Nr'   r   r   )reshapeshapeswapaxes)r}   r~   r   s      rT   permutezLlamaModel.permute   sr     Vy%8%8F7=+;v+E+J_W][\[][]M^___!Q''	)r|   namestrweightscalescale2input_scalec                   | j         r|                     ddgd          }|                     ddgd          }||                    d          r9t                              |||          }t                              |||          }nM|                    d          r8t                              |||          }t                              |||          }t                                          |||||           d S )	Nn_headsr   T)optional
n_kv_headsnum_key_value_headsq_proj.weightk_proj.weight)undo_permutefind_hparamendswithr   r   r-   _repack_nvfp4)	rJ   r   r   r   r   r   r~   	n_kv_headrS   s	           rT   r   zLlamaModel._repack_nvfp4   s    		J%%y2G&HSW%XXF((,8M)NY](^^I!==11 J'//GGF'//vvFFEE]]?33 J'//	JJF'//vyIIEdFE6;GGGGGr|   Nzlist[dict[str, Tensor]] | None_expertsitem tuple[str, Callable[[], Tensor]]return'tuple[str, Callable[[], Tensor]] | Nonec                    |\  }}d|v r|                     dd          }t                                          ||f          S )Nztext_model. )replacer-   filter_tensors)clsr   r   genrS   s       rT   r   zLlamaModel.filter_tensors   sF    	cD  <<r22Dww%%tSk222r|   remote_hf_model_id
str | Nonedict[str, Callable[[], Tensor]]c                   t                                          |          }d| j        v ri | j        | j        d         | _        d| j        v r| j        d         dk    rt          j        d           i }|                                D ]_\  }}|                    d          r d|t          d          d          z   }|||<   :|                    d          rd	|z   }|||<   Z|||<   `|S |S )
Ntransformer_layer_configr   r    r   z=EAGLE-3: renaming midlayer.* / layers.0.* to model.layers.0.*z	midlayer.zmodel.layers.0.z	layers.0.model.)r-   index_tensorsr0   r   r9   items
startswithlen)rJ   r   tensorsnew_tensorsr   r   new_namerS   s          rT   r   zLlamaModel.index_tensors   s   ''''(:;; &55WdlWdl;U.VWDL --$,?R2SWX2X2XKWXXXK$]]__ , ,	c??;// ,04K8H8H8I8I3JJH,/K))__[11 ,'$H,/K))(+K%%r|   
data_torchbidIterable[tuple[str, Tensor]]c           	   #  "  K   t          | dd          r|dk    r||fV  d S |dk    r#t          | d          si | _        || j        |<   d S |dk    rd S |                    d          r+|                     t
          j        j        |          |fV  d S |                     dd	g          }|                     d
dg          }| j	        dk    rd|z   }| j
        rb|                    d          rt                              |||          }|                    d          rt                              |||          }|                    d          dk    r| j        d         }|J | j        #d t!          | j                  D             | _        || j        |         |<   t%          | j        |                   |dz  k    rdD ]}g }t!          |          D ]B}	d| d|	 d| d}
|                    | j        |         |
                    | j        |         |
= Ct)          j        |d          }d| d| d}t-                                          |||          E d {V  d S d S t-                                          |||          E d {V  d S )Nr5   Fz	fc.weightd2t_eagle3_int_tensorst2dz.hidden_norm.weightr   r   r   r   r   r   )r   zq_proj.bias)r   zk_proj.biaszblock_sparse_moe.expertsnum_local_expertsc                    g | ]}i S  r   ).0_s     rT   
<listcomp>z-LlamaModel.modify_tensors.<locals>.<listcomp>   s     E E E E E Er|   r(   )w1w2w3zmodel.layers.z.block_sparse_moe.experts...weightr   dimzlayers.z.feed_forward.experts.)r`   hasattrr   r   format_tensor_namer   MODEL_TENSORATTN_NORM_2r   r/   r   r   r   findr0   r   ranger;   r   appendtorchstackr-   modify_tensors)rJ   r   r   r   r~   r   	n_expertswiddatasxidenamemerged_namerS   s               rT   r   zLlamaModel.modify_tensors   s     4e,, 	{""Z((((u}}t%:;; 2/1D,1;(.u}}}}233 ..t/@/LcRRT^____!!9.C"DEE$$l4I%JKK	<<''d?D 	O}}=>> L'//
FFKK
}}=>> O'//
FINN
 99/00B66%89I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-77- T TC*,E$Y// 6 6 a a as a aUX a a aT]3%7%>??? M#.u55!&U!:!:!:J"SC"S"Ss"S"S"SK$ww55j+sSSSSSSSSSS77))*dC@@@@@@@@@@@r|   c              #    K   | j                             d| j                   x}r|                    dd                                          dk    r|                    dd          }| j                            d          x}| j        d         | j        d	         z  }d
|t	          j        d|dt          j                  |z  z  z  }|                    dd          }|                    dd
          }|                    dd          }|                    dd          }||z  }	||z  }
g }|D ]{}dt          j        z  |z  }||
k     r|	                    d           0||	k    r|	                    |           L||z  |z
  ||z
  z  }|	                    dd|z
  |z  |z   z             || 
                    t          j        j                  t	          j        |t          j                  fV  d S d S d S )Nfull_attention	rope_typer   llama3
rope_thetag     @rw   r*   r   g      ?r   r'   dtypefactorg       @low_freq_factorhigh_freq_factorg      @ original_max_position_embeddingsi    r   )rope_parametersr1   lowerr0   r   arangefloat32mathpir   r   r   r   
ROPE_FREQStensor)rJ   rope_paramsbaser   freqsr   r   r   old_context_lenlow_freq_wavelenhigh_freq_wavelenrope_factorsfreqwavelensmooths                  rT   generate_extra_tensorsz!LlamaModel.generate_extra_tensors  s-     .223CTEYZZZ; 	{B//55778CC"|W==<++J777C@,}5F[9\\CtQQem(T(T(TWZ(Z[\$377"-//2CS"I"I#.??3Es#K#K "-//2TVZ"["[#2_#D $36F$F!  "! R RD$'kD0G!222$++A.... #333$++F3333"1G";o"MRbetRt!u$++A!f*1F1O,PQQQQ..t/@/KLLel[goto|N}N}N}~~~~~~7	 	CCr|   c           
        i }t          | dd          r*|                                 D ]\  }}|dk    r
|j        ||<   t                                                       t          | dd          rt          | d          r| j                                        D ]\  }}|                    ||j                  }|	                    t          j                                                                                  }|dk    r|                    d          }|t          j        |j        t          j                  z   }t          j        |dk     || j        k    z            rt)          d| j                   t          j        |          j        |j        k    rt)          d	          t,          j        j        }d
d                    d t5          |j                  D                        d}t9          j        |dz   dd| d|j         d|            | j                             |||           | j!        6d | j!        D             }tE          |          dk    rt)          d|           d S d S )Nr5   Fr   r   r   r   r   z:EAGLE-3 d2t target ids out of range for target vocab size z)EAGLE-3 d2t contains duplicate target ids{z, c              3  4   K   | ]}t          |          V  d S N)r   )r   ns     rT   	<genexpr>z-LlamaModel.prepare_tensors.<locals>.<genexpr>O  s(      *P*Pa3q66*P*P*P*P*P*Pr|   },z<30 z --> z
, shape = )	raw_dtypec                @    g | ]}|                                 D ]}|S r   )keys)r   dks      rT   r   z.LlamaModel.prepare_tensors.<locals>.<listcomp>U  s-    BBBQBBAqBBBBr|   zUnprocessed experts: )#r`   get_tensorsr   r-   prepare_tensorsr   r   r   r1   tor   int64cpunumpyr   npr   sizeanyrF   rB   uniquer   GGMLQuantizationTypeI64joinreversedr   r   r9   r   r>   
add_tensorr   r   )
rJ   eagle3_original_dtypesr   r   	old_dtypedata
data_qtype	shape_strexpertsrS   s
            rT   r  zLlamaModel.prepare_tensors7  s   !#4e,, 	D$($4$4$6$6 D D j5==3=3C*40!!! 4e,, 	N?T1U1U 	N$($<$B$B$D$D N N j266tZ=MNN	!}}U[115577==??5==<<++D")DIRX"F"F"FFDvtaxDD4J,JKLL @()~fjf|)~)~y+ty88()TUUU!6:
T*P*P8DJ;O;O*P*P*P!P!PTTT	tczfff	ff
ff[dffggg ++D$*+MMMM=$BB$-BBBG7||a !B!B!BCCC	 %$  r|   )r}   r   r~   r   r   r   )
r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   )__name__
__module____qualname__r   r6   LLAMAr8   r   r.   ru   rx   staticmethodr   r   r   __annotations__classmethodr   r   r   r   r  __classcell__rS   s   @rT   r   r      s{         &JL8m 8m 8m 8m 8mt80 80 80t	< 	< 	< 	< 	< ) ) ) \)H H H H H H 04H33333 3 3 3 3 [3      0=A =A =A =A =A =A~   < D  D  D  D  D  D  D  D  Dr|   ArceeForCausalLMc                  6     e Zd Zej        j        Z fdZ xZS )
ArceeModelc                p    t                                                       |                                  d S r   )r-   rx   _try_set_pooling_typerJ   rS   s    rT   rx   zArceeModel.set_gguf_parameters^  s1    ##%%%""$$$$$r|   )	r  r  r  r   r6   ARCEEr8   rx   r  r  s   @rT   r"  r"  Z  sB        &J% % % % % % % % %r|   r"  Llama4ForConditionalGenerationLlama4ForCausalLMc                  V     e Zd Zej        j        ZdZ fdZd Z	 fdZ
d fdZ xZS )Llama4ModelFc                     t                      j        |i | | j        d         | j        d<   | j        d         | j        d<   d S )Nintermediate_sizeintermediate_size_moeintermediate_size_mlp)r-   r.   r0   )rJ   rK   rL   rS   s      rT   r.   zLlama4Model.__init__k  sO    $)&)))04=P0Q,-,0L9P,Q()))r|   c                .    |                                   d S r   )rj   )rJ   s    rT   ru   zLlama4Model.set_vocabq  s    r|   c                r   t                                                       | j                            | j        d                    | j                            | j        d                    d| j        v r@t          d | j        d         D                       r| j                            d           d S d S d S )Ninterleave_moe_layer_stepr-  layer_typesc              3  "   K   | ]
}|d k    V  dS )r   Nr   )r   lts     rT   r   z2Llama4Model.set_gguf_parameters.<locals>.<genexpr>y  s(      PPb2))PPPPPPr|   r   )r-   rx   r>   add_interleave_moe_layer_stepr0   add_expert_feed_forward_lengthalladd_sliding_windowr%  s    rT   rx   zLlama4Model.set_gguf_parameterst  s    ##%%%66t|D_7`aaa77E\8]^^^DL((PPDL4OPPPPP 7 33A66666 )(7 7r|   r   r   r   r   r   r   c              #  B  K   d|v r|                     dd          }|                     dd          }|j        d         dz  }|                    dd                              |d          \  }}t	                                          |||          E d {V  t	                                          |||          E d {V  d S |                    d          r|d	z  }|                    dd          }t	                                          |||          E d {V  d S )
Ngate_up_projzup_proj.weightzgate_proj.weightr   r'   r   	down_projr   )r   r   	transposesplitr-   r   r   )
rJ   r   r   r   name_up	name_gatedim_halfgate_proj_weightup_proj_weightrS   s
            rT   r   zLlama4Model.modify_tensors}  sE     T!!ll>3CDDG^5GHHI!'+q0H/9/C/CB/K/K/Q/QRZ`b/Q/c/c,nww--.>	3OOOOOOOOOww--ngsKKKKKKKKKF==%% 	6ID#--b"55J77))*dC@@@@@@@@@@@r|   )r   r   r   r   r   r   )r  r  r  r   r6   LLAMA4r8   r   r.   ru   rx   r   r  r  s   @rT   r*  r*  c  s        
 'JLR R R R R  7 7 7 7 7A A A A A A A A A Ar|   r*  LlamaBidirectionalModelc                  &    e Zd Zej        j        ZdS )LlamaEmbedNemotronModelN)r  r  r  r   r6   LLAMA_EMBEDr8   r   r|   rT   rG  rG    s        ,JJJr|   rG  SmolLM3ForCausalLMc                  &    e Zd Zej        j        ZdS )SmolLM3ModelN)r  r  r  r   r6   SMOLLM3r8   r   r|   rT   rK  rK    s        (JJJr|   rK  ApertusForCausalLMc                  J     e Zd Zej        j        ZdZi Zi Z	i Z
i Z fdZ xZS )ApertusModelFc              #     K    j         d         }|                    d          r|                    d                                                                           j        |<   t           j                  |k    r8 j                             fdt           j                  D                        d S |                    d          r|                    d                                                                           j
        |<   t           j
                  |k    r8 j                             fdt           j
                  D                        d S |                    d          r|                    d                                                                           j        |<   t           j                  |k    r8 j                             fdt           j                  D                        d S |                    d	          r|                    d                                                                           j        |<   t           j                  |k    r8 j                             fd
t           j                  D                        d S t!                                          |||          E d {V  d S )Nr    z.act_fn.alpha_nr  c                *    g | ]}j         |         S r   )_alpha_nr   r  rJ   s     rT   r   z/ApertusModel.modify_tensors.<locals>.<listcomp>       3d3d3dDM!4D3d3d3dr|   z.act_fn.alpha_pc                *    g | ]}j         |         S r   )_alpha_prS  s     rT   r   z/ApertusModel.modify_tensors.<locals>.<listcomp>  rT  r|   z.act_fn.betac                *    g | ]}j         |         S r   )_betarS  s     rT   r   z/ApertusModel.modify_tensors.<locals>.<listcomp>  s    0[0[0[1A0[0[0[r|   z.act_fn.epsc                *    g | ]}j         |         S r   )_epsrS  s     rT   r   z/ApertusModel.modify_tensors.<locals>.<listcomp>  s    /X/X/X	!/X/X/Xr|   )r0   r   r  floatr   rR  r   r>   add_xielu_alpha_nsortedrV  add_xielu_alpha_prX  add_xielu_betarZ  add_xielu_epsr-   r   )rJ   r   r   r   n_layersrS   s   `    rT   r   zApertusModel.modify_tensors  s     < 34==*++ 	!+u!5!5!;!;!=!=!B!B!D!DDM#DM""h.. 223d3d3d3dfUYUbNcNc3d3d3deeeF==*++ 	!+u!5!5!;!;!=!=!B!B!D!DDM#DM""h.. 223d3d3d3dfUYUbNcNc3d3d3deeeF==(( 	(mmE2288::??AADJsODJ8++ //0[0[0[0[tzHZHZ0[0[0[\\\F=='' 	']]5117799>>@@DIcNDI(** ../X/X/X/XfTYFWFW/X/X/XYYYF77))*dC@@@@@@@@@@@r|   )r  r  r  r   r6   APERTUSr8   r   rR  rV  rX  rZ  r   r  r  s   @rT   rO  rO    sd        (JLHHEDA A A A A A A A Ar|   rO  )
__future__r   rD   r   typingr   r   r   r  r  r   r   r   r	   r
   r   r   registerr   r"  r*  rG  rK  rO  r   r|   rT   <module>rf     s   " " " " " "   4 4 4 4 4 4 4 4 4 4       4 4 4 4 4 4 4 4 4 4 4 4 #% yD yD yD yD yD yD yD yDx	 &''% % % % % % % ('% $ %A %A %A %A %A* %A %A	 %AP -..- - - - -j - - /.- ())) ) ) ) ): ) ) *)) ())!A !A !A !A !A: !A !A *)!A !A !Ar|   