
    ihj                     N   d Z ddlZddlmZ ddlmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6  e(jn                  e8      Z9 e'       rddl:m;Z;  e&d      e G d de                    Z< G d de/      Z= G d de0      Z>	 	 dAd ej~                  d!ej                  d"ej                  d#ej                  d$eej                  d%f   d&eAdz  d'eAdz  d(eBej                  ej                  f   fd)ZC e        ZDeCeDd*<    G d+ d,ej~                        ZE G d- d.e-      ZF G d/ d0ej~                        ZG G d1 d2e      ZH G d3 d4e.      ZI G d5 d6e6      ZJ	 	 	 	 dBd7ej                  eBej                     z  dz  d8eKdz  d9eKdz  d:eKd$ej                  dz  d(ej                  eKz  fd;ZL G d< d=e5      ZM G d> d?e,      ZNg d@ZOy)CzPyTorch Doge model.    N)Callable)Union)strict)nn   )initialization)ACT2FN)Cache)PreTrainedConfig)compile_friendly_flex_attention)GradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)RopeParameters)AttentionInterfacePreTrainedModel)Unpack)TransformersKwargsauto_docstringis_torch_flex_attn_availablelogging)OutputRecorder   )LlamaForSequenceClassificationLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward	repeat_kv)MixtralForCausalLMMixtralModel)	BlockMaskzSmallDoge/Doge-320M)
checkpointc                   :    e Zd ZU dZdZdgZddddddddddddZd	gd
gfddgdgfdgdgfdZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee	z  e
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   d Zee
d!<   dZe	e
d"<   d#Zeez  d#z  e
d$<   d%Ze	e
d&<   d#Ze	d#z  e
d'<   d Zee
d(<   dZed#z  e
d)<   d Zee
d*<   d#Z e	d#z  e
d+<   dZ!e	e
d,<   d Z"ee
d-<   d.Z#e	e
d/<   d0Z$e	e
d1<   d Z%ee
d2<   d Z&ee
d3<   d4Z'ee
d5<   d#Z(e	d#z  e
d6<   d#Z)e	d#z  e
d7<   d#Z*e	e+e	   z  d#z  e
d8<    fd9Z, xZ-S ):
DogeConfiga  
    keep_window_size (`int`, *optional*, defaults to 2048):
        The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
    is_moe (`bool`, *optional*, defaults to `False`):
        Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.

    ```python
    >>> from transformers import DogeConfig, DogeModel

    >>> # Initializing a Doge-320M style configuration
    >>> configuration = DogeConfig()

    >>> # Initializing a model from the Doge-320M style configuration
    >>> model = DogeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```dogepast_key_valuescolwiserowwisecolwise_gather_outputrowwise_split_input)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gatezlayers.*.mlp.down_embedzlayers.*.mlp.up_embed	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormi   
vocab_sizei   hidden_size   intermediate_size    num_hidden_layers        hidden_dropoutsilu
hidden_actg{Gz?initializer_rangegư>rms_norm_epsT	use_cacheFtie_word_embeddingsmax_position_embeddingsNrope_parameters   num_attention_headsnum_key_value_headsattention_biasattention_dropoutmlp_biassliding_windowkeep_window_sizeis_moei @  num_experts@   num_experts_per_toknorm_topk_proboutput_router_logitsgMbP?router_aux_loss_coefpad_token_idbos_token_ideos_token_idc                 ^    | j                   | j                  | _         t        |   di | y )N )rG   rF   super__post_init__)selfkwargs	__class__s     v/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/doge/modular_doge.pyrZ   zDogeConfig.__post_init__   s-    ##+'+'?'?D$''    ).__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr5   int__annotations__r6   r8   r:   r<   floatr>   strr?   r@   rA   boolrB   rC   rD   r   dictrF   rG   rH   rI   rJ   rK   rL   rM   rN   rP   rQ   rR   rS   rT   rU   rV   listrZ   __classcell__r]   s   @r^   r'   r'   :   s   & J#4"5 &/%.%.&/%."+ )"+$;#8!6 &(9:#%568IJ!"_$56 JK!s!s"%NECK%J#u#L%It %%#'S'48O^d*T18  &*t* ND &)ut|)Hd!%NC$J% c FDK!! ND !&$&"'%'#L#*##L#*#+/L#S	/D(/( (r_   r'   c                       e Zd Zy)DogeRMSNormNr`   ra   rb   rX   r_   r^   rr   rr          r_   rr   c                       e Zd Zy)DogeRotaryEmbeddingNrs   rX   r_   r^   rv   rv      rt   r_   rv   modulequerykeyvaluer1   r$   scalingsoftcapreturnc           
      .   d }d t        |t              r|}n|d d d d d d d |j                  d   f   fd}	t        ||||	|d|d      \  }
}|j	                  |j
                        }|
j                  dd      j                         }
|
|fS )Nc                 h    t        j                  | z        z  } | |   |   |   |   z   } | S N)torchtanh)score	batch_idxhead_idxq_idxkv_idxcausal_maskr|   s        r^   	score_modz)flex_attention_forward.<locals>.score_mod   sI    ejj99E"K	28<UCFKKEr_   T)r   
block_mask
enable_gqascale
return_lse   r   )
isinstancer$   shaper   todtype	transpose
contiguous)rw   rx   ry   rz   r1   r{   r|   r\   r   r   attn_outputattention_weightsr   s         `     @r^   flex_attention_forwardr      s     JK.),#
$!!Q?SYYr]?":; &E &"K" *,,U[[9''1-88:K)))r_   doge_flex_attentionc                   r    e Zd Zddededz  f fdZ	 	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  eej                     dz  f   f
d
Z
	 	 ddej                  dej                  dedej                  dz  fdZ xZS )DogeAttentionNconfig	layer_idxc                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j&                  t)        j*                  |j                              | _        t        j                  |j                  | j                  z  |j                  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t3        | j                  |j4                        | _        t3        | j                  |j4                        | _        y )Nhead_dimg      ࿩biaseps)rY   __init__r   r   getattrr6   rF   r   rG   num_key_value_groupsr{   rI   rL   r   LinearrH   q_projk_projv_proj	Parameterr   zerosAdt_projo_projrr   r@   q_normk_normr[   r   r   r]   s      r^   r   zDogeAttention.__init__   s   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9 & 7 7ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ekk&*D*DEFyy&&68R8RY_YnYn
 ii&&68J8JQWQfQf
 "$--V5H5HI!$--V5H5HIr_   r0   position_embeddingsr1   r)   r}   c                    |j                   d d }g |d| j                  }| j                  | j                  |      j	                  |            j                  dd      }| j                  | j                  |      j	                  |            j                  dd      }	| j                  |      j	                  |      j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
| j                  |
j                  dd      j                  |
j                   d   |
j                   d   d            }t        j                  | j                   t#        j$                  |      z        j                  dd      }| j'                  ||| j(                  |      }t+        || j,                        }t.        j1                  | j2                  j4                  t6              } || ||	|
f|| j8                  sdn| j:                  | j<                  d|\  }} |j                  g |d j?                         }| jA                  |      }||fS )	Nr   r   r   r   )r0   	dt_statesrL   r1   r;   )r1   dropoutr{   )!r   r   r   r   viewr   r   r   r   r   updater   r   reshaper   expr   Fsoftplusprepare_dynamic_maskrL   r!   r   ALL_ATTENTION_FUNCTIONSget_interfacer   _attn_implementationr    trainingrI   r{   r   r   )r[   r0   r   r1   r)   r\   input_shapehidden_shapequery_states
key_statesvalue_statescossinr   	attn_maskattention_interfacer   attn_weightss                     r^   forwardzDogeAttention.forward   sV    $))#2.88b8$--8{{4;;}#=#B#B<#PQ[[\]_`a[[]!;!@!@!NOYYZ[]^_
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J LL""1a(001C1CA1FHZHZ[]H^`bc
	 IIdffqzz)'<<=GGBO	--'!22)	 . 
	 i)B)BC	(?(M(MKK,,.E)
 %8		%

 %#}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r_   r   rL   c           	         t        j                  |j                        j                  }|j                  }|dddddddf   j	                  dd|j
                  d   d      }|t        |t              s|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |      |      }|j                  |ddddddd|j
                  d   f   dk7  |      }|j
                  d   |kD  rnt        j                  |||j                        }t        j                  ||ddd	
      j                  }	|j!                  d|	d      }|j                  |dk(  |      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr   r   r;   )devicer   r   r   r   TF)dimlargestsortedg      ?)r   finfor   minexpandr   r   r$   rl   wheretensorr   masked_fill
zeros_liketopkindicesscatter)
r[   r0   r   rL   r1   	min_dtyper   r   active_masktopk_indicess
             r^   r   z"DogeAttention.prepare_dynamic_mask  sg   $ KK 3 3488	##aD!m,33M''*B
	 %j.S##uzz1%++!&"ELL^=R=RZ_$`bk" "--nQ1F[	XZH[F[=[.\`a.aclmI??2!11**9E)JZJZ[K ::i1ArSW`efnnL%--b,DK!--kS.@)LIr_   r   NN)r7   N)r`   ra   rb   r'   rh   r   r   Tensortupler
   r   r   ro   rp   s   @r^   r   r      s    Jz JcDj JD /3(,3)||3) #5<<#=>3) t+	3)
 3) 
u||U\\D0%2E2LL	M3)r !%.2#||# <<# 	#
 t+#r_   r   c                       e Zd Zy)DogeMLPNrs   rX   r_   r^   r   r   ?  rt   r_   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	DogeCDMoEr   c                    t         |           |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                              | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  dz  d      | _        t        j,                  | j                  | j                        | _        t        j,                  | j                  | j                        | _        y )Nr   r   F)rY   r   r6   r8   r	   r>   act_fnrN   mathfloorsqrtnum_keysrP   top_krQ   r   r   rJ   	gate_projup_proj	down_projrouter_gate	Embedding
down_embedup_embedr[   r   r]   s     r^   r   zDogeCDMoE.__init__D  s_   !--!'!9!9V../!--

499T-=-=#>?//
$33 4#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRab 99T%5%5t}}q7HuU ,,t'7'79I9IJT%5%5t7G7GHr_   r0   r}   c                    |j                   \  }}}| j                  |      j                  d||z  d      }|j                  | j                  d      \  \  }}\  }	}
|j                  d      |j                  d      z   }|	j                  d      | j                  z  |
j                  d      z   } |j                  g |j                   d d d } |j                  g |j                   d d d }|j                  | j                  d      \  }}|j                  d|      }t        j                  |d      }| j                  r||j                  dd      z  }| j                  |      }| j                  |      }t        j                  ||j                  ||z  dd            j                  ||z  d      }| j!                  |      |z  }t        j                  |j                  ||z  dd      |      j                  ||d      }| j#                  | j!                  | j%                  |            | j'                  |      z        }||z   }||fS )Nr   r   r   r   T)r   keepdimr   )r   r   r   r   r   	unsqueezer   gatherr   softmaxrQ   sumr   r   r   matmulr   r   r   r   )r[   r0   r\   bszseq_len_router_logitsscores_xscores_y	indices_x	indices_y
all_scoresall_indicesscoresposition_indicesr   routing_weightsr   r   experts_weightsexperts_statess                        r^   r   zDogeCDMoE.forward[  sF   
 (--Wa ((7<<QgrR 8E7I7I$--]_7I7`484y)''+h.@.@.DD
))"-=	@S@STV@WW$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC#-??4::2?#F  $$R)9:))F322r42HHO __W-
==),,z=3E3EcGmUWYZ3[\aabehoboqst++o6Ho&:&:3=!R&PRZ[``adfmoqrt{{4>>-3P'QTXT`T`anTo'op%6m++r_   )	r`   ra   rb   r'   r   r   r   r   ro   rp   s   @r^   r   r   C  s0    Iz I.,||, 
	,r_   r   c                   L    e Zd Zddededz  f fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )DogeDecoderLayerNr   r   c                 *   t         |           |j                  | _        t        |j                  |j
                        | _        t        ||      | _        t        j                  t        j                  |j                              | _        t        |j                  |j
                        | _        |j                  st!        |      n
t#        |      | _        t        j                  t        j                  |j                              | _        y )Nr   )r   r   )rY   r   r<   rr   r6   r@   input_layernormr   	self_attnr   r   r   onesinput_residualpost_attention_layernormrM   r   r   mlppost_attention_residualr   s      r^   r   zDogeDecoderLayer.__init__}  s    $33*6+=+=6CVCVW&f	J ll5::f6H6H+IJ(3F4F4FFL_L_(`%*0--76?Yv=N')||EJJv?Q?Q4R'S$r_   r0   r   r1   position_idsr)   rA   r\   r}   c           
         |}| j                  |      } | j                  d||||||d|\  }}	t        j                  || j                  | j
                        }| j                  |z  |z   }|}| j                  |      }| j                  |      }t        j                  || j                  | j
                        }| j                  |z  |z   }|S )N)r0   r   r1   r  r)   rA   )pr   rX   )
r  r  r   r   r<   r   r  r  r  r  )
r[   r0   r   r1   r  r)   rA   r\   residualself_attn_weightss
             r^   r   zDogeDecoderLayer.forward  s     !,,];+94>> ,
' 3)%+,
 ,
(( 		-43F3FQUQ^Q^_++h6F !55mD/		-43F3FQUQ^Q^_44x?-Or_   r   )NNNNF)r`   ra   rb   r'   rh   r   r   r   r   
LongTensorr
   rl   r   r   FloatTensorr   ro   rp   s   @r^   r  r  |  s    
Tz 
TcDj 
T IM.204(,!& ||  #5<<#=>E  t+	 
 &&-    $;  +,  
u  %(9(95;L;L(L"MPT"TT	U r_   r  c                   ^    e Zd ZdZdZ eed      eedZ	 e
j                         d        Zy)DogePreTrainedModelFr   )index)r  r0   
attentionsc                 z   t        j                  | |       t        |t              r-t	        |d      r t        j                  |j                         yyt        |t              rXt	        |d      rt        j                  |j                         t	        |d      r t        j                  |j                         yyy)zInitialize the weightsr   r  r  N)r   _init_weightsr   r   hasattrinitzeros_r   r  ones_r  r  )r[   rw   s     r^   r$  z!DogePreTrainedModel._init_weights  s     	%%dF3fm,vs#FHH% $ 01v/0

6001v89

699: : 2r_   N)r`   ra   rb   _supports_flash_attn_can_compile_fullgraphr   r   r  r   _can_record_outputsr   no_gradr$  rX   r_   r^   r   r     sB     "'	;)# U]]_
; 
;r_   r   c                       e Zd Zy)	DogeModelNrs   rX   r_   r^   r.  r.    rt   r_   r.  gate_logitsrN   r   r   c                    | t        | t              sy| d   j                  }| d   j                  }g }g }| D ]  }	|	j	                  |      }	|	j                  |d      \  \  }
}\  }}|
j                  d      |j                  d      z   }|j                  d      |z  |j                  d      z   } |j                  g |j                  dd d } |j                  g |j                  dd d }|j                  |d      \  }}|j                  d|      }t        j                  |d      }|j                  |       |j                  |        t        j                  |d      }t        j                  |d      }|}|j                  d      }t        j                  |||      }t        j                   |||      }|j#                  d||      |j                  d   z  }t        j$                  |d      }nD|j                  \  }}t'        |       }|ddddddf   j)                  ||||f      j+                  d      j	                  |      }|j                  d      |j-                            }t        j                  |||      }t        j                   |||      }|j#                  d||      t        j.                  |      z  }|ddddddf   j)                  ||||f      j+                  d|      j	                  |      }t        j.                  ||z  d      t        j.                  |d      z  }t        j.                  ||z        }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   appendr   catr   	ones_likescatter_add_meanlenr   r   rl   r   )r/  rN   r   r   r1   compute_dtypecompute_deviceall_expert_indicesall_routing_weightslayer_gate_logitsr  r  r  r  r  r  r  r
  expert_indicesr  tokens_per_expertpadrouter_prob_per_expert
batch_sizesequence_lengthr:   expert_attention_mask router_per_expert_attention_maskoverall_losss                                r^   load_balancing_loss_funcrE    s{   @ *[%"@N((M ^**N( 4-00@7H7M7Mh\^7M7_484y)''+h.@.@.DD
))"-89;N;Nr;RR$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC(ooeo<$++B0@A))JB7!!.1""?3!4" #51=))$7Q?/44R8!KK=Q_`oo0n]-::1>PRUVYkYqYqrsYtt "',?Q!G&4&:&:#
O, 4At+,V&
OUKLWR[R	 	 044R89N9S9S9UV "KK=Q_`oo0n]-::1>PRUVY^YbYb!Z
 
 4At+,V&
O[QRWR%R	 	) "'+>Aa+agh!ilqlulu,!m
 "
 99.1GGHL+%%r_   c                       e Zd Z fdZ	 	 	 	 	 	 	 	 	 ddej
                  dz  dej                  dz  dej
                  dz  dedz  dej                  dz  dej
                  dz  d	e	dz  d
e
ej                  z  de	dz  dee   defdZ xZS )DogeForCausalLMc                 f    t         |   |       t        |      | _        |j                  | _        y r   )rY   r   r.  modelrN   r   s     r^   r   zDogeForCausalLM.__init__2  s*     v&
!--r_   Nr.   r1   r  r)   r/   labelsrA   logits_to_keeprR   r\   r}   c
           
         |	|	n| j                   j                  }	 | j                  d||||||d|
}|j                  }t	        |t
              rt        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |
}d}|	rt        |j                  | j                  t        j                  t        j                  | j                              | j                   |      }|+|| j"                  |j%                  |j&                        z  z  }t)        ||||j*                  |j,                  |j.                  |j                        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r.   r1   r  r)   r/   rA   )lossaux_losslogitsr)   r0   r"  r  rX   )r   rR   rI  last_hidden_stater   rh   slicelm_headloss_functionr5   rE  r  rN   r   r   r   rP   rS   r   r   r   r)   r0   r"  )r[   r.   r1   r  r)   r/   rJ  rA   rK  rR   r\   outputsr0   slice_indicesrO  rM  rN  s                    r^   r   zDogeForCausalLM.forward7  sm   H %9$D $++JjJj 	
 +5$** +
)%+'+
 +
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  

499T%5%567((H !11HKK4LLL(#33!//))!//
 	
r_   )	NNNNNNNr   N)r`   ra   rb   r   r   r  r   r
   r  rl   rh   r   r   r   r   ro   rp   s   @r^   rG  rG  1  s    . .2.204(,26*.!%-.,0O
##d*O
 t+O
 &&-	O

 O
 ((4/O
   4'O
 $;O
 ell*O
 #TkO
 +,O
 
#O
r_   rG  c                       e Zd Zy)DogeForSequenceClassificationNrs   rX   r_   r^   rW  rW    rt   r_   rW  )r'   rG  r.  r   rW  r   )NNr   N)Prc   r   collections.abcr   typingr   r   torch.nn.functionalr   
functionalr   huggingface_hub.dataclassesr    r   r&  activationsr	   cache_utilsr
   configuration_utilsr   integrations.flex_attentionr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.output_capturingr   llama.modeling_llamar   r   r   r   r   r   r    r!   mixtral.modeling_mixtralr"   r#   
get_loggerr`   logger!torch.nn.attention.flex_attentionr$   r'   rr   rv   Moduler   rj   r   r   r   r   r   r   r  r   r.  rh   rE  rG  rW  __all__rX   r_   r^   <module>rp     sj      $     .  & !   3 J 9 Q 1 A & ^ ^ 4	 	 	 H 
		H	%!; 01L(! L(  2L(^	, 		. 	 ! +*II+*<<+* 
+* <<	+*
 %,,34+* T\+* T\+* 5<<%&+*\ -. 1G - .wBII wt	h 	6,		 6,r-1 -`;. ;.	 	 #*.g&ell 33d:g&tg& Djg& 	g&
 LL4'g& \\Cg&TU
( U
p	$B 	r_   