
    i]                     j   d dl Z d dlmZ d dlmZmZ d dlZd dlmc m	Z
 d dlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7  e/       rd dl8m9Z9  ed       G d dejt                               Z; G d dejt                        Z<d Z= ed      dGd       Z>d ej~                  d!e@d"ej~                  fd#ZA	 dHd$ejt                  d%ej~                  d&ej~                  d'ej~                  d(ej~                  dz  d)eBd*eBd+e*e,   fd,ZC	 	 dId$ejt                  d%ej~                  d&ej~                  d'ej~                  d(eej~                  d-f   d)eBdz  d.eBdz  d"eDej~                  ej~                  f   fd/ZE e'       ZFeEeFd0<    G d1 d2ejt                        ZG G d3 d4ejt                        ZH G d5 d6ejt                        ZI G d7 d8e      ZJe- G d9 d:e(             ZKe- G d; d<eK             ZL	 	 	 	 dJd=ej~                  eDej~                     z  dz  d>e@dz  d?e@dz  d@e@d(ej~                  dz  d"ej~                  e@z  fdAZMe- G dB dCeKe             ZN G dD dEeeK      ZOg dFZPy)K    N)Callable)OptionalUnion)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hubuse_kernel_func_from_hub)compile_friendly_flex_attention)create_causal_mask!create_sliding_window_causal_mask) GenericForSequenceClassificationGradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)AttentionInterfacePreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_flex_attn_available)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs   )
DogeConfig)	BlockMaskRMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	DogeRMSNormepsreturnNc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z:
        DogeRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizer)   	__class__s      w/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/doge/modeling_doge.pyr-   zDogeRMSNorm.__init__7   s1     	ll5::k#:; #    hidden_statesc                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )N   T)keepdim)	dtypetor/   float32powmeanrsqrtr2   r1   )r3   r8   input_dtypevariances       r6   forwardzDogeRMSNorm.forward?   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r7   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler1   shaper2   )r3   s    r6   
extra_reprzDogeRMSNorm.extra_reprF   s*    ))*+6$2G2G1HIIr7   )gư>)
__name__
__module____qualname__floatr-   r/   TensorrE   rI   __classcell__r5   s   @r6   r(   r(   5   s7    $ $$ $;U\\ ;ell ;Jr7   r(   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )DogeRotaryEmbeddinginv_freqNconfigc                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultrS   F)
persistentoriginal_inv_freq)r,   r-   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrT   rope_parametersrV   compute_default_rope_parametersr   attention_scalingregister_bufferclone)r3   rT   devicerope_init_fnrS   r5   s        r6   r-   zDogeRotaryEmbedding.__init__M   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr7   rb   ztorch.deviceseq_lenr*   ztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimN      ?r   r:   r=   rb   r=   )	r]   getattrr4   num_attention_headsr/   arangeint64r>   rM   )rT   rb   rd   basedimattention_factorrS   s          r6   r^   z3DogeRotaryEmbedding.compute_default_rope_parameters]   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r7   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r;   r#   mpscpuF)device_typeenabledr:   rp   ri   )rS   rM   expandrH   r>   rb   
isinstancetypestrr   	transposer/   catcosr_   sinr=   )
r3   xposition_idsinv_freq_expandedposition_ids_expandedru   freqsembr~   r   s
             r6   rE   zDogeRotaryEmbedding.forward{   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$N)NNN)rJ   rK   rL   r/   rN   __annotations__r$   r-   staticmethodr   intrG   rM   r^   no_gradr   rE   rO   rP   s   @r6   rR   rR   J   s    llVz V  $(+/"*T!*(* t* 
~u$	%	* *: U]]_<  <r7   rR   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr;   r:   rw   )rH   r/   r}   )r   x1x2s      r6   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r7   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr~   r   unsqueeze_dimq_embedk_embeds          r6   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGr7   r8   n_repr*   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rH   rx   reshape)r8   r   batchnum_key_value_headsslenrg   s         r6   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr7   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr:   r   r;   )rp   r=   ptrainingr#   )r   num_key_value_groupsr/   matmulr|   r   
functionalsoftmaxr?   r>   r=   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r6   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r7   r%   softcapc           
      .   d }d t        |t              r|}n|d d d d d d d |j                  d   f   fd}	t        ||||	|d|d      \  }
}|j	                  |j
                        }|
j                  dd      j                         }
|
|fS )Nc                 h    t        j                  | z        z  } | |   |   |   |   z   } | S r   )r/   tanh)score	batch_idxhead_idxq_idxkv_idxcausal_maskr   s        r6   	score_modz)flex_attention_forward.<locals>.score_mod   sI    ejj99E"K	28<UCFKKEr7   T)r   
block_mask
enable_gqascale
return_lser#   r:   )ry   r%   rH   r   r>   r=   r|   r   )r   r   r   r   r   r   r   r   r   r   r   attention_weightsr   s         `     @r6   flex_attention_forwardr      s     JK.),#
$!!Q?SYYr]?":; &E &"K" *,,U[[9''1-88:K)))r7   doge_flex_attentionc                   r    e Zd Zddededz  f fdZ	 	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	eej                  ej                  dz  eej                     dz  f   f
d
Z
	 	 ddej                  dej                  dedej                  dz  fdZ xZS )DogeAttentionNrT   	layer_idxc                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j&                  t)        j*                  |j                              | _        t        j                  |j                  | j                  z  |j                  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t3        | j                  |j4                        | _        t3        | j                  |j4                        | _        y )Nrg   g      ࿩biasr)   )r,   r-   rT   r   rk   r4   rl   rg   r   r   r   attention_dropoutkeep_window_sizer   Linearattention_biasq_projk_projv_projr.   r/   zerosAdt_projo_projr(   rms_norm_epsq_normk_normr3   rT   r   r5   s      r6   r-   zDogeAttention.__init__  s   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9 & 7 7ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ekk&*D*DEFyy&&68R8RY_YnYn
 ii&&68J8JQWQfQf
 "$--V5H5HI!$--V5H5HIr7   r8   position_embeddingsr   past_key_valuesr*   c                    |j                   d d }g |d| j                  }| j                  | j                  |      j	                  |            j                  dd      }| j                  | j                  |      j	                  |            j                  dd      }	| j                  |      j	                  |      j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
| j                  |
j                  dd      j                  |
j                   d   |
j                   d   d            }t        j                  | j                   t#        j$                  |      z        j                  dd      }| j'                  ||| j(                  |      }t+        || j,                        }t.        j1                  | j2                  j4                  t6              } || ||	|
f|| j8                  sdn| j:                  | j<                  d|\  }} |j                  g |d j?                         }| jA                  |      }||fS )	Nr;   r#   r:   r   r   )r8   	dt_statesr   r           )r   r   r   )!rH   rg   r   r   viewr|   r   r   r   r   updater   r   r   r/   expr   Fsoftplusprepare_dynamic_maskr   r   r   ALL_ATTENTION_FUNCTIONSget_interfacerT   _attn_implementationr   r   r   r   r   r   )r3   r8   r   r   r   r   input_shapehidden_shapequery_statesr   r   r~   r   r   	attn_maskattention_interfacer   r   s                     r6   rE   zDogeAttention.forward"  sV    $))#2.88b8$--8{{4;;}#=#B#B<#PQ[[\]_`a[[]!;!@!@!NOYYZ[]^_
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J LL""1a(001C1CA1FHZHZ[]H^`bc
	 IIdffqzz)'<<=GGBO	--'!22)	 . 
	 i)B)BC	(?(M(MKK,,.E)
 %8		%

 %#}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r7   r   r   c           	         t        j                  |j                        j                  }|j                  }|dddddddf   j	                  dd|j
                  d   d      }|t        |t              s|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |      |      }|j                  |ddddddd|j
                  d   f   dk7  |      }|j
                  d   |kD  rnt        j                  |||j                        }t        j                  ||ddd	
      j                  }	|j!                  d|	d      }|j                  |dk(  |      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr;   r#   r   rj   r   r=   rb   TF)rp   largestsortedrh   )r/   finfor=   minrx   rH   ry   r%   boolwheretensorrb   masked_fill
zeros_liketopkindicesscatter)
r3   r8   r   r   r   	min_dtyper=   r   active_masktopk_indicess
             r6   r   z"DogeAttention.prepare_dynamic_maskW  sg   $ KK 3 3488	##aD!m,33M''*B
	 %j.S##uzz1%++!&"ELL^=R=RZ_$`bk" "--nQ1F[	XZH[F[=[.\`a.aclmI??2!11**9E)JZJZ[K ::i1ArSW`efnnL%--b,DK!--kS.@)LIr7   r   NN)i   N)rJ   rK   rL   r$   r   r-   r/   rN   rG   r
   rE   r   rO   rP   s   @r6   r   r     s    Jz JcDj JD /3(,3)||3) #5<<#=>3) t+	3)
 3) 
u||U\\D0%2E2LL	M3)r !%.2#||# <<# 	#
 t+#r7   r   c                   $     e Zd Z fdZd Z xZS )DogeMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _	        t        j                  | j                  | j                  |j                        | _
        t        |j                     | _        y )Nr   )r,   r-   rT   r4   intermediate_sizer   r   mlp_bias	gate_projup_proj	down_projr	   
hidden_actact_fnr3   rT   r5   s     r6   r-   zDogeMLP.__init__~  s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r7   c                     | j                  | j                  | j                  |            | j                  |      z        }|S r   )r
  r  r  r	  )r3   r   r
  s      r6   rE   zDogeMLP.forward  s6    NN4;;t~~a/@#ADLLQRO#ST	r7   )rJ   rK   rL   r-   rE   rO   rP   s   @r6   r  r  }  s    0r7   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	DogeCDMoErT   c                    t         |           |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                              | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  dz  d      | _        t        j,                  | j                  | j                        | _        t        j,                  | j                  | j                        | _        y )Nr   r:   F)r,   r-   r4   r  r	   r  r  num_expertsmathfloorsqrtnum_keysnum_experts_per_toktop_knorm_topk_probr   r   r  r  r	  r
  router_gate	Embedding
down_embedup_embedr  s     r6   r-   zDogeCDMoE.__init__  s_   !--!'!9!9V../!--

499T-=-=#>?//
$33 4#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRab 99T%5%5t}}q7HuU ,,t'7'79I9IJT%5%5t7G7GHr7   r8   r*   c                    |j                   \  }}}| j                  |      j                  d||z  d      }|j                  | j                  d      \  \  }}\  }	}
|j                  d      |j                  d      z   }|	j                  d      | j                  z  |
j                  d      z   } |j                  g |j                   d d d } |j                  g |j                   d d d }|j                  | j                  d      \  }}|j                  d|      }t        j                  |d      }| j                  r||j                  dd      z  }| j                  |      }| j                  |      }t        j                  ||j                  ||z  dd            j                  ||z  d      }| j!                  |      |z  }t        j                  |j                  ||z  dd      |      j                  ||d      }| j#                  | j!                  | j%                  |            | j'                  |      z        }||z   }||fS )Nr:   r;   rw   r   T)rp   r<   r#   )rH   r  r   r   r  r   r  gatherr   r   r  sumr  r  r/   r   r  r
  r  r	  )r3   r8   r   bszrd   _router_logitsscores_xscores_y	indices_x	indices_y
all_scoresall_indicesscoresposition_indicesr   routing_weightsr  r  experts_weightsexperts_statess                        r6   rE   zDogeCDMoE.forward  sF   
 (--Wa ((7<<QgrR 8E7I7I$--]_7I7`484y)''+h.@.@.DD
))"-=	@S@STV@WW$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC#-??4::2?#F  $$R)9:))F322r42HHO __W-
==),,z=3E3EcGmUWYZ3[\aabehoboqst++o6Ho&:&:3=!R&PRZ[``adfmoqrt{{4>>-3P'QTXT`T`anTo'op%6m++r7   )	rJ   rK   rL   r$   r-   r/   rN   rE   rO   rP   s   @r6   r  r    s0    Iz I.,||, 
	,r7   r  c                   L    e Zd Zddededz  f fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )DogeDecoderLayerNrT   r   c                 *   t         |           |j                  | _        t        |j                  |j
                        | _        t        ||      | _        t        j                  t        j                  |j                              | _        t        |j                  |j
                        | _        |j                  st!        |      n
t#        |      | _        t        j                  t        j                  |j                              | _        y )Nr   )rT   r   )r,   r-   hidden_dropoutr(   r4   r   input_layernormr   	self_attnr   r.   r/   r0   input_residualpost_attention_layernormis_moer  r  mlppost_attention_residualr   s      r6   r-   zDogeDecoderLayer.__init__  s    $33*6+=+=6CVCVW&f	J ll5::f6H6H+IJ(3F4F4FFL_L_(`%*0--76?Yv=N')||EJJv?Q?Q4R'S$r7   r8   r   r   r   r   	use_cacher   r*   c           
         |}| j                  |      } | j                  d||||||d|\  }}	t        j                  || j                  | j
                        }| j                  |z  |z   }|}| j                  |      }| j                  |      }t        j                  || j                  | j
                        }| j                  |z  |z   }|S )N)r8   r   r   r   r   r:  r    )
r3  r4  r   r   r2  r   r5  r6  r8  r9  )
r3   r8   r   r   r   r   r:  r   residualself_attn_weightss
             r6   rE   zDogeDecoderLayer.forward  s     !,,];+94>> ,
' 3)%+,
 ,
(( 		-43F3FQUQ^Q^_++h6F !55mD/		-43F3FQUQ^Q^_44x?-Or7   r   )NNNNF)rJ   rK   rL   r$   r   r-   r/   rN   rG   
LongTensorr
   r   r   r   FloatTensorrE   rO   rP   s   @r6   r0  r0    s    
Tz 
TcDj 
T IM.204(,!& ||  #5<<#=>E  t+	 
 &&-    $;  +,  
u  %(9(95;L;L(L"MPT"TT	U r7   r0  c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed      eed	Z ej(                          fd
       Z xZS )DogePreTrainedModelrT   modelTr0  r   Fr#   )index)r#  r8   
attentionsc                 n   t         |   |       t        |t              r-t	        |d      r t        j                  |j                         yyt        |t              rXt	        |d      rt        j                  |j                         t	        |d      r t        j                  |j                         yyy)zInitialize the weightsr   r5  r9  N)r,   _init_weightsry   r   hasattrinitzeros_r   r0  ones_r5  r9  )r3   r   r5   s     r6   rG  z!DogePreTrainedModel._init_weights  s     	f%fm,vs#FHH% $ 01v/0

6001v89

699: : 2r7   )rJ   rK   rL   r$   r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr!   r  r0  r   _can_record_outputsr/   r   rG  rO   rP   s   @r6   rB  rB    sx    &*#+,#4"5 N""&'	;)# U]]_
; 
;r7   rB  c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd                     Z xZS )	DogeModelrT   c           	         t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t#        |      | _        d| _        | j)                          y c c}w )Nr   rT   F)r,   r-   pad_token_idpadding_idx
vocab_sizer   r  r4   embed_tokens
ModuleListrangenum_hidden_layersr0  layersr(   r   normrR   
rotary_embgradient_checkpointing	post_initr   s      r6   r-   zDogeModel.__init__  s     !.. ++LL):):F<N<NPTP`P`ammBGH`H`BabYfi0b
   2 28K8KL	-V<&+# 	 cs   DN	input_idsr   r   r   inputs_embedsr:  r   r*   c           
      z   |d u |d uz  rt        d      |r|t        | j                        }|| j                  |      }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }| j                  j                  t        nt        }	 |	| j                  ||||      }
|}| j                  ||      }| j                  d | j                  j                   D ]  } ||f|
||||d|} | j!                  |      }t#        ||	      S )
Nz:You must specify exactly one of input_ids or inputs_embedsrY  r   r#   )rb   )rT   rg  r   r   r   )r   )r   r   r   r:  r   )last_hidden_stater   )
ValueErrorr   rT   r]  get_seq_lengthr/   rm   rH   rb   r   sliding_windowr   r   rc  ra  r`  rb  r   )r3   rf  r   r   r   rg  r:  r   past_seen_tokensmask_functionr   r8   r   decoder_layers                 r6   rE   zDogeModel.forward(  sd    -t";<YZZ0*$++>O  --i8MCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L.2kk.H.H.P*Vw#;;')+%
 &"oom,oW![[)H4;;+H+HI 		M)*) /#$7 M		 		-0%++
 	
r7   )NNNNNN)rJ   rK   rL   r$   r-   r    r"   r   r/   r?  rN   r
   r@  r   r   r   r   rE   rO   rP   s   @r6   rW  rW    s    z     .2.204(,26!%4
##d*4
 t+4
 &&-	4

 4
 ((4/4
 $;4
 +,4
 
 4
    4
r7   rW  gate_logitsr  r  r  c                    | t        | t              sy| d   j                  }| d   j                  }g }g }| D ]  }	|	j	                  |      }	|	j                  |d      \  \  }
}\  }}|
j                  d      |j                  d      z   }|j                  d      |z  |j                  d      z   } |j                  g |j                  dd d } |j                  g |j                  dd d }|j                  |d      \  }}|j                  d|      }t        j                  |d      }|j                  |       |j                  |        t        j                  |d      }t        j                  |d      }|}|j                  d      }t        j                  |||      }t        j                   |||      }|j#                  d||      |j                  d   z  }t        j$                  |d      }nD|j                  \  }}t'        |       }|ddddddf   j)                  ||||f      j+                  d      j	                  |      }|j                  d      |j-                            }t        j                  |||      }t        j                   |||      }|j#                  d||      t        j.                  |      z  }|ddddddf   j)                  ||||f      j+                  d|      j	                  |      }t        j.                  ||z  d      t        j.                  |d      z  }t        j.                  ||z        }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r;   rw   r   r   )ry   rG   r=   rb   r>   r   r   r   rH   r  r   r   appendr/   r}   r   	ones_likescatter_add_rA   lenrx   r   r   r   )rp  r  r  r  r   compute_dtypecompute_deviceall_expert_indicesall_routing_weightslayer_gate_logitsr$  r%  r&  r'  r(  r)  r"  r+  expert_indicesr,  tokens_per_expertpadrouter_prob_per_expert
batch_sizesequence_lengthr`  expert_attention_mask router_per_expert_attention_maskoverall_losss                                r6   load_balancing_loss_funcr  b  s{   @ *[%"@N((M ^**N( 4-00@7H7M7Mh\^7M7_484y)''+h.@.@.DD
))"-89;N;Nr;RR$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC(ooeo<$++B0@A))JB7!!.1""?3!4" #51=))$7Q?/44R8!KK=Q_`oo0n]-::1>PRUVYkYqYqrsYtt "',?Q!G&4&:&:#
O, 4At+,V&
OUKLWR[R	 	 044R89N9S9S9UV "KK=Q_`oo0n]-::1>PRUVY^YbYb!Z
 
 4At+,V&
O[QRWR%R	 	) "'+>Aa+agh!ilqlulu,!m
 "
 99.1GGHL+%%r7   c                   N    e Zd ZddiZddiZddgdgfiZ fdZee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  dedz  dee   defd              Z xZS )DogeForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr8   logitsc                 N   t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                          y )NFr   )r,   r-   rW  rC  r\  r   r   r4   r  router_aux_loss_coefr  r  re  r  s     r6   r-   zDogeForCausalLM.__init__  s     v&
 ++yy!3!3V5F5FUS$*$?$?!!--#)#=#=  	r7   Nrf  r   r   r   rg  labelsr:  logits_to_keepoutput_router_logitsr   r*   c
           
         |	|	n| j                   j                  }	 | j                  d||||||d|
}|j                  }t	        |t
              rt        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |
}d}|	rt        |j                  | j                  t        j                  t        j                  | j                              | j                   |      }|+|| j"                  |j%                  |j&                        z  z  }t)        ||||j*                  |j,                  |j.                  |j                        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rf  r   r   r   rg  r:  )lossaux_lossr  r   r8   rE  r#  r<  )rT   r  rC  ri  ry   r   slicer  loss_functionr\  r  r#  r  r  r  r  r  r  r>   rb   r   r   r8   rE  )r3   rf  r   r   r   rg  r  r:  r  r  r   outputsr8   slice_indicesr  r  r  s                    r6   rE   zDogeForCausalLM.forward  sm   L %9$D $++JjJj 	
 +5$** +
)%+'+
 +
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  

499T%5%567((H !11HKK4LLL(#33!//))!//
 	
r7   )	NNNNNNNr   N)rJ   rK   rL   _tied_weights_keys_tp_plan_pp_planr-   r   r   r/   r?  rN   r
   r@  r   r   r   r   r   rE   rO   rP   s   @r6   r  r    s5   *,GH23H_-z:;H
  .2.204(,26*.!%-.,0O
##d*O
 t+O
 &&-	O

 O
 ((4/O
   4'O
 $;O
 ell*O
 #TkO
 +,O
 
#O
  O
r7   r  c                       e Zd Zy)DogeForSequenceClassificationN)rJ   rK   rL   r<  r7   r6   r  r  2  s    r7   r  )r  rW  rB  r  )r#   )r   r  )NNr:   N)Qr  collections.abcr   typingr   r   r/   torch.nn.functionalr   r   r    r   rI  activationsr	   cache_utilsr
   r   
generationr   integrationsr   r   integrations.flex_attentionr   masking_utilsr   r   modeling_layersr   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r    utils.output_capturingr!   r"   configuration_doger$   !torch.nn.attention.flex_attentionr%   Moduler(   rR   r   r   rN   r   r   rM   r   rG   r   r   r   r  r  r0  rB  rW  r  r  r  __all__r<  r7   r6   <module>r     s0  .  $ "     & ! . ) Q J R [ Q K A & g g G E *  !; Y'J")) J (J(><")) ><B( *+ ,2	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%> ! +*II+*<<+* 
+* <<	+*
 %,,34+* T\+* T\+* 5<<%&+*\ -. 1G - .wBII wtbii  6,		 6,r-1 -` ;/ ; ;> H
# H
 H
Z #*.g&ell 33d:g&tg& Djg& 	g&
 LL4'g& \\Cg&T b
)? b
 b
J	$DFY 	 cr7   