
    iW                        d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.  G d dej^                        Z0 G d dejb                        Z2 G d dejb                        Z3 G d dejb                        Z4d Z5 ed      d=d        Z6d!ejn                  d"e8d#ejn                  fd$Z9	 d>d%ejb                  d&ejn                  d'ejn                  d(ejn                  d)ejn                  dz  d*e:d+e:d,e#e%   fd-Z; ee6       G d. d/ejb                               Z< G d0 d1e      Z=e& G d2 d3e!             Z>e& G d4 d5e>             Z?e& G d6 d7e>e             Z@ G d8 d9ee>      ZA G d: d;ee>      ZBg d<ZCy)?    )Callable)OptionalN)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_func_from_hubuse_kernelized_func)create_causal_mask) GenericForSequenceClassificationGenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)maybe_autocastmerge_with_config_defaults)capture_outputs   )GemmaConfigc            	       Z     e Zd ZdZd	dedededef fdZdej                  f fdZ	 xZ
S )
GemmaTextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    num_embeddingsembedding_dimpadding_idxembed_scalec                     t         |   |||       || _        | j                  dt	        j
                  |      d       y )Nr&   F
persistent)super__init__scalar_embed_scaleregister_buffertorchtensor)selfr#   r$   r%   r&   	__class__s        y/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/gemma/modeling_gemma.pyr+   z%GemmaTextScaledWordEmbedding.__init__7   s;    D"-]ELL,ERWX    	input_idsc                     t         |   |      | j                  j                  | j                  j
                        z  S N)r*   forwardr&   toweightdtype)r0   r4   r1   s     r2   r7   z$GemmaTextScaledWordEmbedding.forward<   s2    wy)D,<,<,?,?@Q@Q,RRRr3   )      ?)__name__
__module____qualname____doc__intfloatr+   r.   Tensorr7   __classcell__r1   s   @r2   r"   r"   2   sG    Ys Y3 YS Y_d Y
S S Sr3   r"   c                   <     e Zd Zddedef fdZd Zd Zd Z xZ	S )GemmaRMSNormdimepsc                     t         |           || _        t        j                  t        j                  |            | _        y r6   )r*   r+   rH   r   	Parameterr.   zerosr9   )r0   rG   rH   r1   s      r2   r+   zGemmaRMSNorm.__init__A   s.    ll5;;s#34r3   c                     |t        j                  |j                  d      j                  dd      | j                  z         z  S )N   T)keepdim)r.   rsqrtpowmeanrH   )r0   xs     r2   _normzGemmaRMSNorm._normF   s4    5;;quuQx}}R}>IJJJr3   c                     | j                  |j                               }|d| j                  j                         z   z  }|j                  |      S )Nr;   )rT   rA   r9   type_as)r0   rS   outputs      r2   r7   zGemmaRMSNorm.forwardI   sC    AGGI& 3!2!2!445~~a  r3   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler9   shaperH   )r0   s    r2   
extra_reprzGemmaRMSNorm.extra_reprP   s'    ))*+6$((<<r3   )gư>)
r<   r=   r>   r@   rA   r+   rT   r7   r[   rC   rD   s   @r2   rF   rF   @   s&    5C 5e 5
K!=r3   rF   c                   $     e Zd Z fdZd Z xZS )GemmaMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFbias)r*   r+   confighidden_sizeintermediate_sizer   Linear	gate_projup_proj	down_projr   
hidden_actact_fnr0   rb   r1   s     r2   r+   zGemmaMLP.__init__U   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../r3   c                     | j                  | j                  | j                  |            | j                  |      z        }|S r6   )rh   rj   rf   rg   )r0   rS   rh   s      r2   r7   zGemmaMLP.forward_   s6    NN4;;t~~a/@#ADLLQRO#ST	r3   )r<   r=   r>   r+   r7   rC   rD   s   @r2   r]   r]   T   s    0r3   r]   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )GemmaRotaryEmbeddinginv_freqNrb   c                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultro   Fr(   original_inv_freq)r*   r+   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrb   rope_parametersrq   compute_default_rope_parametersr   attention_scalingr-   clone)r0   rb   devicerope_init_fnro   r1   s        r2   r+   zGemmaRotaryEmbedding.__init__g   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr3   r{   ztorch.deviceseq_lenreturnztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNr;   r   rM   r:   )r{   r:   )	rw   getattrrc   num_attention_headsr.   arangeint64r8   rA   )rb   r{   r}   baserG   attention_factorro   s          r2   rx   z4GemmaRotaryEmbedding.compute_default_rope_parametersw   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r3   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   rN   r   mpscpuF)device_typeenabledrM   rG   r   )ro   rA   expandrZ   r8   r{   
isinstancetypestrr   	transposer.   catcosry   sinr:   )
r0   rS   position_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r2   r7   zGemmaRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$r6   NNN)r<   r=   r>   r.   rB   __annotations__r    r+   staticmethodr   r@   rY   rA   rx   no_gradr   r7   rC   rD   s   @r2   rn   rn   d   s    llV{ V  %)+/"*d"*(* t* 
~u$	%	* *: U]]_<  <r3   rn   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..NrN   rM   r   )rZ   r.   r   )rS   x1x2s      r2   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r3   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   unsqueeze_dimq_embedk_embeds          r2   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGr3   hidden_statesn_repr~   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rZ   r   reshape)r   r   batchnum_key_value_headsslenr   s         r2   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr3   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )NrM   r   rN   )rG   r:   )ptrainingr   )r   num_key_value_groupsr.   matmulr   r   
functionalsoftmaxfloat32r8   r:   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r2   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r3   c                        e Zd ZdZdedef fdZ	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fdZ xZS )GemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrb   	layer_idxc                 |   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        t	        |dd       | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        y )Nr   g      use_bidirectional_attentionFr`   )r*   r+   rb   r   r   rc   r   r   r   r   r   attention_dropout	is_causalr   re   attention_biasq_projk_projv_projo_projr0   rb   r   r1   s      r2   r+   zGemmaAttention.__init__   sZ   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9$V-JERRii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r3   Nr   position_embeddingsr   past_key_valuesr   r~   c                 
   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )NrN   r   rM           )r   r   )rZ   r   r   viewr   r   r   r   updater   r   get_interfacerb   _attn_implementationr   r   r   r   r   r   r   )r0   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   r   r   attention_interfacer   r   s                   r2   r7   zGemmaAttention.forward  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r3   r   )r<   r=   r>   r?   r    r@   r+   r.   rB   rY   r	   r   r   r7   rC   rD   s   @r2   r   r      s    G
{ 
s 
4 IM.2(,&)||&) #5<<#=>E&) t+	&)
 &) +,&) 
u||U\\)	*&)r3   r   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )GemmaDecoderLayerrb   r   c                     t         |           |j                  | _        t        ||      | _        t        |      | _        t        |j                  |j                        | _	        t        |j                  |j                        | _
        y )N)rb   r   rH   )r*   r+   rc   r   	self_attnr]   mlprF   rms_norm_epsinput_layernormpost_attention_layernormr   s      r2   r+   zGemmaDecoderLayer.__init__0  sl    !--'vKF#+F,>,>FDWDWX(4V5G5GVM`M`(a%r3   Nr   r   r   r   	use_cacher   r   r~   c           
          |}| j                  |      } | j                  d||||||d|\  }}	||z   }|}| j                  |      }| j                  |      }||z   }|S )N)r   r   r   r   r   r    )r   r   r   r   )
r0   r   r   r   r   r   r   r   residual_s
             r2   r7   zGemmaDecoderLayer.forward:  s     !,,];)4>> 
')%+ 3
 
q !=0 !55mD/ =0r3   )NNNFN)r<   r=   r>   r    r@   r+   r.   rB   
LongTensorr	   boolrY   r   r   r7   rC   rD   s   @r2   r   r   /  s    b{ bs b /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
r3   r   c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedZ ej$                          fd       Z xZS )GemmaPreTrainedModelrb   modelTr   r   )r   
attentionsc                 
   t         |   |       d|j                  j                  v r t	        j
                  |j                         y t        |t              r+t	        j                  |j                  |j                         y y )NRMSNorm)r*   _init_weightsr1   r<   initzeros_r9   r   r"   	constant_r&   r,   )r0   r   r1   s     r2   r   z"GemmaPreTrainedModel._init_weightsl  s^    f%((111KK& <=NN6--v/H/HI >r3   )r<   r=   r>   r    r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   r   _can_record_outputsr.   r   r   rC   rD   s   @r2   r   r   Z  sp    &*#,-#4"5N!"&*$
 U]]_J Jr3   r   c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd                     Z xZS )
GemmaModelrb   c           	      *   t         |   |       |j                  | _        |j                  | _        t        |j                  |j                  | j                  | j                  j                  dz        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                         | _        t%        |      | _        d| _        | j+                          y c c}w )Ng      ?)r&   r   rb   F)r*   r+   pad_token_idr%   
vocab_sizer"   rc   rb   embed_tokensr   
ModuleListrangenum_hidden_layersr   layersrF   r   normrn   
rotary_embgradient_checkpointing	post_initr   s      r2   r+   zGemmaModel.__init__x  s     !.. ++8v1143C3CQUQ\Q\QhQhjmQm
 mmCHIaIaCbcivy1c
 !!3!39L9LM	.f=&+# 	 ds   "DNr4   r   r   r   inputs_embedsr   r   r~   c           
      N   |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        | j                  ||||      }	|}
| j                  |
|      }| j                  d | j                  j                   D ]  } ||
f|	||||d|}
 | j                  |
      }
t        |
|r|	      S d 	      S )
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )r{   )rb   r  r   r   r   )r   )r   r   r   r   r   )last_hidden_stater   )
ValueErrorr  r
   rb   get_seq_lengthr.   r   rZ   r{   r   r   r  r  r  r  r   )r0   r4   r   r   r   r  r   r   past_seen_tokenscausal_maskr   r   decoder_layers                r2   r7   zGemmaModel.forward  s[    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L(;;')+%
 &"oom,oW![[)H4;;+H+HI 		M)*) /#$7 M		 		-0&+/8O
 	
>B
 	
r3   )NNNNNN)r<   r=   r>   r    r+   r   r   r   r.   r   rB   r	   FloatTensorr   r   r   r   r7   rC   rD   s   @r2   r	  r	  v  s    { $   .2.204(,26!%2
##d*2
 t+2
 &&-	2

 2
 ((4/2
 $;2
 +,2
 
!2
    2
r3   r	  c                   B    e Zd ZddiZddiZddgdgfiZ fdZee	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  dee   defd              Z xZS )GemmaForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr   logitsc                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y r_   )
r*   r+   r	  r   r  r   re   rc   r"  r  rk   s     r2   r+   zGemmaForCausalLM.__init__  sU     '
 ++yy!3!3V5F5FUS 	r3   Nr4   r   r   r   r  labelsr   logits_to_keepr   r~   c	           
      x    | j                   d||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|* | j                  d||| j                  j                  d|	}t        |||
j                  |
j                  |
j                        S )a|  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, GemmaForCausalLM

        >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r4   r   r   r   r  r   N)r$  r&  r  )lossr$  r   r   r   r   )r   r  r   r@   slicer"  loss_functionrb   r  r   r   r   r   )r0   r4   r   r   r   r  r&  r   r'  r   outputsr   slice_indicesr$  r)  s                  r2   r7   zGemmaForCausalLM.forward  s    > ,64:: ,
)%+',
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD%#33!//))
 	
r3   )NNNNNNNr   )r<   r=   r>   _tied_weights_keys_tp_plan_pp_planr+   r   r   r.   r   rB   r	   r  r   r@   r   r   r   r7   rC   rD   s   @r2   r!  r!    s   *,GH23H_-z:;H  .2.204(,26*.!%-.6
##d*6
 t+6
 &&-	6

 6
 ((4/6
   4'6
 $;6
 ell*6
 +,6
 
 6
  6
r3   r!  c                       e Zd Zy)GemmaForSequenceClassificationNr<   r=   r>   r   r3   r2   r2  r2        r3   r2  c                       e Zd Zy)GemmaForTokenClassificationNr3  r   r3   r2   r6  r6    r4  r3   r6  )r	  r!  r2  r6  r   )r   )r   )Dcollections.abcr   typingr   r.   r    r   r   activationsr   cache_utilsr	   r
   
generationr   integrationsr   r   masking_utilsr   modeling_layersr   r   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   utils.output_capturingr   configuration_gemmar    	Embeddingr"   ModulerF   r]   rn   r   r   rB   r@   r   rA   r   r   r   r   r	  r!  r2  r6  __all__r   r3   r2   <module>rK     s  . %    & ! . ) I / 
 P K F & I I G 5 ,S2<< S=299 =(ryy  ><299 ><B( *+ ,2	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%2 )*@)RYY @) +@)F(2 (V J? J J6 H
% H
 H
V F
+_ F
 F
R	%EG[ 		"?AU 	r3   