
    i(                        d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ  ej>                  e       Z!dZ"dZ# G d dejH                        Z% G d de      Z&d&dZ' G d dejH                        Z( G d de      Z) G d de      Z* G d d e      Z+ G d! d"e      Z, G d# d$e      Z-g d%Z.y)'zPyTorch Phi-3 model.    )CallableN)nn   )ACT2FN)Cache)GenerationMixin)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half)PhiRotaryEmbedding   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Phi3MLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr   	__class__s     v/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr   dim)r#   chunkr&   r$   )r'   r+   	up_statesgates       r)   forwardzPhi3MLP.forward9   sL    %%m4	#//!/4i 2 24 88	~~i((r*   )__name__
__module____qualname__r   torchFloatTensorr4   __classcell__r(   s   @r)   r   r   0   s'    7)U%6%6 )5;L;L )r*   r   c                       e Zd Zy)Phi3RotaryEmbeddingNr5   r6   r7    r*   r)   r=   r=   B       r*   r=   c                 `   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}t        j                  ||z  t	        |      |z  z   |gd      }
t        j                  ||z  t	        |      |z  z   |	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r.   .Nr/   )	unsqueezeshaper8   catr   )qkcossinunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r)   apply_rotary_pos_embrQ   F   s    $ --
&C
--
&C2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6Eii%#++e*<s*BCVLRTUGii%#++e*<s*BCVLRTUGGr*   c                       e Zd ZdZddededz  f fdZ	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  e	ej                     dz  f   fdZ xZS )Phi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   	layer_idxc                 |   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        |j                  | j                  z  d|j                  | j                  z  z  z   }t        j                  |j                  | j                  z  |j
                  d      | _        t        j                  |j
                  |d      | _        y )Nhead_dimg      Tr   Fr   )r   r   r   rT   getattrr!   num_attention_headsrV   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r    o_projqkv_proj)r'   r   rT   op_sizer(   s       r)   r   zPhi3Attention.__init__g   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr*   r+   position_embeddingsattention_maskpast_key_valueskwargsr,   c           
         |j                   d d }g |d| j                  }| j                  |      }| j                  j                  | j                  z  }	|dd |	f   }
|d|	|	| j
                  | j                  z  z   f   }|d|	| j
                  | j                  z  z   d f   }|
j                  |      j                  dd      }
|j                  |      j                  dd      }|j                  |      j                  dd      }|\  }}t        |
|||      \  }
}| |j                  ||| j                        \  }}t        j                  | j                  j                  t              } || |
|||f| j                  sdn| j                   | j"                  t%        | j                  dd       d|\  }} |j&                  g |d j)                         }| j+                  |      }||fS )Nr.   .r   r   g        sliding_window)dropoutr[   rf   )rC   rV   r_   r   rX   rY   view	transposerQ   updaterT   r
   get_interface_attn_implementationr   trainingr\   r[   rW   reshape
contiguousr^   )r'   r+   ra   rb   rc   rd   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrG   rH   attention_interfaceattn_outputattn_weightss                     r)   r4   zPhi3Attention.forwardv   s    $))#2.88b8$--8mmM*KK33dmmC	3

?+i)d6N6NQUQ^Q^6^*^^^_
3	D,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ *k));;;;FFHkk+.L((r*   )N)r5   r6   r7   __doc__r   intr   r8   Tensortupler   r   r	   r4   r:   r;   s   @r)   rS   rS   d   s    GKz KcDj K( )--)||-) #5<<#=>-) t+	-)
 -) -.-) 
u||U\\D0%2E2LL	M-)r*   rS   c                   D    e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )Phi3DecoderLayerr   rT   c                    t         |   ||       || _        t        ||      | _        t        |      | _        t        j                  |j                        | _
        t        j                  |j                        | _        y )N)r   rT   )r   r   r   rS   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r'   r   rT   r(   s      r)   r   zPhi3DecoderLayer.__init__   s`    +&f	J6?"$**V-?-?"@!#F,>,>!?r*   Nr+   rb   position_idsrc   	use_cachera   rd   r,   c           
          |}| j                  |      } | j                  d||||||d|\  }}	|| j                  |      z   }|}| j                  |      }| j	                  |      }|| j                  |      z   }|S )N)r+   rb   r   rc   r   ra   r?   )input_layernormr   r   post_attention_layernormr   r   )
r'   r+   rb   r   rc   r   ra   rd   residualself_attn_weightss
             r)   r4   zPhi3DecoderLayer.forward   s     !,,];+94>> ,
')%+ 3,
 ,
(( !4#:#:=#II 55mD/ 4#9#9-#HHr*   )NNNFN)r5   r6   r7   r   r{   r   r8   r|   
LongTensorr   boolr}   r   r	   r9   r4   r:   r;   s   @r)   r   r      s    @z @c @ /304(,!&HL|| t+ &&-	
  $; #5<<#=>E -. 
u  %(9(95;L;L(L"MPT"TT	Ur*   r   c                       e Zd ZdZy)Phi3PreTrainedModelz0.0.5N)r5   r6   r7   _versionr?   r*   r)   r   r      s    Hr*   r   c                        e Zd Z	 	 	 	 	 	 ddZy)Phi3ForCausalLMNc                    |rjt        | j                  d      rT|j                  d   | j                  j                  dz   k\  r+|j	                         }	|	| j                  j                  k  rd }t        j                  | f|||||||d|}
|
S )N original_max_position_embeddingsr   )	input_idsrc   rb   inputs_embedsr   r   logits_to_keep)hasattrr   rC   r   get_seq_lengthr   prepare_inputs_for_generation)r'   r   rc   rb   r   r   r   r   rd   past_lengthmodel_inputss              r)   r   z-Phi3ForCausalLM.prepare_inputs_for_generation   s    " %GH"dkk&R&RUV&VV)88:KdkkJJJ"&&DD

+)'%)

 

 r*   )NNNNTN)r5   r6   r7   r   r?   r*   r)   r   r      s     $r*   r   c                       e Zd Zy)Phi3ForSequenceClassificationNr>   r?   r*   r)   r   r      r@   r*   r   c                       e Zd Zy)Phi3ForTokenClassificationNr>   r?   r*   r)   r   r      r@   r*   r   )r   	Phi3Modelr   r   r   )r   )/rz   collections.abcr   r8   r   activationsr   cache_utilsr   
generationr   modeling_flash_attention_utilsr	   modeling_utilsr
   processing_utilsr   utilsr   mistral.modeling_mistralr   r   r   r   r   r   r   phi.modeling_phir   configuration_phi3r   
get_loggerr5   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   r=   rQ   rS   r   r   r   r   r   __all__r?   r*   r)   <module>r      s     $   !   ) B 5 &    2 * 
		H	%8 )bii )$	, 	<?)BII ?)D%* %P0 %( %P	$D 		!> 	r*   