
    ik                        d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2  e!jf                  e4      Z5e G d de             Z6 G d dejn                        Z8 G d de,      Z9 G d de(      Z: G d d e)      Z; G d! d"e'      Z< G d# d$ejn                        Z= G d% d&e      Z> G d' d(e6      Z? G d) d*e      Z@ G d+ d,e6      ZA ed-.       G d/ d0e6             ZB ed1.       G d2 d3e6e2             ZCg d4ZDy)5zPyTorch Dia model.    )CallableN)nn   )initialization)DynamicCacheEncoderDecoderCache)create_bidirectional_maskcreate_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)merge_with_config_defaults)capture_outputs   )LlamaAttentionLlamaRMSNormLlamaRotaryEmbeddingeager_attention_forward)Phi3MLP   )	DiaConfigDiaDecoderConfigDiaEncoderConfig)DiaGenerationMixinc                   N     e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZ fdZ xZS )DiaPreTrainedModelconfigmodelT	input_idsDiaEncoderLayerDiaDecoderLayerc                 &   t         |   |       t        |t              rqt	        j
                  | j                  j                  t        j                        | j                  j                  z  }t        j                  |j                  |       y y )Ndtype)super_init_weights
isinstanceDiaMultiChannelEmbeddingtorcharanger(   num_channelslong
vocab_sizeinitcopy_offsets)selfmoduler;   	__class__s      t/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/dia/modular_dia.pyr1   z DiaPreTrainedModel._init_weights?   sb    f%f67ll4;;#;#;5::NQUQ\Q\QgQggGJJv~~w/ 8    )__name__
__module____qualname__r"   __annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraphmain_input_name_no_split_modulesr1   __classcell__r>   s   @r?   r'   r'   3   sG    &*#N!!O*,=>0 0r@   r'   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )r3   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r(   c                 ~   t         |           t        j                  |j                  |j
                  z  |j                        | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                        |j                  z  }| j                  d|d       y )Nr.   r;   F)
persistent)r0   __init__r   	Embeddingr8   r6   hidden_sizeembedr4   r5   r7   register_buffer)r<   r(   r;   r>   s      r?   rR   z!DiaMultiChannelEmbedding.__init__T   s    \\&"3"3f6I6I"I6K]K]^
!--"//,,v22%**EHYHYYYEBr@   audio_codesreturnc                 "   || j                   j                  |j                        z   j                  d      }| j	                  |      j                  |j                  d   |j                  d   d| j                        }|j                  d      S )Nr!   r   r   )dim)	r;   todevicesqueezerU   viewshaperT   sum)r<   rW   tokensembedss       r?   forwardz DiaMultiChannelEmbedding.forward\   su    0B0B CCLLQOF#((a+:K:KA:NPRTXTdTdezzaz  r@   )
rA   rB   rC   __doc__r#   rR   r4   Tensorrd   rM   rN   s   @r?   r3   r3   F   s2    C/ C!5<< !ELL !r@   r3   c                       e Zd Zy)DiaMLPNrA   rB   rC    r@   r?   rh   rh   b       r@   rh   c                       e Zd Zy)
DiaRMSNormNri   rj   r@   r?   rm   rm   f   rk   r@   rm   c                       e Zd Zy)DiaRotaryEmbeddingNri   rj   r@   r?   ro   ro   j   rk   r@   ro   c                   ,    e Zd ZdZddeez  dedefdZy)DiaSelfAttention=Multi-headed attention from 'Attention Is All You Need' paperr(   	layer_idx	is_causalc                    t         j                  j                  |        || _        || _        |j
                  | _        | j                  j                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _	        t        |d|j
                  | j                  z        | _        d| _        d| _        || _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j
                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j
                  d      | _        y )Nhead_dimr!           Fbias)r   ModulerR   r(   rs   rT   num_attention_heads	num_headsnum_key_value_headsnum_key_value_groupsgetattrrv   scalingattention_dropoutrt   Linearq_projk_projv_projo_proj)r<   r(   rs   rt   s       r?   rR   zDiaSelfAttention.__init__q   sL   
		4 "!--88#';;#B#B#Tdnn $(NNd6N6N$N!
F4F4F$..4XY!$"ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]r@   N)F)	rA   rB   rC   re   r$   r#   intboolrR   rj   r@   r?   rq   rq   n   s+    G^/2BB ^s ^_c ^r@   rq   c                        e Zd ZdZdedef fdZ	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fdZ xZS )DiaCrossAttentionrr   r(   rs   c                 f   t         |           || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j                  | j                  z  | j                  d      | _        y )Nr!   rw   Frx   )r0   rR   r(   rs   rT   cross_hidden_sizecross_num_attention_headsr|   cross_num_key_value_headsr}   r~   cross_head_dimrv   r   r   rt   r   r   r   r   r   r   r<   r(   rs   r>   s      r?   rR   zDiaCrossAttention.__init__   s?   "!--!'!9!9>>#';;#H#H $(NNd6N6N$N!--!$ii 0 0$..4==2PW\]ii 6 68P8PSWS`S`8`glmii 6 68P8PSWS`S`8`glmii >@P@PW\]r@   Nhidden_statescross_attention_statesattention_maskpast_key_valueskwargsrX   c                 F   |j                   d d }g |d| j                  }g |j                   d d d| j                  }| j                  |      j                  |      j	                  dd      }	|%|j
                  j                  | j                        nd}
|]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }|C|j                  j                  ||| j                        \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&              } || |	|||fd| j(                  i|\  }}|j+                  g |d      j-                         }| j/                  |      }||fS )NrZ   r!   r   FTr   )r`   rv   r   r_   	transpose
is_updatedgetrs   cross_attention_cachelayerskeysvaluesr   r   updater   get_interfacer(   _attn_implementationr   r   reshape
contiguousr   )r<   r   r   r   r   r   input_shapehidden_shapecross_shapequery_statesr   
key_statesvalue_statesattention_interfaceattn_outputattn_weightss                   r?   rd   zDiaCrossAttention.forward   s    $))#2.88b8$--8M.44Sb9M2Mt}}M{{=166|DNNqRSTGVGb_//33DNNChm
&:(>>EEdnnUZZJ*@@GGW^^L%;<AA+NXXYZ\]^J;;'=>CCKPZZ[\^_`L*+:+P+P+W+W NN,(
L >B**4>>:(?(M(MKK,,.E)
 %8%
 LL%
 %
!\ "))*<K*<*<=HHJkk+.L((r@   NN)rA   rB   rC   re   r#   r   rR   r4   rf   r   r   r   tuplerd   rM   rN   s   @r?   r   r      s    G^/ ^C ^. /36:1)||1) !&1) t+	1)
 -t31) -.1) 
u||U\\D00	11)r@   r   c                        e Zd Zdedef fdZ	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
Z xZS )r+   r(   rs   c                     t         |           t        |j                  |j                        | _        t        ||d      | _        t        |j                  |j                        | _        t        |      | _
        y )NepsFrt   )r0   rR   rm   rT   norm_epspre_sa_normrq   self_attentionpost_sa_normrh   mlpr   s      r?   rR   zDiaEncoderLayer.__init__   s\    %f&8&8fooN.vyER&v'9'9vO&>r@   Nr   position_embeddingsr   r   rX   c                     |}| j                  |      } | j                  |f||d|\  }}||z   }|}| j                  |      }| j                  |      }	||	z   }|S )N)r   r   )r   r   r   r   )
r<   r   r   r   r   residualnormed_statesself_attn_output_mlp_outs
             r?   rd   zDiaEncoderLayer.forward   s     !((71d11
 3)
 	
! !#33 ))-8((=) 7*r@   r   )rA   rB   rC   r$   r   rR   r4   rf   r   r   r   rd   rM   rN   s   @r?   r+   r+      s    "/ "C " IM.2	|| #5<<#=>E t+	
 -. 
u||U\\D00	1r@   r+   c                        e Zd ZeedZdef fdZee	e
	 d
dej                  dej                  dz  dee   defd	                     Z xZS )
DiaEncoderr   
attentionsr(   c           	         t         |   |       || _        t        j                  |j
                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w Nr   r(   )r0   rR   r(   r   rS   r8   rT   	embedding
ModuleListrangenum_hidden_layersr+   r   rm   r   normro   
rotary_emb	post_initr   s      r?   rR   zDiaEncoder.__init__   s     f&7&79K9KLmmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   -CNr*   r   r   rX   c                 Z   | j                  |      }t        j                  |j                  d   |j                        d d d f   }t        | j                  ||      }| j                  ||      }| j                  D ]  } ||f|||d|} | j                  |      }t        |      S )NrZ   r]   )r(   inputs_embedsr   position_ids)r   r   r   )last_hidden_state)r   r4   r5   r`   r]   r	   r(   r   r   r   r   )r<   r*   r   r   r   r   r   encoder_layers           r?   rd   zDiaEncoder.forward  s     y1
 ||IOOB$7	@P@PQRVXYRYZ2;;')

 #oom,oW![[ 	M)-)$7	
 M	 		-0??r@   N)rA   rB   rC   r+   rq   _can_record_outputsr$   rR   r   r   r   r4   rf   r   r   r   rd   rM   rN   s   @r?   r   r      s    (&
/    /3@<<@ t+@ +,	@
 
@    @r@   r   c                   L    e Zd Zdedef fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  deej                  ej                  dz  ej                  dz  f   fdZ
 xZS )r,   r(   rs   c                    t         |           |j                  | _        t	        ||d      | _        t        ||      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        y )NTr   r   )r0   rR   rT   	embed_dimrq   r   r   cross_attentionrm   r   r   pre_ca_normpre_mlp_normrh   r   r   s      r?   rR   zDiaDecoderLayer.__init__(  s    ++.vyDQ0C%f&8&8fooN%f&8&8fooN&v'9'9vO&>r@   Nr   r   r   encoder_hidden_statesencoder_attention_maskr   rX   c                 Z   |}t        |t              r|j                  }|}	| j                  |      }
 | j                  |
|||fi |\  }}|	|z   }|}	| j                  |      }
 | j                  |
|f||d|\  }}|	|z   }|}	| j                  |      }
| j                  |
      }|	|z   }|S )N)r   r   )	r2   r   self_attention_cacher   r   r   r   r   r   )r<   r   r   r   r   r   r   r   self_attn_cacher   r   r   r   cross_statesr   s                  r?   rd   zDiaDecoderLayer.forward2  s     *o':;-BBO ((71d11 
 
! !#33 ((7.$..!
 2+	

 
a !</ ))-8((=) 7*r@   NNNNN)rA   rB   rC   r#   r   rR   r4   rf   r   r   rd   rM   rN   s   @r?   r,   r,   '  s    "/ "C " IM.2596:6:+||+ #5<<#=>E+ t+	+
  %||d2+ !&t 3+ -t3+ 
u||U\\D0%,,2EE	F+r@   r,   c                       e Zd ZdZeeegdZdef fdZ	e
ee	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dedz  dee   deez  fd                     Z xZS )
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.r   r(   c           	         t         |   |       |j                  | _        |j                  | _        t	        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w r   )r0   rR   r6   r8   r3   
embeddingsr   r   r   r   r,   r   rm   rT   r   r   ro   r   r   r   s      r?   rR   zDiaDecoder.__init__h  s     "// ++26:mmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   )CNr*   r   r   r   r   r   r   rX   c                 T   |j                         dd \  }}	||j                         nd}
|5t        j                  |	|j                        |
z   }|j                  d      }| j                  |      }|1t               s'|
|	z   }t        j                  |||j                        }t        | j                  |||      }t        | j                  |||      }| j                  ||      }| j                  D ]  } |||||f|||d|} | j                  |      }t        ||	      S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        NrZ   r   r   )r(   r   r   r   )r(   r   r   r   r   )r   r   r   )r   r   )sizeget_seq_lengthr4   r5   r]   	unsqueezer   r   onesr
   r(   r	   r   r   r   r   )r<   r*   r   r   r   r   r   r   
batch_size
seq_lengthpast_key_values_lengthr   mask_seq_lengthr   layers                  r?   rd   zDiaDecoder.forwardu  sW   ( "+!1#2!6
JETE`!?!?!Afg <<
9;K;KLOeeL'11!4L 	2!*B*D4zAO"ZZ
OIL\L\]N+;;')+	
 ";;;'1"7	"
 #oom,oW[[ 	E! $% (> /) M	 		-08++
 	
r@   r   )rA   rB   rC   re   r,   rq   r   r   r#   rR   r   r   r   r4   rf   
LongTensorFloatTensorr   r   r   r   r   rd   rM   rN   s   @r?   r   r   `  s    7 )'):;
/    15.2:>:>6:A
<<A
 &&-A
 t+	A

  %0047A
 !& 0 04 7A
 -t3A
 +,A
 
3U	:A
    A
r@   r   z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )custom_introc                       e Zd Zdef fdZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd              Z xZS )DiaModelr(   c                     t         |   |       || _        t        |j                        | _        t        |j                        | _        | j                          y r   )
r0   rR   r(   r   encoder_configencoderr   decoder_configdecoderr   r<   r(   r>   s     r?   rR   zDiaModel.__init__  sE     !&"7"78!&"7"78r@   Nr*   r   decoder_input_idsdecoder_position_idsdecoder_attention_maskencoder_outputsr   	use_cacherX   c	                    ||t        d      | j                  r%| j                  r|rt        j	                  d       d}|r6|4t        t        | j                        t        | j                              }| | j                  d||d|	}nGt        |t              s7t        |d   t        |      dkD  r|d   ndt        |      d	kD  r|d	   nd
      }|d   j                  d   d| j                  j                  j                  }}}
|Ct        j                   |
d|f| j                  j                  j"                  | j$                        }|j&                  d	k(  r#|j)                  |
||      j+                  dd	      } | j,                  d||||d   |||d|	}t/        |j0                  |j2                  |j4                  |j6                  |j8                  |d   |j4                  |j6                        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   )r*   r   r   r!   r   )r   r   r   rZ   )r   
fill_valuer]   )r*   r   r   r   r   r   r   )r   r   decoder_hidden_statesdecoder_attentionscross_attentionsencoder_last_hidden_stater   encoder_attentionsrj   )
ValueErroris_gradient_checkpointingtrainingloggerwarning_oncer   r   r(   r   r2   r   lenr`   r   r6   r4   fullbos_token_idr]   ndimr   r   r   r   r   r   r   r   r  )r<   r*   r   r   r   r   r   r   r   r   bszseq_lenchannelsdecoder_outputss                 r?   rd   zDiaModel.forward  s   H !8j  ))dmm##p "	01,dkk2RT`hlhshsTtuO"*dll #- O O_=-"1!"4474H14Loa0RV14_1E1I?1-tO #2!"4":":1"=r4;;C]C]CjCjhW$ %

1h'DKK4N4N4[4[dhdodo! !!Q& 1 9 9#x Q [ [\]_` a&$,, 	
'-1"1!"4#1+	
 	
 "-??+;;"1"?"?.99,==&5a&8"1"?"?.99	
 		
r@   )NNNNNNNN)rA   rB   rC   r"   rR   r   r   r4   r   r   r   r   r   r   rd   rM   rN   s   @r?   r   r     s    y   .226598<:>:>6:!%]
##d*]
 ((4/]
 !++d2	]

 $..5]
 !& 0 04 7]
 )5047]
 -t3]
 $;]
 
#	#]
  ]
r@   r   zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   >    e Zd ZdZdZdef fdZee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  de	j                  dz  deez  fd              Z xZS )DiaForConditionalGenerationr)   )audior(   c                 |   t         |   |       || _        t        |      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d      | _        d| _        | j                          y )NFrx   ForMaskedLM)r0   rR   r(   r   r)   r   r6   r8   r   r   rT   logits_dense	loss_typer   r   s     r?   rR   z$DiaForConditionalGeneration.__init__4  s     f%
"11>> //::II!!--0A0ADOO0S[`
 ' 	r@   Nr*   r   r   r   r   r   r   r   labelsrX   c
                 X    | j                   d	||||||||d|
}|d   }|j                  d   }| j                  |      j                  |d| j                  | j
                  f      j                  dd      j                         j                  || j                  z  d| j
                        }d}|	  | j                  d	||	| j
                  d|
}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   	      S )
a  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r*   r   r   r   r   r   r   r   r   rZ   r!   r   N)logitsr  r8   )	lossr  r   r  r  r  r  r   r  rj   )r)   r`   r  r_   r6   r8   r   r   loss_functionr   r   r  r  r  r  r   r  )r<   r*   r   r   r   r   r   r   r   r  r   outputsr   r   audio_logitsr  s                   r?   rd   z#DiaForConditionalGeneration.forwardC  s?   R $** 

)/!5#9++

 

 $AJ&,,Q/
 /0T:r4#4#4dooFGYq!_Z\T*t000"dooF 	 %4%%o\&UYUdUdohnoD#33")"?"?&99$55&-&G&G")"?"?&99

 
	
r@   )	NNNNNNNNN)rA   rB   rC   rE   output_modalitiesr"   rR   r   r   r4   r   r   r   r   r   r   rd   rM   rN   s   @r?   r  r  +  s     "y   .226598<:>:>6:!%*.L
##d*L
 ((4/L
 !++d2	L

 $..5L
 !& 0 04 7L
 )5047L
 -t3L
 $;L
   4'L
 
	 L
  L
r@   r  )r   r'   r  )Ere   collections.abcr   r4   r    r   r9   cache_utilsr   r   masking_utilsr	   r
   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   utils.output_capturingr   llama.modeling_llamar   r   r   r   phi3.modeling_phi3r    configuration_diar"   r#   r$   generation_diar%   
get_loggerrA   r  r'   rz   r3   rh   rm   ro   rq   r   r+   r   r,   r   r   r  __all__rj   r@   r?   <module>r6     sy    $   & < J B 9  G & l l 7 5  ) L L . 
		H	% 0 0 0$!ryy !8	W 		 		- 	^~ ^,G)		 G)T0 B5@# 5@p60 6rY
# Y
x 
g
! g

g
T 
a
"46H a

a
H Lr@   