
    i                     Z   d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7  e,jp                  e9      Z:e) G d de$             Z; G d dejx                        Z= G d dejx                        Z> ed       G d dejx                               Z? G d  d!ejx                        Z@d" ZA ed#      dFd$       ZBd%ej                  d&eDd'ej                  fd(ZE	 dGd)ejx                  d*ej                  d+ej                  d,ej                  d-ej                  dz  d.eFd/eFd0e&e(   fd1ZG eeB       G d2 d3ejx                               ZH G d4 d5ejx                        ZI G d6 d7e      ZJ G d8 d9e;      ZK G d: d;e      ZL G d< d=e;      ZM e)d>?       G d@ dAe;             ZN e)dB?       G dC dDe;e7             ZOg dEZPy)H    )Callable)OptionalN)nn   )initialization)ACT2FN)CacheDynamicCacheEncoderDecoderCache)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_bidirectional_maskcreate_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)maybe_autocastmerge_with_config_defaults)capture_outputs   )	DiaConfigDiaDecoderConfigDiaEncoderConfig)DiaGenerationMixinc                   N     e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZ fdZ xZS )DiaPreTrainedModelconfigmodelT	input_idsDiaEncoderLayerDiaDecoderLayerc                 &   t         |   |       t        |t              rqt	        j
                  | j                  j                  t        j                        | j                  j                  z  }t        j                  |j                  |       y y )Ndtype)super_init_weights
isinstanceDiaMultiChannelEmbeddingtorcharanger+   num_channelslong
vocab_sizeinitcopy_offsets)selfmoduler>   	__class__s      u/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/dia/modeling_dia.pyr4   z DiaPreTrainedModel._init_weightsA   sb    f%f67ll4;;#;#;5::NQUQ\Q\QgQggGJJv~~w/ 8    )__name__
__module____qualname__r%   __annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraphmain_input_name_no_split_modulesr4   __classcell__rA   s   @rB   r*   r*   5   sG    &*#N!!O*,=>0 0rC   r*   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )r6   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r+   c                 ~   t         |           t        j                  |j                  |j
                  z  |j                        | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                        |j                  z  }| j                  d|d       y )Nr1   r>   F
persistent)r3   __init__r   	Embeddingr;   r9   hidden_sizeembedr7   r8   r:   register_buffer)r?   r+   r>   rA   s      rB   rV   z!DiaMultiChannelEmbedding.__init__V   s    \\&"3"3f6I6I"I6K]K]^
!--"//,,v22%**EHYHYYYEBrC   audio_codesreturnc                 "   || j                   j                  |j                        z   j                  d      }| j	                  |      j                  |j                  d   |j                  d   d| j                        }|j                  d      S )Nr$   r      dim)	r>   todevicesqueezerY   viewshaperX   sum)r?   r[   tokensembedss       rB   forwardz DiaMultiChannelEmbedding.forward^   su    0B0B CCLLQOF#((a+:K:KA:NPRTXTdTdezzaz  rC   )
rD   rE   rF   __doc__r&   rV   r7   Tensorrj   rP   rQ   s   @rB   r6   r6   H   s2    C/ C!5<< !ELL !rC   r6   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )DiaMLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr_   Fbias)r3   rV   r+   r   LinearrX   intermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fnr?   r+   rA   s     rB   rV   zDiaMLP.__init__e   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56rC   hidden_statesr\   c                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr_   r^   r`   )rt   chunkrw   ru   )r?   ry   	up_statesgates       rB   rj   zDiaMLP.forwardm   sL    %%m4	#//!/4i 2 24 88	~~i((rC   )rD   rE   rF   rV   r7   FloatTensorrj   rP   rQ   s   @rB   rn   rn   d   s'    7)U%6%6 )5;L;L )rC   rn   RMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	
DiaRMSNormepsr\   Nc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z9
        DiaRMSNorm is equivalent to T5LayerNorm
        N)r3   rV   r   	Parameterr7   onesweightvariance_epsilon)r?   rX   r   rA   s      rB   rV   zDiaRMSNorm.__init__x   s1     	ll5::k#:; #rC   ry   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )Nr_   r^   T)keepdim)	r2   rb   r7   float32powmeanrsqrtr   r   )r?   ry   input_dtypevariances       rB   rj   zDiaRMSNorm.forward   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::rC   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler   rf   r   )r?   s    rB   
extra_reprzDiaRMSNorm.extra_repr   s*    ))*+6$2G2G1HIIrC   )gư>)
rD   rE   rF   floatrV   r7   rl   rj   r   rP   rQ   s   @rB   r   r   v   s7    $ $$ $;U\\ ;ell ;JrC   r   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )DiaRotaryEmbeddinginv_freqNr+   c                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultr   FrT   original_inv_freq)r3   rV   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr+   rope_parametersr   compute_default_rope_parametersr   attention_scalingrZ   clone)r?   r+   rc   rope_init_fnr   rA   s        rB   rV   zDiaRotaryEmbedding.__init__   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUrC   rc   ztorch.deviceseq_lenr\   ztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNg      ?r   r_   r1   )rc   r2   )	r   getattrrX   num_attention_headsr7   r8   int64rb   r   )r+   rc   r   basera   attention_factorr   s          rB   r   z2DiaRotaryEmbedding.compute_default_rope_parameters   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))rC   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r^   r$   mpscpuF)device_typeenabledr_   r`   r1   )r   r   expandrf   rb   rc   r5   typestrr!   	transposer7   catcosr   sinr2   )
r?   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             rB   rj   zDiaRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$NNNN)rD   rE   rF   r7   rl   rG   r%   rV   staticmethodr   intr   r   r   no_gradr   rj   rP   rQ   s   @rB   r   r      s    llVy V  #'+/"*D *(* t* 
~u$	%	* *: U]]_<  <rC   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr^   r_   r`   )rf   r7   r   )r   x1x2s      rB   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rC   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   unsqueeze_dimq_embedk_embeds          rB   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGrC   ry   n_repr\   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)rf   r   reshape)ry   r   batchnum_key_value_headsslenr   s         rB   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTrC   r@   querykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr_   r   r^   )ra   r2   )ptrainingr$   )r   num_key_value_groupsr7   matmulr   r   
functionalsoftmaxr   rb   r2   r   r   
contiguous)r@   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               rB   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$rC   c                       e Zd ZdZddeez  dedef fdZ	 	 	 dde	j                  dee	j                  e	j                  f   dz  d	e	j                  dz  d
edz  dee   dee	j                  e	j                  f   fdZ xZS )DiaSelfAttention=Multi-headed attention from 'Attention Is All You Need' paperr+   	layer_idx	is_causalc                    t         |           || _        || _        |j                  | _        | j                  j
                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _        t        |d|j                  | j                  z        | _
        d| _        d| _        || _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        y )Nr   r$           Frp   )r3   rV   r+   r   rX   r   	num_headsr   r   r   r   r   attention_dropoutr   r   rr   q_projk_projv_projo_proj)r?   r+   r   r   rA   s       rB   rV   zDiaSelfAttention.__init__  sF   "!--88#';;#B#B#Tdnn $(NNd6N6N$N!
F4F4F$..4XY!$"ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]rC   Nry   position_embeddingsr   past_key_valuesr   r\   c                 
   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nr^   r$   r_   r   )r   r   )rf   r   r   re   r   r   r   r   updater   r   get_interfacer+   _attn_implementationr   r   r   r   r   r   r   )r?   ry   r   r   r   r   input_shapehidden_shapequery_statesr   r   r   r   attention_interfacer   r   s                   rB   rj   zDiaSelfAttention.forward(  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((rC   )Fr   )rD   rE   rF   rk   r'   r&   r   boolrV   r7   rl   r   r	   r   r   rj   rP   rQ   s   @rB   r   r     s    G^/2BB ^s ^_c ^* IM.2(,&)||&) #5<<#=>E&) t+	&)
 &) +,&) 
u||U\\)	*&)rC   r   c                        e Zd ZdZdedef fdZ	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fdZ xZS )DiaCrossAttentionr   r+   r   c                 f   t         |           || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j
                  | j                  | j                  z  d      | _        t!        j"                  | j                  | j                  z  | j                  d      | _        y )Nr$   r   Frp   )r3   rV   r+   r   rX   cross_hidden_sizecross_num_attention_headsr   cross_num_key_value_headsr   r   cross_head_dimr   r   r   r   r   rr   r   r   r   r   r?   r+   r   rA   s      rB   rV   zDiaCrossAttention.__init__T  s?   "!--!'!9!9>>#';;#H#H $(NNd6N6N$N!--!$ii 0 0$..4==2PW\]ii 6 68P8PSWS`S`8`glmii 6 68P8PSWS`S`8`glmii >@P@PW\]rC   Nry   cross_attention_statesr   r   r   r\   c                 F   |j                   d d }g |d| j                  }g |j                   d d d| j                  }| j                  |      j                  |      j	                  dd      }	|%|j
                  j                  | j                        nd}
|]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }|C|j                  j                  ||| j                        \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&              } || |	|||fd| j(                  i|\  }}|j+                  g |d      j-                         }| j/                  |      }||fS )Nr^   r$   r_   FTr   )rf   r   r   re   r   
is_updatedgetr   cross_attention_cachelayerskeysvaluesr   r   r   r   r   r+   r   r   r   r   r   r   )r?   ry   r  r   r   r   r   r   cross_shaper   r  r   r   r  r   r   s                   rB   rj   zDiaCrossAttention.forwardg  s    $))#2.88b8$--8M.44Sb9M2Mt}}M{{=166|DNNqRSTGVGb_//33DNNChm
&:(>>EEdnnUZZJ*@@GGW^^L%;<AA+NXXYZ\]^J;;'=>CCKPZZ[\^_`L*+:+P+P+W+W NN,(
L >B**4>>:(?(M(MKK,,.E)
 %8%
 LL%
 %
!\ "))*<K*<*<=HHJkk+.L((rC   NN)rD   rE   rF   rk   r&   r   rV   r7   rl   r   r   r   r   rj   rP   rQ   s   @rB   r  r  Q  s    G^/ ^C ^. /36:1)||1) !&1) t+	1)
 -t31) -.1) 
u||U\\D00	11)rC   r  c                        e Zd Zdedef fdZ	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
Z xZS )r.   r+   r   c                     t         |           t        |j                  |j                        | _        t        ||d      | _        t        |j                  |j                        | _        t        |      | _
        y )Nr   Fr   )r3   rV   r   rX   norm_epspre_sa_normr   self_attentionpost_sa_normrn   mlpr
  s      rB   rV   zDiaEncoderLayer.__init__  s\    %f&8&8fooN.vyER&v'9'9vO&>rC   Nry   r   r   r   r\   c                     |}| j                  |      } | j                  |f||d|\  }}||z   }|}| j                  |      }| j                  |      }	||	z   }|S )N)r   r   )r  r  r  r  )
r?   ry   r   r   r   residualnormed_statesself_attn_output_mlp_outs
             rB   rj   zDiaEncoderLayer.forward  s     !((71d11
 3)
 	
! !#33 ))-8((=) 7*rC   r  )rD   rE   rF   r'   r   rV   r7   rl   r   r   r   rj   rP   rQ   s   @rB   r.   r.     s    "/ "C " IM.2	|| #5<<#=>E t+	
 -. 
u||U\\D00	1rC   r.   c                        e Zd ZeedZdef fdZee	e
	 d
dej                  dej                  dz  dee   defd	                     Z xZS )
DiaEncoderry   
attentionsr+   c           	         t         |   |       || _        t        j                  |j
                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w Nr  r+   )r3   rV   r+   r   rW   r;   rX   	embedding
ModuleListrangenum_hidden_layersr.   r  r   r  normr   
rotary_emb	post_initr
  s      rB   rV   zDiaEncoder.__init__  s     f&7&79K9KLmmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   -CNr-   r   r   r\   c                 Z   | j                  |      }t        j                  |j                  d   |j                        d d d f   }t        | j                  ||      }| j                  ||      }| j                  D ]  } ||f|||d|} | j                  |      }t        |      S )Nr^   rc   )r+   inputs_embedsr   r   )r   r   r   )last_hidden_state)r+  r7   r8   rf   rc   r   r+   r0  r  r/  r   )r?   r-   r   r   ry   r   r   encoder_layers           rB   rj   zDiaEncoder.forward  s     y1
 ||IOOB$7	@P@PQRVXYRYZ2;;')

 #oom,oW![[ 	M)-)$7	
 M	 		-0??rC   r   )rD   rE   rF   r.   r   _can_record_outputsr'   rV   r"   r#   r   r7   rl   r   r   r   rj   rP   rQ   s   @rB   r%  r%    s    (&
/    /3@<<@ t+@ +,	@
 
@    @rC   r%  c                   L    e Zd Zdedef fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  deej                  ej                  dz  ej                  dz  f   fdZ
 xZS )r/   r+   r   c                    t         |           |j                  | _        t	        ||d      | _        t        ||      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        y )NTr  r  )r3   rV   rX   	embed_dimr   r  r  cross_attentionr   r  r  pre_ca_normpre_mlp_normrn   r  r
  s      rB   rV   zDiaDecoderLayer.__init__  s    ++.vyDQ0C%f&8&8fooN%f&8&8fooN&v'9'9vO&>rC   Nry   r   r   encoder_hidden_statesencoder_attention_maskr   r\   c                 Z   |}t        |t              r|j                  }|}	| j                  |      }
 | j                  |
|||fi |\  }}|	|z   }|}	| j                  |      }
 | j                  |
|f||d|\  }}|	|z   }|}	| j                  |      }
| j                  |
      }|	|z   }|S )N)r   r   )	r5   r   self_attention_cacher  r  r=  r<  r>  r  )r?   ry   r   r   r?  r@  r   r   self_attn_cacher  r   r!  r"  cross_statesr#  s                  rB   rj   zDiaDecoderLayer.forward  s     *o':;-BBO ((71d11 
 
! !#33 ((7.$..!
 2+	

 
a !</ ))-8((=) 7*rC   NNNNN)rD   rE   rF   r&   r   rV   r7   rl   r   r   rj   rP   rQ   s   @rB   r/   r/     s    "/ "C " IM.2596:6:+||+ #5<<#=>E+ t+	+
  %||d2+ !&t 3+ -t3+ 
u||U\\D0%,,2EE	F+rC   r/   c                       e Zd ZdZeeegdZdef fdZ	e
ee	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dedz  dee   deez  fd                     Z xZS )
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.r&  r+   c           	         t         |   |       |j                  | _        |j                  | _        t	        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t!        |      | _        | j%                          y c c}w r)  )r3   rV   r9   r;   r6   
embeddingsr   r,  r-  r.  r/   r  r   rX   r  r/  r   r0  r1  r
  s      rB   rV   zDiaDecoder.__init__5  s     "// ++26:mmAFvG_G_A`aI_VY/a
 v11vG	,F; bs   )CNr-   r   r   r?  r@  r   r   r\   c                 T   |j                         dd \  }}	||j                         nd}
|5t        j                  |	|j                        |
z   }|j                  d      }| j                  |      }|1t               s'|
|	z   }t        j                  |||j                        }t        | j                  |||      }t        | j                  |||      }| j                  ||      }| j                  D ]  } |||||f|||d|} | j                  |      }t        ||	      S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        Nr^   r   r3  )r+   r4  r   r   )r+   r4  r   r?  r5  )r@  r   r   )r6  r   )sizeget_seq_lengthr7   r8   rc   r   rI  r   r   r   r+   r   r0  r  r/  r   )r?   r-   r   r   r?  r@  r   r   
batch_size
seq_lengthpast_key_values_lengthry   mask_seq_lengthr   layers                  rB   rj   zDiaDecoder.forwardB  sW   ( "+!1#2!6
JETE`!?!?!Afg <<
9;K;KLOeeL'11!4L 	2!*B*D4zAO"ZZ
OIL\L\]N+;;')+	
 ";;;'1"7	"
 #oom,oW[[ 	E! $% (> /) M	 		-08++
 	
rC   rE  )rD   rE   rF   rk   r/   r   r  r8  r&   rV   r"   r#   r   r7   rl   
LongTensorr~   r   r   r   r   r   rj   rP   rQ   s   @rB   rG  rG  -  s    7 )'):;
/    15.2:>:>6:A
<<A
 &&-A
 t+	A

  %0047A
 !& 0 04 7A
 -t3A
 +,A
 
3U	:A
    A
rC   rG  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )custom_introc                       e Zd Zdef fdZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd              Z xZS )DiaModelr+   c                     t         |   |       || _        t        |j                        | _        t        |j                        | _        | j                          y r   )
r3   rV   r+   r%  encoder_configencoderrG  decoder_configdecoderr1  rx   s     rB   rV   zDiaModel.__init__  sE     !&"7"78!&"7"78rC   Nr-   r   decoder_input_idsdecoder_position_idsdecoder_attention_maskencoder_outputsr   	use_cacher\   c	                    ||t        d      | j                  r%| j                  r|rt        j	                  d       d}|r6|4t        t        | j                        t        | j                              }| | j                  d||d|	}nGt        |t              s7t        |d   t        |      dkD  r|d   ndt        |      d	kD  r|d	   nd
      }|d   j                  d   d| j                  j                  j                  }}}
|Ct        j                   |
d|f| j                  j                  j"                  | j$                        }|j&                  d	k(  r#|j)                  |
||      j+                  dd	      } | j,                  d||||d   |||d|	}t/        |j0                  |j2                  |j4                  |j6                  |j8                  |d   |j4                  |j6                        S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr*  )r-   r   r   r$   r_   )r6  ry   r'  r^   )rK  
fill_valuerc   )r-   r   r   r?  r@  r   r_  )r6  r   decoder_hidden_statesdecoder_attentionscross_attentionsencoder_last_hidden_stater?  encoder_attentions )
ValueErroris_gradient_checkpointingr   loggerwarning_oncer   r
   r+   rX  r5   r   lenrf   rY  r9   r7   fullbos_token_idrc   ndimr   r   rZ  r   r6  r   ry   r'  rd  )r?   r-   r   r[  r\  r]  r^  r   r_  r   bszr   channelsdecoder_outputss                 rB   rj   zDiaModel.forward  s   H !8j  ))dmm##p "	01,dkk2RT`hlhshsTtuO"*dll #- O O_=-"1!"4474H14Loa0RV14_1E1I?1-tO #2!"4":":1"=r4;;C]C]CjCjhW$ %

1h'DKK4N4N4[4[dhdodo! !!Q& 1 9 9#x Q [ [\]_` a&$,, 	
'-1"1!"4#1+	
 	
 "-??+;;"1"?"?.99,==&5a&8"1"?"?.99	
 		
rC   )NNNNNNNN)rD   rE   rF   r%   rV   r   r   r7   rR  r   r   r   r  r   rj   rP   rQ   s   @rB   rU  rU    s    y   .226598<:>:>6:!%]
##d*]
 ((4/]
 !++d2	]

 $..5]
 !& 0 04 7]
 )5047]
 -t3]
 $;]
 
#	#]
  ]
rC   rU  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   >    e Zd ZdZdZdef fdZee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  de	j                  dz  deez  fd              Z xZS )DiaForConditionalGenerationr,   )audior+   c                 |   t         |   |       || _        t        |      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d      | _        d| _        | j                          y )NFrp   ForMaskedLM)r3   rV   r+   rU  r,   rY  r9   r;   r   rr   rX   logits_dense	loss_typer1  rx   s     rB   rV   z$DiaForConditionalGeneration.__init__  s     f%
"11>> //::II!!--0A0ADOO0S[`
 ' 	rC   Nr-   r   r[  r\  r]  r^  r   r_  labelsr\   c
                 X    | j                   d	||||||||d|
}|d   }|j                  d   }| j                  |      j                  |d| j                  | j
                  f      j                  dd      j                         j                  || j                  z  d| j
                        }d}|	  | j                  d	||	| j
                  d|
}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   	      S )
a  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r-   r   r[  r\  r]  r^  r   r_  r   r^   r$   r_   N)logitsrz  r;   )	lossr|  r   rb  rc  rd  re  r?  rf  rg  )r,   rf   rx  re   r9   r;   r   r   loss_functionr   r   rb  rc  rd  re  r?  rf  )r?   r-   r   r[  r\  r]  r^  r   r_  rz  r   outputsr6  rM  audio_logitsr}  s                   rB   rj   z#DiaForConditionalGeneration.forward  s?   R $** 

)/!5#9++

 

 $AJ&,,Q/
 /0T:r4#4#4dooFGYq!_Z\T*t000"dooF 	 %4%%o\&UYUdUdohnoD#33")"?"?&99$55&-&G&G")"?"?&99

 
	
rC   )	NNNNNNNNN)rD   rE   rF   rH   output_modalitiesr%   rV   r   r   r7   rR  r   r   r   r  r   rj   rP   rQ   s   @rB   rt  rt    s     "y   .226598<:>:>6:!%*.L
##d*L
 ((4/L
 !++d2	L

 $..5L
 !& 0 04 7L
 )5047L
 -t3L
 $;L
   4'L
 
	 L
  L
rC   rt  )rU  r*   rt  )r$   )r   )Qcollections.abcr   typingr   r7   r    r   r<   activationsr   cache_utilsr	   r
   r   integrationsr   r   r   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r    utils.genericr!   r"   utils.output_capturingr#   configuration_diar%   r&   r'   generation_diar(   
get_loggerrD   rj  r*   Moduler6   rn   r   r   r   r   rl   r   r   r   r   r   r  r.   r%  r/   rG  rU  rt  __all__rg  rC   rB   <module>r     sn  * %    & ! C C f f J B 9  L F & l l G 5 L L . 
		H	% 0 0 0$!ryy !8)RYY )$ Y'J J (J(>< ><B( *+ ,2	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%2 )*;)ryy ;) +;)|G)		 G)T0 B5@# 5@p60 6rY
# Y
x 
g
! g

g
T 
a
"46H a

a
H LrC   