
    i                     
   d dl mZ d dlmZ ddlmZmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZ  ed	
      e G d de                    Z G d de      Z G d de
      Z G d de      Z G d de      Z G d de      Zg dZy)    N)strict   )auto_docstringcan_return_tuple   )LlamaConfig)LlamaDecoderLayerLlamaForCausalLM
LlamaModelLlamaPreTrainedModel)NemotronMLPzinceptionai/Jais-2-8B-Chat)
checkpointc                       e Zd ZU dddddddZdZeed<   dZeed<   dZeed	<   d
Z	eed<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeee   z  dz  ed<   dZeed<   dZeed<    e       Z e       Zy)Jais2Configcolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proji K 
vocab_sizei   hidden_sizei h  intermediate_size   num_attention_headsrelu2
hidden_acti    max_position_embeddingsgh㈵>layer_norm_epsr   Nbos_token_idiJ eos_token_idTattention_biasmlp_bias)__name__
__module____qualname__base_model_tp_planr   int__annotations__r   r   r   r   strr   r   floatr   r   listr   boolr   AttributeErrorrms_norm_epspretraining_tp     x/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/jais2/modular_jais2.pyr   r      s     &/%.%.%. )"+ JK"s"!!J#'S' NE  L#* +1L#S	/D(1NDHd!#L#%Nr.   r   c                       e Zd Zy)Jais2MLPNr    r!   r"   r-   r.   r/   r1   r1   9       r.   r1   c                   (     e Zd Zdedef fdZ xZS )Jais2DecoderLayerconfig	layer_idxc                     t         |   ||       t        j                  |j                  |j
                        | _        t        j                  |j                  |j
                        | _        y N)eps)super__init__nn	LayerNormr   r   input_layernormpost_attention_layernorm)selfr6   r7   	__class__s      r/   r<   zJais2DecoderLayer.__init__>   sP    +!||F,>,>FDYDYZ(*V5G5GVMbMb(c%r.   )r    r!   r"   r   r$   r<   __classcell__rB   s   @r/   r5   r5   =   s    d{ ds d dr.   r5   c                       e Zd Zy)Jais2PreTrainedModelNr2   r-   r.   r/   rF   rF   D   r3   r.   rF   c                   $     e Zd Zdef fdZ xZS )
Jais2Modelr6   c                     t         |   |       t        j                  |j                  |j
                        | _        y r9   )r;   r<   r=   r>   r   r   norm)rA   r6   rB   s     r/   r<   zJais2Model.__init__I   s.     LL!3!39N9NO	r.   )r    r!   r"   r   r<   rC   rD   s   @r/   rH   rH   H   s    P{ P Pr.   rH   c                   2     e Zd Zee fd              Z xZS )Jais2ForCausalLMc                 "    t        |   di |S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Jais2ForCausalLM

        >>> model = Jais2ForCausalLM.from_pretrained("inceptionai/Jais-2-8B-Chat")
        >>> tokenizer = AutoTokenizer.from_pretrained("inceptionai/Jais-2-8B-Chat")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```r-   )r;   forward)rA   super_kwargsrB   s     r/   rN   zJais2ForCausalLM.forwardO   s    ( w...r.   )r    r!   r"   r   r   rN   rC   rD   s   @r/   rL   rL   N   s    /  /r.   rL   )r   rH   rL   rF   )torch.nnr=   huggingface_hub.dataclassesr   utilsr   r   llama.configuration_llamar   llama.modeling_llamar	   r
   r   r   nemotron.modeling_nemotronr   r   r1   r5   rF   rH   rL   __all__r-   r.   r/   <module>rW      s      . 5 3  5 78&+ &  9&2	{ 	d) d	/ 	P P/' /0r.   