
    i#                        d dl Z d dlmZmZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6  e,jn                  e8      Z9 e+d      e G d de                    Z: G d dejv                        Z< G d dejv                        Z= G d d e5      Z> ed!      d=d"       Z? ee?       G d# d$ejv                               Z@ G d% d&e      ZAe+ G d' d(e&             ZBe+ G d) d*eB             ZC G d+ d,ejv                        ZD e+d-.       G d/ d0eB             ZE e+d1.       G d2 d3eB             ZF e+d4.       G d5 d6eB             ZGe+ G d7 d8eB             ZH e+d9.       G d: d;eB             ZIg d<ZJy)>    N)LiteralOptional)strict)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FN)PreTrainedConfig)use_kernel_func_from_hubuse_kernelized_func)create_bidirectional_mask(create_bidirectional_sliding_window_mask)GradientCheckpointingLayer)BaseModelOutputMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )eager_attention_forward)Gemma3RotaryEmbeddingrotate_halfzanswerdotai/ModernBERT-base)
checkpointc                       e Zd ZU dZdZdgZdddZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee   z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee   dz  e	d)<   dZ e!e"d*   e!f   dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4   e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<    fd>Z0d? Z1 fd@Z2e3dA        Z4e4jj                  dB        Z4 xZ6S )CModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
modernbertpast_key_valuesg     Ag     @)globallocali  
vocab_sizei   hidden_sizei  intermediate_size   num_hidden_layers   num_attention_headsgeluhidden_activationi    max_position_embeddingsg{Gz?initializer_range       @initializer_cutoff_factorgh㈵>norm_epsF	norm_biasik  Npad_token_idij  eos_token_idii  bos_token_idcls_token_idsep_token_idattention_bias        attention_dropoutlayer_typesfull_attentionsliding_attentionrope_parameters   local_attentionembedding_dropoutmlp_biasmlp_dropoutTdecoder_biascls)rP   meanclassifier_poolingclassifier_dropoutclassifier_biasclassifier_activationdeterministic_flash_attnsparse_predictionisparse_pred_ignore_indextie_word_embeddingsc                     |j                  dd      }| j                  8t        | j                        D cg c]  }t	        ||z        rdnd c}| _        t        |   di | y c c}w )Nglobal_attn_every_n_layersr
   rH   rG    )getrE   ranger2   boolsuper__post_init__)selfkwargsr[   i	__class__s       /var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/modernbert/modular_modernbert.pyra   zModernBertConfig.__post_init__   st    %+ZZ0La%P"# t556  (,A0J,J'K#Qaa D
 	'' s   A(c                    |j                  dd       }ddiddid}| j                  | j                  n|| _        |<| j                  d   j                  |       | j                  d   j                  |       | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d	                | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d
| j
                  d                | j                          |S )Nrope_scaling	rope_typedefault)rH   rG   rG   rH   
rope_thetaglobal_rope_thetar,   local_rope_thetar-   )poprI   updater]   
setdefaultdefault_thetastandardize_rope_params)rb   rc   rh   default_rope_paramss       rf   convert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict   sc   zz.$7
 #.y!9*I6
 8<7K7K7Wt33]p#  !1299,G  !45<<\J ##$45=6A95MD  !12-.99&**%8$:L:LX:VW	
 ##$78@9Di8PD  !4501<<&**%79K9KG9TU	

 	$$&    c                 H    t         |          }|j                  dd        |S )Nreference_compile)r`   to_dictrn   )rb   outputre   s     rf   rx   zModernBertConfig.to_dict   s#    "

&-ru   c                      | j                   dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r#   rK   rb   s    rf   sliding_windowzModernBertConfig.sliding_window   s     ##q((ru   c                     |dz  | _         y)z<Set sliding_window by updating local_attention to 2 * value.r#   Nr{   rb   values     rf   r}   zModernBertConfig.sliding_window   s      %qyru   )7__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencerq   r.   int__annotations__r/   r0   r2   r4   r6   strr7   r8   floatr:   r;   r<   r_   r=   r>   listr?   r@   rA   rB   rD   rE   rI   dictr   rK   rL   rM   rN   rO   rR   rS   rT   rU   rV   rW   rX   rY   ra   rt   rx   propertyr}   setter__classcell__re   s   @rf   r)   r)   3   s   (T J#4"5(8<MJK!s!s!!#s##'S'#u#'*u*HeIt$L#*$+0L#S	/D(0$L#*$$L#*$$L#*$ ND %(us{($(KcT!(Y]OT'"GH$NORVV]OS%(us{(Hd"K"L$16.6&))!OT!!'3'%*d*#t#$(c( $$	(<
 ) ) ) )ru   r)   c                        e Zd ZdZdef fdZ	 d	dej                  dz  dej                  dz  dej                  fdZ	 xZ
S )
ModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    configc                 d   t         |           || _        t        j                  |j
                  |j                  |j                        | _        t        j                  |j                  |j                  |j                        | _        t        j                  |j                        | _        y )N)padding_idxepsbias)r`   __init__r   r   	Embeddingr.   r/   r=   tok_embeddings	LayerNormr;   r<   normDropoutrL   droprb   r   re   s     rf   r   zModernBertEmbeddings.__init__   sw     ll6+<+<f>P>P^d^q^qrLL!3!3vO_O_`	JJv778	ru   N	input_idsinputs_embedsreturnc                     |"| j                  | j                  |            }|S | j                  | j                  | j                  |                  }|S N)r   r   r   )rb   r   r   hidden_statess       rf   forwardzModernBertEmbeddings.forward   sS     $ IIdii&>?M  !IIdii0C0CI0N&OPMru   NN)r   r   r   r   r)   r   torch
LongTensorTensorr   r   r   s   @rf   r   r      sR    9/ 9 _c))D0HMW[H[	ru   r   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )ModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r   c                    t         |           || _        t        j                  |j
                  t        |j                        dz  |j                        | _	        t        |j                     | _        t        j                  |j                        | _        t        j                  |j                  |j
                  |j                        | _        y )Nr#   r   )r`   r   r   r   Linearr/   r   r0   rM   Wir   r6   actr   rN   r   Wor   s     rf   r   zModernBertMLP.__init__   s    ))F..F4L4L0MPQ0QX^XgXgh&223JJv112	))F44f6H6Hv_ru   r   r   c                     | j                  |      j                  dd      \  }}| j                  | j                  | j	                  |      |z              S )Nr#   dim)r   chunkr   r   r   )rb   r   inputgates       rf   r   zModernBertMLP.forward   sI    ggm,221"2=twwtyy%4!7899ru   )
r   r   r   r   r)   r   r   r   r   r   r   s   @rf   r   r      s2    `/ `:U\\ :ell :ru   r   c                   |     e Zd Zddef fdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
 fd
       Z xZS )ModernBertRotaryEmbeddingNr   c                 &    t         |   ||       y r   )r`   r   )rb   r   devicere   s      rf   r   z"ModernBertRotaryEmbedding.__init__   s    (ru   r   ztorch.deviceseq_len
layer_typer   ztorch.Tensorc                 (    t         |   | |||      S r   )r`   compute_default_rope_parameters)r   r   r   r   re   s       rf   r   z9ModernBertRotaryEmbedding.compute_default_rope_parameters   s     w6vvwPZ[[ru   r   NNNN)r   r   r   r)   r   staticmethodr   r   r   tupler   r   r   r   s   @rf   r   r      s    )/ ) *.+/"!%	\ 4'\(\ t\ $J	\
 
~u$	%\ \ru   r   rotary_pos_embc                 b   | j                   }|j                  |      }|j                  |      }| j                         |z  t        | j                               |z  z   }|j                         |z  t        |j                               |z  z   }|j	                  |      |j	                  |      fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )dtype	unsqueezer   r&   to)qkcossinunsqueeze_dimoriginal_dtypeq_embedk_embeds           rf   apply_rotary_pos_embr      s    & WWN
--
&C
--
&Cwwy3;qwwy#9C#?@Gwwy3;qwwy#9C#?@G::n%wzz.'AAAru   c                        e Zd ZdZddededz  f fdZ	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
e   d
e	ej                  ej                  dz  f   f
dZ xZS )ModernBertAttentiona  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr   	layer_idxc                 R   t         |           || _        || _        |j                  |j
                  z  dk7  r&t        d|j                   d|j
                   d      |j                  | _        |j                  | _        |j                  |j
                  z  | _	        t        j                  |j                  d| j                  z  |j
                  z  |j                        | _        |j                  |   dk(  r|j                  dz   | _        nd | _        d	| _        t        j                  |j                  |j                  |j                        | _        |j                  d
kD  r%t        j$                  |j                        | _        y t        j&                         | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r
   r   rH      FrC   )r`   r   r   r   r/   r4   
ValueErrorrD   rV   head_dimr   r   rB   WqkvrE   r}   	is_causalr   r   Identityout_droprb   r   r   re   s      rf   r   zModernBertAttention.__init__  sz   " : ::a?#F$6$6#77mnt  oI  oI  nJ  JK  L  "(!9!9(.(G(G%**f.H.HHIIDMM 1F4N4N NU[UjUj
	 i(,?? #)"7"7!";D"&D))F..0B0BI^I^_@F@X@X[^@^

6#;#;<dfdododqru   r   position_embeddingsattention_maskrc   r   c                    |j                   d d }| j                  |      } |j                  g |dd| j                   }|j	                  d      \  }}}	|j                  dd      }|j                  dd      }|	j                  dd      }	|\  }
}t        |||
|d      \  }}t        }| j                  j                  dk7  rt        | j                  j                     } || |||	|f| j                  r| j                  nd	| j                  d
z  | j                  | j                  d|\  }} |j                  g |d j!                         }| j#                  | j%                  |            }||fS )Nr   r
   r   r   r#   )r   eagerrC         )dropoutscalingr}   deterministic)shaper   viewr   unbind	transposer   r$   r   _attn_implementationr   trainingrD   r}   rV   reshape
contiguousr   r   )rb   r   r   r   rc   input_shapeqkvquery_states
key_statesvalue_statesr   r   attention_interfaceattn_outputattn_weightss                  rf   r   zModernBertAttention.forward:  s    $))#2.ii&chh::Q::DMM:141C.j,#--a3))!Q/
#--a3&S#7jRUWZjk#l j5;;++w6"9$++:Z:Z"[$7%
 /3mmD**MM4'..77%
 %
!\ *k));;;;FFHmmDGGK$89L((ru   r   r   )r   r   r   r   r)   r   r   r   r   r   r   r   r   r   r   s   @rf   r   r     s    r/ rC$J r@ IM.2	')||') #5<<#=>E') t+	')
 +,') 
u||U\\D00	1')ru   r   c                        e Zd Zddededz  f fdZ	 	 ddej                  dej                  dz  dej                  dz  dee	   d	ej                  f
d
Z
 xZS )ModernBertEncoderLayerNr   r   c                    t         |           || _        || _        |dk(  rt	        j
                         | _        n;t	        j                  |j                  |j                  |j                        | _        t        ||      | _        t	        j                  |j                  |j                  |j                        | _        t        |      | _        |j                   |   | _        y )Nr   r   )r   r   )r`   r   r   r   r   r   	attn_normr   r/   r;   r<   r   attnmlp_normr   mlprE   attention_typer   s      rf   r   zModernBertEncoderLayer.__init__e  s    ">[[]DN\\&*<*<&//X^XhXhiDN'vK	V%7%7V__SYScScd ($00;ru   r   r   r   rc   r   c                      | j                   | j                  |      f||d|\  }}||z   }|| j                  | j                  |            z   }|S )N)r   r   )r   r   r   r   )rb   r   r   r   rc   r   _s          rf   r   zModernBertEncoderLayer.forwardr  sg     #NN=)
 3)
 	
Q &3%}1M(NNru   r   r   )r   r   r   r)   r   r   r   r   r   r   r   r   r   s   @rf   r   r   d  sx    </ <C$J <  /337	|| t+ #\\D0	
 +, 
ru   r   c                       e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedZ ej                          dej$                  fd       Zy	)
ModernBertPreTrainedModelr   modelTr   r   )r   
attentionsmodulec                    | j                   j                  ddt        j                  dt        ffd}| j                   j
                  | j                   j
                  t        j                  d| j                   j                  z        z  | j                   j
                  | j                   j                  dz  d}t        |t              r ||j                  |d          y t        |t              r- ||j                  |d	           ||j                  |d
          y t        |t               r- ||j"                  |d	           ||j                  |d
          y t        |t$              r ||j&                  |d
          y t        |t(              r ||j*                  |d
          y t        |t,        t.        t0        t2        f      r ||j4                  |d          y t        |t        j6                        rLt9        j:                  |j<                         |j>                   t9        j@                  |j>                         y y t        |tB              r|jD                  D ]  }|jF                  }|jH                  |   dk7  rtJ        |jH                  |      } ||j                   |      \  }}t9        jL                  tO        || d      |       t9        jL                  tO        || d      |        y y )Nr
   r  stdc                     t        j                  | j                  d| |z  |z         t        | t        j
                        r-| j                   t        j                  | j                         y y y )NrC   )rQ   r
  ab)inittrunc_normal_weight
isinstancer   r   r   zeros_)r  r
  cutoff_factors     rf   init_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weight  sd     .3&#% &")),;;*KK, + -ru   r9   r   )inout	embedding	final_outr  r  r  r  rj   )r   	_inv_freq_original_inv_freq)(r   r:   r   Moduler   r8   mathsqrtr2   r/   r  r   r   r   r   r   r   r   ModernBertPredictionHeaddenseModernBertForMaskedLMdecoder#ModernBertForSequenceClassificationModernBertForMultipleChoice ModernBertForTokenClassificationModernBertForQuestionAnswering
classifierr   r  ones_r  r   r  r   rE   r   ri   r   copy_getattr)	rb   r  r  stdsr   rope_init_fncurr_inv_freqr  r  s	           @rf   _init_weightsz'ModernBertPreTrainedModel._init_weights  sd   == M	-		 	- 	- ++//;;00499S4;;C`C`=`3aa6600$6	
 f23--tK/@A.		4:.		4;/ 34T$Z0		4;/ 89d5k2 56U43+0.	
 ))4+<=-JJv}}%{{&FKK( ' 9:$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ ;ru   N)r   r   r   r)   r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr   r   _can_record_outputsr   no_gradr   r  r-  r\   ru   rf   r  r    sq    &*#/1IJN"& 0)
 U]]_:^BII :^ :^ru   r  c                        e Zd Zdef fdZd Zd Zeee		 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	e
j                  dz  d
ee   defd                     Z xZS )ModernBertModelr   c           	         t         |   |       || _        t        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _
        t        j                  |j                  |j                  |j                        | _        t!        |      | _        d| _        | j'                          y c c}w )Nr   )r   F)r`   r   r   r   
embeddingsr   
ModuleListr^   r2   r   layersr   r/   r;   r<   
final_normr   
rotary_embgradient_checkpointing	post_initr   s      rf   r   zModernBertModel.__init__  s     .v6mmHMfNfNfHgh9#FI6h
 ,,v'9'9vU[UeUef36B&+# is   Cc                 .    | j                   j                  S r   r:  r   r|   s    rf   get_input_embeddingsz$ModernBertModel.get_input_embeddings  s    ---ru   c                 &    || j                   _        y r   rB  r   s     rf   set_input_embeddingsz$ModernBertModel.set_input_embeddings  s    ).&ru   Nr   r   position_idsr   rc   r   c                    |d u |d uz  rt        d      ||j                  d   n|j                  d   }||j                  n|j                  }|&t        j                  ||      j                  d      }| j                  ||      }t        |x}	t              s'| j                  ||d}
t        d
i |
t        d
i |
d}	i }| j                  j                  D ]  }| j                  |||      ||<    | j                  D ](  } ||f|	|j                     ||j                     d|}* | j!                  |      }t#        |	      S )Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )r   r   )r   r   r   rF   )r   r   )last_hidden_stater\   )r   r   r   r   aranger   r:  r  r   r   r   r   rE   r>  r<  r  r=  r   )rb   r   r   rF  r   rc   r   r   r   attention_mask_mappingmask_kwargsr   r   encoder_layers                 rf   r   zModernBertModel.forward  sw    -t";<YZZ,9,E-%%a(9??[\K]%.%:!!@T@T <<?II!LL)=YNB0DI++!."0K #<"Jk"J%M%\P[%\&"
 !++11 	gJ.2oom\[e.f
+	g "[[ 	M)5m6R6RS$78T8T$U 	M	 6??ru   r   )r   r   r   r)   r   rC  rE  r!   r"   r   r   r   r   r   r   r   r   r   r   s   @rf   r8  r8    s    
/ 
./   .2.204-1,@##d*,@ t+,@ &&-	,@
 ||d*,@ +,,@ 
,@    ,@ru   r8  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r  r   c                 J   t         |           || _        t        j                  |j
                  |j
                  |j                        | _        t        |j                     | _
        t        j                  |j
                  |j                  |j                        | _        y )Nr   )r`   r   r   r   r   r/   rT   r  r   rU   r   r   r;   r<   r   r   s     rf   r   z!ModernBertPredictionHead.__init__  sq    YYv1163E3EvG]G]^
&667LL!3!3vO_O_`	ru   r   r   c                 `    | j                  | j                  | j                  |                  S r   )r   r   r  )rb   r   s     rf   r   z ModernBertPredictionHead.forward   s#    yy$**]";<==ru   )	r   r   r   r)   r   r   r   r   r   r   s   @rf   r  r    s-    a/ a>U\\ >ell >ru   r  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )custom_introc                   >    e Zd ZddiZdef fdZd Zdej                  fdZ	e
e	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dee   deej                     ez  fd              Z xZS )r   zdecoder.weightz&model.embeddings.tok_embeddings.weightr   c                 t   t         |   |       || _        t        |      | _        t        |      | _        t        j                  |j                  |j                  |j                        | _        | j                  j                  | _        | j                  j                  | _        | j                          y )Nr   )r`   r   r   r8  r  r  headr   r   r/   r.   rO   r!  rW   rX   r@  r   s     rf   r   zModernBertForMaskedLM.__init__,  s     $V,
,V4	yy!3!3V5F5FVM`M`a!%!>!>(,(L(L% 	ru   c                     | j                   S r   r!  r|   s    rf   get_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings9  s    ||ru   new_embeddingsc                     || _         y r   rV  )rb   rX  s     rf   set_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings<  s	    %ru   Nr   r   rF  r   labelsrc   r   c                     | j                   d||||d|}|d   }| j                  rK|I|j                  d      }|j                  |j                  d   d      }|| j                  k7  }	||	   }||	   }| j                  | j                  |            }
d }|* | j                  |
|fd| j                  j                  i|}t        ||
|j                  |j                        S )Nr   r   rF  r   r   r   r.   losslogitsr   r  r\   )r  rW   r   r   rX   r!  rT  loss_functionr   r.   r   r   r  )rb   r   r   rF  r   r[  rc   outputsrI  mask_tokensr`  r_  s               rf   r   zModernBertForMaskedLM.forward?  s	    $** 
)%'	

 
 $AJ!!f&8[[_F 1 6 6v||A K !D$A$AAK 1+ >K(Fdii(9:;%4%%ffbAWAWb[abD!//))	
 	
ru   NNNNN)r   r   r   _tied_weights_keysr)   r   rW  r   r   rZ  r    r   r   r   r   r   r   r   r   r   r   r   s   @rf   r   r   $  s     +,TU/ &BII &  .2.2,0-1&*'
##d*'
 t+'
 llT)	'

 ||d*'
 t#'
 +,'
 
u||	~	-'
  '
ru   r   z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r"  r   c                 n   t         |   |       |j                  | _        || _        t	        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j!                          y r   )r`   r   
num_labelsr   r8  r  r  rT  r   r   r   rS   r   r   r/   r&  r@  r   s     rf   r   z,ModernBertForSequenceClassification.__init__q  s      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJ 	ru   Nr   r   rF  r   r[  rc   r   c                 d    | j                   d||||d|}|d   }| j                  j                  dk(  r
|dddf   }n| j                  j                  dk(  rw|=t        j                  |j
                  dd |j                  t        j                        }||j                  d      z  j                  d	
      |j                  d	d      z  }| j                  |      }	| j                  |	      }	| j                  |	      }
d}|| j                  j                  | j                  d	k(  rd| j                  _        nl| j                  d	kD  rL|j                  t        j                   k(  s|j                  t        j"                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt%               }| j                  d	k(  r& ||
j'                         |j'                               }n ||
|      }n| j                  j                  dk(  r=t)               } ||
j+                  d| j                        |j+                  d            }n,| j                  j                  dk(  rt-               } ||
|      }t/        ||
|j0                  |j2                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r]  r   rP   NrQ   r#   )r   r   r   r   r   Tr   keepdim
regressionsingle_label_classificationmulti_label_classificationr^  r\   )r  r   rR   r   onesr   r   r_   r   sumrT  r   r&  problem_typerh  r   longr   r	   squeezer   r   r   r   r   r  )rb   r   r   rF  r   r[  rc   rb  rI  pooled_outputr`  r_  loss_fcts                rf   r   z+ModernBertForSequenceClassification.forward~  se   " $** 
)%'	

 
 $AJ;;))U2 1!Q$ 7[[++v5%!&%++BQ/8I8P8PX]XbXb" "3^5M5Mb5Q!Q V V[\ V ]`n`r`rt as a ! 		"34		-0/{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./'!//))	
 	
ru   rd  )r   r   r   r)   r   r    r   r   r   r   r   r   r   r   r   r   r   s   @rf   r"  r"  k  s    /   .2.2,0-1&*C
##d*C
 t+C
 llT)	C

 ||d*C
 t#C
 +,C
 
u||	7	7C
  C
ru   r"  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r$  r   c                 `   t         |   |       |j                  | _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   r`   r   rh  r8  r  r  rT  r   r   r   rS   r   r   r/   r&  r@  r   s     rf   r   z)ModernBertForTokenClassification.__init__  s{      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJ 	ru   Nr   r   rF  r   r[  rc   r   c                 f    | j                   d||||d|}|d   }| j                  |      }| j                  |      }| j                  |      }	d}
|<t	               } ||	j                  d| j                        |j                  d            }
t        |
|	|j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r]  r   Nr   r^  r\   )
r  rT  r   r&  r   r   rh  r   r   r  )rb   r   r   rF  r   r[  rc   rb  rI  r`  r_  ru  s               rf   r   z(ModernBertForTokenClassification.forward  s     $** 
)%'	

 
 $AJ II&78 II&78!23')HFKKDOO<fkk"oND$!//))	
 	
ru   rd  )r   r   r   r)   r   r    r   r   r   r   r   r   r   r   r   r   r   s   @rf   r$  r$    s    
/ 
  .2.2,0-1&*$
##d*$
 t+$
 llT)	$

 ||d*$
 t#$
 +,$
 
u||	4	4$
  $
ru   r$  c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eej                     ez  fd              Z xZS )r%  r   c                 `   t         |   |       |j                  | _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   rx  r   s     rf   r   z'ModernBertForQuestionAnswering.__init__  sy      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJru   Nr   r   rF  start_positionsend_positionsrc   r   c                     | j                   |f||d|}|d   }| j                  |      }| j                  |      }| j                  |      }	|	j	                  dd      \  }
}|
j                  d      j                         }
|j                  d      j                         }d }|| | j                  |
|||fi |}t        ||
||j                  |j                        S )N)r   rF  r   r   r   r   )r_  start_logits
end_logitsr   r  )r  rT  r   r&  splitrs  r   ra  r   r   r  )rb   r   r   rF  r|  r}  rc   rb  rI  r`  r  r  r_  s                rf   r   z&ModernBertForQuestionAnswering.forward  s    $**
)%
 	
 $AJ II&78 II&78!23#)<<r<#: j#++B/::<''+668
&=+D%4%%lJQ^ibhiD+%!!//))
 	
ru   rd  )r   r   r   r)   r   r    r   r   r   r   r   r   r   r   r   r   s   @rf   r%  r%    s    	/ 	  *..2,0/3-1#
<<$&#
 t+#
 llT)	#

 ,#
 ||d*#
 +,#
 
u||	;	;#
  #
ru   r%  z
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r#  r   c                 8   t         |   |       || _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  d      | _        | j                          y )Nr   )r`   r   r   r8  r  r  rT  r   r   r   rS   r   r   r/   r&  r@  r   s     rf   r   z$ModernBertForMultipleChoice.__init__<  so     $V,
,V4	HH$$V%>%>?	))F$6$6: 	ru   Nr   r   rF  r   r[  rc   r   c                    ||j                   d   n|j                   d   }|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|1|j                  d|j                  d      |j                  d            nd} | j                  d||||d|}|d   }	| j                  j
                  dk(  rt        j                  |	j                   d   |	j                        }
|,|j                  d	      j                  |	j                        }n0t        j                  dt        j                  |	j                  
      }|	|
|f   }	nS| j                  j
                  dk(  r:|j                  dd      }|	|j                  d      z  j                  d	      |z  }	| j                  |	      }| j!                  |      }| j#                  |      }|j                  d|      }d}|t%        j&                         } |||      }t)        |||j*                  |j,                        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        Nr   r   r]  r   rP   rH  r   )r   r   rQ   Trj  r^  r\   )r   r   sizer  r   rR   r   rJ  r   argmaxr   tensorrr  rp  r   rT  r   r&  r   r   r   r   r  )rb   r   r   rF  r   r[  rc   num_choicesrb  rI  	indices_0cls_masknum_non_pad_tokensrt  r`  reshaped_logitsr_  ru  s                     rf   r   z#ModernBertForMultipleChoice.forwardH  sn     -6,Aiooa(}GZGZ[\G]>G>SINN2y~~b'9:Y]	M[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei ( r=#5#5b#9=;M;Mb;QR 	 $** 
)%'	

 
 $AJ ;;))U2%6%<%<Q%?HYH`H`aI))00R08;;<M<T<TU !<<DUD\D\] 1)X2E F [[++v5!/!3!34!3!H!2^5M5Mb5Q!Q V V[\ V ]`r r		"34		-0/ ++b+6**,HOV4D("!//))	
 	
ru   rd  )r   r   r   r)   r   r    r   r   r   r   r   r   r   r   r   r   r   s   @rf   r#  r#  6  s    
/ 
  .2.2,0-1&*C
##d*C
 t+C
 llT)	C

 ||d*C
 t#C
 +,C
 
u||	8	8C
  C
ru   r#  )r)   r8  r  r   r"  r$  r%  r#  )r   )Kr  typingr   r   r   huggingface_hub.dataclassesr   r   torch.nnr   r   r	    r   r  activationsr   configuration_utilsr   integrationsr   r   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr    r!   utils.output_capturingr"   align.modeling_alignr$   gemma3.modeling_gemma3r%   r&   
get_loggerr   loggerr)   r  r   r   r   r   r   r   r  r8  r  r   r"  r$  r%  r#  __all__r\   ru   rf   <module>r     s4     $  .  A A & ! 3 I ` 9  7 F & @ @ I 5 : G 
		H	% 89G)' G)  :G)T299 ,:BII :(\ 5 \ *+B ,B4 )*N)")) N) +N)b7 @ J^ J^ J^Z B@/ B@ B@J	>ryy 	> 
?
5 ?

?
D 
S
*C S

S
l 
3
'@ 3

3
l 1
%> 1
 1
h 
R
"; R

R
j	ru   