
    iq                        d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: ddl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF ddlGmHZH ddlImJZJmKZKmLZL  e/j                  eN      ZO e-d      e
 G d de:e                    ZP e-d      e
 G d de9                    ZQ e-d      e
 G d d e:e                    ZR e-d      e
 G d! d"e                    ZS G d# d$e@      ZT G d% d&e=      ZU G d' d(eA      ZV G d) d*e<      ZW G d+ d,e<      ZXdMd-eYd.efd/ZZ G d0 d1eK      Z[ G d2 d3eK      Z\ G d4 d5eL      Z] G d6 d7eJ      Z^ G d8 d9e>      Z_ G d: d;eB      Z`e- G d< d=e?             Za G d> d?ea      Zb G d@ dAea      Zc G dB dCea      Zde- G dD dEea             Ze G dF dGeae      Zfe- G dH dIea             Zge- G dJ dKea             Zhg dLZiy)N    N)Callable)AnyOptional)strict   )initialization)DynamicCacheEncoderDecoderCacheStaticCache)PreTrainedConfig)GenerationConfigGenerationMixinGenerationMode)create_bidirectional_mask)FlashAttentionKwargs)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPoolingSeq2SeqLMOutputSeq2SeqModelOutputSequenceClassifierOutputTokenClassifierOutput)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleloggingtorch_compilable_check)merge_with_config_defaults)OutputRecordercapture_outputs   )	AutoModel)Gemma3ConfigGemma3TextConfig)Gemma3Attention	Gemma3MLPGemma3MultiModalProjectorGemma3PreTrainedModelGemma3RMSNormGemma3RotaryEmbeddingGemma3TextScaledWordEmbeddingapply_rotary_pos_embcreate_causal_mask!create_sliding_window_causal_maskeager_attention_forward)SiglipVisionConfig)T5GemmaClassificationHeadT5GemmaEncoderLayerT5GemmaLMHeadzgoogle/t5gemma-2-270m-270m)
checkpointc                   (    e Zd ZdZdZ e       Zd Zy)T5Gemma2TextConfigt  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        Scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    t5gemma2_textc                     |j                  dd      }| j                  ;t        | j                        D cg c]  }t	        |dz   |z        rdnd c}| _        t        j                  di | y c c}w Nsliding_window_pattern      sliding_attentionfull_attention poplayer_typesrangenum_hidden_layersboolr   __post_init__selfkwargs_sliding_window_patternis       ~/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/t5gemma2/modular_t5gemma2.pyrK   z T5Gemma2TextConfig.__post_init__[   x    "(**-Eq"I# t556  (,QU6M,M'N#Tdd D
 	&&00    A0N__name__
__module____qualname____doc__
model_typeAttributeErroruse_bidirectional_attentionrK   rD       rQ   r:   r:   L   s     !J"0"2	1r\   r:   c                       e Zd ZdZeedZy)T5Gemma2EncoderConfigt5gemma2_encoder)text_configvision_configN)rU   rV   rW   rY   r:   r4   sub_configsrD   r\   rQ   r^   r^   g   s     $J *+Kr\   r^   c                   (    e Zd ZdZdZ e       Zd Zy)T5Gemma2DecoderConfigr;   t5gemma2_decoderc                     |j                  dd      }| j                  ;t        | j                        D cg c]  }t	        |dz   |z        rdnd c}| _        t        j                  di | y c c}w r>   rE   rL   s       rQ   rK   z#T5Gemma2DecoderConfig.__post_init__   rR   rS   NrT   rD   r\   rQ   rd   rd   r   s     $J"0"2	1r\   rd   c                       e Zd ZU dZdZdgZeedZdddZ	dZ
eeeef   z  dz  ed	<   dZeeeef   z  dz  ed
<   dZeed<   dZeez  ed<   dZeez  ed<   dZeez  ed<   dZeed<   dZeed<   dZedz  ed<   dZeed<    fdZd Z xZS )T5Gemma2Configa`  
    encoder (`Union[T5Gemma2EncoderConfig, dict]`, optional, *optional*):
        Configuration for the encoder.
    decoder (`Union[T5Gemma2DecoderConfig, dict]`, optional, *optional*):
        Configuration for the decoder.
    eoi_token_index (`int`, *optional*):
        The end-of-image token index to wrap the image prompt. Will be same as
        `self.encoder.eoi_token_index`

    ```python
    >>> from transformers import T5Gemma2Config, T5Gemma2Model
    >>> t5gemma2_config = T5Gemma2Config.from_pretrained("google/t5gemma-270m-270m")
    >>> model = T5Gemma2Model(t5gemma2_config)
    ```
    t5gemma2past_key_values)encoderdecoderimage_token_indexeoi_token_index)image_token_ideoi_token_idNrk   rl   Tis_encoder_decoder        dropout_rateattention_dropoutclassifier_dropout_rateg{Gz?initializer_rangei tie_word_embeddingsc                    t        | j                  t              rt        di | j                  | _        n0| j                  $t               | _        t        j                  d       t        | j                  t              rt        di | j                  | _        n0| j                  $t               | _        t        j                  d       | j                  | j                  j                  _        | j                  | j                  j                  _
        | j                  | j                  j                  _
        | j                  | j                  _        | j                  | j                  _        | j                  | j                  _
        | j                  j                  | _        dD ]   }||vst        | j                  |      ||<   " t        | @  di | y )NzDencoder is None, using default T5Gemma2EncoderConfig encoder config.zDdecoder is None, using default T5Gemma2DecoderConfig decoder config.)bos_token_idpad_token_ideos_token_id
vocab_sizerD   )
isinstancerk   dictr^   loggerinforl   rd   rs   r`   rt   ra   rm   rn   getattrsuperrK   )rM   rN   special_token_key	__class__s      rQ   rK   zT5Gemma2Config.__post_init__   sZ   dllD)0@4<<@DL\\!02DLKK^_dllD)0@4<<@DL\\!02DLKK^_040A0A  -595K5K  27;7M7M""4)-)?)?&$($5$5!)-)?)?&#||;;!_ 	U .,3DLLBS,T()	U 	''r\   c                    | j                   j                  j                  | j                  j                  k7  rDt	        d| j                   j                  j                   d| j                  j                   d      | j
                  st	        d      | j                   j                  j                  | j                  j                  k7  rDt	        d| j                   j                  j                   d| j                  j                   d      y)zOPart of `@strict`-powered validation. Validates the architecture of the config.zBImbalanced encoder-decoder is not supported in T5Gemma2: encoder (z) vs decoder (z).z4T5Gemma2Model only support encoder-decoder modeling.zRImbalanced encoder-decoder vocabulary size is not supported in T5Gemma2: encoder (N)rk   r`   hidden_sizerl   
ValueErrorrq   r|   rM   s    rQ   validate_architecturez$T5Gemma2Config.validate_architecture   s    <<##//4<<3K3KK LL44@@APTP\P\PhPhOiikm 
 &&STT<<##..$,,2I2II LL44??@t||OfOfNggik  Jr\   )rU   rV   rW   rX   rY   keys_to_ignore_at_inferencer^   rd   rb   attribute_maprk   r~   strr   __annotations__rl   rq   rJ   rs   floatintrt   ru   rv   rm   rn   rw   rK   r   __classcell__r   s   @rQ   rh   rh      s      J#4"5 )(K .)M
 >BG"T#s(^3d:A=AG"T#s(^3d:A## #L%#+#%(us{(+.US[.#u#$s$"&OS4Z& $$(8r\   rh   c                       e Zd Zy)T5Gemma2RMSNormNrU   rV   rW   rD   r\   rQ   r   r          r\   r   c                   *     e Zd Zdef fdZd Z xZS )T5Gemma2MLPconfigc                 l    t         |   |       t        j                  |j                        | _        y N)r   __init__nnDropoutrs   dropoutrM   r   r   s     rQ   r   zT5Gemma2MLP.__init__   s&     zz&"5"56r\   c                     | j                  | j                  |            | j                  |      z  }| j                  |      }| j	                  |      }|S r   )act_fn	gate_projup_projr   	down_proj)rM   xhidden_statesr   s       rQ   forwardzT5Gemma2MLP.forward   sH    DNN1$56aH]3NN=1	r\   )rU   rV   rW   r:   r   r   r   r   s   @rQ   r   r      s    71 7r\   r   c                   |     e Zd Zddef fdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
 fd
       Z xZS )T5Gemma2RotaryEmbeddingNr   c                 &    t         |   ||       y r   r   r   )rM   r   devicer   s      rQ   r   z T5Gemma2RotaryEmbedding.__init__   s    (r\   r   ztorch.deviceseq_len
layer_typereturnztorch.Tensorc                 (    t         |   | |||      S r   )r   compute_default_rope_parameters)r   r   r   r   r   s       rQ   r   z7T5Gemma2RotaryEmbedding.compute_default_rope_parameters   s     w6vvwPZ[[r\   r   )NNNN)rU   rV   rW   r:   r   staticmethodr   r   r   tupler   r   r   r   s   @rQ   r   r      s    )1 ) ,0+/"!%	\"T)\(\ t\ $J	\
 
~u$	%\ \r\   r   c                   (     e Zd Zdedef fdZ xZS )T5Gemma2SelfAttentionr   	layer_idxc                 4    t         |   ||       d| _        y NFr   r   	is_causalrM   r   r   r   s      rQ   r   zT5Gemma2SelfAttention.__init__      +r\   )rU   rV   rW   r:   r   r   r   r   s   @rQ   r   r     s    1 c  r\   r   c                   .    e Zd ZdZdedef fdZ	 ddej                  de	ej                  ej                  f   dej                  dz  d	ej                  d
e
dz  dee   de	ej                  ej                  dz  e	ej                     dz  f   fdZ xZS )T5Gemma2MergedAttentionz6Merged self-attention and cross-attention for decoder.r   r   c                 4    t         |   ||       d| _        y r   r   r   s      rQ   r   z T5Gemma2MergedAttention.__init__  r   r\   Nr   position_embeddingsmerged_attention_maskencoder_hidden_statesrj   rN   r   c                    |j                   d d }g |d| j                  }|j                   d d }	g |	d| j                  }
| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      }| j                  |      }|\  }}t        ||||      \  }}|]|j                  }|j                  ||| j                        \  }}|j                  j                  | j                        }|j                  }|s| j                  |      j                  |
      j	                  dd      }| j                  |      j                  |
      j	                  dd      }| j                  |      }|j                  ||| j                        \  }}d|j                  | j                  <   nFj                   | j                     j"                  }|j                   | j                     j$                  }|}|	d   }t'        j(                  ||gd      }t'        j(                  ||gd      }t+        j,                  | j.                  j0                  t2              } || ||||f| j4                  r| j6                  nd| j8                  d|\  }} |j:                  g |d j=                         }| j?                  |      }||dd | f   }|d| d f   }nd	\  }}|||fS )
NrA   r%   Tdimrr   )r   scaling.)NN) shapehead_dimq_projview	transposek_projv_projq_normk_normr0   self_attention_cacheupdater   
is_updatedgetcross_attention_cachelayerskeysvaluestorchcatr   get_interfacer   _attn_implementationr3   trainingrt   r   reshape
contiguouso_proj)rM   r   r   r   r   rj   rN   input_shapehidden_shapecross_input_shapecross_hidden_shapequery_states
key_statesvalue_statescossinr   r   r   cross_key_statescross_value_statescross_key_sizeattention_interfaceattn_outputattn_weightsself_attn_weightscross_attn_weightss                              rQ   r   zT5Gemma2MergedAttention.forward  sv    $))#2.88b8$--8177<D0D"DdmmD {{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST{{<0[[,
&S#7jRUWZ#[ j&#2#G#G ';'B'B:|]a]k]k'l$J )3377GJ$3$I$I!"*#{{+@AFFGYZddefhij!%-B!C!H!HI[!\!f!fghjk!l#{{+;<*7L7S7S$&8$..84 "4 >B**4>>:4;;DNNKPP!6!=!=dnn!M!T!T $*1-YY
,<=1E
yy,0B!CK(?(M(MKK,,.E)
 %8!	%
 /3mmD**LL	%
 	%
!\ *k));;;;FFHkk+. # ,S2BN?2B-B C!-cN?3C.C!D4>11-/AAAr\   r   )rU   rV   rW   rX   r:   r   r   r   Tensorr   r
   r   r   r   r   r   s   @rQ   r   r   
  s    @1 c  7;TB ||TB #5<<#=>	TB
  %||d2TB  %||TB -t3TB -.TB 
u||U\\D0%2E2LL	MTBr\   r   sliding_windowr   c           
      T     dt         dt         dt         dt         dt        f
 fd}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    	batch_idxhead_idxq_idxkv_idxr   c                 t    	r
d}}n
dz   dz  
dz  dz   }}||z
  }|dk\  ||k  z  }|dk  | |k  z  }||z  S )Nr   rA   r%   rD   )r   r   r   r   left_window_sizeright_window_sizedist	left_mask
right_maskr   r   s            rQ   
inner_maskz0sliding_window_mask_function.<locals>.inner_maskm  sp    2@!/4BQ4F13L~bcNcfgNg/v~QY4*:#:;	QhD5+<#<=
:%%r\   )r   rJ   )r   r   r   s   `` rQ   sliding_window_mask_functionr   h  s3    
	&c 	&S 	& 	&c 	&d 	& r\   c                       e Zd Zy)T5Gemma2EncoderLayerNr   rD   r\   rQ   r  r  {  r   r\   r  c                       e Zd ZdZdef fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  fdZ xZS )T5Gemma2DecoderLayerzFDecoder sub-layer: merged attention instead of vanilla self-attention.r   c                 J    t         |   ||       t        ||      | _        y )N)r   r   )r   r   r   	self_attnr   s      rQ   r   zT5Gemma2DecoderLayer.__init__  s&    + 1
r\   Nr   r   r   position_idsrj   	use_cacher   r   c                 D   |}	| j                  |      } | j                  d|||||||d|\  }}
}
| j                  |      }|	| j                  |      z   }|}	| j	                  |      }| j                  |      }| j                  |      }|	| j                  |      z   }|S )N)r   r   r   r  rj   r  r   rD   )pre_self_attn_layernormr  post_self_attn_layernormr   pre_feedforward_layernormmlppost_feedforward_layernorm)rM   r   r   r   r  rj   r  r   rN   residual_s              rQ   r   zT5Gemma2DecoderLayer.forward  s     !44]C,dnn 	
' 3"7%+"7	
 	
q! 55mD 4<<#>> 66}E/77F 4<<#>>r\   )NNNFN)rU   rV   rW   rX   r   r   r   r   r   
LongTensorr
   rJ   FloatTensorr   r   r   s   @rQ   r  r    s    P
# 
 6:046:!&59 ||  #5<<#=>   %||d2	 
 &&-  -t3  $;   %||d2  
		 r\   r  c                       e Zd Zy)T5Gemma2LMHeadNr   rD   r\   rQ   r  r    r   r\   r  c                       e Zd Zy)T5Gemma2ClassificationHeadNr   rD   r\   rQ   r  r    r   r\   r  c                   $     e Zd Zdef fdZ xZS )T5Gemma2MultiModalProjectorr   c                 $    t         |   |       y r   r   r   s     rQ   r   z$T5Gemma2MultiModalProjector.__init__  s     r\   )rU   rV   rW   r^   r   r   r   s   @rQ   r  r    s    !4 ! !r\   r  c                   b     e Zd ZdZ	 	 d
dededededef
 fdZdej                  f fd	Z	 xZ
S )T5Gemma2TextScaledWordEmbeddingzCT5Gemma2 Embedding: override to add eoi token embedding separately.num_embeddingsembedding_dimpadding_idxembed_scalern   c                     t         |   ||||       || _        t        j                  t        j                  | j                              | _        y r   )	r   r   rn   r   	Parameterr   zerosr  eoi_embedding)rM   r  r  r  r  rn   r   s         rQ   r   z(T5Gemma2TextScaledWordEmbedding.__init__  s@     	[Q.\\%++d6H6H*IJr\   	input_idsc                     t         |   |      | j                  j                  | j                  j
                        z  }| j                  j                  |j
                        ||| j                  k(  <   |S r   )r   r   r  toweightdtyper#  rn   )rM   r$  input_embeddingsr   s      rQ   r   z'T5Gemma2TextScaledWordEmbedding.forward  sf     7?958H8H8K8KDKKL]L]8^^>B>P>P>S>STdTjTj>kd&:&::;r\   )g      ?  )rU   rV   rW   rX   r   r   r   r   r   r   r   r   s   @rQ   r  r    s^    M !&
K
K 
K 	
K
 
K 
K     r\   r  c                       e Zd ZU eed<   dZdZdZdZg dZ	e
eg eedd       eedd       eed	d
      gdZd Zd Zy)T5Gemma2PreTrainedModelr   modelTF)r  r  SiglipVisionEmbeddingsSiglipEncoderLayer#SiglipMultiheadAttentionPoolingHeadrA   r  )index
layer_namer%   
cross_attn)r   
attentionsc                    t        j                  | |       t        |t              r t	        j
                  |j                         y t        |t              rJt	        j
                  |j                         t	        j                  |j                  |j                         y t        |t              r|j                  j                  j                  d   dz  }t	        j                   |j                  j                  d| j"                  j$                  |z         t'        |j                  d      rA|j                  j(                  *t	        j
                  |j                  j(                         y y y d|j*                  j,                  v r t	        j
                  |j                         y t        |t.              r|j0                  D ]  }|j2                  }|j4                  |   dk7  rt6        |j4                  |      } ||j"                  |      \  }}t	        j8                  t;        || d	      |       t	        j8                  t;        || d
      |        y y )Nr   g      rr   )meanstdbiasRMSNormdefault)r   	_inv_freq_original_inv_freq)r   _init_weightsr}   r  initzeros_mm_input_projection_weightr  r#  	constant_r  scalar_embed_scaler  out_projr'  r   normal_r   rv   hasattrr8  r   rU   r   rG   r   	rope_typer   copy_r   )rM   modulescaler   rope_init_fncurr_inv_freqr  s          rQ   r=  z%T5Gemma2PreTrainedModel._init_weights  s   %%dF3f9:KK99: ?@KK,,-NN6--v/H/HI :;OO**003t;ELL//ct{{?\?\_d?dev/FOO4H4H4TFOO001 5U/ &**333KK& 78$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 9r\   c                 <   | j                   j                  }|j                  }|j                  }|t	        d      |j                  |j                        }|dddf   j                         |dddf<   ||d<   |t	        d      |j                  |dk(  |       |S )	z
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .r   rA   ).r   z9self.model.config.decoder.pad_token_id has to be defined.i)	r   rl   ry   rz   r   	new_zerosr   clonemasked_fill_)rM   r$  decoder_configdecoder_start_token_idrz   shifted_input_idss         rQ   %prepare_decoder_input_ids_from_labelsz=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labels  s     ,,!/!<!<%22!)YZZ &//	@%.sCRCx%8%>%>%@#qr'"$:&!XYY 	&&'8D'@,O  r\   N)rU   rV   rW   rh   r   base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_flex_attn_no_split_modulesr  r  r#   r   r   _can_record_outputsr=  rS  rD   r\   rQ   r,  r,    st    &*# ! /0DE0kR2!T2!U
^0!r\   r,  c                       e Zd ZU eed<   eedZ	 ddedef fdZ	e
ee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dee   defd                     Z xZS )T5Gemma2TextEncoderr   )r4  r   rn   c           	      ^   t         |   |       |j                  | _        |j                  | _        t        |j                  |j                  | j                  |j                  dz  |      | _        t        |j                  |j                        | _
        d| _        t        j                  t        |j                        D cg c]  }t!        ||       c}      | _        t        j$                  |j&                        | _        t+        |      | _        | j/                          y c c}w Ng      ?)r  rn   )epsF)r   r   rz   r  r|   r  r   embed_tokensr   rms_norm_epsnormgradient_checkpointingr   
ModuleListrH   rI   r  r   r   rs   r   r   
rotary_emb	post_initrM   r   rn   r   r   s       rQ   r   zT5Gemma2TextEncoder.__init__'  s    
 	 !.. ++;**C/+
 $F$6$6F<O<OP	&+#mmFKFLdLdFef!&)4f
 zz&"5"561&9 	 g   D*Nr$  attention_maskr  inputs_embedstoken_type_idsrN   r   c           
      |   |d u |d uz  rt        d      |j                  dd        || j                  |      }|>t        j                  d|j
                  d   |j                        j                  d      }t        |x}t              sJ| j                  ||d}t        di |t        di |dt        | j                  j                  d	      id
}|}	i }
| j                  j                  D ]  }| j                  |	||      |
|<    | j!                  |	      }	t#        | j$                  d | j                  j&                         D ]E  \  }} ||	|
| j                  j                  |      || j                  j                  |      |fi |}	G | j)                  |	      }	| j!                  |	      }	t+        |	      S )N:You must specify exactly one of input_ids or inputs_embedsrj   r   rA   r   )r   ri  rh  and_mask_functionF)r   rC   rB   )last_hidden_staterD   )r   rF   r_  r   aranger   r   	unsqueezer}   r~   r   r   r   r   rG   rd  r   	enumerater   rI   ra  r   )rM   r$  rh  r  ri  rj  rN   self_attn_mask_mappingmask_kwargsr   r   r   rP   layer_modules                 rQ   r   zT5Gemma2TextEncoder.forwardC  s    -t";<YZZ 	

$d+  --i8M <<=+>+>q+A-J^J^_iijklLNB0DI++!."0K #<"Jk"J%> &!&&B4;;C]C]in&o&&" & !++11 	gJ.2oom\[e.f
+	g ]3(5Tt{{7T7T)UV 	OA|(#DKK$;$;A$>?&t{{'>'>q'AB	
 M	 		-0]3+
 	
r\   r*  )NNNNN)rU   rV   rW   r:   r   r   r  rY  r   r   r"   r$   r   r   r  r   r  r   r   r   r   r   r   s   @rQ   r[  r[     s    +-  '" 8   .2.20426.2<
##d*<
 t+<
 &&-	<

 ((4/<
 t+<
 +,<
 
<
    <
r\   r[  c                       e Zd ZU eed<   	 ddedef fdZd Zd Ze	e
dej                  dee   deez  fd	              Zd
ej$                  dz  dej&                  dz  dej&                  fdZe
	 	 	 	 	 	 dd
ej$                  dz  dej                  dz  dej$                  dz  dej&                  dz  dej&                  dz  dej                  dz  dee   defd       Z xZS )T5Gemma2Encoderr   rn   c                     t         |   |       t        j                  |j                  |      | _        t        j                  |j                        | _	        t        |      | _        | j                          y )N)rn   r   )r   r   r[  _from_configr`   
text_modelr&   from_configra   vision_towerr  multi_modal_projectorre  )rM   r   rn   r   s      rQ   r   zT5Gemma2Encoder.__init__  sb    
 	 -::6;M;M_n:o%119M9MN%@%H" 	r\   c                 6    | j                   j                         S r   )r}  get_input_embeddingsr   s    rQ   r  z$T5Gemma2Encoder.get_input_embeddings  s    3355r\   c                 8    | j                   j                  |      S r   )r}  set_input_embeddingsrM   new_embeddingss     rQ   r  z$T5Gemma2Encoder.set_input_embeddings  s    33NCCr\   pixel_valuesrN   r   c                 x     | j                   d|dd|}|j                  }| j                  |      }||_        |S )NT)r  return_dictrD   )r  rp  r  pooler_output)rM   r  rN   vision_outputsrp  image_featuress         rQ   get_image_featuresz"T5Gemma2Encoder.get_image_features  sM     +**aRVaZ`a*<<334EF'5$r\   r$  Nri  r  c                 D   | j                   j                  }|f|t        d      | | j                         t	        j
                  |t        j                  |j                              k(  }|j                  d      }n||k(  }|j                         }|j                  d      j                  |      j                  |j                        }|j                  d   |j                  d   z  }t        ||   j                         |j                         k(  d| d|        |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        z9Either `input_ids` or `inputs_embeds` has to be provided.)r(  r   r   r   rA   z6Image features and image tokens do not match: tokens: z, features )r   ro   r   r  r   tensorlongr   allsumrr  	expand_asr&  r   r!   numel)rM   r$  ri  r  ro   special_image_maskn_image_tokensn_image_featuress           rQ   get_image_placeholder_maskz*T5Gemma2Encoder.get_image_placeholder_mask  s"    33$ !\]]!.2M$2K2K2M^5::mFZFZ[3 " "4!7!7!;!*n!<+//1/99"=GGVYYZgZnZno)//2^5I5I!5LL,-3359M9M9OOD^DTT_`p_qr	
 "!r\   rh  r  rj  c                 j   |d u |d uz  rt        d      || j                  j                  |      }|i| j                  |d      j                  }|j                  |j                  |j                        }| j                  |||      }	|j                  |	|      } | j                  d|||d|}
|
S )Nrl  T)r  )ri  r  )ri  rh  r  rD   )
r   r}  r_  r  r  r&  r   r(  r  masked_scatter)rM   r$  rh  r  ri  r  rj  rN   r  
image_maskoutputss              rQ   r   zT5Gemma2Encoder.forward  s     -t";<YZZ  OO88CM#!44\t4TbbN+..}/C/C]EXEXYN88~ 9 J *88^TM!$// 
')%
 	
 r\   rw  )NNNNNN)rU   rV   rW   r^   r   r   r   r  r  r   r   r   r   r   r   r   r   r  r  r  r  r   r   r   r   s   @rQ   ry  ry    sk   !!
  '% 6D 
!LL
4:;M4N
	+	+
  
"##d*" ((4/" ))	"<  .2.2042615.2!##d*! t+! &&-	!
 ((4/! ''$.! t+! +,! 
! !r\   ry  c                   p    e Zd ZU eed<    eed       eed      edZddede	f fdZ
eee	 	 	 	 	 	 	 	 dd	ej                  dz  d
ej                   dz  dej                  dz  dedz  dej$                  dz  dedz  dej                   dz  dej                   dz  dee   defd                     Z xZS )T5Gemma2Decoderr   rA   )r1  r%   )r4  cross_attentionsr   rn   c           	      ^   t         |   |       |j                  | _        |j                  | _        t        |j                  |j                  |j                  |j                  dz  |      | _        t        |j                  |j                        | _
        d| _        t        j                  t        |j                        D cg c]  }t!        ||       c}      | _        t        j$                  |j&                        | _        t+        |      | _        | j/                          y c c}w r]  )r   r   rz   r  r|   r  r   r_  r   r`  ra  rb  r   rc  rH   rI   r  r   r   rs   r   r   rd  re  rf  s       rQ   r   zT5Gemma2Decoder.__init__  s     !.. ++;**C/+
 $F$6$6F<O<OP	&+#mmFKFLdLdFef!&)4f
 zz&"5"561&9	 grg  Nr$  rh  r  rj   ri  r  r   encoder_attention_maskrN   r   c	           
         |d u |d uz  rt        d      |t        d      || j                  |      }| j                  s,|r*|(t        t	        | j
                        t	                     }|V||j                         nd}
t        j                  |j                  d   |j                        |
z   }|j                  d      }t        |x}t              s;d }| j
                  ||||j                  nd ||d}t        di |t!        di |d	}t        |x}t              sd
t#        | j
                  |||      i}t        j$                  |d
   |d
   gd      t        j$                  |d   |d
   gd      d	}|}i }| j
                  j&                  D ]  }| j)                  |||      ||<    | j+                  |      }t-        | j.                  d | j
                  j0                         D ]H  \  }} |||| j
                  j&                  |      || j
                  j&                  |      ||||fi |	}J | j3                  |      }| j+                  |      }t5        ||      S )Nrl  z0`encoder_hidden_states` must be given in decoderr{  r   rA   rm  c                  L    t        j                  dt         j                        S )NT)r(  )r   r  rJ   )argss    rQ   <lambda>z)T5Gemma2Decoder.forward.<locals>.<lambda>-  s    ELLUZZ4X r\   )r   ri  rh  rj   r  rn  ro  rC   )r   ri  rh  r   rn  r   r   rB   )rp  rj   rD   )r   r_  r   r
   r	   r   get_seq_lengthr   rq  r   r   rr  r}   r~   r   r1   r2   r   r   rG   rd  r   rs  r   rI   ra  r   )rM   r$  rh  r  rj   ri  r  r   r  rN   past_seen_tokensrt  dummy_and_mask_functionru  cross_attn_mask_mappingmerged_attn_mask_mappingr   r   r   rP   rv  s                        rQ   r   zT5Gemma2Decoder.forward  s    -t";<YZZ (OPP  --i8M}}/F1,dkk2RT`TbcOCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4LNB0DI 'Y#++!."0KZKf?#G#Glp ,%<K #5"C{"C%F%U%U&"
 5KK1TR ";;;"/#9*?&=#'# $ii'(89;RSc;dekm "''(;<>UVf>ghnp"	$
  & !++11 	gJ.2oom\[e.f
+	g ]3(5Tt{{7T7T)UV 
	OA|(#DKK$;$;A$>?()@)@)CD%	 	M
	 		-0]38++
 	
r\   rw  )NNNNNNNN)rU   rV   rW   rd   r   r#   r   r  rY  r   r   r"   r$   r   r   r  r   r
   r  rJ   r   r   r   r   r   r   s   @rQ   r  r    s5   !!$%<AF*+B!L-4 s ,   .2.2046:26!%596:]
##d*]
 t+]
 &&-	]

 -t3]
 ((4/]
 $;]
  %||d2]
 !&t 3]
 +,]
 
3]
    ]
r\   r  c                       e Zd ZdddZdef fdZd Zd Zd Zd	 Z	e
e	 	 	 	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  ded
z  dej$                  d
z  dej$                  d
z  ded
z  dee   defd              Z xZS )T5Gemma2Modelz&encoder.text_model.embed_tokens.weightz-encoder.text_model.embed_tokens.eoi_embedding)zdecoder.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingr   c                     t         |   |       t        |j                  |j                        | _        t        |j                  |j                        | _        | j                          y r   )r   r   ry  rk   rn   r  rl   re  r   s     rQ   r   zT5Gemma2Model.__init__u  sL      'v~~v7M7MN&v~~v7M7MNr\   c                     | j                   S r   )rk   r   s    rQ   get_encoderzT5Gemma2Model.get_encoder~      ||r\   c                     | j                   S r   rl   r   s    rQ   get_decoderzT5Gemma2Model.get_decoder  r  r\   c                 6    | j                   j                         S r   )rk   r  r   s    rQ   r  z"T5Gemma2Model.get_input_embeddings  s    ||0022r\   c                 8    | j                   j                  |      S r   )rk   r  r  s     rQ   r  z"T5Gemma2Model.set_input_embeddings  s    ||00@@r\   Nr$  r  rh  r  decoder_input_idsdecoder_attention_maskdecoder_position_idsencoder_outputsrj   ri  decoder_inputs_embedsr  rN   r   c                 N   | | j                   d||||
|dd|}|j                  } | j                  d|||||	|||dd	|}t        |j                  |j                  |j
                  |j                  |j                  |j                  |j
                  |j                        S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        T)r$  rh  r  ri  r  r  )	r$  rh  r  ri  rj   r   r  r  r  )rp  rj   decoder_hidden_statesdecoder_attentionsr  encoder_last_hidden_stater   encoder_attentionsrD   )rk   rp  rl   r   rj   r   r4  r  )rM   r$  r  rh  r  r  r  r  r  rj   ri  r  r  rN   r   decoder_outputss                   rQ   r   zT5Gemma2Model.forward  s    6 "*dll #-)+)  O !0 A A '$,, 
'1-/+"7#1
 
 "-??+;;"1"?"?.99,==&5&G&G"1"?"?.99	
 		
r\   )NNNNNNNNNNNN)rU   rV   rW   _tied_weights_keysrh   r   r  r  r  r  r   r   r   r  r  
BoolTensorr   r
   r   rJ   r   r   r   r   r   r   s   @rQ   r  r  n  sv    (P.]
~ 3A  .215370459:>8<266:-159!%!=
 ##d*=
 ''$.	=

 ))D0=
 &&-=
 !++d2=
 !& 0 04 7=
 $..5=
 )4/=
 -t3=
 ||d*=
  %||d2=
  $;!=
" +,#=
$ 
%=
  =
r\   r  c            $           e Zd ZddiZddiZddgdgfiZdef fdZd	 Zd
 Z	d Z
d Zd Zd Zeedej"                  dee   deez  fd              Zed        Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d)dej2                  dz  dej4                  dz  dej4                  dz  dej2                  dz  dej2                  dz  dej6                  dz  dej2                  dz  dedz  dedz  dej4                  dz  dej4                  dz  dej2                  dz  d edz  d!eej"                  z  dee   deej4                     e z  f d"              Z!d#e"d$e#d%e$d&ed'edef fd(Z% xZ&S )* T5Gemma2ForConditionalGenerationzlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projcolwise_gather_outputr   logitsr   c                    t         |   |       t        |      | _        |j                  j
                  | _        t        |j                  j                  | j
                        | _        d| _	        | j                          y )NForMaskedLM)r   r   r  r-  rl   r|   r  r   lm_head	loss_typere  r   s     rQ   r   z)T5Gemma2ForConditionalGeneration.__init__  sZ     "6*
 ..33%fnn&@&@$//R&r\   c                 &    || j                   _        y r   r  rC  r  s     rQ   set_output_embeddingsz6T5Gemma2ForConditionalGeneration.set_output_embeddings  s     .r\   c                 .    | j                   j                  S r   r  r   s    rQ   get_output_embeddingsz6T5Gemma2ForConditionalGeneration.get_output_embeddings  s    ||$$$r\   c                 6    | j                   j                         S r   r-  r  r   s    rQ   r  z5T5Gemma2ForConditionalGeneration.get_input_embeddings      zz..00r\   c                 :    | j                   j                  |       y r   r-  r  rM   values     rQ   r  z5T5Gemma2ForConditionalGeneration.set_input_embeddings      

''.r\   c                 6    | j                   j                         S r   )r-  r  r   s    rQ   r  z,T5Gemma2ForConditionalGeneration.get_encoder      zz%%''r\   c                 6    | j                   j                         S r   )r-  r  r   s    rQ   r  z,T5Gemma2ForConditionalGeneration.get_decoder  r  r\   r  rN   r   c                 D     | j                         j                  |fi |S r   )r  r  )rM   r  rN   s      rQ   r  z3T5Gemma2ForConditionalGeneration.get_image_features  s%    
 5t!44\LVLLr\   c                 6    | j                         j                  S r   )r  r  r   s    rQ   r  z-T5Gemma2ForConditionalGeneration.vision_tower  s    !...r\   Nr$  rh  r  r  r  r  r  rj   ri  r  labelsr  logits_to_keepc                    |||| j                  |      } | j                  d|||||||||	|
||d|}|j                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }| j                  j                  }|j                  3||j                  z  }t        j                  |      }||j                  z  }d}| | j                  ||| j                  fi |}t        |||j                  |j                   |j"                  |j$                  |j&                  |j(                  |j*                  	      S )a  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)r$  r  rh  r  r  r  r  r  rj   ri  r  r  )	lossr  rj   r  r  r  r  r   r  rD   )rS  r-  rp  r}   r   slicer  r   rl   final_logit_softcappingr   tanhloss_functionr|   r   rj   r  r  r  r  r   r  )rM   r$  r  rh  r  r  r  r  r  rj   ri  r  r  r  r  rN   r  r   slice_indicesr  rP  r  s                         rQ   r   z(T5Gemma2ForConditionalGeneration.forward  sr   B "3";@U@] $ J J6 R.8djj /
%)%/#9!5++'"7/
 /
  (998B>SV8W~ot4]kmA}a,?@A,,11=nDDDFZZ'FnDDDF%4%%ffdooPPD+;;"1"G"G.AA,==&5&O&O"1"G"G.AA

 
	
r\   generation_configmodel_kwargsgeneration_mode
batch_sizemax_cache_lengthc           	      P   t         |   |||||       |j                  du ry|j                  }|d}nd|j                  v }t	        j
                  | j                  j                  d            }|`|`	||d}	|j                  d      }
|
t        |
t              st        d      t        |
j                        d	kD  r|
j                  j                  d	      ryt!        |
j"                        }|t$        k(  r|d
   d	   j&                  d   |	d<    |di |	|
_        n=t        t)        di | j                  j                  d      |dt)                     |d<   t+        | d      r=| j,                  0t        | j,                  t              st        d      |d   | _        yyy)zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FN	offloadedTr  )r   
offloadingrj   zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.r   r  rA   max_cache_len_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.rD   )r   _prepare_cache_for_generationr  cache_implementationcopydeepcopyr   get_text_configr   rG   r   r}   r
   r   lenr   typer   r   r   r	   rE  r  )rM   r  r  r  r  r  r  offload_cachecross_attn_configcross_attn_cache_kwargsrj   cross_attn_clsr   s               rQ   r  z>T5Gemma2ForConditionalGeneration._prepare_cache_for_generationK  s    	-	
 &&%/0EE'!M'+<+Q+QQM !MM$++*E*Ed*E*ST ,) ('#

 '**+<=&o/BC w 
 ?--.27Q7Q7U7UVW7X!/"G"GHN,;GHY;Z[\;];c;cde;f'84B4]E\4]O1 /B "&++"="=d"="K&3 /L*+ 4"t{{'>dkk+>? !opp&'89DK	 (?"r\   )NNNNNNNNNNNNNr   )'rU   rV   rW   r  _tp_plan_pp_planrh   r   r  r  r  r  r  r  r   r   r   r   r   r   r   r   r  propertyr  r  r  r  r   r
   rJ   r   r   r   r   r~   r   r  r   r   s   @rQ   r  r    s   !#Q #$;<H"o%6
$CDH~ /%1/(( M!LLM4:;M4NM	+	+M  M
 / /  .215370459:>8<266:26:>*.!%-.%M
 ##d*M
 ''$.	M

 ))D0M
 &&-M
 !++d2M
 !& 0 04 7M
 $..5M
 )4/M
 -t3M
 ((4/M
  %0047M
    4'!M
" $;#M
$ ell*%M
& +,'M
( 
u  	!O	3)M
  M
^I:+I: I: (	I:
 I: I: 
I: I:r\   r  c                       e Zd Zdef fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  de	j                  dz  dee   defd              Z xZS )!T5Gemma2ForSequenceClassificationr   c                 "   t         |   |       |j                  | _        |j                  j                  | _        t        |      | _        t        |dd      }t        | j                  | j                  |      | _	        | j                          y Nru   g?r   r   
num_labelsrl   r   r  r-  r   r  scorere  rM   r   classifier_dropoutr   s      rQ   r   z*T5Gemma2ForSequenceClassification.__init__  sp      ++!>>55"6*
$V-FL/0@0@$//Sef
r\   c                 6    | j                   j                         S r   r  r   s    rQ   r  z6T5Gemma2ForSequenceClassification.get_input_embeddings  r  r\   c                 :    | j                   j                  |       y r   r  r  s     rQ   r  z6T5Gemma2ForSequenceClassification.set_input_embeddings  r  r\   Nr$  r  rh  r  r  r  r  r  ri  r  r  rN   r   c                 v   |	|
#t        d| j                  j                   d      |t        d      || j	                  |      } | j
                  |f||||||||	|
dd
|}|j                  }|j                  }|j                  }| j                  |      }|j                  d   }|| j                  j                  k7  j                  |j                  t        j                         }t        j"                  |j                  d   |j                  t        j                   	      }||z  j%                  d      }t        j&                  ||j                  d   d
z
        }|t        j"                  ||j                        |f   }d}|| j)                  |||| j                        }t+        ||||      S )  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N8Passing input embeddings is currently not supported for .You have to specify input_idsF
r  rh  r  r  r  r  r  ri  r  r  r   r   )r   r(  rA   )maxrm  )r  r  pooled_logitsr   r  r  r   r4  )NotImplementedErrorr   rU   r   rS  r-  rp  r  r  r  r   r   rz   r&  r   r   int32rq  argmaxclampr  r   )rM   r$  r  rh  r  r  r  r  r  ri  r  r  rN   r  rp  r   r4  r  r  non_pad_masktoken_indiceslast_non_pad_tokenr  r  s                           rQ   r   z)T5Gemma2ForSequenceClassification.forward  s   4 $(=(I%J4>>KbKbJccde  <==$ $ J J9 U&0djj'
%)%/#9!5+'"7'
 '
 $5555//
-.__Q'
)T[[-E-EEII&--Y^YdYde%6%<%<R%@^c^i^ij+l:BB2F"[[);ARAXAXY[A\_`A`au||Jv}}MOaab%%VFR_hlhshs%tD' '!	
 	
r\   NNNNNNNNNNN)rU   rV   rW   rh   r   r  r  r   r   r   r  r  r   r   r   r   r   r   r   r   s   @rQ   r  r    s\   	~ 	1/  .215.204596:8<2626:>*.J
##d*J
 ''$.J
 t+	J

 &&-J
 !++d2J
 !&t 3J
 $..5J
 )4/J
 ((4/J
  %0047J
   4'J
 +,J
 
"J
  J
r\   r  c                       e Zd Zdef fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  de	j                  dz  dee   defd              Z xZS )T5Gemma2ForTokenClassificationr   c                 "   t         |   |       |j                  | _        |j                  j                  | _        t        |      | _        t        |dd      }t        | j                  | j                  |      | _	        | j                          y r  r  r  s      rQ   r   z'T5Gemma2ForTokenClassification.__init__  sp      ++!>>55"6*
$V-FL/0@0@$//Sef
r\   c                 6    | j                   j                         S r   r  r   s    rQ   r  z3T5Gemma2ForTokenClassification.get_input_embeddings  r  r\   c                 :    | j                   j                  |       y r   r  r  s     rQ   r  z3T5Gemma2ForTokenClassification.set_input_embeddings
  r  r\   Nr$  r  rh  r  r  r  r  r  ri  r  r  rN   r   c                    |	|
#t        d| j                  j                   d      |t        d      || j	                  |      } | j
                  |f||||||||	|
dd
|}|j                  }|j                  }|j                  }| j                  |      }d}|| j                  ||| j                        }t        ||||      S )r
  Nr  r  r  Fr  r  )r  r   rU   r   rS  r-  rp  r  r  r  r  r   r   )rM   r$  r  rh  r  r  r  r  r  ri  r  r  rN   r  rp  r   r4  r  r  s                      rQ   r   z&T5Gemma2ForTokenClassification.forward  s   4 $(=(I%J4>>KbKbJccde  <==$ $ J J9 U&0djj'
%)%/#9!5+'"7'
 '
 $5555//
-.%%ffdkkBD$'!	
 	
r\   r  )rU   rV   rW   rh   r   r  r  r   r   r   r  r  r   r   r   r   r   r   r   r   s   @rQ   r  r    s\   
~ 
1/  .215.204596:8<2626:>*.@
##d*@
 ''$.@
 t+	@

 &&-@
 !++d2@
 !&t 3@
 $..5@
 )4/@
 ((4/@
  %0047@
   4'@
 +,@
 
@
  @
r\   r  )
rh   r:   r^   rd   r  r  ry  r,  r  r  )T)jr  collections.abcr   typingr   r   r   torch.nnr   huggingface_hub.dataclassesr    r   r>  cache_utilsr	   r
   r   configuration_utilsr   
generationr   r   r   masking_utilsr   modeling_flash_attention_utilsr   modeling_outputsr   r   r   r   r   r   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r    r!   utils.genericr"   utils.output_capturingr#   r$   autor&   gemma3.configuration_gemma3r'   r(   gemma3.modeling_gemma3r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   siglipr4   t5gemma.modeling_t5gemmar5   r6   r7   
get_loggerrU   r   r:   r^   rd   rh   r   r   r   r   r   r   r   r  r  r  r  r  r  r,  r[  ry  r  r  r  r  r  __all__rD   r\   rQ   <module>r8     s    $     . & I I 3 K K 6 B   7 F &  8 E  H    (  
		H	% 781)+; 1  912 78L   9 781,.> 1  912 78T% T  9Tn	m 		) 	\3 \O [Bo [B|  &	. 	,. ,^	] 		!: 	!"; !
 &C  * L!3 L! L!^b
1 b
Je- eP~
- ~
B Z
+ Z
 Z
zH:'> H:V ^
(? ^
 ^
B U
%< U
 U
pr\   