
    i              	          d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej>                  e       Z!e ed       G d de                    Z"e ed       G d de                    Z#e ed       G d de                    Z$e ed       G d de                    Z%d Z&d Z' G d d ejP                        Z) G d! d"ejP                        Z* G d# d$ejP                        Z+dJd%ejX                  d&e-d'e.d(ejX                  fd)Z/ G d* d+ejP                        Z0 G d, d-ejP                        Z1 G d. d/ejP                        Z2 G d0 d1ejP                        Z3 G d2 d3ejP                        Z4 G d4 d5ejP                        Z5 G d6 d7ejP                        Z6 G d8 d9e      Z7 G d: d;ejP                        Z8e G d< d=e             Z9e G d> d?e9             Z: ed@       G dA dBe9             Z; edC       G dD dEe9             Z< edF       G dG dHee9             Z=g dIZ>y)KzPyTorch Swin Transformer model.    N)	dataclass)nn   )initialization)ACT2FN)BackboneMixinfilter_output_hidden_states)GradientCheckpointingLayer)BackboneOutput)PreTrainedModel)ModelOutputauto_docstringlogging	torch_int)can_return_tuple   )
SwinConfigzN
    Swin encoder's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   dZ
e	ej                  df   dz  ed<   dZe	ej                  df   dz  ed<   y)SwinEncoderOutputa  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   tupler   r        w/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/swin/modeling_swin.pyr   r   (   s}     37u((4/6:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr$   r   zV
    Swin model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinModelOutputa  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   pooler_output.r   r   r   )r   r   r   r   r   r   r    r!   r(   r   r"   r   r   r#   r$   r%   r'   r'   >   s    	 37u((4/6.2M5$$t+2:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr$   r'   z*
    Swin masked image model outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinMaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlossreconstruction.r   r   r   )r   r   r   r   r+   r   r    r!   r,   r   r"   r   r   r#   r$   r%   r*   r*   W   s     &*D%

d
")/3NE%%,3:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr$   r*   z0
    Swin outputs for image classification.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr+   logits.r   r   r   )r   r   r   r   r+   r   r    r!   r/   r   r"   r   r   r#   r$   r%   r.   r.   r   s     &*D%

d
")'+FE$+:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr$   r.   c                     | j                   \  }}}}| j                  |||z  |||z  ||      } | j                  dddddd      j                         j                  d|||      }|S )z2
    Partitions the given input into windows.
    r   r   r            shapeviewpermute
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowss          r%   window_partitionrA      s}     /<.A.A+J|!&&Fk);8Lk[gM ##Aq!Q15@@BGGKYdfrsGNr$   c                     | j                   d   }| j                  d||z  ||z  |||      } | j                  dddddd      j                         j                  d|||      } | S )z?
    Merges windows to produce higher resolution features.
    r4   r   r   r   r1   r2   r3   r5   )r@   r;   r=   r>   r?   s        r%   window_reverserC      sn     ==$Lll2v4e{6JKYdfrsGooaAq!Q/::<AA"feUabGNr$   c            
            e Zd ZdZd fd	Zdej                  dededej                  fdZ	 	 dd	ej                  dz  d
ej                  dz  dedeej                     fdZ xZS )SwinEmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    c                 ~   t         |           t        |      | _        | j                  j                  }| j                  j
                  | _        |r4t        j                  t        j                  dd|j                              nd | _        |j                  r=t        j                  t        j                  d|dz   |j                              | _        nd | _        t        j                  |j                        | _        t        j"                  |j$                        | _        |j(                  | _        || _        y )Nr   )super__init__SwinPatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr   zeros	embed_dim
mask_tokenuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)selfrZ   use_mask_tokenrK   	__class__s       r%   rH   zSwinEmbeddings.__init__   s     3F ;++77//99O]",,u{{1a9I9I'JKcg))')||EKK;QR?TZTdTd4e'fD$'+D$LL!1!12	zz&"<"<= ++r$   
embeddingsr=   r>   returnc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr4         ?r   r   r1   bicubicF)sizemodealign_cornersdim)r6   rS   r   jit
is_tracingrY   r   reshaper8   r   
functionalinterpolater7   cat)r[   r^   r=   r>   rK   num_positionsclass_pos_embedpatch_pos_embedrg   
new_height	new_widthsqrt_num_positionss               r%   interpolate_pos_encodingz'SwinEmbeddings.interpolate_pos_encoding   s`    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr$   Npixel_valuesbool_masked_posrt   c                    |j                   \  }}}}| j                  |      \  }}	| j                  |      }|j                         \  }
}}|K| j                  j                  |
|d      }|j                  d      j                  |      }|d|z
  z  ||z  z   }| j                  (|r|| j                  |||      z   }n|| j                  z   }| j                  |      }||	fS )Nr4   g      ?)r6   rJ   rU   rc   rQ   expand	unsqueezetype_asrS   rt   rX   )r[   ru   rv   rt   _r?   r=   r>   r^   output_dimensionsr<   seq_lenmask_tokensmasks                 r%   forwardzSwinEmbeddings.forward   s     *6););&<(,(=(=l(K%
%YYz*
!+!2
GQ&//00WbIK",,R088ED#sTz2[45GGJ##/''$*G*G
TZ\a*bb
'$*B*BB
\\*-
,,,r$   )FNF)r   r   r   r   rH   r   Tensorintrt   r    
BoolTensorboolr"   r   __classcell__r]   s   @r%   rE   rE      s    &&D5<< &D &DUX &D]b]i]i &DV 48).	-''$.- ))D0- #'	-
 
u||	-r$   rE   c                   v     e Zd ZdZ fdZd Zdej                  dz  deej                  ee
   f   fdZ xZS )rI   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        |d   |d   z  |d   |d   z  f| _        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)rG   rH   
image_sizerY   r?   rP   
isinstancecollectionsabcIterablerK   rL   r   Conv2d
projection)r[   rZ   r   rY   r?   hidden_sizerK   r]   s          r%   rH   zSwinPatchEmbeddings.__init__  s    !'!2!2F4E4EJ
$*$7$79I9Ik#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&$Q-:a=8*Q-:VW=:XY))L+:^hir$   c                 n   || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|S )Nr   r   )rY   r   rk   pad)r[   ru   r=   r>   
pad_valuess        r%   	maybe_padzSwinPatchEmbeddings.maybe_pad  s    4??1%%*T__Q/%$//!:L2LLMJ==,,\:FLDOOA&&!+Q4??1#5QRAS8S#STJ==,,\:FLr$   ru   Nr_   c                     |j                   \  }}}}| j                  |||      }| j                  |      }|j                   \  }}}}||f}|j                  d      j	                  dd      }||fS )Nr1   r   )r6   r   r   flatten	transpose)r[   ru   r{   r?   r=   r>   r^   r|   s           r%   r   zSwinPatchEmbeddings.forward  s}    )5););&<~~lFEB__\2
(..1fe#UO''*44Q:
,,,r$   )r   r   r   r   rH   r   r   r    r"   r   r   r   r   r   s   @r%   rI   rI      sF    j	-E$5$5$< 	-u||UZ[^U_G_A` 	-r$   rI   c            	            e Zd ZdZej
                  fdee   dedej                  ddf fdZ	d Z
d	ej                  d
eeef   dej                  fdZ xZS )SwinPatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutionrg   
norm_layerr_   Nc                     t         |           || _        || _        t	        j
                  d|z  d|z  d      | _         |d|z        | _        y )Nr2   r1   Fbias)rG   rH   r   rg   r   Linear	reductionrU   )r[   r   rg   r   r]   s       r%   rH   zSwinPatchMerging.__init__7  sI     01s7AG%@q3w'	r$   c                     |dz  dk(  xs |dz  dk(  }|r.ddd|dz  d|dz  f}t         j                  j                  ||      }|S )Nr1   r   r   )r   rk   r   )r[   r:   r=   r>   
should_padr   s         r%   r   zSwinPatchMerging.maybe_pad>  sU    qjAo:519>
Q519a!<JMM--mZHMr$   r:   input_dimensionsc                    |\  }}|j                   \  }}}|j                  ||||      }| j                  |||      }|d d dd ddd dd d f   }|d d dd ddd dd d f   }	|d d dd ddd dd d f   }
|d d dd ddd dd d f   }t        j                  ||	|
|gd      }|j                  |dd|z        }| j                  |      }| j                  |      }|S )Nr   r1   r   r4   r2   )r6   r7   r   r   rm   rU   r   )r[   r:   r   r=   r>   r<   rg   r?   input_feature_0input_feature_1input_feature_2input_feature_3s               r%   r   zSwinPatchMerging.forwardF  s   ((5(;(;%
C%**:vulS}feD'14a4Aq(89'14a4Aq(89'14a4Aq(89'14a4Aq(89		?O_Ve"fhjk%**:r1|;KL		-0}5r$   )r   r   r   r   r   rT   r"   r   ModulerH   r   r   r   r   r   r   s   @r%   r   r   *  sr    
 XZWcWc (s (# (299 (hl (U\\ U3PS8_ Y^YeYe r$   r   input	drop_probtrainingr_   c                    |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                        z   }|j                          | j                  |      |z  }|S )zc
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

            r   r   )r   dtypedevice)r6   ndimr   randr   r   floor_div)r   r   r   	keep_probr6   random_tensoroutputs          r%   	drop_pathr   a  s    
 CxII[[^

Q 77E

5ELL YYMYYy!M1FMr$   c                   x     e Zd ZdZd	dedz  ddf fdZdej                  dej                  fdZde	fdZ
 xZS )
SwinDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   r_   c                 0    t         |           || _        y N)rG   rH   r   )r[   r   r]   s     r%   rH   zSwinDropPath.__init__t  s    "r$   r   c                 D    t        || j                  | j                        S r   )r   r   r   r[   r   s     r%   r   zSwinDropPath.forwardx  s    FFr$   c                      d| j                    S )Nzp=)r   r[   s    r%   
extra_reprzSwinDropPath.extra_repr{  s    DNN#$$r$   r   )r   r   r   r   floatrH   r   r   r   strr   r   r   s   @r%   r   r   q  sG    b#%$, #$ #GU\\ Gell G%C %r$   r   c            
            e Zd Z fdZ	 	 d	dej
                  dej                  dz  dedz  deej
                     fdZ	d Z
 xZS )
SwinSelfAttentionc                    t         |           ||z  dk7  rt        d| d| d      || _        t	        ||z        | _        | j                  | j
                  z  | _        t        |t        j                  j                        r|n||f| _        t        j                  t        j                  d| j                  d   z  dz
  d| j                  d   z  dz
  z  |            | _        | j#                  d| j%                                t        j&                  | j                  | j                  |j(                        | _        t        j&                  | j                  | j                  |j(                        | _        t        j&                  | j                  | j                  |j(                        | _        t        j0                  |j2                        | _        y )	Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r1   r   relative_position_indexr   )rG   rH   
ValueErrornum_attention_headsr   attention_head_sizeall_head_sizer   r   r   r   r;   r   rN   r   rO   relative_position_bias_tableregister_buffercreate_relative_position_indexr   qkv_biasquerykeyvaluerV   attention_probs_dropout_probrX   r[   rZ   rg   	num_headsr;   r]   s        r%   rH   zSwinSelfAttention.__init__  s   ?a#C5(^_h^iijk  $- #&sY#7 !558P8PP%k;??3K3KLKS^`kRl 	 -/LLKKT--a0014T=M=Ma=P9PST9TUW`a-
) 	68[8[8]^YYt1143E3EFOO\
99T//1C1C&//ZYYt1143E3EFOO\
zz&"E"EFr$   Nr   attention_maskoutput_attentionsr_   c                    |j                   \  }}}||d| j                  f}| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
t        j                  ||	j	                  dd            }|t        j                  | j                        z  }| j                  | j                  j                  d         }|j                  | j                  d   | j                  d   z  | j                  d   | j                  d   z  d      }|j                  ddd      j                         }||j!                  d      z   }|r|j                   d   }|j                  ||z  || j"                  ||      }||j!                  d      j!                  d      z   }|j                  d| j"                  ||      }t$        j&                  j)                  |d      }| j+                  |      }t        j                  ||
      }|j                  dddd      j                         }|j-                         d d | j.                  fz   }|j                  |      }|r||f}|S |f}|S )Nr4   r   r1   r   rf   r   )r6   r   r   r7   r   r   r   r   matmulmathsqrtr   r   r;   r8   r9   ry   r   r   rk   softmaxrX   rc   r   )r[   r   r   r   r<   rg   r?   hidden_shapequery_layer	key_layervalue_layerattention_scoresrelative_position_bias
mask_shapeattention_probscontext_layernew_context_layer_shapeoutputss                     r%   r   zSwinSelfAttention.forward  s    )6(;(;%
C"CT-E-EFjj/44\BLLQPQRHH]+00>HHAN	jj/44\BLLQPQR !<<Y5H5HR5PQ+dii8P8P.QQ!%!B!B4C_C_CdCdegCh!i!7!<!<Q$"2"21"55t7G7G7JTM]M]^_M`7`bd"
 "8!?!?1a!H!S!S!U+.D.N.Nq.QQ%'--a0J/44j(*d6N6NPSUX   0.2J2J12M2W2WXY2ZZ/44R9Q9QSVX[\ --//0@b/I ,,7_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r$   c                    t        j                  | j                  d         }t        j                  | j                  d         }t        j                  t        j                  ||gd            }t        j
                  |d      }|d d d d d f   |d d d d d f   z
  }|j                  ddd      j                         }|d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   d| j                  d   z  dz
  z  cc<   |j                  d      }|S )Nr   r   ij)indexingr1   r4   )	r   aranger;   stackmeshgridr   r8   r9   sum)r[   coords_hcoords_wcoordscoords_flattenrelative_coordsr   s          r%   r   z0SwinSelfAttention.create_relative_position_index  s-   << 0 0 34<< 0 0 34U^^Xx,@4PQvq1(At4~aqj7QQ)11!Q:EEG1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? "1"5"5b"9&&r$   r   )r   r   r   rH   r   r   r    r   r"   r   r   r   r   s   @r%   r   r     s^    G: 48).	1||1 ))D01  $;	1
 
u||	1f'r$   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )SwinSelfOutputc                     t         |           t        j                  ||      | _        t        j
                  |j                        | _        y r   )rG   rH   r   r   denserV   r   rX   r[   rZ   rg   r]   s      r%   rH   zSwinSelfOutput.__init__  s6    YYsC(
zz&"E"EFr$   r   input_tensorr_   c                 J    | j                  |      }| j                  |      }|S r   r   rX   )r[   r   r   s      r%   r   zSwinSelfOutput.forward  s$    

=1]3r$   r   r   r   rH   r   r   r   r   r   s   @r%   r   r     s2    G
U\\  RWR^R^ r$   r   c            
            e Zd Z fdZ	 	 ddej
                  dej                  dz  dedz  deej
                     fdZ	 xZ
S )	SwinAttentionc                 j    t         |           t        ||||      | _        t	        ||      | _        y r   )rG   rH   r   r[   r   r   r   s        r%   rH   zSwinAttention.__init__  s.    %fc9kJ	$VS1r$   Nr   r   r   r_   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r[   r   )r[   r   r   r   self_outputsattention_outputr   s          r%   r   zSwinAttention.forward  sE     yy@QR;;|AF#%QR(88r$   r   )r   r   r   rH   r   r   r    r   r"   r   r   r   s   @r%   r  r    sW    2 48).		||	 ))D0	  $;		
 
u||		r$   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )SwinIntermediatec                    t         |           t        j                  |t	        |j
                  |z              | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )rG   rH   r   r   r   	mlp_ratior   r   
hidden_actr   r   intermediate_act_fnr   s      r%   rH   zSwinIntermediate.__init__  sa    YYsC(8(83(>$?@
f''-'-f.?.?'@D$'-'8'8D$r$   r   r_   c                 J    | j                  |      }| j                  |      }|S r   )r   r  r   s     r%   r   zSwinIntermediate.forward  s&    

=100?r$   r  r   s   @r%   r
  r
    s#    9U\\ ell r$   r
  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )
SwinOutputc                     t         |           t        j                  t	        |j
                  |z        |      | _        t        j                  |j                        | _	        y r   )
rG   rH   r   r   r   r  r   rV   rW   rX   r   s      r%   rH   zSwinOutput.__init__  sF    YYs6#3#3c#9:C@
zz&"<"<=r$   r   r_   c                 J    | j                  |      }| j                  |      }|S r   r  r   s     r%   r   zSwinOutput.forward  s$    

=1]3r$   r  r   s   @r%   r  r  
  s#    >
U\\ ell r$   r  c                        e Zd Zd fd	Zd Zd Zd Z	 	 ddej                  de	e
e
f   dedz  d	edz  d
e	ej                  ej                  f   f
dZ xZS )	SwinLayerc                    t         |           |j                  | _        || _        |j                  | _        || _        t        j                  ||j                        | _	        t        |||| j                        | _        |dkD  rt        |      nt        j                         | _        t        j                  ||j                        | _        t!        ||      | _        t%        ||      | _        y )Neps)r;   r   )rG   rH   chunk_size_feed_forward
shift_sizer;   r   r   rT   layer_norm_epslayernorm_beforer  	attentionr   Identityr   layernorm_afterr
  intermediater  r   )r[   rZ   rg   r   r   drop_path_rater  r]   s          r%   rH   zSwinLayer.__init__  s    '-'E'E$$!-- 0 "Sf6K6K L&vsI4K[K[\9G#9Mn5SUS^S^S`!||CV5J5JK,VS9 -r$   c                    t        |      | j                  k  rgt        d      | _        t        j
                  j                         r(t	        j                   t	        j                  |            n
t        |      | _        y y Nr   )minr;   r   r  r   rh   ri   tensor)r[   r   s     r%   set_shift_and_window_sizez#SwinLayer.set_shift_and_window_size$  s\     D$4$44'lDO=BYY=Q=Q=S		%,,'789Y\]mYn  5r$   c           	         | j                   dkD  rht        j                  d||df||      }t        d| j                         t        | j                   | j                          t        | j                    d       f}t        d| j                         t        | j                   | j                          t        | j                    d       f}d}|D ]  }	|D ]  }
||d d |	|
d d f<   |dz  }  t        || j                        }|j                  d| j                  | j                  z        }|j                  d      |j                  d      z
  }|j                  |dk7  d      j                  |dk(  d      }|S d }|S )Nr   r   r   r4   r1   g      Yr   )	r  r   rO   slicer;   rA   r7   ry   masked_fill)r[   r=   r>   r   r   img_maskheight_sliceswidth_slicescountheight_slicewidth_slicemask_windows	attn_masks                r%   get_attn_maskzSwinLayer.get_attn_mask,  s   ??Q{{Avua#8fUHa$***+t'''$//)9:t&-M a$***+t'''$//)9:t&-L
 E - #/ K@EHQk1<=QJE
 ,Hd6F6FGL',,R1A1ADDTDT1TUL$..q1L4J4J14MMI!--i1nfEQQR[_`R`befI  Ir$   c                     | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||      }||fS r#  )r;   r   rk   r   )r[   r   r=   r>   	pad_right
pad_bottomr   s          r%   r   zSwinLayer.maybe_padH  s    %%0@0@(@@DDTDTT	&&$2B2B)BBdFVFVV
Ay!Z8
))-Dj((r$   r   r   r   Nalways_partitionr_   c                    |s| j                  |       n	 |\  }}|j                         \  }}}	|}
| j                  |      }|j                  ||||	      }| j	                  |||      \  }}|j
                  \  }}}}| j                  dkD  r1t        j                  || j                   | j                   fd      }n|}t        || j                        }|j                  d| j                  | j                  z  |	      }| j                  |||j                  |j                        }| j                  |||      }|d   }|j                  d| j                  | j                  |	      }t        || j                  ||      }| j                  dkD  r/t        j                  || j                  | j                  fd      }n|}|d   dkD  xs |d   dkD  }|r|d d d |d |d d f   j!                         }|j                  |||z  |	      }|
| j#                  |      z   }| j%                  |      }| j'                  |      }|| j)                  |      z   }|r	||d	   f}|S |f}|S )
Nr   )r   r1   )shiftsdimsr4   r   )r   r   r3   r   )r&  rc   r  r7   r   r6   r  r   rollrA   r;   r2  r   r   r  rC   r9   r   r  r   r   )r[   r   r   r   r6  r=   r>   r<   r{   channelsshortcutr   
height_pad	width_padshifted_hidden_stateshidden_states_windowsr1  attention_outputsr  attention_windowsshifted_windows
was_paddedlayer_outputlayer_outputss                           r%   r   zSwinLayer.forwardO  s     **+;<("/"4"4"6
Ax --m<%**:vuhO %)NN=&%$P!z&3&9&9#:y!??Q$)JJ}tFVY]YhYhXhEipv$w!$1! !11FHXHX Y 5 : :2t?O?ORVRbRb?bdl m&&	)<)<EZEaEa ' 
	 !NN+@)_pNq,Q/,11"d6F6FHXHXZbc():D<L<LjZcd ??Q %

?DOOUYUdUdCelr s /]Q&;*Q-!*;
 1!WfWfufa2G H S S U-22:v~xX 4>>2C#DD++M:((6$t{{<'@@@Q'8';< YeWfr$   )r   r   FF)r   r   r   rH   r&  r2  r   r   r   r"   r   r   r   r   r   s   @r%   r  r    sz    .8) */(->||>  S/>  $;	>
 +> 
u||U\\)	*>r$   r  c                        e Zd Z fdZ	 	 d	dej
                  deeef   dedz  dedz  deej
                     f
dZ	 xZ
S )
	SwinStagec                 h   t         	|           || _        || _        t	        j
                  t        |      D cg c]-  }t        ||||||   |dz  dk(  rdn|j                  dz        / c}      | _	        |& |||t        j                        | _        d| _        y d | _        d| _        y c c}w )Nr1   r   )rZ   rg   r   r   r!  r  )rg   r   F)rG   rH   rZ   rg   r   
ModuleListranger  r;   blocksrT   
downsamplepointing)
r[   rZ   rg   r   depthr   r   rN  ir]   s
            r%   rH   zSwinStage.__init__  s    mm u
  !%5'#,Q<%&UaZqf6H6HA6M

 !()9sr||\DO  #DO'
s   2B/r   r   r   Nr6  r_   c                    |\  }}t        | j                        D ]  \  }} |||||      }	|	d   } |}
| j                  )|dz   dz  |dz   dz  }}||||f}| j                  |
|      }n||||f}||
|f}|r|	dd  z  }|S )Nr   r   r1   )	enumeraterM  rN  )r[   r   r   r   r6  r=   r>   rQ  layer_modulerF  !hidden_states_before_downsamplingheight_downsampledwidth_downsampledr|   stage_outputss                  r%   r   zSwinStage.forward  s     )(5 	-OA|(8HJ[]mnM)!,M	-
 -:)??&5;aZA4EPQ	VWGW 1!'0BDU V OO,MO_`M!' >&(IK\]]12..Mr$   rG  )r   r   r   rH   r   r   r"   r   r   r   r   r   s   @r%   rI  rI    sb    < */(-||  S/  $;	
 + 
u||	r$   rI  c                        e Zd Z fdZ	 	 	 	 	 ddej
                  deeef   dedz  dedz  dedz  dedz  d	edz  d
ee	z  fdZ
 xZS )SwinEncoderc                    t         |           t        |j                        | _        || _        t        j                  d|j                  t        |j                        d      D cg c]  }|j                          }}t        j                  t        | j                        D cg c]  }t        |t        |j                   d|z  z        |d   d|z  z  |d   d|z  z  f|j                  |   |j"                  |   |t        |j                  d |       t        |j                  d |dz           || j                  dz
  k  rt$        nd        c}      | _        d| _        y c c}w c c}w )Nr   cpu)r   r1   r   )rZ   rg   r   rP  r   r   rN  F)rG   rH   lendepths
num_layersrZ   r   linspacer!  r   itemr   rK  rL  rI  r   rP   r   r   layersgradient_checkpointing)r[   rZ   rL   xdpri_layerr]   s         r%   rH   zSwinEncoder.__init__  sM   fmm,!&63H3H#fmmJ\ej!klAqvvxllmm  %T__5  !F,,q'z9:&/lq'z&BIaLUVX_U_D`%a --0$..w7!#fmmHW&=">V]]S`U\_`U`EaAbc4;dooPQ>Q4Q/X\
 ',#! ms   )E&(B*E+r   r   r   Noutput_hidden_states(output_hidden_states_before_downsamplingr6  return_dictr_   c                    |rdnd }|rdnd }	|rdnd }
|rE|j                   \  }}} |j                  |g|| }|j                  dddd      }||fz  }|	|fz  }	t        | j                        D ]  \  }} |||||      }|d   }|d   }|d   }|d   |d   f}|rP|rN|j                   \  }}} |j                  |g|d   |d   f| }|j                  dddd      }||fz  }|	|fz  }	nI|rG|sE|j                   \  }}} |j                  |g|| }|j                  dddd      }||fz  }|	|fz  }	|s|
|dd  z  }
 |st        d |||
fD              S t        |||
|		      S )
Nr#   r   r   r   r1   r   r4   c              3   &   K   | ]	  }||  y wr   r#   ).0vs     r%   	<genexpr>z&SwinEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   )r   r   r   r   )r6   r7   r8   rS  rb  r"   r   )r[   r   r   r   rg  rh  r6  ri  all_hidden_statesall_reshaped_hidden_statesall_self_attentionsr<   r{   r   reshaped_hidden_staterQ  rT  rF  rU  r|   s                       r%   r   zSwinEncoder.forward  s0    #7BD+?RT"$5b4)6)<)<&J;$6M$6$6z$bDT$bVa$b!$9$A$A!Q1$M!-!11&+@*BB&(5 	9OA|(8HJ[]mnM)!,M0=a0@- -a 0 1" 57H7LM#(P-N-T-T*
A{ )O(I(N(N)"3A"68I!8L!M)OZ)% )>(E(EaAq(Q%!&G%II!*/D.FF*%.V-:-@-@*
A{(:(:(::(fHX(fZe(f%(=(E(EaAq(Q%!m%55!*/D.FF* #}QR'88#9	9< m]4EGZ$[mmm ++*#=	
 	
r$   )FFFFT)r   r   r   rH   r   r   r"   r   r   r   r   r   r   s   @r%   rZ  rZ    s    ,4 */,1@E(-#'<
||<
  S/<
  $;	<

 #Tk<
 37+<
 +<
 D[<
 
"	"<
r$   rZ  c                   h     e Zd ZU eed<   dZdZdZdZdgZ	 e
j                          fd       Z xZS )SwinPreTrainedModelrZ   swinru   )imageTrI  c                    t         |   |       t        |t              rX|j                  t        j                  |j                         |j                   t        j                  |j                         yyt        |t              rNt        j                  |j                         t        j                  |j                  |j                                yy)zInitialize the weightsN)rG   _init_weightsr   rE   rQ   initzeros_rS   r   r   copy_r   r   )r[   moduler]   s     r%   rx  z!SwinPreTrainedModel._init_weights&  s     	f%fn-  ,F--.))5F667 6 12KK;;<JJv55v7\7\7^_ 3r$   )r   r   r   r   r!   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modulesr   no_gradrx  r   r   s   @r%   rt  rt    sD    $O!&*#$U]]_
` 
`r$   rt  c                        e Zd Zd fd	Zd Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  de	d	e	dz  d
e
ez  fd       Z xZS )	SwinModelc                    t         |   |       || _        t        |j                        | _        t        |j                  d| j
                  dz
  z  z        | _        t        ||      | _
        t        || j                  j                        | _        t        j                  | j                  |j                         | _        |rt        j$                  d      nd| _        | j)                          y)a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r1   r   )r\   r  N)rG   rH   rZ   r]  r^  r_  r   rP   num_featuresrE   r^   rZ  rM   encoderr   rT   r  	layernormAdaptiveAvgPool1dpooler	post_init)r[   rZ   add_pooling_layerr\   r]   s       r%   rH   zSwinModel.__init__6  s     	 fmm, 0 0119L3M MN(O"64??+E+EFd&7&7V=R=RS1Bb**1- 	r$   c                 .    | j                   j                  S r   r^   rJ   r   s    r%   get_input_embeddingszSwinModel.get_input_embeddingsK      ///r$   Nru   rv   r   rg  rt   ri  r_   c                 @   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  |||      \  }}	| j                  ||	|||      }
|
d   }| j                  |      }d}| j                  7| j                  |j                  dd            }t        j                  |d      }|s||f|
dd z   }|S t        |||
j                  |
j                  |
j                        S )	z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rv   rt   )r   rg  ri  r   r   r1   )r   r(   r   r   r   )rZ   r   rg  ri  r   r^   r  r  r  r   r   r   r'   r   r   r   )r[   ru   rv   r   rg  rt   ri  kwargsembedding_outputr   encoder_outputssequence_outputpooled_outputr   s                 r%   r   zSwinModel.forwardN  sJ    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY?@@-1__/Tl .= .
** ,,/!5# ' 
 *!,..9;;" KK(A(A!Q(GHM!MM-;M%}58KKFM-')77&11#2#I#I
 	
r$   )TFNNNNFN)r   r   r   rH   r  r   r   r    r   r   r"   r'   r   r   r   s   @r%   r  r  4  s    *0  2637)-,0).#'6
''$.6
 ))D06
  $;	6

 #Tk6
 #'6
 D[6
 
	 6
 6
r$   r  ad  
    Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
       Z xZS )SwinForMaskedImageModelingc                    t         |   |       t        |dd      | _        t	        |j
                  d|j                  dz
  z  z        }t        j                  t        j                  ||j                  dz  |j                  z  d      t        j                  |j                              | _        | j                          y )NFT)r  r\   r1   r   )in_channelsout_channelsr   )rG   rH   r  ru  r   rP   r_  r   
Sequentialr   encoder_strider?   PixelShuffledecoderr  )r[   rZ   r  r]   s      r%   rH   z#SwinForMaskedImageModeling.__init__  s     fdS	6++aF4E4E4I.JJK}}II(v7L7La7ORXReRe7est OOF112	
 	r$   Nru   rv   r   rg  rt   ri  r_   c                    ||n| j                   j                  }| j                  ||||||      }|d   }	|	j                  dd      }	|	j                  \  }
}}t        j                  |dz        x}}|	j                  |
|||      }	| j                  |	      }d}|| j                   j                  | j                   j                  z  }|j                  d||      }|j                  | j                   j                  d      j                  | j                   j                  d      j                  d      j                         }t        j                  j!                  ||d	      }||z  j#                         |j#                         d
z   z  | j                   j$                  z  }|s|f|dd z   }||f|z   S |S t'        |||j(                  |j*                  |j,                        S )a  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, SwinForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swin-base-simmim-window6-192")
        >>> model = SwinForMaskedImageModeling.from_pretrained("microsoft/swin-base-simmim-window6-192")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```N)rv   r   rg  rt   ri  r   r   r1   ra   r4   none)r   gh㈵>)r+   r,   r   r   r   )rZ   ri  ru  r   r6   r   floorrj   r  r   rY   repeat_interleavery   r9   r   rk   l1_lossr   r?   r*   r   r   r   )r[   ru   rv   r   rg  rt   ri  r  r   r  r<   r?   sequence_lengthr=   r>   reconstructed_pixel_valuesmasked_im_lossrc   r   reconstruction_lossr   s                        r%   r   z"SwinForMaskedImageModeling.forward  s   P &1%<k$++BYBY))+/!5%=#  
 "!*)33Aq94C4I4I1
L/OS$899)11*lFTYZ &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN02WQR[@F3A3M^%.YSYY,5!//))#*#A#A
 	
r$   r  )r   r   r   rH   r   r   r    r   r   r"   r*   r   r   r   s   @r%   r  r    s       2637)-,0).#'S
''$.S
 ))D0S
  $;	S

 #TkS
 #'S
 D[S
 
.	.S
 S
r$   r  a  
    Swin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune Swin on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
       Z xZS )SwinForImageClassificationc                 >   t         |   |       |j                  | _        t        |      | _        |j                  dkD  r4t        j                  | j                  j                  |j                        nt        j                         | _	        | j                          y r#  )rG   rH   
num_labelsr  ru  r   r   r  r  
classifierr  )r[   rZ   r]   s     r%   rH   z#SwinForImageClassification.__init__  sx      ++f%	 EKDUDUXYDYBIIdii,,f.?.?@_a_j_j_l 	
 	r$   Nru   labelsr   rg  rt   ri  r_   c                 V   ||n| j                   j                  }| j                  |||||      }|d   }	| j                  |	      }
d}|| j	                  ||
| j                         }|s|
f|dd z   }||f|z   S |S t        ||
|j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   rg  rt   ri  r   r1   )r+   r/   r   r   r   )	rZ   ri  ru  r  loss_functionr.   r   r   r   )r[   ru   r  r   rg  rt   ri  r  r   r  r/   r+   r   s                r%   r   z"SwinForImageClassification.forward  s    " &1%<k$++BYBY))/!5%=#  
  
/%%ffdkkBDY,F)-)9TGf$EvE(!//))#*#A#A
 	
r$   r  )r   r   r   rH   r   r   r    
LongTensorr   r"   r.   r   r   r   s   @r%   r  r    s      26*.)-,0).#',
''$.,
   4',
  $;	,

 #Tk,
 #',
 D[,
 
*	*,
 ,
r$   r  zM
    Swin backbone, to be used with frameworks like DETR and MaskFormer.
    c                        e Zd Zdef fdZd Zee	 	 	 ddej                  de
dz  de
dz  de
dz  d	ef
d
              Z xZS )SwinBackbonerZ   c           	          t         |   |       |j                  gt        t	        |j
                              D cg c]  }t        |j                  d|z  z         c}z   | _        t        |      | _	        t        || j                  j                        | _        i }t        | j                  | j                        D ]  \  }}t!        j"                  |      ||<    t!        j$                  |      | _        | j)                          y c c}w )Nr1   )rG   rH   rP   rL  r]  r^  r   r  rE   r^   rZ  rM   r  zipout_featuresr;  r   rT   
ModuleDicthidden_states_normsr  )r[   rZ   rQ  r  stager?   r]   s         r%   rH   zSwinBackbone.__init__O  s     #--.X]^abhbobo^pXq1rST#f6F6FA6M2N1rr(0"64??+E+EF !#&t'8'8$--#H 	DE<)+l)C&	D#%==1D#E  	 2ss   "Dc                 .    | j                   j                  S r   r  r   s    r%   r  z!SwinBackbone.get_input_embeddings_  r  r$   Nru   rg  r   ri  r_   c           	         ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |      \  }}| j                  |||dddd      }|j                  }	d}
t        | j                  |	      D ]  \  }}|| j                  v s|j                  \  }}}}|j                  dddd      j                         }|j                  |||z  |      } | j                  |   |      }|j                  ||||      }|j                  dddd      j                         }|
|fz  }
 |s|
f}|r||j                  fz  }|S t!        |
|r|j                  nd|j"                  	      S )
a  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("shi-labs/nat-mini-in1k-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swin-tiny-patch4-window7-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 7, 7]
        ```NT)r   rg  rh  r6  ri  r#   r   r1   r   r   )feature_mapsr   r   )rZ   ri  rg  r   r^   r  r   r  stage_namesr  r6   r8   r9   r7   r  r   r   r   )r[   ru   rg  r   ri  r  r  r   r   r   r  r  hidden_stater<   r?   r=   r>   r   s                     r%   r   zSwinBackbone.forwardb  s   J &1%<k$++BYBY$8$D $++JjJj 	 2C1N-TXT_T_TqTq-1__\-J**,,/!%59!  
  66#&t'7'7#G 	0E<))):F:L:L7
L&%+33Aq!Q?JJL+00Ve^\Z>t77>|L+00VULY+33Aq!Q?JJL/	0 "_F#70022M%3G'//T))
 	
r$   )NNN)r   r   r   r   rH   r  r   r	   r   r   r   r   r   r   r   s   @r%   r  r  I  s    z  0   -1)-#'L
llL
 #TkL
  $;	L

 D[L
 
L
 ! L
r$   r  )r  r  r  rt  r  )r   F)?r   collections.abcr   r   dataclassesr   r   r    r   ry  activationsr   backbone_utilsr   r	   modeling_layersr
   modeling_outputsr   modeling_utilsr   utilsr   r   r   r   utils.genericr   configuration_swinr   
get_loggerr   loggerr   r'   r*   r.   rA   rC   r   rE   rI   r   r   r   r   r   r   r   r   r  r
  r  r  rI  rZ  rt  r  r  r  r  __all__r#   r$   r%   <module>r     s   &   !   & ! H 9 . - D D - * 
		H	% 
H H H  
Hk H H& 
HK H H* 
H H H*	Y-RYY Y-x(-")) (-V3ryy 3nU\\ e T V[VbVb  %299 %Z'		 Z'z
RYY 
BII $ryy 	 	w		 wt4* 4nS
")) S
l `/ ` `, P
# P
 P
f 	e
!4 e
e
P <
!4 <
<
~ 
b
="5 b

b
Jr$   