idZddlZddlmZddlmZddlZddlmZddlm Z ddl m Z dd l mZmZdd lmZdd lmZdd lmZdd lmZddlmZmZmZmZddlmZmZddl m!Z!ddl"m#Z#ddl$m%Z%m&Z&m'Z'm(Z(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2e(jfe4Z5ee'dGdde%Z6dejndefdZ8e+dd d!" d^d#ed!ejnd$ejndzd%edzd&ejndzd'ejndzd(ejrdzd)e:d*e:dzde;fd+Z<Gd,d-ejzZ>Gd.d/ejzZ?Gd0d1ejzZ@d2e?iZAGd3d4ejzZBGd5d6ejzZCGd7d8ejzZDGd9d:eZEGd;deZGGd?d@ejzZHGdAdBejzZI d_dCejzdDejndEejndFejnd$ejndzdGeJdHeJfdIZKGdJdKejzZLGdLdMeZMGdNdOejzZNGdPdQejzZOe'dRGdSdTeGZPGdUdVejzZQe'dWGdXdYeGZRe'dZGd[d\eGeZSgd]ZTy)`zPyTorch GIT model.N)Callable) dataclass)nn)initialization)ACT2FN)Cache DynamicCache)PreTrainedConfig)GenerationMixin)create_masks_for_generate)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPastBaseModelOutputWithPoolingCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging torch_int)deprecate_kwarg)merge_with_config_defaults)capture_outputs) GitConfigGitVisionConfigz} Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. ) custom_introceZdZUdZdZej dzed<dZej dzed<dZ e ej dfdzed<dZ e ej dfdzed<y)GitVisionModelOutputz image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The image embeddings obtained by applying the projection layer to the pooler_output. N image_embedslast_hidden_state. hidden_states attentions) __name__ __module__ __qualname____doc__r%torch FloatTensor__annotations__r&r'tupler(u/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/transformers/models/git/modeling_git.pyr$r$8sr .2L%##d*126u((4/6:>M5**C/047>7;Je'',-4;r2r$ group_idsreturnc Pdtdtdtdtdtf fd }|S)au This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths, not start and end indices. Args: group_ids (`torch.Tensor`): A tensor of shape `(bs, len)` assigning each token to a vision group. Tokens with the same group come from the same input image. Text is denoted by `-1`. batch_idxhead_idxq_idxkv_idxr5c jd}|j|dz }|j|dz } ||f} ||f}tj||k|d}tj||k|d}||k(|dk\zS)Nr)maxr)shapeclampr-where) r7r8r9r: seq_length q_idx_clampedkv_idx_clampedq_groupkv_groupr4s r3 inner_maskz0token_type_ids_mask_function..inner_maskVs__R(  Q 7 *q.9I}45Y67++ej0'2>;;v 2HbA8#1 55r2)intbool)r4rFs` r3token_type_ids_mask_functionrILs3 6c 6S 6 6c 6d 6 r2 input_embedsz5.6.0 inputs_embeds)versionnew_nameconfigattention_maskpast_key_values position_idstoken_type_ids pixel_values is_trainingis_first_iterationc  |r | td|j||||d} ||n|duxs|j xs|du}||r|dk(j|j} t j j| ddddddf} | | z} tj| jd dz }tj| |d}t|| d <td i| S) a Overwrites the base `create_masks_for_generate` with `token_type_ids` masking to create the causal mask mapping for all kinds of forward passes. Gemma3 uses a bidirectional mask for images. Uses `pixel_values` as an optional input to disambiguate edge cases. Nz;`token_type_ids` is required as a model input when training)rNrKrOrPrQr)rrr)valuer<dimor_mask_functionr1) ValueErrorget_text_configis_initializedtodevicer functionalpadr-cumsumrGr@rIr )rNrKrOrPrQrRrSrTrUkwargs mask_kwargsis_imageis_previous_imagenew_image_startr4s r3create_causal_mask_mappingrhgs(~-VWW((*&(*$ K  ) %g_-K-K)Kg|cgOg !&8 #a'++M,@,@AMM--ha-HCRCP"&7%77LL!4!4!6A>B KK)R8 *Fy*Q &' $ 3{ 33r2c eZdZdZfdZ d dej dzdej dzdejdzdedejf d Z xZ S) GitEmbeddingsz;Construct the embeddings from word and position embeddings.ct|tj|j|j |j |_tj|j|j |_ tj|j |j|_ tj|j|_|jdt!j"|jj%ddy)N) padding_idxepsrQrr<F persistent)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferr-arangeexpandselfrN __class__s r3rszGitEmbeddings.__init__s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  r2N input_idsrQrKpast_key_values_lengthr5c,||j}n|jdd}|d}||jdd|||zf}||j|}n|}|j|}||z }|j |}|j |}|S)Nr<r)sizerQrxrzr{r) rrrQrKr input_shaperA embeddingsrzs r3forwardzGitEmbeddings.forwards  #..*K',,.s3K ^  ,,Q0FVlIl0l-lmL  --i8J&J"66|D)) ^^J/ \\*- r2)NNNr) r)r*r+r,rsr- LongTensorr.rGTensorr __classcell__rs@r3rjrjssE  .20426&' ##d*&&-((4/  !$   r2rjc eZdZd fd Z d dej dej dzdedzdee de ej f dZ xZ S) GitSelfAttentionNct||j|jzdk7r2t |ds&t d|jd|jd||_|-tjd|jjd|j|_t|j|jz |_ |j|jz|_ t|jj|jj z dzd z|_|j$|xj"|j$zc_t'j(|j|j|_t'j(|j|j|_t'j(|j|j|_t'j0|j2|_y) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.r)rrrsrvnum_attention_headshasattrr[ layer_idxlogger warning_oncerr)rGattention_head_size all_head_size vision_config image_size patch_sizeimage_patch_tokensnum_image_with_embeddingrLinearquerykeyrWr}attention_probs_dropout_probrrrNrrs r3rszGitSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8 #      !8!8 9:,,  $*#=#= #&v'9'9Frrview transposerrWupdaterr-matmulmathsqrtrr`softmaxrpermute contiguousrr)rr'rOrPrcr hidden_shape query_layer key_layer value_layerattention_scoresattention_probs context_layernew_context_layer_shapes r3rzGitSelfAttention.forwards$))#2. CCbC$*B*BC jj/44\BLLQPQR HH]+00>HHAN jj/44\BLLQPQR  &%4%;%;I{TXTbTb%c "I{!<< Y5H5HR5PQ+dii8P8P.QQ  %/.@ --//0@b/I,,7 _kB %--aAq9DDF "/"4"4"6s";t?Q?Q>S"S%**+BC o--r2NNN r)r*r+rsr-rr.r rrr0rrrs@r3rrshG>48(, %.||%.))D0%. %. +, %. u||  %.r2rcneZdZfdZdej dej dej fdZxZS) GitSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNrm) rrrsrrrvdenser{r|r}r~rrs r3rszGitSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r2r' input_tensorr5cr|j|}|j|}|j||z}|Srrrr{rr'rs r3rzGitSelfOutput.forward7 =1  ]3 }|'CD r2r)r*r+rsr-rrrrs@r3rr1> U\\RWR^R^r2reagerc eZdZd fd Z d dej dej dzdedzdee de ej f dZ xZ S) GitAttentionNct|t|j|||_t ||_y)Nr)rrrsGIT_SELF_ATTENTION_CLASSES_attn_implementationrroutputrs r3rszGitAttention.__init__$s4 .v/J/JKF^gh #F+ r2r'rOrPrcr5c Z|j|||fi|\}}|j||}|Sr)rr)rr'rOrPrc attn_output_attention_outputs r3rzGitAttention.forward)sD#       Q  ;;{MBr2rrrrs@r3rr#sg,48(,  || ))D0   +,  u||   r2rcVeZdZfdZdej dej fdZxZS)GitIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) rrrsrrrvintermediate_sizer isinstance hidden_actstrrintermediate_act_fnrs r3rszGitIntermediate.__init__<s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r2r'r5cJ|j|}|j|}|Sr)rrrr's r3rzGitIntermediate.forwardDs& =1 00? r2rrs@r3rr;s#9U\\ellr2rcneZdZfdZdej dej dej fdZxZS) GitOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) rrrsrrrrvrr{r|r}r~rrs r3rszGitOutput.__init__Ls` YYv779K9KL f&8&8f>S>STzz&"<"<= r2r'rr5cr|j|}|j|}|j||z}|Srrrs r3rzGitOutput.forwardRrr2rrs@r3rrKrr2rc eZdZd fd Z d dej dej dzdedzdee de ej f dZ d Z xZ S) GitLayerNct||j|_d|_t |||_t ||_t||_ y)Nrr) rrrschunk_size_feed_forward seq_len_dimr attentionr intermediaterrrs r3rszGitLayer.__init__ZsK '-'E'E$%f B+F3' r2r'rOrPrcr5c |j||fd|i|}t|j|j|j|}|S)NrP)rrfeed_forward_chunkrr)rr'rOrPrcr layer_outputs r3rzGitLayer.forwardbs_*4>>   ,   1  # #T%A%A4CSCSUe r2cL|j|}|j||}|Sr)rr)rrintermediate_outputrs r3rzGitLayer.feed_forward_chunkus,"//0@A{{#68HI r2rr)r)r*r+rsr-rr.r rrr0rrrrs@r3rrYsl(48(, ||))D0  +,  u||  &r2rc eZdZfdZ d dej dej dzdedzdedzde e de f d Z xZ S) GitEncoderc t|||_tjt |j Dcgc]}t||c}|_d|_ ycc}wNF) rrrsrNr ModuleListrangenum_hidden_layersrlayergradient_checkpointing)rrNirs r3rszGitEncoder.__init__|sP  ]]vG_G_A`#aAHVQ$7#ab &+#$bsA$Nr'rOrP use_cachercr5c T|jD] }||||fi|}t||S)Nr&rP)rr)rr'rOrPrrc layer_modules r3rzGitEncoder.forwardsI!JJ L( M '++  r2)NNN)r)r*r+rsr-rr.r rHrrrrrrs@r3rr{so,48(,!%  || ))D0   $;  +,   ! r2rcReZdZUeed<dZdZdZejdZ y)GitPreTrainedModelrNgit)imagetextTct|tr tj|jd|j j tj|jj|j j tj|jj|j j tj|jtj|jjdjdt|t j"rctj|jd|j j |j$ tj&|j$yyt|t j(rtj|jd|j j |j*Et-|jdds-tj&|j|j*yyyt|t j.r?tj&|j$tj0|jyt|t2rZtj|jtj|jjdjdyy) zInitialize the weights)meanstd)rr<roN_is_hf_initializedF)rGitVisionEmbeddingsinitnormal_class_embeddingrNinitializer_rangepatch_embeddingweightposition_embeddingcopy_rQr-rr>rrrbiaszeros_rtrlgetattrr{ones_rj)rmodules r3 _init_weightsz GitPreTrainedModel._init_weightss f1 2 LL//ct{{?\?\ ] LL//66DKK7j-  - KK $ JJv}} %  . JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h/r2N) r)r*r+r r/base_model_prefixinput_modalitiessupports_gradient_checkpointingr-no_gradrr1r2r3rrs6 (&*#U]]_iir2rceZdZdeffd Zdej dededej fdZd dejdej fd Z xZ S) rrNct|||_|j|_|j |_|j |_tjtj|j|_ tj|j|j|j |j d|_|j |j zdz|_|jdz|_tj"|j |j|_|j'dtj(|j j+ddy)NF) in_channels out_channels kernel_sizestriderrrrQrorp)rrrsrNrv embed_dimrrr Parameterr-randnr Conv2d num_channelsr  num_patches num_positionsrtrrrrrs r3rszGitVisionEmbeddings.__init__s   ++ ++ ++!||EKK,GH!yy++??  !OOt>1D!--1"$,,t/A/A4>>"R ^U\\$:L:L-M-T-TU\-]jopr2rheightwidthr5c|jddz }|jjjd}|jddz }tj j s%||k(r ||k(r|j|jS|ddddf}|ddddf}|jd} ||jz} ||jz} t|dz} |jd| | | }|jdddd}tjj|| | fdd }|jddddjdd| }t j ||fd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rrNr<g?rrbicubicF)rmode align_cornersrX)r>rr unsqueezer-jit is_tracingrQrrreshaperrr` interpolatercat) rrr(r)r&rr'class_pos_embedpatch_pos_embedrY new_height new_widthsqrt_num_positionss r3interpolate_pos_encodingz,GitVisionEmbeddings.interpolate_pos_encodingsv!&&q)A- !44;;EEaH*003a7 yy##%+*F6UZ?**4+<+<= =,QU3,QU3r"t. T__, &}c'9:)11!5GI[]`a)11!Q1=--33 i( 4 *11!Q1=BB1b#Nyy/?;CCr2rSc `|j\}}}}|sJ||jk7s||jk7r,td|d|d|jd|jd |jjj }|j|j |}|jdjdd}|jj|dd} tj| |gd } |r| |j| ||z} | S| |j|jz} | S) NzInput image size (*z) doesn't match model ().dtyperrr<rX)r>rr[r rr>r^flattenrr rr-r3r9rrQ) rrSr9 batch_sizerr(r) target_dtype patch_embeds class_embedsrs r3rzGitVisionEmbeddings.forwards6'3'9'9$ Avu'Vt-F%SWSbSbJb$VHAeW4KDOOK\\]^b^m^m]nnpq ++2288 ++LOO,O,OP #++A.88A> ++22:q"E YY l;C ##d&C&CJPVX]&^^J$d&=&=d>O>O&PPJr2)F) r)r*r+r!rsr-rrGr9r.rrrs@r3rrsdqq,'D5<<'D'DUX'D]b]i]i'DRE$5$5Z_ZfZfr2rcVeZdZfdZdej dej fdZxZS) GitVisionMLPct|||_t|j|_t j|j|j|_ t j|j|j|_ yr) rrrsrNrr activation_fnrrrvrfc1fc2rs r3rszGitVisionMLP.__init__ sd  #F$5$5699V//1I1IJ99V55v7I7IJr2r'r5cl|j|}|j|}|j|}|Sr)rHrGrIrs r3rzGitVisionMLP.forwards4/ **=9 / r2rrs@r3rErE s$KU\\ellr2rErrrrWscalingrc tj||jdd|z}|||z}tjj |dtj j|j}tjj|||j}tj||} | jddj} | |fS)Nr<r)rYr>)ptrainingrr) r-rrrr`rfloat32r^r>rrNr) rrrrWrOrKrrc attn_weightsrs r3eager_attention_forwardrQs<<s}}R'<=GL!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K  $$r2c eZdZdZfdZ d dej dej dzdeede ej ej dzffdZ xZ S) GitVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperct|||_|j|_|j |_|j|j z|_|j|j z|jk7r&td|jd|j d|jdz|_ |j|_ d|_ tj|j|j|_tj|j|j|_tj|j|j|_tj|j|j|_y)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: r<gF)rrrsrNrvr!r num_headshead_dimr[scaleattention_dropoutr is_causalrrk_projv_projq_projout_projrs r3rszGitVisionAttention.__init__5s  ++33$..8 ==4>> )T^^ ;MdnnM]^NN#2' ]]D( // ii? ii? ii?  $..$..A r2Nr'rOrcr5c |jdd}g|d|j}|j|j|j dd}|j |j|j dd}|j |j|j dd}tj|jjt} | |||||f|j|j|jsdn |jd|\} } | j g|dj#} |j%| } | | fS)z#Input shape: Batch x Time x ChannelNr<rrr)rYrKr)r>rVr\rrrZr[r get_interfacerNrrQrYrWrNrr1rr]) rr'rOrcrrquerieskeysvaluesattention_interfacerrPs r3rzGitVisionAttention.forwardIsM$))#2. 88b8$--8 ++m,11,?II!QO{{=)..|<FFq!L]+00>HHAN(?(M(M KK , ,.E) %8      % nnJJ#}}C$,, %  % ! \*k));;;;FFH mmK0 L((r2r) r)r*r+r,rsr-rrrr0rrrs@r3rSrS2sfGB./3)||) t+)+, ) u||U\\D00 1 )r2rSc eZdZdeffd Zdej dej deede ejej dzffdZ xZ S) GitVisionEncoderLayerrNcDt||j|_t ||_t j|j|j|_ t||_ t j|j|j|_ yr) rrrsrvr!rS self_attnrr{r| layer_norm1rEmlp layer_norm2rs r3rszGitVisionEncoderLayer.__init__msm +++F3<<F> ')   q !=0  ((7 /  =0 r2) r)r*r+r!rsr-rrrr0r.rrrs@r3rerelscSS|| +,  u  %,,"55 6 r2recfeZdZdZdeffd Z d dejdzdee de e zfdZ xZ S) GitVisionEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`GitVisionEncoderLayer`]. Args: config: GitVisionConfig rNct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) rrrsrNrrrrrelayersr)rrNrrs r3rszGitVisionEncoder.__init__sP  mmERXRjRjLk$lq%:6%B$lm &+#%msA#NrOrcr5c T|}|jD] }|||fi|}t|S)a8 Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) r&)rpr)rrKrOrcr' encoder_layers r3rzGitVisionEncoder.forwardsH*& ![[ M)M +  r2r)r)r*r+r,r!rsr-rrrr0rrrrs@r3rnrnsP,,/3  t+ +,    r2rnc reZdZdeffd Ze d dejdzdedzde e de fdZ xZ S) GitVisionTransformerrNc t|||_|j}t ||_t j||j|_ t||_ t j||j|_ yr) rrrsrNrvrrrr{r| pre_layrnormrnencoderpost_layernorm)rrNr!rs r3rszGitVisionTransformer.__init__sj  && -f5LL8M8MN'/  ll9&:O:OPr2NrSr9rcr5c | td|j||}|j|}|jdd|i|}|j}|j |}t |S)Nz You have to specify pixel_valuesr9rKrrr1)r[rrwrxr&ryr)rrSr9rcr'encoder_outputsr&s r3rzGitVisionTransformer.forwards  ?@ @ Ogh ))-8 &$,, '   ,== //0AB/  r2r)r)r*r+r!rsrr-r.rHrrrrrrs@r3rurushQQ2605 ''$. #'+ +,     r2ruzY The vision model from CLIP, used in GIT, without any head or projection on top. c eZdZUeed<dZdZeedZ deffd Z de jfdZ eed e ddej$d zd ed eedeezfd ZxZS)GitVisionModelrNrS)rr'r(cdt||t||_|j yr)rrrsru vision_model post_initrs r3rszGitVisionModel.__init__s'  08 r2r5cB|jjjSr)rrr rs r3get_input_embeddingsz#GitVisionModel.get_input_embeddingss  ++;;;r2F)tie_last_hidden_statesNr9rcc ,|jd||d|S)a Examples: ```python >>> from PIL import Image >>> import httpx >>> from io import BytesIO >>> from transformers import AutoProcessor, GitVisionModel >>> processor = AutoProcessor.from_pretrained("microsoft/git-base") >>> model = GitVisionModel.from_pretrained("microsoft/git-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state ```)rSr9r1)r)rrSr9rcs r3rzGitVisionModel.forwards.<!t   %%=   r2r)r)r*r+r!r/main_input_namerrerS_can_record_outputsrsrModulerrrrr-r.rHrrr0rrrrs@r3r~r~s $O!.(  z$GitModel.__init__..Fs;; U[[Av/C/C/O/OPQ;sAA )rrrsrNrjrr~r image_encoderrrxrrrr ParameterListrimg_temporal_embeddingrrs `r3rszGitModel.__init__;s   '/+F,@,@A!&) !.v!6  * * 6*,*:*:;v>>?;+D ' r2c.|jjSrrrxrs r3rzGitModel.get_input_embeddingsNs...r2c&||j_yrr)rrWs r3set_input_embeddingszGitModel.set_input_embeddingsQs*/'r2NrrOrQrSrKrPrr9rcr5c N|du|duz r td|r|t|j}d} |0t|ts|j n|j } |||j ddk(r|| z}|j|||| } tj| tjd} ||jd k(r|j|| j} n|jd k(rg} t|j dD]O}|j|dd|ddddf| j}||j|z }| j!|Qtj"| d } n td |j%| }|j'| j)d|j)dzdd}tj"|| fd } tj*|tjd}tj"|| gd } |tj"tj*||gd }n{|y|j ddk(rgtj,|j d| |j dz dzf|j.|j0}tj"||gd }t3|j| ||d| |}| }|j4|f|||d| }t7|j|j8S)a Examples: ```python >>> from transformers import AutoProcessor, AutoModel >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> processor = AutoProcessor.from_pretrained("microsoft/git-base") >>> model = AutoModel.from_pretrained("microsoft/git-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> text = "this is an image of two cats" >>> inputs = processor(images=image, text=text, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state ```Nz:You must specify exactly one of input_ids or inputs_embeds)rNrr)rrQrKrr=).rr{rXz#pixel_values must be of rank 4 or 5r<)r>r_)rOrPrr)r[r rNrr get_seq_lengthr>rr- zeros_likerGndimrr&rrappendr3rrepeatr ones_likeonesr>r_rhrxrrP)rrrOrQrSrKrPrr9rcrembedding_outputrRvisual_features frame_idxvisual_features_frameprojected_visual_featuresimage_token_type_idsextended_attention_mask causal_maskr'r|s r3rzGitModel.forwardTswL  -t"; <YZ Z 0*$++>O"#  &"/59 ..0$335 #  O$?IOOTUDVZ[D['*@@L??%'#9 + ))*:%))LVT  #  A%"&"4"4 ;S#5### ""a'"$!&|'9'9!',>$Q 1a%78Sk-?-''**T-H-H-SS)#**+@A B#())O"C!!FGG(,(>(>(O %)B(H(H %%a(,E,J,J1,MMqRS) % %yy*CEU)V\]^ #(??3LTYT]T]#^_e#f "YY(CWCWXYCZ*Z]^*^_$**%,,' # #YY(?'PVXYN1 KK       ) 3?4<< 4 &+ 4  4 '-??+;;  r2)NNNNNNNF)r)r*r+rrrrsrrrrrr-rr rHrrr0rrrrs@r3rr0s"& &/0 *..2,0,0-1(,!%).F <<$&F  t+F llT) F llT) F ||d* F F $;F #'F +,F  u|| 9 9F  F r2rz` GIT Model with a `language modeling` head on top for autoregressive language modeling. ceZdZddiZfdZdZdZeee dde jdzde jdzd e jdzd e jdzd e jdzd e jdzd e dzde dzde dee jzdeedee jezfdZ dfd ZxZS)GitForCausalLMz output.weightz%git.embeddings.word_embeddings.weightct||t||_t j |j |j|_|jyr) rrrsrrrrrvrurrrs r3rszGitForCausalLM.__init__sF  F#ii 2 2F4E4EF  r2c|jSrrrs r3get_output_embeddingsz$GitForCausalLM.get_output_embeddingss {{r2c||_yrr)rnew_embeddingss r3set_output_embeddingsz$GitForCausalLM.set_output_embeddingss $ r2NrrOrQrSrKlabelsrPrr9logits_to_keeprcr5c |d}|j|f||||||| d| } | j} t| tr t | dn| }|j | dd|ddf}d}||jj jdjjj}|dd|dddfj}|ddddfj}|j|jd|jj|jdfd|jji| }t!||| j"| j$| j&S) a0 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` Examples: Image captioning example: ```python >>> from transformers import AutoProcessor, AutoModelForCausalLM >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-coco") >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50) >>> generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] >>> print(generated_caption) two cats sleeping on a pink blanket next to remotes. ``` Visual question answering (VQA) example: ```python >>> from transformers import AutoProcessor, AutoModelForCausalLM >>> from huggingface_hub import hf_hub_download >>> from PIL import Image >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-textvqa") >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-textvqa") >>> file_path = hf_hub_download(repo_id="nielsr/textvqa-sample", filename="bus.png", repo_type="dataset") >>> image = Image.open(file_path).convert("RGB") >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values >>> question = "what does the front of the bus say at the top?" >>> input_ids = processor(text=question, add_special_tokens=False).input_ids >>> input_ids = [processor.tokenizer.cls_token_id] + input_ids >>> input_ids = torch.tensor(input_ids).unsqueeze(0) >>> generated_ids = model.generate(pixel_values=pixel_values, input_ids=input_ids, max_length=50) >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True)) ['what does the front of the bus say at the top? special'] ``` Video captioning example: ```python >>> import av >>> import numpy as np >>> from PIL import Image >>> from huggingface_hub import hf_hub_download >>> from transformers import AutoProcessor, AutoModelForCausalLM >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-vatex") >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-vatex") >>> # set seed for reproducibility >>> np.random.seed(45) >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len): ... ''' ... Sample a given number of frame indices from the video. ... Args: ... clip_len (`int`): Total number of frames to sample. ... frame_sample_rate (`int`): Sample every n-th frame. ... seg_len (`int`): Maximum allowed index of sample's last frame. ... Returns: ... indices (`list[int]`): List of sampled frame indices ... ''' ... converted_len = int(clip_len * frame_sample_rate) ... end_idx = np.random.randint(converted_len, seg_len) ... start_idx = end_idx - converted_len ... indices = np.linspace(start_idx, end_idx, num=clip_len) ... indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64) ... return indices >>> # load video >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample frames >>> num_frames = model.config.num_image_with_embedding >>> indices = sample_frame_indices( ... clip_len=num_frames, frame_sample_rate=4, seg_len=container.streams.video[0].frames ... ) >>> frames = read_video_pyav(container, indices) >>> pixel_values = processor(images=list(frames), return_tensors="pt").pixel_values >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50) >>> print("Generated caption:", processor.batch_decode(generated_ids, skip_special_tokens=True)) Generated caption: ['a woman is sitting at a table and she is talking about the food she is holding.'] ``` NF)rOrQrSrKrPrr9rr<rru)losslogitsrPr'r()rr&rrGslicerrxrrrrr loss_functionrrNrurrPr'r()rrrOrQrSrKrrPrr9rrcoutputsr' slice_indicesrrnum_image_tokensshifted_logitss r3rzGitForCausalLM.forwards~l  I+3488  , )%%'+%= ,  ,  11 8B>SV8W~ot4]k ]1mQ+>?@  #xx//55a8BBGGZZ #A'7':A$=>IIKNAqrE]--/F%4%%##B (>(>? B ;;11 D&#33!//))   r2c Dt ||f||||d|}|s|s||d<|S)N)rPrOrrUrS)rrprepare_inputs_for_generation) rrrPrSrOrrUrc model_inputsrs r3rz,GitForCausalLM.prepare_inputs_for_generationsIw<  +)1     Y+7L (r2) NNNNNNNNFr)NNNNF)r)r*r+_tied_weights_keysrsrrrrrr-rr rHrGrrr0rrrrrs@r3rrs` *+RS% *..2,0,0-1&*(,!%).-.z <<$&z  t+z llT) z llT) z ||d* z  t#z z $;z #'z ell*z +,z  u|| 5 5z  z ~ r2r)rrrr~)NNFN)r)Ur,rcollections.abcr dataclassesrr-rrr  activationsr cache_utilsr r configuration_utilsr generationr masking_utilsr modeling_layersrmodeling_outputsrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrrrrutils.deprecationr utils.genericrutils.output_capturingrconfiguration_gitr r! get_loggerr)rr$rrIr.rHdictrhrrjrrrrrrrrrrrEfloatrQrSrernrur~rrr__all__r1r2r3rs $! &!.3)69 G&61759   H %  <; <   <ELLX6?K+/-1&*14 14<<14LL4'14T\ 14 ,,% 14 LL4' 14##d*1414t 14 14L14h*BII*ZB.ryyB.L BII    299 0 bii   )D  :iii>P"))Pf 299 .% II% <<% % << % LL4' %  %%.6)6)t6D. ryy. b$ 299$ N 5 '5  5 p 2BII 2 h !h  h V i'i  iX Qr2