iKFUdZddlmZddlZddlZddlmZerddlmZddl m Z ddl m Z ddl mZmZddlZdd lmZd d lmZd gZdad ed<dad ed<e GddZe d*dZ d+ d,dZd-dZd.dZ d/dZ d0dZ d1dZ edZ!d2dZ"d3dZ# d4 d5dZ$ d6 d7dZ% d8ddddddd d9d Z&ddddddd d:d!Z'dddd" d;d#Z( ddd$ d?d&Z*dd$ d@d'Z+ejXd(e)y)Az PROTOTYPE! Flash Attention 3 implementation. For fp8: only supports forward pass right now. For fp16/bf16: supports forward and backward pass. ) annotationsN) TYPE_CHECKING)Callable) dataclass)cache) TypeVarTupleUnpack)Library) _registryregister_flash_attention_fa3zCallable | None _FA3_CUDA_FWD _FA3_CUDA_BWDc eZdZUded<ddZy) _FA3HandlezLibrary | NonelibrarycPd|_tjjdy)NF)rtorch_C_set_sdp_use_fa3)selfs h/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/torch/nn/attention/_fa3.pyremovez_FA3Handle.remove*s  !!%(N)returnNone)__name__ __module__ __qualname____annotations__rrrrr&s )rrcJtjj|\}}|SN)rcudaget_device_capability)devicemajor_s r_get_device_majorr)0szz//7HE1 Lrc|t|tjjdt t S)z Register FA3 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA3 implementation. T)_fa3_import_modulerrrr_fa3_register_kernels module_paths rr r 6s/{# HHd# +- ..rctj|ttjdst d|dttjj dst d|dttjj dst d|dtjj jatjj ja y)N flash_attn_3zModule 'z' does not expose FA3 kernelsfwdz%' does not expose FA3 forward kernelsbwdz&' does not expose FA3 backward kernels) importlib import_modulehasattrrops RuntimeErrorr0r1rr2rr-s rr+r+Gs K( 599n -Xk]2OPQQ 599))5 1{m#H I   599))5 1{m#I J  II**..MII**..Mrc4tddd}|jdtd|jdtd|jdtd|jdt d|jdt d|jd td|S) NatenIMPLCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantized_flash_attention_forward#_scaled_dot_product_flash_attention_flash_attention_backward,_scaled_dot_product_flash_attention_backward)r impl!_fa3_flash_attention_forward_impl4_fa3_scaled_dot_product_flash_attention_forward_impl)_fa3_flash_attention_forward_impl_default<_fa3_scaled_dot_product_flash_attention_forward_impl_default"_fa3_flash_attention_backward_impl5_fa3_scaled_dot_product_flash_attention_backward_impl)libs rr,r,Xs &&& )CHH,.OQWHH7< HH"$MvHH-D HH (*LfUHH6= Jrc|dk7rytd|Dsyt|Dchc]}|jc}dk7ry|jtj k(r |||t jdt||jdk7ry ||jd k7ry tjjsy t|jd k7ryycc}w)Nzdropout_p must be 0c34K|]}|jywr#)is_cuda.0ts r z,_fa3_common_support_error..s*Qqyy*szinputs must be CUDA tensorsr inputs must share devicezWhen using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.zdense query must be 4Dzragged query must be 3DzCUDA not available z#FA3 requires compute capability 9.0) alllenr&dtyper float8_e4m3fnwarningswarn UserWarningdimr$ is_availabler))querytensors dropout_p cum_seq_q q_descale k_descale v_descalerNs r_fa3_common_support_errorrdtsC$ *'* *, g &AHH &'1,) {{e)))Y.)2C  +   UYY[A-'!1( :: " " $#&!+4 ) 'sC#c  |ry|y|+|jtjk7ry|jsytjtj tj f t fd|||hDsd St|||hD chc]} | jc} dk7ryt||||f|||| | } | | d k(ry | Sycc} w) Nzreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc3:K|]}|jvywr#rVrMrNsupported_dtypess rrOz-_fa3_forward_support_error..sHqqww**Hinputs must be one of r #all inputs must have the same dtyperPz(query, key, value must be on same device) rVrint32rKrWfloat16bfloat16rTrUrd)r]keyvaluer_return_debug_mask alibi_slopes seqused_kr`rarbrcrNerrorris @r_fa3_forward_support_errorrvs0+ ??ekk ),  +++U]]ENNK HUC4GH H'(8'9:: eS%0 1AGG 12a74 % U E  . .=  2s C c  |jtjk(r y|jtjk7rytjtj f t fd|||||hDsd St|||||hD chc]} | jc} dk7ryt|||||||f||ddd} | | Sycc} w)NzHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c3:K|]}|jvywr#rgrhs rrOz._fa3_backward_support_error..sWqqww**Wrjrkr rl) rVrrWfloat32rnrorTrUrd) grad_outr]rprqout logsumexpr_r`window_size_leftwindow_size_rightrNruris @r_fa3_backward_support_errorrs {{e))) V %--'0 u~~6 WXuc5RU4VW W'(8'9:: hsE3? @AGG @AQF4 % 5#uc95    E   As CTsc&td|DS)Nc3@K|]}|jddyw)r N) transposerLs rrOz#_transpose_dense..s4qQ"4s)tuple)r^s r_transpose_densers 4G4 44rcR|$|jddk7r|jS|S)z2Ensure tensor is contiguous in the last dimension.r )stride contiguous)xs r_maybe_contiguousrs&]qxx|q/@1<<>GaGrct tdt|}t|}|jtj k(r8|j ddk7r$|j ddk7r|jn t|}t|}t|}t| } tg|||ddd| ||dd| ||dddddd| ||||| | nd| | ndddddt jrdnddtjjxsd\} }}}| |jfS) zF Run the FA3 forward pass by calling the C++ kernel directly. NFA3 not registeredrr rrIT) rr7rrVrrWrr$are_deterministic_algorithms_enabledr_get_sm_carveout_experimental)r]rprqcu_seq_qcu_seq_kmax_qmax_kscale is_causalr}r~rtr{rarbrcqkv cu_seqlens_q cu_seqlens_k softmax_lse out_accumsoftmax_lse_accums r_fa3_run_forwardrsM*/00% A#A ;;%-- - LL  ! LL  ! u % %X.L$X.L!),I5B#6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 #6 !#6" ##6$ %#6& '#6( )#6* +#6, -#6. /#60 1#62 3#64-8b5#66/:7#68 9#6: ;#6< =#6> ?#6@ 7 7 9qA#6B C#6D ..05AE#62Ci!2H  &&( ((rcRt tdt|}|jddk7r|j n|}|jddk7r|j n|}|jddk7r|j n|}t|}t|}t j |}t j |}t j |}t|||||||||||dd|| | | | | d|t jjxsd|||fS)Nrrr rIr) rr7rrrr empty_likerr)rzr]rprqr{r|rr max_seqlen_q max_seqlen_krrr}r~ deterministicdoutrrrolsedqdkdvs r_fa3_run_backwardrCs/"/00 X &D#ll2.!3AJJrNa/SA#ll2.!3A#A I &C   ! B   ! B   ! B              ..05A-0 r2:rrrr}r~rtrsr{c t||||| |||| | | }|td|t|||||||| |||||| | | \}}tjdtj |j }tjdtj |j }tjd|j|j }|||||fS)Nz)FA3 flash_attention forward unsupported: )r)rVr&r!r) rvr7rrzerosuint64r&emptyrV)r]rprqr` cum_seq_krrr_rrrrarbrcrr}r~rtrsr{rur rng_state philox_offset debug_masks rrArA~s, '    E FugNOO       !HC$ D U\\JIKK%,,u||LMQekk%,,GJ Y z 99rc >t|||||||||| ddd| | | | ||S)Nr)rA)r]rprqr`rrrr_rrrrr}r~rtrsr{s rrCrCsJ& -        )+! ' r)rr}r~ct||||||| ||| }|td|tj}t |||||||||| || ||nd||nd|\}}}|||fS)z0FA3 implementation of _flash_attention_backward.z*FA3 flash_attention backward unsupported: r)rr7rrr)rzr]rprqr{r|r`rrrr_rrunusedrr}r~rurrrrs rrErEs* (     E GwOPP>>@M"       ,8b.:JBB" r2:rrc t|||||ddd||| } | td| t|||\} } } |jtj k(rtj n |j}t j||}|jdd}| jd}| jd}t| | | dd|||||| ||||\}}}}}|jd}|jd}||dd|||||f S)NzFA3 SDPA forward unsupported: rgr r)rr{rarbrc) rvr7rrVrrWrorrsizerA)r]rprqrarbrcr_rrrrrurrr out_dtypeout_bhsdout_bshd max_q_flash max_k_flashr(rrrrrrs rrBrB sA '       E ;E7CDDuc51GAq! #(++1D1D"D%++IY7H!!!Q'H&&)K&&)K3T      40AsI}j" JJqME HHQKE       rc ,t|||ddd|||| S)Nr)rB)r]rprqr_rrrrs rrDrDgs0 @        rct||||||| ddd }|td|t|||||\}}}}}t||||||dd|| | | | | |\}}}t|||\}}}|||fS)zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: r)rr7rrE)rzr]rprqr{r|r`rrrr_r philox_seedrrru grad_out_tq_tk_tv_tout_trrrdq_outdk_outdv_outs rrFrFs& (%eS)YdD E rs[# (!2 !#  "& %!% % )) ) .///"/"8" " %""# " # " # "#""J( ( ( ( (  ( & (#(#(#(#(#((V## # #  #  #  ####!#"##L$5H$ $%)%)%)!J) J) J) J)" J) " J)  J) J) J)J)!J)"J)#J) J)#J)#J) #!J)"'#J)x 88 8 8  8  8  8"8"888 88888 5!8L&*%)%)::%)(,#):: :: :: ::# :: # ::  :: ::::::::#::#::#:: :: !::"#::$#%::&&'::( )::R%)(,##' ' ' '# ' # '  ' '''' '''#' &!'" #'t#'$(%88 8 8  8  8  8#8#8 8 8888 8 !8"!#8$"%8~&*%)%)#DD D D D# D # D # DDDD DV#          P!2"2" 2" 2"  2"  2"  2"#2"#2" 2" 2"2"2"2" 2" !2"j( '';WXr