o m9:jR @sUdZddlmZddlZddlZddlmZerddlmZddl m Z ddl m Z ddl mZmZddlZdd lmZd d lmZd gZdad ed<dad ed<e GdddZe dyddZ dzd{dd Zd|ddZd}d d!Zd~d.d/Zdd6d7Zdd>d?Z ed@Z!ddCdDZ"ddFdGZ#      dddRdSZ$ TdddYdZZ%   ddd[d[ddddd\dd] dd`daZ&dd[d[dddddbddcddZ'dd[d[ddddddeddfdgZ(ddddhddkdlZ)    m T TdddnddodpZ* m T TdddnddqdrZ+ddnddudvZ,ej-dwedxdS)z PROTOTYPE! Flash Attention 3 implementation. For fp8: only supports forward pass right now. For fp16/bf16: supports forward and backward pass. ) annotationsN) TYPE_CHECKING)Callable) dataclass)cache) TypeVarTupleUnpack)Library) _registryregister_flash_attention_fa3zCallable | None _FA3_CUDA_FWD _FA3_CUDA_BWDc@s eZdZUded<dddZdS) _FA3HandlezLibrary | NonelibraryreturnNonecCsd|_tjddS)NF)rtorch_C_set_sdp_use_fa3)selfr^/home/nk/hobo-godmode/plappi-mvp/.venv/lib/python3.10/site-packages/torch/nn/attention/_fa3.pyremove*sz_FA3Handle.removeN)rr)__name__ __module__ __qualname____annotations__rrrrrr&s rdevice torch.devicerintcCstj|\}}|SN)rcudaget_device_capability)rmajor_rrr_get_device_major0sr&flash_attn_interface module_pathstrcCst|tjdttS)z Register FA3 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA3 implementation. T)_fa3_import_modulerrrr_fa3_register_kernelsr(rrrr 6s  rcCszt|ttjdstd|dttjjds"td|dttjjds1td|dtjjjatjjj a dS)N flash_attn_3zModule 'z' does not expose FA3 kernelsfwdz%' does not expose FA3 forward kernelsbwdz&' does not expose FA3 backward kernels) importlib import_modulehasattrrops RuntimeErrorr-r.r r/rr,rrrr*Gs     r*r cCsrtddd}|dtd|dtd|dtd|dtd|dtd|d td|d td|S) NatenIMPLCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantized_flash_attention_forward+_flash_attention_forward_no_dropout_inplace#_scaled_dot_product_flash_attention_flash_attention_backward,_scaled_dot_product_flash_attention_backward) r impl!_fa3_flash_attention_forward_impl4_fa3_scaled_dot_product_flash_attention_forward_impl)_fa3_flash_attention_forward_impl_default4_fa3_flash_attention_forward_no_dropout_inplace_impl<_fa3_scaled_dot_product_flash_attention_forward_impl_default"_fa3_flash_attention_backward_impl5_fa3_scaled_dot_product_flash_attention_backward_impl)librrrr+Xs: r+query torch.Tensortensorstuple[torch.Tensor, ...] dropout_pfloat cum_seq_qtorch.Tensor | None q_descale k_descale v_descale str | NonecCs|dkrdStdd|DsdStdd|Ddkrd S|jtjkr6|dus0|dus0|dur6td t|durB|d krBd S|durN|d krNdStj sUdSt |j dkr^dSdS)Nzdropout_p must be 0css|]}|jVqdSr!)is_cuda.0trrr sz,_fa3_common_support_error..zinputs must be CUDA tensorscSh|]}|jqSr)rrTrrr z,_fa3_common_support_error..r inputs must share devicezWhen using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.zdense query must be 4Dzragged query must be 3DzCUDA not available z#FA3 requires compute capability 9.0) alllendtyper float8_e4m3fnwarningswarn UserWarningdimr" is_availabler&r)rFrHrJrLrNrOrPrrr_fa3_common_support_errorys*   rhkeyvaluereturn_debug_maskbool alibi_slopes seqused_kc s|rdS|dur dS|dur|jtjkrdS|jsdStjtjtjftfdd|||hDs6dStdd |||hDd krFd St ||||f|||| | } | dur_| d kr]d S| SdS)Nzreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc3|]}|jvVqdSr!rarTsupported_dtypesrrrWz-_fa3_forward_support_error..inputs must be one of cSrXrrprTrrrrYrZz-_fa3_forward_support_error..r #all inputs must have the same dtyper[z(query, key, value must be on same device) rarint32rSrbfloat16bfloat16r_r`rh) rFrirjrJrkrmrnrLrNrOrPerrorrrqr_fa3_forward_support_errors8    rzgrad_outout logsumexpwindow_size_left int | Nonewindow_size_rightc s|jtjkr dS|jtjkrdStjtjftfdd|||||hDs,dStdd|||||hDdkr>d St|||||||f||ddd} | durT| SdS) NzHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c3ror!rprTrqrrrWrsz._fa3_backward_support_error..rtcSrXrrprTrrrrYrZz._fa3_backward_support_error..r ru) rarrbfloat32rwrxr_r`rh) r{rFrirjr|r}rJrLr~rryrrqr_fa3_backward_support_errors,       rTs Unpack[Ts]tuple[Unpack[Ts]]cGstdd|DS)Ncss|] }|ddVqdS)r N) transposerTrrrrWsz#_transpose_dense..)tuple)rHrrr_transpose_densesrxcCs"|dur|ddkr|S|S)z2Ensure tensor is contiguous in the last dimension.Nr )stride contiguous)rrrr_maybe_contiguouss"rcu_seq_qcu_seq_kmax_qmax_kscale float | None is_causal block_table num_splits!tuple[torch.Tensor, torch.Tensor]cCsPtdurtdt|}t|}|jtjkr(|ddkr(|ddkr(|nt|}t|}t|}t| } t|}tg|||ddd| ||dd| |||ddddd| ||||| durv| nd| dur~| nddddd|ptrdnddtj pdR\} }}}| |fS) zF Run the FA3 forward pass by calling the C++ kernel directly. NFA3 not registeredrr rrRT) r r4rrarrbrr$are_deterministic_algorithms_enabledr_get_sm_carveout_experimental)rFrirjrrrrrrr~rrnr|rNrOrPrrqkv cu_seqlens_q cu_seqlens_k softmax_lse out_accumsoftmax_lse_accumrrr_fa3_run_forwards       " # %rF max_seqlen_q max_seqlen_k deterministic/tuple[torch.Tensor, torch.Tensor, torch.Tensor]cCstdurtdt|}|ddkr|n|}|ddkr$|n|}|ddkr1|n|}t|}t|}t|}t|}t|}t|||||||||||dd|| | | | | d|tjpfd|||fS)Nrrr rRr) rr4rrrr empty_likerr)r{rFrirjr|r}rrrrrrr~rrdoutrrrolsedqdkdvrrr_fa3_run_backwardLsH     rrT rr~rrnrmr|rcompute_auxiliaryr cum_seq_krc Cst||||| |||| | | }|durtd|t|||||||| |||||| | | ||\}}|rQtjdtj|jd}tjdtj|jd}tjd|j|jd}nd}d}d}|||||fS)Nz)FA3 flash_attention forward unsupported: )r)rarrr) rzr4rrzerosuint64remptyra)rFrirjrLrrrrJrrkrNrOrPrr~rrnrmr|rrrryr rng_state philox_offset debug_maskrrrr>sV r>)rr~rrnrmrrc CsFt||||||||| | dddf | | | ||||d|d \}}}}}|S)NFrr>)r|rFrirjrLrrrrJrrkrr~rrnrmrrr%rrrrrAs4rA)rr~rrnrmrr|rc Cs2t|||||||||| ddd| | | | ||||dS)N)rr~rrnrmr|rrr)rFrirjrLrrrrJrrkrr~rrnrmrr|rrrrr@s.r@)rr~rrunusedcCst||||||| ||| }|durtd|t}t|||||||||| || |dur/|nd|dur6|nd|\}}}|||fS)z0FA3 implementation of _flash_attention_backward.Nz*FA3 flash_attention backward unsupported: r)rr4rrr)r{rFrirjr|r}rLrrrrJrrrrr~rryrrrrrrrrC,sB  rCrRrc Cst|||||ddd||| } | durtd| t|||\} } } |jtjkr+tjn|j}tj||d}|dd}| d}| d}t | | | dd|||||| ||||d\}}}}}| d}| d}||dd|||||f S)NzFA3 SDPA forward unsupported: rpr r)rr|rNrOrP) rzr4rrarrbrxrrsizer>)rFrirjrNrOrPrJrrkrryrrr out_dtypeout_bhsdout_bshd max_q_flash max_k_flashr%rrrrrrrrrr?gsd       r?c Cst|||ddd||||d S)Nr)r?)rFrirjrJrrkrrrrrBs rB philox_seedrcCst||||||| ddd }|durtd|t|||||\}}}}}t||||||dd|| | | | | |d\}}}t|||\}}}|||fS)zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: r)rr4rrC)r{rFrirjr|r}rLrrrrJrrrrry grad_out_tq_tk_tv_tout_trrrdq_outdk_outdv_outrrrrDs6   rDFA3) register_fn)rrrr )r')r(r)rr)r(r)rr)rr )rFrGrHrIrJrKrLrMrNrMrOrMrPrMrrQ)rFrGrirGrjrGrJrKrkrlrmrMrnrMrLrMrNrMrOrMrPrMrrQ)r{rGrFrGrirGrjrGr|rGr}rGrJrKrLrMr~rrrrrQ)rHrrr)rrMrrM)NNNNNN)&rFrGrirGrjrGrrMrrMrr rr rrrrlr~rrrrnrMr|rMrNrMrOrMrPrMrrMrrrr)F) r{rGrFrGrirGrjrGr|rGr}rGrrMrrMrrrrrrrrlr~r rr rrlrr)NNN),rFrGrirGrjrGrLrMrrMrr rr rJrKrrlrkrlrNrMrOrMrPrMrrr~r rr rnrMrmrMr|rMrrMrrlrr)$r|rGrFrGrirGrjrGrLrMrrMrr rr rJrKrrlrkrlrrr~r rr rnrMrmrMrrMrr)$rFrGrirGrjrGrLrMrrMrr rr rJrKrrlrkrlrrr~r rr rnrMrmrMrrMr|rMrr)"r{rGrFrGrirGrjrGr|rGr}rGrLrMrrMrr rr rJrKrrlrrGrrGrrr~rrr)NNNrRFF)rFrGrirGrjrGrNrMrOrMrPrMrJrKrrlrkrlrr)rRFF)rFrGrirGrjrGrJrKrrlrkrlrr)r{rGrFrGrirGrjrGr|rGr}rGrLrMrrMrr rr rJrKrrlrrGrrGrr).__doc__ __future__rr0rctypingrcollections.abcr dataclassesr functoolsrtyping_extensionsrrr torch.libraryr r __all__r rrrr&r r*r+rhrzrrrrrrr>rAr@rCr?rBrDregister_flash_attention_implrrrrs             ! % +&   ` FT<>? K(5