
    
3jڦ                      S SK Jr  S SKrS SKrS SKJr  S SKrS SKJs  J	r
  S SKJr  SSKJr  SSKJrJrJr  SSKJrJrJr  SS	KJrJr  \R0                  " \5      r\" 5       (       a  S SKr\" 5       (       a	  S SKrS SKrOSr\" 5       (       a  \" S
S5      (       a  S SKJr  S SK J!r!  Sr"OSr"\ " S S\RF                  5      5       r$ " S S\RF                  5      r% " S S\RF                  5      r& " S S\RF                  5      r' " S S5      r( " S S5      r) " S S\RF                  5      r* " S S5      r+ " S  S!5      r, " S" S#5      r- " S$ S%5      r. " S& S'5      r/ " S( S)5      r0 " S* S+5      r1 " S, S-5      r2 " S. S/5      r3 " S0 S15      r4 " S2 S35      r5 " S4 S55      r6 " S6 S75      r7 " S8 S95      r8 " S: S;5      r9 " S< S=5      r: " S> S?5      r; " S@ SA5      r< " SB SC5      r= " SD SE5      r> " SF SG5      r? " SH SI5      r@ " SJ SK5      rA " SL SM5      rB " SN SO5      rC " SP SQ\RF                  5      rD " SR SS\RF                  5      rE " ST SU5      rF " SV SW5      rG " SX SY\RF                  5      rH " SZ S[\RF                  5      rI " S\ S]\R                  RF                  5      rJ " S^ S_\R                  RF                  5      rK " S` Sa\R                  RF                  5      rL " Sb Sc5      rM " Sd Se5      rN " Sf Sg5      rO " Sh Si5      rP " Sj Sk5      rQ " Sl Sm5      rR " Sn So5      rS " Sp Sq5      rT " Sr Ss5      rU " St Su5      rV " Sv Sw5      rW " Sx Sy5      rX " Sz S{5      rY " S| S}5      rZ " S~ S5      r[ " S S5      r\ " S S5      r]\+\G\,\74r^\)\:\8\F\I\J\Z4r_\)\*-  \+-  \,-  \--  \.-  \/-  \0-  \2-  \3-  \4-  \W-  \[-  \Y-  \\-  \5-  \6-  \7-  \8-  \;-  \9-  \:-  \<-  \(-  \=-  \>-  \?-  \@-  \A-  \B-  \C-  \D-  \E-  \F-  \G-  \T-  \U-  \V-  \&-  \O-  \%-  \I-  \J-  \K-  \L-  \M-  \N-  \P-  \Q-  \R-  \S-  r`g)    )annotationsN)Callable)nn   )IPAdapterMaskProcessor)	deprecateis_torch_xla_availablelogging)is_torch_npu_availableis_torch_xla_versionis_xformers_available)is_torch_versionmaybe_allow_in_graph>z2.2)flash_attention)is_spmdTFc                    ^  \ rS rSrSr                             S                                                       SU 4S jjjr  S     SS jjrSS jr S     SS jjrSS jr	SS jr
SSS	 jjr  S       SS
 jjrSS jrS S!S jjr S       S"S jjr S          S#S jjrS$S jr\R&                  " 5       S%S j5       rSrU =r$ )&	Attention3   a  
A cross attention layer.

Parameters:
    query_dim (`int`):
        The number of channels in the query.
    cross_attention_dim (`int`, *optional*):
        The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.
    heads (`int`,  *optional*, defaults to 8):
        The number of heads to use for multi-head attention.
    kv_heads (`int`,  *optional*, defaults to `None`):
        The number of key and value heads to use for multi-head attention. Defaults to `heads`. If
        `kv_heads=heads`, the model will use Multi Head Attention (MHA), if `kv_heads=1` the model will use Multi
        Query Attention (MQA) otherwise GQA is used.
    dim_head (`int`,  *optional*, defaults to 64):
        The number of channels in each head.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
    bias (`bool`, *optional*, defaults to False):
        Set to `True` for the query, key, and value linear layers to contain a bias parameter.
    upcast_attention (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the attention computation to `float32`.
    upcast_softmax (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the softmax computation to `float32`.
    cross_attention_norm (`str`, *optional*, defaults to `None`):
        The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`.
    cross_attention_norm_num_groups (`int`, *optional*, defaults to 32):
        The number of groups to use for the group norm in the cross attention.
    added_kv_proj_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the added key and value projections. If `None`, no projection is used.
    norm_num_groups (`int`, *optional*, defaults to `None`):
        The number of groups to use for the group norm in the attention.
    spatial_norm_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the spatial normalization.
    out_bias (`bool`, *optional*, defaults to `True`):
        Set to `True` to use a bias in the output linear layer.
    scale_qk (`bool`, *optional*, defaults to `True`):
        Set to `True` to scale the query and key by `1 / sqrt(dim_head)`.
    only_cross_attention (`bool`, *optional*, defaults to `False`):
        Set to `True` to only use cross attention and not added_kv_proj_dim. Can only be set to `True` if
        `added_kv_proj_dim` is not `None`.
    eps (`float`, *optional*, defaults to 1e-5):
        An additional value added to the denominator in group normalization that is used for numerical stability.
    rescale_output_factor (`float`, *optional*, defaults to 1.0):
        A factor to rescale the output by dividing it with this value.
    residual_connection (`bool`, *optional*, defaults to `False`):
        Set to `True` to add the residual connection to the output.
    _from_deprecated_attn_block (`bool`, *optional*, defaults to `False`):
        Set to `True` if the attention block is loaded from a deprecated state dict.
    processor (`AttnProcessor`, *optional*, defaults to `None`):
        The attention processor to use. If `None`, defaults to `AttnProcessor2_0` if `torch 2.x` is used and
        `AttnProcessor` otherwise.
c                  > [         T#U ]  5         SSKJnJn Jn!  Ub  UOXS-  U l        Uc  U R                  OXT-  U l        Xl        Xpl	        US LU l
        Ub  UOUU l        Xl        Xl        UU l        UU l        X`l        SU l        Ub  UOUU l        Ub  UOUU l        UU l        UU l        UU l        UU l        UU l        U R0                  (       a  US-  OSU l        Ub  UU-  OUU l        X0l        Xl        UU l        U R8                  c  U R:                  (       a  [=        S5      eUb  [>        R@                  " XUSS9U l!        OS U l!        Ub  [E        UUS	9U l#        OS U l#        Uc  S U l$        S U l%        GOUS
:X  a7  [>        RL                  " UUUS9U l$        [>        RL                  " UUUS9U l%        OUS:X  a  U" USSUS9U l$        U" USSUS9U l%        OUS:X  a9  [>        RL                  " XS-  US9U l$        [>        RL                  " XT-  US9U l%        OtUS:X  a  U!" UUUS9U l$        U!" UUUS9U l%        OSUS:X  a  U!" XS-  US9U l$        U!" XT-  US9U l%        O0US:X  a  U " SSUS9U l$        U " SSUS9U l%        O[=        SU S35      eU
c  S U l'        OzU
S
:X  a&  [>        RL                  " U R                  5      U l'        ONU
S:X  a9  U R8                  b  Un"OU R                  n"[>        R@                  " U"USSS9U l'        O[=        SU
 S35      e[>        RP                  " XR                  US9U l)        U R:                  (       d_  [>        RP                  " U R                  U R                  US9U l*        [>        RP                  " U R                  U R                  US9U l+        OS U l*        S U l+        Xl,        U R8                  bz  [>        RP                  " XR                  US9U l-        [>        RP                  " XR                  US9U l.        U R(                  b$  [>        RP                  " XR                  US9U l/        OS U l/        S U l-        S U l.        U R*                  (       d  [>        R`                  " / 5      U l1        U Rb                  Re                  [>        RP                  " U R                  U R$                  US95        U Rb                  Re                  [>        Rf                  " U5      5        OS U l1        U R(                  bA  U R(                  (       d0  [>        RP                  " U R                  U R&                  US9U l4        OS U l4        Ub  Ub  US
:X  a7  [>        RL                  " UUUS9U l5        [>        RL                  " UUUS9U l6        O{US:X  a  U" USSUS9U l5        U" USSUS9U l6        OXUS:X  a  U!" UUS9U l5        U!" UUS9U l6        O9US:X  a  S U l5        U!" XT-  US9U l6        O[=        SU S35      eS U l5        S U l6        Uc:  [o        [p        S5      (       a  U R0                  (       a
  [s        5       O	[u        5       nU Rw                  U5        g )N   )FP32LayerNormLpNormRMSNormFg            ?z`only_cross_attention` can only be set to True if `added_kv_proj_dim` is not None. Make sure to set either `only_cross_attention=False` or define `added_kv_proj_dim`.Tnum_channels
num_groupsepsaffine)
f_channelszq_channels
layer_norm)r   elementwise_affinefp32_layer_norm)r$   biasr   layer_norm_across_heads)r   rms_normrms_norm_across_headsl2r   )pdimr   zunknown qk_norm: z. Should be one of None, 'layer_norm', 'fp32_layer_norm', 'layer_norm_across_heads', 'rms_norm', 'rms_norm_across_heads', 'l2'.
group_normh㈵>zunknown cross_attention_norm: z.. Should be None, 'layer_norm' or 'group_norm'r&   zC. Should be one of `None,'layer_norm','fp32_layer_norm','rms_norm'`scaled_dot_product_attention)<super__init__normalizationr   r   r   	inner_diminner_kv_dim	query_dimuse_biasis_cross_attentioncross_attention_dimupcast_attentionupcast_softmaxrescale_output_factorresidual_connectiondropoutfused_projectionsout_dimout_context_dimcontext_pre_onlypre_only	is_causal_from_deprecated_attn_blockscale_qkscaleheadssliceable_head_dimadded_kv_proj_dimonly_cross_attention
ValueErrorr   	GroupNormr.   SpatialNormspatial_normnorm_qnorm_k	LayerNorm
norm_crossLinearto_qto_kto_vadded_proj_bias
add_k_proj
add_v_proj
add_q_proj
ModuleListto_outappendDropout
to_add_outnorm_added_qnorm_added_khasattrFAttnProcessor2_0AttnProcessorset_processor)$selfr7   r:   rI   kv_headsdim_headr?   r&   r;   r<   cross_attention_normcross_attention_norm_num_groupsqk_normrK   rY   norm_num_groupsspatial_norm_dimout_biasrG   rL   r   r=   r>   rF   	processorrA   rB   rC   rD   r$   rE   r   r   r   norm_cross_num_channels	__class__s$                                      ^/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/attention_processor.pyr3   Attention.__init__k   s   B 	 	BA$+$7X=M.6.>DNNHDW""5T"A:M:Y#6_h  0,%:"#6 !&")"5w92A2MS\ 0 " ,G( '+}}Xt^#
,3,?W(U
 #(!2$8!!!)d.G.G y  & ll	cfostDO"DO' +yN^ _D $D?DKDK$,,xSM_`DK,,xSM_`DK))'UQV\_`DK'UQV\_`DK11,,x'7SADK,,x':DDK
"!(HZ[DK!(HZ[DK//!("2<DK!("53?DK_ 1"#6DK 1"#6DK#G9  -l  m   '"DO!\1 ll4+C+CDDO!\1%%1 +<'*.*B*B' ll4A`fjswDO 01E0FFtu  IIidC	((		$":":D<M<MTXYDI		$":":D<M<MTXYDIDIDI.!!- ii(9;L;LSbcDO ii(9;L;LSbcDO$$0"$)),=~~Tc"d"DO"DO"DO}}--+DKKKryyHUVKKrzz'23DK  ,T5J5J ii8L8LS[\DO"DO#4#@,&$&LLsWi$j!$&LLsWi$j!--$1(u[`fi$j!$1(u[`fi$j!J&$+H#$>!$+H#$>!33 %)!$+H,?S$I! 'y0st  !%D $D &-a1O&P&PUYUbUb "huhw  	9%    c                j   U(       aa  [         (       d  Se[        SS5      (       a  Se[        5       (       a  [        SS5      (       a  SeU(       a  [        U5      nOF[	        U5      nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)	a3  
Set whether to use xla flash attention from `torch_xla` or not.

Args:
    use_xla_flash_attention (`bool`):
        Whether to use pallas flash attention kernel from `torch_xla` or not.
    partition_spec (`tuple[]`, *optional*):
        Specify the partition specification if using SPMD. Otherwise None.
ztorch_xla is not available<2.3zEflash attention pallas kernel is supported from torch_xla version 2.32.4zPflash attention pallas kernel using SPMD is supported from torch_xla version 2.4r1   N)r	   r   r   XLAFluxFlashAttnProcessor2_0XLAFlashAttnProcessor2_0rd   re   rG   rf   rg   rh   )ri   use_xla_flash_attentionpartition_specis_fluxrr   s        ru   set_use_xla_flash_attention%Attention.set_use_xla_flash_attention7  s     #))22%c511]]3C??hh <^ LI 8 HI '.a1O&P&PUYUbUb "huhw  	9%rw   c                    U(       a  [        5       nO:[        [        S5      (       a  U R                  (       a
  [	        5       O	[        5       nU R                  U5        g)zB
Set whether to use npu flash attention from `torch_npu` or not.

r1   N)AttnProcessorNPUrd   re   rG   rf   rg   rh   )ri   use_npu_flash_attentionrr   s      ru   set_use_npu_flash_attention%Attention.set_use_npu_flash_attentionX  sI    
 #(*I '.a1O&P&PUYUbUb "huhw  	9%rw   c                   [        U S5      =(       a%    [        U R                  [        [        [
        45      n[        U S5      =(       a*    [        U R                  [        [        [        [        45      n[        U S5      =(       a%    [        U R                  [        [        [        45      n[        U S5      =(       a     [        U R                  [        [        45      nU(       Ga  U(       a  U(       a  [        SU R                   35      e[!        5       (       d
  [#        SSS9e[$        R&                  R)                  5       (       d  [+        S5      e SnUb  Uu  pUR,                  tpz[$        R.                  " SS	US
9n[0        R2                  R5                  XU5      n
U(       a  [	        U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOU(       a   [L        RO                  S5        [        US9nGOwU(       Ga	  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  GOfU(       a  [        US9nGOT[Y        US9nGOIU(       a  [        [Z        S5      (       a  [
        O[        nU" U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOIU(       Ga  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  O:[        [Z        S5      (       a  U R\                  (       a
  [_        5       O	[a        5       nU Rc                  U5        g! [6         a  nUeSnAff = f)ax  
Set whether to use memory efficient attention from `xformers` or not.

Args:
    use_memory_efficient_attention_xformers (`bool`):
        Whether to use memory efficient attention from `xformers` or not.
    attention_op (`Callable`, *optional*):
        The attention operation to use. Defaults to `None` which uses the default attention operation from
        `xformers`.
rr   zhMemory efficient attention is currently not supported for custom diffusion for attention processor type zeRefer to https://github.com/facebookresearch/xformers for more information on how to install xformersxformers)namezvtorch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only available for GPU N)r   r   (   cudadevicedtype)train_kvtrain_q_outhidden_sizer:   attention_opto_k_custom_diffusionzMemory efficient attention with `xformers` might currently not work correctly if an attention mask is required for the attention operation.r   )r   r:   
num_tokensrH   r   to_k_ipr   r1   )r   r   r   r:   )r   r:   r   rH   )2rd   
isinstancerr   CustomDiffusionAttnProcessor$CustomDiffusionXFormersAttnProcessorCustomDiffusionAttnProcessor2_0AttnAddedKVProcessorAttnAddedKVProcessor2_0SlicedAttnAddedKVProcessorXFormersAttnAddedKVProcessorIPAdapterAttnProcessorIPAdapterAttnProcessor2_0IPAdapterXFormersAttnProcessorJointAttnProcessor2_0XFormersJointAttnProcessorNotImplementedErrorr   ModuleNotFoundErrortorchr   is_availablerM   SUPPORTED_DTYPESrandnr   opsmemory_efficient_attention	Exceptionr   r   r   r:   load_state_dict
state_dicttor   weightr   loggerinfor   rH   r   r   XFormersAttnProcessorre   rG   rf   rg   rh   )ri   'use_memory_efficient_attention_xformersr   is_custom_diffusionis_added_kv_processoris_ip_adapteris_joint_processorr   op_fwop_bw_qerr   attn_processor_classs                  ru   +set_use_memory_efficient_attention_xformers5Attention.set_use_memory_efficient_attention_xformersi  s    &dK8 
ZNN)+OQpq>
 !(k : !
zNN$'*,	@
  k2 
zNN#%>@^_8
 %T;7 
JNN%*=
 3$)<)~  @D  @N  @N  O  P  )**)$ $  ZZ,,.. / 
	 E#/'3$)$:$:	JvUKA ??aHA #@!^^44 $ : : $ : :(,(J(J!-	 ))$..*C*C*EF4>>+BCCLL!E!E!L!L!S!ST&
  b 9lS	: $ : :(,(J(J#~~88....!-	 ))$..*C*C*EF4>>955LL#~~55a8??FFdnnNdNdefNgNnNnNtNt !  $6LQ	1|L	" q"@AA 45 %
 1!^^44 $ : : $ : :(,(J(J		 ))$..*C*C*EF4>>+BCCLL!E!E!L!L!S!ST5 $ : :(,(J(J#~~88....		 ))$..*C*C*EF4>>955LL#~~55a8??FFdnnNdNdefNgNnNnNtNt !  q"@AAdmm %&&  	9%_ ! Gs   AX6 6
Y YYc                |   Ub+  XR                   :  a  [        SU SU R                    S35      eUb  U R                  b  [        U5      nOaUb  [	        U5      nORU R                  b  [        5       nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)z
Set the slice size for attention computation.

Args:
    slice_size (`int`):
        The slice size for attention computation.
Nzslice_size z has to be smaller or equal to .r1   )rJ   rM   rK   r   SlicedAttnProcessorr   rd   re   rG   rf   rg   rh   )ri   
slice_sizerr   s      ru   set_attention_sliceAttention.set_attention_slice  s     !j3J3J&J{:,6UVZVmVmUnnopqq!d&<&<&H2:>I#+J7I##/,.I '.a1O&P&PUYUbUb "huhw  	9%rw   c                j   [        U S5      (       a  [        U R                  [        R                  R
                  5      (       ai  [        U[        R                  R
                  5      (       d@  [        R                  SU R                   SU 35        U R                  R                  S5        Xl        g)zu
Set the attention processor to use.

Args:
    processor (`AttnProcessor`):
        The attention processor to use.
rr   z-You are removing possibly trained weights of z with N)
rd   r   rr   r   r   Moduler   r   _modulespop)ri   rr   s     ru   rh   Attention.set_processor  sx     D+&&4>>588??;;y%((//::KKGGWW]^g]hijMMk*"rw   c                *    U(       d  U R                   $ g)z
Get the attention processor in use.

Args:
    return_deprecated_lora (`bool`, *optional*, defaults to `False`):
        Set to `True` to return the deprecated LoRA attention processor.

Returns:
    "AttentionProcessor": The attention processor in use.
Nrr   )ri   return_deprecated_loras     ru   get_processorAttention.get_processor+  s     &>>! &rw   c                2   [        [        R                  " U R                  R                  5      R
                  R                  5       5      nSS1nUR                  5        VVs/ s H  u  pxXu;  d  M  Xv;  d  M  UPM     n	nn[        U	5      S:  a:  [        R                  SU	 SU R                  R                  R                   S35        UR                  5        VV
s0 s H  u  pzXu;   d  M  Xz_M     nnn
U R                  " U U4UUS.UD6$ s  snnf s  sn
nf )a   
The forward method of the `Attention` class.

Args:
    hidden_states (`torch.Tensor`):
        The hidden states of the query.
    encoder_hidden_states (`torch.Tensor`, *optional*):
        The hidden states of the encoder.
    attention_mask (`torch.Tensor`, *optional*):
        The attention mask to use. If `None`, no mask is applied.
    **cross_attention_kwargs:
        Additional keyword arguments to pass along to the cross attention.

Returns:
    `torch.Tensor`: The output of the attention layer.
ip_adapter_masksip_hidden_statesr   zcross_attention_kwargs z are not expected by z and will be ignored.encoder_hidden_statesattention_mask)setinspect	signaturerr   __call__
parameterskeysitemslenr   warningrt   __name__)ri   hidden_statesr   r   cross_attention_kwargsattn_parametersquiet_attn_parameterskr   unused_kwargsws              ru   forwardAttention.forward9  s"   6 g//0G0GHSSXXZ[!35G H0668
8$!A<TAYZYwA8 	 
 }!NN)-8MdnnNfNfNoNoMp  qF  G 4J3O3O3Q!j3Q41UVUi$!$3Q!j~~
 #8)	

 %
 	

 "ks   #D2D9DD-Dc                    U R                   nUR                  u  p4nUR                  X2-  X$U5      nUR                  SSSS5      R                  X2-  XEU-  5      nU$ )a+  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size // heads, seq_len, dim * heads]`. `heads`
is the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.

Returns:
    `torch.Tensor`: The reshaped tensor.
r   r   r      )rI   shapereshapepermute)ri   tensor	head_size
batch_sizeseq_lenr-   s         ru   batch_to_head_dimAttention.batch_to_head_dimg  s_     JJ	#)<< 
S
 7SQ1a+33J4KW\eVefrw   c                   U R                   nUR                  S:X  a  UR                  u  pEnSnOUR                  u  pGpVUR                  XEU-  X6U-  5      nUR	                  SSSS5      nUS:X  a  UR                  XC-  XW-  Xc-  5      nU$ )a  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size, seq_len, heads, dim // heads]` `heads` is
the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.
    out_dim (`int`, *optional*, defaults to `3`): The output dimension of the tensor. If `3`, the tensor is
        reshaped to `[batch_size * heads, seq_len, dim // heads]`.

Returns:
    `torch.Tensor`: The reshaped tensor.
r   r   r   r   )rI   ndimr   r   r   )ri   r   rA   r   r   r   r-   	extra_dims           ru   head_to_batch_dimAttention.head_to_batch_dimx  s     JJ	;;!'-||$JI28,,/J7
i,?S\L\]1a+a<^^J$:G<OQTQabFrw   c                $   UR                   nU R                  (       a   UR                  5       nUR                  5       nUcV  [        R                  " UR
                  S   UR
                  S   UR
                  S   UR                   UR                  S9nSnOUnSn[        R                  " UUUR                  SS5      UU R                  S9nAU R                  (       a  UR                  5       nUR                  SS9nAUR                  U5      nU$ )a+  
Compute the attention scores.

Args:
    query (`torch.Tensor`): The query tensor.
    key (`torch.Tensor`): The key tensor.
    attention_mask (`torch.Tensor`, *optional*): The attention mask to use. If `None`, no mask is applied.

Returns:
    `torch.Tensor`: The attention probabilities/scores.
r   r   r   r   r+   )betaalphar-   )r   r;   floatr   emptyr   r   baddbmm	transposerH   r<   softmaxr   )	ri   querykeyr   r   baddbmm_inputr   attention_scoresattention_probss	            ru   get_attention_scoresAttention.get_attention_scores  s       KKME))+C!!KKAA		!EKKX]XdXdM D*MD ==MM"b!**
 /557*22r2:),,U3rw   c                ^   U R                   nUc  U$ UR                  S   nXb:w  a  UR                  R                  S:X  a_  UR                  S   UR                  S   U4n[        R
                  " XqR                  UR                  S9n[        R                  " X/SS9nO[        R                  " USU4SS	9nUS
:X  a8  UR                  S   X5-  :  a!  UR                  USUR                  S   U-  S9nU$ US:X  a2  UR                  S5      nUR                  USUR                  S   U-  S9nU$ )a)  
Prepare the attention mask for the attention computation.

Args:
    attention_mask (`torch.Tensor`):
        The attention mask to prepare.
    target_length (`int`):
        The target length of the attention mask. This is the length of the attention mask after padding.
    batch_size (`int`):
        The batch size, which is used to repeat the attention mask.
    out_dim (`int`, *optional*, defaults to `3`):
        The output dimension of the attention mask. Can be either `3` or `4`.

Returns:
    `torch.Tensor`: The prepared attention mask.
r+   mpsr   r   r   r   r           )valuer   r-   output_size   )rI   r   r   typer   zerosr   catre   padrepeat_interleave	unsqueeze)	ri   r   target_lengthr   rA   r   current_lengthpadding_shapepaddings	            ru   prepare_attention_mask Attention.prepare_attention_mask  sQ   & JJ	!!!,2226*$$))U2 "0!5!5a!8.:N:Nq:QS` a++m;O;OXfXmXmn!&N+D!!L "#~=7IQT!Ua<##A&)??!/!A!A1.2F2Fq2II2U "B "  \+55a8N+==qn.B.B1.E	.Q > N rw   c                f   U R                   c   S5       e[        U R                   [        R                  5      (       a  U R                  U5      nU$ [        U R                   [        R                  5      (       a7  UR                  SS5      nU R                  U5      nUR                  SS5      nU$  e)a  
Normalize the encoder hidden states. Requires `self.norm_cross` to be specified when constructing the
`Attention` class.

Args:
    encoder_hidden_states (`torch.Tensor`): Hidden states of the encoder.

Returns:
    `torch.Tensor`: The normalized encoder hidden states.
zGself.norm_cross must be defined to call self.norm_encoder_hidden_statesr   r   )rT   r   r   rS   rN   r  )ri   r   s     ru   norm_encoder_hidden_states$Attention.norm_encoder_hidden_states  s     *u,uu*door||44$(OO4I$J! %$ 66 %:$C$CAq$I!$(OO4I$J!$9$C$CAq$I! %$ 5rw   c                	   U R                   R                  R                  R                  nU R                   R                  R                  R                  nU R
                  (       Gd  [        R                  " U R                   R                  R                  U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R                  R                  R                  U5        U R                  (       a  [        R                  " U R                   R                   R                  U R                  R                   R                  U R                  R                   R                  /5      nU R                  R                   R                  U5        GOF[        R                  " U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R"                  R                  R                  U5        U R                  (       ay  [        R                  " U R                  R                   R                  U R                  R                   R                  /5      nU R"                  R                   R                  U5        [%        U SS 5      Gb  [%        U SS 5      Gb  [%        U SS 5      Gb  [        R                  " U R&                  R                  R                  U R(                  R                  R                  U R*                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R,                  X#S9U l        U R.                  R                  R                  U5        U R,                  (       a  [        R                  " U R&                  R                   R                  U R(                  R                   R                  U R*                  R                   R                  /5      nU R.                  R                   R                  U5        Xl        g )Nr   r   )r&   r   r   r\   rZ   r[   )rV   r   datar   r   r9   r   r  rW   rX   r   r   rU   r8   to_qkvcopy_r&   to_kvgetattrr\   rZ   r[   rY   to_added_qkvr@   )ri   fuser   r   concatenated_weightsin_featuresout_featuresconcatenated_biass           ru   fuse_projectionsAttention.fuse_projections  sg   !!&&--		  %%++&&&#(99dii.>.>.C.CTYYEUEUEZEZ\`\e\e\l\l\q\q-r#s .44Q7K/55a8L ))KDMMZ`nDKKK$$%9:}}$)IItyy~~/B/BDIINNDWDWY]YbYbYgYgYlYl.m$n!  &&'89 $)99dii.>.>.C.CTYYEUEUEZEZ-[#\ .44Q7K/55a8L;4==Y_mDJJJ##$89}}$)IItyy~~/B/BDIINNDWDW.X$Y!

%%&78 D,-9lD1=lD1=#(99'',,doo.D.D.I.I4??KaKaKfKfg$  /44Q7K/55a8L "		0D0DV!D $$**+?@##$)II__))..0D0D0I0I4??K_K_KdKde%! !!&&,,->?!%rw   ),rF   rZ   r\   r[   rK   rY   rC   r:   r?   r@   r.   rI   r5   r6   rE   r9   rc   rb   rT   rR   rQ   rL   rB   rA   rD   rr   r7   r=   r>   rH   rG   rJ   rP   ra   r)  rW   r'  r^   rV   r%  rX   r;   r<   r8   )N   N@   r  FFFN    NNTNNTTFr/   r   FFNNNNFTF)8r7   intr:   
int | NonerI   r4  rj   r5  rk   r4  r?   r  r&   boolr;   r6  r<   r6  rl   
str | Nonerm   r4  rn   r7  rK   r5  rY   zbool | Nonero   r5  rp   r5  rq   r6  rG   r6  rL   r6  r   r  r=   r  r>   r6  rF   r6  rr   z'AttnProcessor' | NonerA   r4  rB   r4  r$   r6  rE   r6  )NF)r~   r6  r   tuple[str | None, ...] | NonereturnNone)r   r6  r9  r:  N)r   r6  r   Callable | Noner9  r:  )r   r4  r9  r:  )rr   z'AttnProcessor'r9  r:  )F)r   r6  r9  z'AttentionProcessor'NN)r   torch.Tensorr   torch.Tensor | Noner   r?  r9  r>  )r   r>  r9  r>  )r   )r   r>  rA   r4  r9  r>  )r  r>  r  r>  r   r?  r9  r>  )
r   r>  r  r4  r   r4  rA   r4  r9  r>  )r   r>  r9  r>  )T)r   
__module____qualname____firstlineno____doc__r3   r   r   r   r   rh   r   r   r   r   r  r  r!  r   no_gradr/  __static_attributes____classcell__rt   s   @ru   r   r   3   s   4r +/#!&$+//1"(,'+&*'+%*'*$),1,0##'?J&J& (J& 	J&
 J& J& J& J& J& J& )J& *-J& J& &J& %J&  $!J&" %#J&$ %J&& 'J&( #)J&* +J&,  %-J&. "/J&0 &*1J&2 *3J&4 5J&6 7J&< !=J&> ?J& J&^ 9=	&!%& 6&
 
&B&$ ^bP&7;P&KZP&	P&d&8#("" 6:.2	,
#,
  3,
 ,	,
 
,
\"8 ]a-!-(4-FY-	-` ab1*1;>1LO1Z]1	1f%: ]]_2& 2&rw   r   c                  F   ^  \ rS rSr        SU 4S jjrSS jrSrU =r$ )!SanaMultiscaleAttentionProjectioniH  c           
        > [         TU ]  5         SU-  n[        R                  " UUUUS-  USS9U l        [        R                  " XDSSSSU-  SS9U l        g )Nr   r   F)r  groupsr&   r   r   )rK  r&   )r2   r3   r   Conv2dproj_inproj_out)ri   in_channelsnum_attention_headskernel_sizechannelsrt   s        ru   r3   *SanaMultiscaleAttentionProjection.__init__I  sc     	{?yy1$
 		(aAaJ]F]dijrw   c                J    U R                  U5      nU R                  U5      nU$ r;  rM  rN  ri   r   s     ru   r   )SanaMultiscaleAttentionProjection.forward\  s$    ]3m4rw   rU  )rO  r4  rP  r4  rQ  r4  r9  r:  r   r>  r9  r>  r   r@  rA  rB  r3   r   rE  rF  rG  s   @ru   rI  rI  H  s=    kk !k 	k
 
k& rw   rI  c                     ^  \ rS rSrSr       S                 S	U 4S jjjrS
S jrS
S jrSS jrSr	U =r
$ )SanaMultiscaleLinearAttentionib  z(Lightweight multi-scale linear attentionc
                x  > [         TU ]  5         SSKJn
  Xl        X@l        X`l        Xl        Uc  [        X-  U-  5      OUnX4-  n[        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " 5       U l        U H(  nU R                  R!                  [#        XU5      5        M*     [        R$                  " 5       U l        [        R                  " US[)        U5      -   -  USS9U l        U
" XbS9U l        [/        5       U l        g )Nr   )get_normalizationFr0   )num_features)r2   r3   r4   r]  r   attention_head_dim	norm_typer>   r4  r   rU   rV   rW   rX   r]   to_qkv_multiscaler_   rI  ReLUnonlinearityr   r^   norm_outSanaMultiscaleAttnProcessor2_0rr   )ri   rO  out_channelsrP  r_  multr`  kernel_sizesr   r>   r]  r5   rQ  rt   s                ru   r3   &SanaMultiscaleLinearAttention.__init__e  s    	 	5"4"#6  >Q=XC1D89^q 	 (<	IIk5A	IIk5A	IIk5A	!#'K""))1)R]^ (
 GGIii	Q\1B-B C\X]^))O79rw   c                :   [         R                  " USSSS9n[        R                  " X2R	                  SS5      5      n[        R                  " XA5      nUR                  [        R                  S9nUS S 2S S 2S S24   US S 2S S 2SS 24   U R                  -   -  nU$ )Nr   r   r   r   constantr   moder  r+   r   r   )re   r  r   matmulr  r   float32r   ri   r  r  r  scoresr   s         ru   apply_linear_attention4SanaMultiscaleLinearAttention.apply_linear_attention  s    e\
!De]]2r%:;V3%((u}}(=%aCRCi0M!Q)4Ltxx4WXrw   c                8   [         R                  " UR                  SS5      U5      nUR                  [         R                  S9nU[         R
                  " USSS9U R                  -   -  n[         R                  " X4R                  UR                  5      5      nU$ )Nr+   r   ro  r   T)r-   keepdim)r   rp  r  r   rq  sumr   r   rr  s         ru   apply_quadratic_attention7SanaMultiscaleLinearAttention.apply_quadratic_attention  sr    cmmB3U;/599VDADHHLMUIIekk,BCrw   c                $    U R                  X5      $ r;  r   rV  s     ru   r   %SanaMultiscaleLinearAttention.forward  s    ~~d22rw   )r_  r   rc  rd  r`  rr   r>   rW   r^   rV   ra  rX   )Nr1  r   
batch_norm)   V瞯<F)rO  r4  rf  r4  rP  r5  r_  r4  rg  r  r`  strrh  ztuple[int, ...]r   r  r>   r6  )r  r>  r  r>  r  r>  r9  r>  rX  )r   r@  rA  rB  rC  r3   rt  ry  r   rE  rF  rG  s   @ru   r[  r[  b  s    3 +/"#%(,$)):): ): (	):
  ): ): ): &): ): "): ):V3 3rw   r[  c                     ^  \ rS rSr          S                         SU 4S jjjr  S     SS jjrSrU =r$ )	MochiAttentioni  c                  > [         TU ]  5         SSKJn  U	b  U	OXT-  U l        U	b  U	OUU l        U
(       a  U
OUU l        Xl        U	b  X-  OUU l        U" X]S5      U l	        U" X]S5      U l
        U" X]S5      U l        U" X]S5      U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " X R                  US9U l        [        R                  " X R                  US9U l        U R                  b$  [        R                  " X R                  US9U l        [        R*                  " / 5      U l        U R,                  R/                  [        R                  " U R                  U R
                  US95        U R,                  R/                  [        R0                  " U5      5        U R                  (       d/  [        R                  " U R                  U R                  US9U l        X0l        g )Nr   )MochiRMSNormTr0   )r2   r3   r4   r  r5   rA   rB   rC   rI   rQ   rR   rb   rc   r   rU   rV   rW   rX   rZ   r[   r\   r]   r^   r_   r`   ra   rr   )ri   r7   rK   rr   rI   rk   r?   r&   rY   rA   rB   rq   rC   r   r  rt   s                  ru   r3   MochiAttention.__init__  s     	/$+$7X=M")"5w92Ay 0,3,?W(U
"8$7"8$7(=(=IIidC	IIidC	IIidC	))$5~~O\))$5~~O\  , ii(9>>P_`DOmmB'299T^^T\\QR2::g./$$ ii8L8LS[\DO"rw   c                0    U R                   " U U4UUS.UD6$ )Nr   r   )ri   r   r   r   kwargss        ru   r   MochiAttention.forward  s2     ~~
 #8)	

 
 	
rw   )rZ   r\   r[   rC   rI   r5   rc   rb   rR   rQ   rB   rA   rr   ra   rW   r^   rV   rX   )
r1  r2  r  FTNNTFr/   )r7   r4  rK   r4  rr   z'MochiAttnProcessor2_0'rI   r4  rk   r4  r?   r  r&   r6  rY   r6  rA   r5  rB   r5  rq   r6  rC   r6  r   r  r=  )r   r>  r   r?  r   r?  rY  rG  s   @ru   r  r    s      $"&*!&/#/# /# +	/#
 /# /# /# /# /# /# $/# /# /# /# /#h 6:.2	
#
  3
 ,	
 
rw   r  c                  D    \ rS rSrSrS r S           SS jjrSrg)	MochiAttnProcessor2_0i  z"Attention processor used in Mochi.c                D    [        [        S5      (       d  [        S5      eg )Nr1   zUMochiAttnProcessor2_0 requires PyTorch 2.0. To use it, please upgrade PyTorch to 2.0.rd   re   ImportErrorri   s    ru   r3   MochiAttnProcessor2_0.__init__  !    q899uvv :rw   Nc           	     :   UR                  U5      nUR                  U5      nUR                  U5      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR
                  b  UR                  U5      nUR                  b  UR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SUR                  S45      n	U
R                  SUR                  S45      n
UR                  SUR                  S45      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
Ub  S nU" U/UQ76 nU" U/UQ76 nUR                  SS5      UR                  SS5      UR                  SS5      pnU	R                  SS5      U
R                  SS5      UR                  SS5      pn	UR                  S5      nU	R                  S5      nX-   nUR                  u  nnnn/ n[        U5       GH&  nUU   S S S 24   n[         R"                  " UR%                  5       SS9R%                  5       nU	UUS-   2S S 2US S 24   nU
UUS-   2S S 2US S 24   nUUUS-   2S S 2US S 24   n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[(        R*                  " UUUSSS	9nUR                  S5      n[(        R,                  " US
S
S
UU-
  45      nUR/                  U5        GM)     [         R&                  " US
S9nUR                  SS5      R%                  SS5      nUR1                  X4SS9u  p#UR2                  S
   " U5      nUR2                  S   " U5      n[5        US5      (       a  UR7                  U5      nX#4$ )Nr   r+   c                4   U SSS S24   R                  5       nU SSS S24   R                  5       nX1-  XB-  -
  R                  U R                  5      nX2-  XA-  -   R                  U R                  5      n[        R                  " XV/SS9R                  S5      $ )N.r   r   r   r+   r   r   )r  r   r   r   stackflatten)x	freqs_cos	freqs_sinx_evenx_oddcossins          ru   apply_rotary_emb8MochiAttnProcessor2_0.__call__.<locals>.apply_rotary_emb  s    319++-#qt!t)**,)E,==AA!''J)E,==AA!''J{{C:26>>rBBrw   r   F)as_tupler   r  	dropout_prE   r   r   ra   )rV   rW   rX   	unflattenrI   rQ   rR   r\   rZ   r[   rb   rc   r  sizer   ranger   nonzeror  r  re   r1   r  r_   split_with_sizesr^   rd   ra   ) ri   attnr   r   r   image_rotary_embr  r  r  encoder_queryencoder_keyencoder_valuer  sequence_lengthencoder_sequence_lengthtotal_lengthr   rI   r   r-   attn_outputsidxmaskvalid_prompt_token_indicesvalid_encoder_queryvalid_encoder_keyvalid_encoder_valuevalid_query	valid_keyvalid_valueattn_outputvalid_sequence_lengths                                    ru   r   MochiAttnProcessor2_0.__call__  s,    		-(ii&		-(DJJ#34mmA

B/0DJJ#34;;"KK&E;;"++c"C(=>oo&;<(=>%//DJJ3CD!++A

B/?@%//DJJ3CD( --m<M(++K8K'C %U>-=>E"3:)9:C!OOAq13==A3FXY[\H]E##Aq)!!!Q'##Aq) %2  **Q-"/"4"4Q"7&@$)KK!
E1c$C!#&tQw/D).t||~PU)V)^)^)`&"/cAgqB\^_0_"` +C#'M1>XZ[,[ \"/cAgqB\^_0_"`))U3q%9;N$OUVWK		3sS1W#57H"IqQI))U3q%9;N$OUVWK88YseK %0$4$4Q$7!%%aA|F[7[-\]K,# %& 		,A6%//15==aC/</M/M6A 0N 0
,
 A}5A}54&&$(OO4I$J!33rw    r;  )r  z'MochiAttention'r   r>  r   r>  r   r>  r  r?  r9  r>  r   r@  rA  rB  rC  r3   r   rE  r  rw   ru   r  r    s\    ,w 15_4_4 $_4  ,	_4
 %_4 ._4 
_4 _4rw   r  c                  B    \ rS rSrSr   S           SS jjrSrg)rg   iO  zB
Default processor for performing attention-related computations.
Nc                   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUR                  XOU5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR!                  UUU5      n["        R$                  " UU5      nUR'                  U5      nUR(                  S   " U5      nUR(                  S   " U5      nU
S:X  a$  UR                  SS	5      R+                  UWWW5      nUR,                  (       a  X)-   nX!R.                  -  nU$ )
Nr   rH   The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.1.0.0r  r   r   r+   r   )r   getr   rP   r   r   viewr  r  r.   rV   rT   r!  rW   rX   r   r  r   bmmr   r^   r   r>   r=   )ri   r  r   r   r   tembargsr  deprecation_messageresidual
input_ndimr   channelheightwidthr  r   r  r  r  r
  s                        ru   r   AttnProcessor.__call__T  s3    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q 44^V`a??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-$$S)&&u-33E3O		/59..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  NNNr  r   r   r>  r   r?  r   r?  r  r?  r9  r>  r   r@  rA  rB  rC  r   rE  r  rw   ru   rg   rg   O  s^     6:.2$(@@ $@  3	@
 ,@ "@ 
@ @rw   rg   c                  z   ^  \ rS rSrSr      S           SU 4S jjjr  S         S	S jjrSrU =r$ )
r   i  a  
Processor for implementing attention for the Custom Diffusion method.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
c                L  > [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g NFr0   r2   r3   r   r   r   r:   r   rU   r   to_v_custom_diffusionto_q_custom_diffusionr]   to_out_custom_diffusionr_   r`   ri   r   r   r   r:   rq   r?   rt   s          ru   r3   %CustomDiffusionAttnProcessor.__init__       	 &&#6  ==)+3F3U+Wbin)oD&)+3F3U+Wbin)oD&)+;RW)XD&+-==+<D(((//		+Ya0bc((//

70CD	 rw   c                ^   UR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       a  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR%                  U5      nUR%                  U
5      n
UR%                  U5      nUR'                  XU5      n[        R(                  " X5      nUR+                  U5      nU R                  (       a,  U R,                  S   " U5      nU R,                  S   " U5      nU$ UR.                  S   " U5      nUR.                  S   " U5      nU$ )NFTr   r  r   )r   r  r   r  r   rV   r   r   rT   r!  r   r   r  rW   rX   r   	ones_likedetachr   r  r  r   r  r^   )ri   r  r   r   r   r   r  r   r  	crossattnr  r  r  r
  s                 ru   r   %CustomDiffusionAttnProcessor.__call__  s    *7)<)<&
Q44^V`a..}=@@AQAQAWAWXEIIm..tyy/?/?/E/EFGE (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE&&u-$$S)&&u-33EO		/9..}= 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrw   r:   r   r   r  r  r  r   r   TTNNTr  r   r6  r   r6  r   r5  r:   r5  rq   r6  r?   r  r=  
r  r   r   r>  r   r?  r   r?  r9  r>  	r   r@  rA  rB  rC  r3   r   rE  rF  rG  s   @ru   r   r     s    (  "&*.EE E  	E
 (E E E E< 6:.288 $8  3	8
 ,8 
8 8rw   r   c                  <    \ rS rSrSr  S         SS jjrSrg)r   i  z{
Processor for performing attention-related computations with extra learnable key and value matrices for the text
encoder.
Nc                   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pnUR                  XJU	5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dr  UR                  U5      nUR!                  U5      nUR                  U5      nUR                  U5      n["        R$                  " X/SS9n["        R$                  " UU/SS9nOUnUnUR'                  XU5      n["        R(                  " UU5      nUR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS	5      R/                  UR                  5      nX(-   nU$ )
Nr   rH   r  r  r   r+   r   r   r   )r   r  r   r  r   r  r  rT   r!  r.   rV   r   rZ   r[   rL   rW   rX   r   r  r  r  r   r^   r   )ri   r  r   r   r   r  r  r  r  r   r  r   r  encoder_hidden_states_key_proj encoder_hidden_states_value_projr  r  r
  s                     ru   r   AttnAddedKVProcessor.__call__  sI    t9q=FJJw5A #Ugw(;< %**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E33EO		/59..}= A}5A}5%//B7??O%0rw   r  r=  r  r  r  rw   ru   r   r     sI     6:.2:: $:  3	:
 ,: 
: :rw   r   c                  B    \ rS rSrSrS r  S         SS jjrSrg)	r   iB  z
Processor for performing scaled dot-product attention (enabled by default if you're using PyTorch 2.0), with extra
learnable key and value matrices for the text encoder.
c                D    [        [        S5      (       d  [        S5      eg )Nr1   zWAttnAddedKVProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3    AttnAddedKVProcessor2_0.__init__H  %    q899i  :rw   Nc           	        [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pnUR                  XJU	SS	9nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  USS	9nUR                  U5      nUR                  U5      nUR                  USS	9nUR                  USS	9nUR                  (       dp  UR                  U5      nUR!                  U5      nUR                  USS	9nUR                  USS	9n["        R$                  " X/SS
9n["        R$                  " UU/SS
9nOUnUn[&        R(                  " XUUSSS9nUR                  SS5      R+                  U	SUR                  S   5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS5      R+                  UR                  5      nX(-   nU$ )Nr   rH   r  r  r   r+   r   r  )rA   r   r  F	attn_maskr  rE   r   )r   r  r   r  r   r  r  rT   r!  r.   rV   r   rZ   r[   rL   rW   rX   r   r  re   r1   r   r^   )ri   r  r   r   r   r  r  r  r  r   r  r   r  r  r  r  r  s                    ru   r    AttnAddedKVProcessor2_0.__call__N  s    t9q=FJJw5A #Ugw(;< %**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`jk4l ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&ua&8)-9N)O&+/??;P+Q()-)?)?@^hi)?)j&+/+A+ABblm+A+n((())M*CIIm,E((a(8C**5!*<E));AqICII?GQOE0C4E 663RW
 &//15==j"hnn]^N_` A}5A}5%//B7??O%0rw   r  r=  r  r  r  rw   ru   r   r   B  sN    
 6:.2== $=  3	=
 ,= 
= =rw   r   c                  B    \ rS rSrSrS r  S         SS jjrSrg)	r   i  YAttention processor used typically in processing the SD3-like self-attention projections.c                D    [        [        S5      (       d  [        S5      eg )Nr1   zUJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   JointAttnProcessor2_0.__init__  r  rw   Nc                   UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb<  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  US S 2S UR                   S   24   US S 2UR                   S   S 24   p2UR*                  (       d  UR-                  U5      nUR.                  S   " U5      nUR.                  S   " U5      nUb  X#4$ U$ )	Nr   r+   r   r   r   r  Fr  )r   rV   rW   rX   rI   r  r  rQ   rR   r\   rZ   r[   rb   rc   r   r  re   r1   r   r   r   rC   ra   r^   )ri   r  r   r   r   r  r  r  r   r  r  r  r5   head_dim encoder_hidden_states_query_projr  r  s                    ru   r   JointAttnProcessor2_0.__call__  s%    !"((+
 		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/O/T/TB

H0i1o - .L-P-PB

H.i1o + 0P/T/TB

H0i1o -   ,373D3DEe3f0  ,151B1BCa1b.IIuGQOE))SAqICIIu&FGQOE66u5TWchi%//15==j"djj[cNcd%((5 , a!48>>!#4!445a!2!445 1 (((,8M(N% A}5A}5 , 77  rw   r  r=  
r  r   r   torch.FloatTensorr   r  r   torch.FloatTensor | Noner9  r  r  r  rw   ru   r   r     sT    cw 4837L!L! )L!  1	L!
 1L! 
L! L!rw   r   c                  B    \ rS rSrSrS r  S         SS jjrSrg)	PAGJointAttnProcessor2_0i  r  c                D    [        [        S5      (       d  [        S5      eg )Nr1   zXPAGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   !PAGJointAttnProcessor2_0.__init__  s%    q899j  :rw   Nc           	     T   UnUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUR                  S   nUR	                  S5      u  pUR	                  S5      u  nnUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      n[        R                  " UUUSSS	9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   pUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR-                  S5      n[        R.                  " UU4UR0                  UR$                  S9n [3        S5      U S U2S U24'   U S U2S U24   R5                  S5        U R7                  S5      R7                  S5      n [        R                  " UUUU SSS9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      n[        R                  " X/5      n[        R                  " UU/5      nX#4$ )Nr  r   r   r   r   r+   r  Fr  r   r   -infr  )r   r   r  r  chunkrV   rW   rX   r\   rZ   r[   r   r  rI   re   r1   r   r   r   r^   rC   ra   r  r  r   r  fill_diagonal_r  )!ri   r  r   r   r   r  r  r   r  r  r  context_input_ndimidentity_block_sizehidden_states_orghidden_states_ptbencoder_hidden_states_orgencoder_hidden_states_ptb	query_orgkey_org	value_org$encoder_hidden_states_org_query_proj"encoder_hidden_states_org_key_proj$encoder_hidden_states_org_value_projr5   r  	query_ptbkey_ptb	value_ptb$encoder_hidden_states_ptb_query_proj"encoder_hidden_states_ptb_key_proj$encoder_hidden_states_ptb_value_projr   	full_masks!                                    ru   r   !PAGJointAttnProcessor2_0.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s! ,11!4 0=/B/B1/E,?T?Z?Z[\?]<!#< /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	::w	SE
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)% /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	 ..#KK' 29;K;KS\SbSbc	 AFf	&&&(<)<(<<= 	&&&(<)<(<<=LLQO ''*44Q7	::w	Y#Y^
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)%
 		#4"HI %		+DF_*` a33rw   r  r=  r  r  r  rw   ru   r  r    sS    c 4837P4P4 )P4  1	P4
 1P4 
P4 P4rw   r  c                  B    \ rS rSrSrS r  S         SS jjrSrg)	PAGCFGJointAttnProcessor2_0i  r  c                D    [        [        S5      (       d  [        S5      eg )Nr1   z[PAGCFGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   $PAGCFGJointAttnProcessor2_0.__init__  %    q899m  :rw   Nc           	        UnUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S   nUR	                  S5      u  nnn[
        R                  " UU/5      nUR	                  S5      u  nnn[
        R                  " UU/5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n UR                  U5      n!UR                  U5      n"[
        R                  " UU /SS9n[
        R                  " UU!/SS9n[
        R                  " UU"/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR-                  S5      n#[
        R.                  " U#U#4UR0                  UR$                  S9n$[3        S5      U$S U2S U24'   U$S U2S U24   R5                  S5        U$R7                  S5      R7                  S5      n$[        R                  " UUUU$SS	S9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      n[
        R                  " UU/5      n[
        R                  " UU/5      nX#4$ )Nr  r   r   r   r   r   r+   r  Fr  r   r   r  r  )r   r   r  r  r  r   r  rV   rW   rX   r\   rZ   r[   rI   re   r1   r   r   r   r^   rC   ra   r  r  r   r  r  r  )%ri   r  r   r   r   r  r  r  r  r   r  r  r  r  r  hidden_states_uncondr  r  encoder_hidden_states_uncondr  r	  r
  r  r  r  r  r  r5   r  r  r  r  r  r  r  r   r  s%                                        ru   r   $PAGCFGJointAttnProcessor2_0.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s!+11

 FSEXEXYZE[B/1B!II';=N&OP "''*		
(%%$)II/KMf.g$h! /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	::w	SE
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)% /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	 ..#KK' 29;K;KS\SbSbc	 AFf	&&&(<)<(<<= 	&&&(<)<(<<=LLQO ''*44Q7	::w	Y#Y^
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)%
 		#46G"HI %		+DF_*` a33rw   r  r=  r  r  r  rw   ru   r  r    sS    c 4837Y4Y4 )Y4  1	Y4
 1Y4 
Y4 Y4rw   r  c                  B    \ rS rSrSrS r  S         SS jjrSrg)	FusedJointAttnProcessor2_0i%  r  c                D    [        [        S5      (       d  [        S5      eg Nr1   zPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   #FusedJointAttnProcessor2_0.__init__(  !    q899pqq :rw   Nc                n   UnUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S   n	UR	                  U5      nUR                  S   S-  n[
        R                  " XSS9u  nnnUR                  U5      nUR                  S   S-  n[
        R                  " UUSS9u  nnn[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R                  U	SUR                  U-  5      nUR                  UR                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   p2UR                  S   " U5      nUR                  S   " U5      nUR                   (       d  UR#                  U5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nX#4$ )Nr  r   r   r   r+   r   r   r  Fr  r   )r   r   r  r  r%  r   splitr)  r  rI   re   r1   r   r   r   r^   rC   ra   )ri   r  r   r   r   r  r  r  r  r   r  r  r  r  qkv
split_sizer  r  r  encoder_qkvr  r  r  r5   r  s                            ru   r   #FusedJointAttnProcessor2_0.__call__,  s1    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s!*003
 kk-(YYr]a'
!KKR@sE ''(=> &&r*a/

 KKZR8		
,*, 		5"BCKii<=1E		5"BCKIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP66uc5TWchi%//15==j"djj[cNcd%((5 !0x~~a0001!X^^A.001 - A}5A}5$$$(OO4I$J!?)33B;CCJPWY_afgM"$9$C$CB$K$S$ST^`gioqv$w!33rw   r  r=  r  r  r  rw   ru   r"  r"  %  sT    cr 4837E4E4 )E4  1	E4
 1E4 
E4 E4rw   r"  c                  J    \ rS rSrSrSSS jjr  S	         S
S jjrSrg)r   it    
Processor for implementing memory efficient attention using xFormers.

Args:
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
Nc                    Xl         g r;  r   ri   r   s     ru   r3   #XFormersJointAttnProcessor.__init__      (rw   c           	        UnUR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      R	                  5       nUR                  U	5      R	                  5       n	UR                  U
5      R	                  5       n
UR
                  b  UR                  U5      nUR                  b  UR                  U	5      n	UGb  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  R!                  XXU R"                  UR$                  S9nUR'                  UR(                  5      nUR+                  U5      nUbQ  US S 2S UR,                  S   24   US S 2UR,                  S   S 24   p2UR.                  (       d  UR1                  U5      nUR2                  S   " U5      nUR2                  S   " U5      nUb  X#4$ U$ )Nr   r   	attn_biasoprH   r   )rV   rW   rX   r   
contiguousrQ   rR   r\   rZ   r[   rb   rc   r   r  r   r   r   r   rH   r   r   r   r   rC   ra   r^   )ri   r  r   r   r   r  r  r  r  r  r  r  r  r  s                 ru   r   #XFormersJointAttnProcessor.__call__  s    ! 		-(ii&		-(&&u-88:$$S)446&&u-88:;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/3/E/EFf/g/r/r/t,-1-C-CDb-c-n-n-p*/3/E/EFf/g/r/r/t,  ,373D3DEe3f0  ,151B1BCa1b.IIuGQOE))SAqICIIuGQOE ??D<M<MUYU_U_ @ 
 &((5..}= , a!48>>!#4!445a!2!445 1 (((,8M(N% A}5A}5 , 77  rw   r   r;  r   r<  r=  r  r  r  rw   ru   r   r   t  sU    	) 4837C!C! )C!  1	C!
 1C! 
C! C!rw   r   c                  N    \ rS rSrSrS r    S             SS jjrSrg)	AllegroAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Allegro model. It applies a normalization layer and rotary embedding on the query and key vector.
c                D    [        [        S5      (       d  [        S5      eg )Nr1   zWAllegroAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3    AllegroAttnProcessor2_0.__init__  r  rw   Nc           	     4   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XMU	5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUb6  UR                  (       d%  SSKJn  U" XS   US   5      nU" UUS   US   5      n[         R"                  " UUUUSSS	9nUR	                  SS5      R%                  U	SUR                  U-  5      nUR'                  UR(                  5      nUR*                  S   " U5      nUR*                  S   " U5      nUS:X  a$  UR	                  SS
5      R%                  U	W
WW5      nUR,                  (       a  X'-   nX!R.                  -  nU$ )Nr  r   r   r+   )apply_rotary_emb_allegror   r  Fr  r   )rP   r   r   r  r  r  rI   r.   rV   rT   r!  rW   rX   r9   
embeddingsr?  re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r   r  r  r  r  r   r  r  r  r5   r  r?  s                        ru   r    AllegroAttnProcessor2_0.__call__  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP '0G0G<,UQ4GIYZ[I\]E*30@0CEUVWEXYC 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  NNNNr  r   r   r>  r   r?  r   r?  r  r?  r  r?  r9  r>  r  r  rw   ru   r;  r;    sq    
 6:.2$(04OO $O  3	O
 ,O "O .O 
O Orw   r;  c                  <    \ rS rSrSrS r S       SS jjrSrg)	AuraFlowAttnProcessor2_0i'  z;Attention processor used typically in processing Aura Flow.c                h    [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )Nr1   ry   2.1zAuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. rd   re   r   r  r  s    ru   r3   !AuraFlowAttnProcessor2_0.__init__*  s:    q899>NsTY>Z>Z v  ?[9rw   Nc           	     D   UR                   S   nUR                  U5      nUR                  U5      nUR                  U5      n	Ub3  UR	                  U5      n
UR                  U5      nUR                  U5      nUR                   S   nXR                  -  nUR                  USUR                  U5      nUR                  USUR                  U5      nU	R                  USUR                  U5      n	UR                  b  UR                  U5      nUR                  b  UR                  U5      nUb  W
R                  USUR                  U5      n
WR                  USUR                  U5      nWR                  USUR                  U5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9n[        R                  " X/SS9n	UR                  SS5      nUR                  SS5      nU	R                  SS5      n	[         R"                  " XxU	SUR$                  SS9nUR                  SS5      R'                  USUR                  U-  5      nUR)                  UR*                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR,                  S   " U5      nUR,                  S   " U5      nUb  UR/                  U5      nUb  X#4$ U$ )	Nr   r+   r   r   r   r  Fr  rH   rE   )r   rV   rW   rX   r\   rZ   r[   rI   r  rQ   rR   rb   rc   r   r  r  re   r1   rH   r   r   r   r^   ra   )ri   r  r   r   r  r  r   r  r  r  r  r  r  r5   r  s                  ru   r   !AuraFlowAttnProcessor2_0.__call__0  s    #((+
 		-(ii&		-( !,/3?T/U,-1__=R-S*/3?T/U, IIbM	

*

:r4::x@hhz2tzz8<

:r4::x@ ;;"KK&E;;"++c"C !,/O/T/TB

H0, .L-P-PQ[]_aeakakmu-v*/O/T/TB

H0,   ,373D3DEe3f0  ,151B1BCa1b.II?GQOE));AqICII?GQOE1%mmAq!1% 66DJJ%
 &//15==j"djj[cNcd%((5 !,a!6!<!<Q!?!AABa!A#8#>#>q#A!AAB 1 A}5A}5 ,$(OO4I$J! , 77  rw   r  r;  r  r   r   r  r   r  r9  r  r  r  rw   ru   rE  rE  '  sE    E 48	Q!Q! )Q!  1	Q! 
Q! Q!rw   rE  c                  <    \ rS rSrSrS r S       SS jjrSrg)	FusedAuraFlowAttnProcessor2_0i  zRAttention processor used typically in processing Aura Flow with fused projections.c                h    [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )Nr1   ry   rG  zFusedAuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. rH  r  s    ru   r3   &FusedAuraFlowAttnProcessor2_0.__init__  s:    q899>NsTY>Z>Z {  ?[9rw   Nc           	     h   UR                   S   nUR                  U5      nUR                   S   S-  n[        R                  " XxSS9u  pnUb<  UR	                  U5      nUR                   S   S-  n[        R                  " XSS9u  nnnU
R                   S   nUUR
                  -  nU	R                  USUR
                  U5      n	U
R                  USUR
                  U5      n
UR                  USUR
                  U5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
Ub  WR                  USUR
                  U5      nWR                  USUR
                  U5      nWR                  USUR
                  U5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  " X/SS9nU	R                  SS5      n	U
R                  SS5      n
UR                  SS5      n[        R                  " XUSUR                  SS	9nUR                  SS5      R!                  USUR
                  U-  5      nUR#                  U	R$                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR&                  S   " U5      nUR&                  S   " U5      nUb  UR)                  U5      nUb  X#4$ U$ )
Nr   r+   r   r   r   r   r  FrK  )r   r%  r   r(  r)  rI   r  rQ   rR   rb   rc   r  r  re   r1   rH   r   r   r   r^   ra   )ri   r  r   r   r  r  r   r)  r*  r  r  r  r+  r  r  r  r5   r  s                     ru   r   &FusedAuraFlowAttnProcessor2_0.__call__  s;    #((+
 kk-(YYr]a'
!KKR@E !,++,ABK$**2.!3J
 K<	0.0 IIbM	

*

:r4::x@hhz2tzz8<

:r4::x@ ;;"KK&E;;"++c"C !,/O/T/TB

H0, .L-P-PQ[]_aeakakmu-v*/O/T/TB

H0,   ,373D3DEe3f0  ,151B1BCa1b.II?GQOE));AqICII?GQOE1%mmAq!1% 66DJJ%
 &//15==j"djj[cNcd%((5 !,a!6!<!<Q!?!AABa!A#8#>#>q#A!AAB 1 A}5A}5 ,$(OO4I$J! , 77  rw   r  r;  rM  r  r  rw   ru   rO  rO    sE    \ 48	U!U! )U!  1	U! 
U! U!rw   rO  c                  F    \ rS rSrSrS r  S           SS jjrSrg)	CogVideoXAttnProcessor2_0i  
Processor for implementing scaled dot-product attention for the CogVideoX model. It applies a rotary embedding on
query and key vectors, but does not include spatial normalization.
c                D    [        [        S5      (       d  [        S5      eg Nr1   zVCogVideoXAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   "CogVideoXAttnProcessor2_0.__init__  !    q899vww :rw   Nc           	        UR                  S5      n[        R                  " X2/SS9nUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
UR                  U5      nUR                  U5      nUR                  S   nXR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      nUbY  SSKJn  U" U
S S 2S S 2US 24   U5      U
S S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [         R"                  " XXSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUR)                  XbR                  S5      U-
  /SS9u  p2X#4$ )
Nr   r   r+   r   r  r  Fr  r   )r  r   r  r   r  r  rI   rV   rW   rX   r  rQ   rR   r@  r  r9   re   r1   r   r^   r(  )ri   r  r   r   r   r  text_seq_lengthr   r  r   r  r  r  r5   r  r  s                   ru   r   "CogVideoXAttnProcessor2_0.__call__  sk    044Q7		#8"HaP)6)<)<&
Q%!88ZdeN+00ZZ^MaMabdMefN		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '4,<U1aIYCY=Z\l,mE!Q(()**.>s1aIYCY?Z\l.mAq/**+663RW
 &//15==j"djj[cNcd A}5A}5/</B/B003oEFA 0C 0
, 33rw   r  r=  r  r   r   r>  r   r>  r   r?  r  r?  r9  r>  r  r  rw   ru   rU  rU    sY    
x /3048484 $84  ,	84
 ,84 .84 
84 84rw   rU  c                  F    \ rS rSrSrS r  S           SS jjrSrg)	FusedCogVideoXAttnProcessor2_0i*	  rV  c                D    [        [        S5      (       d  [        S5      eg rX  r  r  s    ru   r3   'FusedCogVideoXAttnProcessor2_0.__init__0	  rZ  rw   Nc           	     0   UR                  S5      n[        R                  " X2/SS9nUc  UR                  OUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
U
R                  S   S-  n[        R                  " XSS9u  pnUR                  S   nXR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUbY  SSKJn  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [        R                   " XXSSS	9nUR                  SS5      R#                  USUR                  U-  5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR                  XbR                  S5      U-
  /SS9u  p2X#4$ )Nr   r   r+   r   r   r\  r  Fr  r   )r  r   r  r   r  r  rI   r%  r(  r  rQ   rR   r@  r  r9   re   r1   r   r^   )ri   r  r   r   r   r  r]  r   r  r   r)  r*  r  r  r  r5   r  r  s                     ru   r   'FusedCogVideoXAttnProcessor2_0.__call__4	  s    044Q7		#8"HaP $9#@MF[FaFa 	'
Q %!88ZdeN+00ZZ^MaMabdMefNkk-(YYr]a'
!KKR@EIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '4,<U1aIYCY=Z\l,mE!Q(()**.>s1aIYCY?Z\l.mAq/**+663RW
 &//15==j"djj[cNcd A}5A}5/</B/B003oEFA 0C 0
, 33rw   r  r=  r_  r  r  rw   ru   ra  ra  *	  sY    
x /304:4:4 $:4  ,	:4
 ,:4 .:4 
:4 :4rw   ra  c                  J    \ rS rSrSrSSS jjr  S	         S
S jjrSrg)r   iq	  r.  Nc                    Xl         g r;  r   r0  s     ru   r3   %XFormersAttnAddedKVProcessor.__init__}	  r2  rw   c           	        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pgnUR                  XGU5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n	UR                  U	5      n	UR                  U5      n
UR                  U5      nUR                  U
5      n
UR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9nOU
nUn[         R"                  R%                  XXU R&                  UR(                  S9nUR+                  U	R,                  5      nUR/                  U5      nUR0                  S   " U5      nUR0                  S   " U5      nUR                  SS5      R3                  UR                  5      nX%-   nU$ )Nr   r   r+   r   r   r4  r   )r  r   r  r  rT   r!  r.   rV   r   rZ   r[   rL   rW   rX   r   r  r   r   r   r   rH   r   r   r   r^   r   )ri   r  r   r   r   r  r   r  r   r  r  r  r  r  s                 ru   r   %XFormersAttnAddedKVProcessor.__call__	  s:    !%**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E ??D<M<MUYU_U_ @ 
 &((5..}= A}5A}5%//B7??O%0rw   r   r;  r9  r=  r  r  r  rw   ru   r   r   q	  sN    	) 6:.255 $5  3	5
 ,5 
5 5rw   r   c                  P    \ rS rSrSrSSS jjr   S	           S
S jjrSrg)r   i	  r.  Nc                    Xl         g r;  r   r0  s     ru   r3   XFormersAttnProcessor.__init__	  r2  rw   c           	     z   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUR                  XOU5      nUb#  UR
                  u  nnnUR                  SUS5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       n[$        R&                  R)                  UUUX@R*                  UR,                  S	9nUR/                  UR0                  5      nUR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nU
S:X  a$  UR                  SS
5      R7                  UWWW5      nUR8                  (       a  X)-   nX!R:                  -  nU$ )Nr   rH   r  r  r  r   r   r+   r4  r   )r   r  r   rP   r   r   r  r  r  expandr.   rV   rT   r!  rW   rX   r   r7  r   r   r   r   rH   r   r   r   r^   r   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r  r   r  r  r  
key_tokensr   query_tokensr  r  r  s                        ru   r   XFormersAttnProcessor.__call__	  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	"
 44^Q[\% "/!4!4A|Q+222|RHN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-88:$$S)446&&u-88: ??3<M<MUYU_U_ @ 
 &((5..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r   r;  r9  r  r  r  r  rw   ru   r   r   	  sc    	) 6:.2$(LL $L  3	L
 ,L "L 
L Lrw   r   c                  H    \ rS rSrSrS r   S           SS jjrSrg)	r   i
  z
Processor for implementing flash attention using torch_npu. Torch_npu supports only fp16 and bf16 data types. If
fp32 is used, F.scaled_dot_product_attention will be used for computation, but the acceleration effect on NPU is
not significant.

c                8    [        5       (       d  [        S5      eg )NzTAttnProcessorNPU requires torch_npu extensions and is supported only on npu devices.)r   r  r  s    ru   r3   AttnProcessorNPU.__init__
  s    %''tuu (rw   Nc                   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  SSUR
                  S   S5      nUR                  [        R                  :X  a%  [        R                  " UR                  5       5      nOUR                  5       nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR!                  U5      nUc  UnO"UR"                  (       a  UR%                  U5      nUR'                  U5      nUR)                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  [        R*                  [        R,                  4;   aR  [.        R0                  " UUUUR                  S	S US
[2        R4                  " UR
                  S   5      -  SSS
SSS9S   nO[6        R8                  " UUUUSSS9nUR                  SS5      R;                  USUR                  U-  5      nUR=                  UR                  5      nUR>                  S   " U5      nUR>                  S   " U5      nU
S:X  a$  UR                  SS5      R;                  UWWW5      nUR@                  (       a  X)-   nX!RB                  -  nU$ )Nr   rH   r  r  r  r   r   r+   BNSDr   i   F)	input_layoutpse
atten_maskrH   pre_tockensnext_tockens	keep_probsyncinner_preciser  r  r   )"r   r  r   rP   r   r   r  r  r  rI   repeatr   r   r6  logical_notr.   rV   rT   r!  rW   rX   float16bfloat16	torch_npunpu_fusion_attentionmathsqrtre   r1   r   r   r^   r>   r=   ri   r  r   r   r   r  r  r  r  r  r  r   r  r  r  r  r   r  r  r  r5   r  s                         ru   r   AttnProcessorNPU.__call__"
  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN+221a9L9LQ9OQRSN##uzz1!&!2!2>3F3F3H!I!/!4!4!6??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ;;5==%..99%::

#)DIIekk"o66!" M" ::sE^sV[M &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  r  r  r  r  rw   ru   r   r   
  sd    v 6:.2$(cc $c  3	c
 ,c "c 
c crw   r   c                  H    \ rS rSrSrS r   S           SS jjrSrg)	rf   i
  k
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
c                D    [        [        S5      (       d  [        S5      eg r$  r  r  s    ru   r3   AttnProcessor2_0.__init__
  r&  rw   Nc           	        [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      n[$        R&                  " UUUUS	S
S9nUR                  SS5      R)                  USUR                  U-  5      nUR+                  UR,                  5      nUR.                  S   " U5      nUR.                  S   " U5      nU
S:X  a$  UR                  SS5      R)                  UWWW5      nUR0                  (       a  X)-   nX!R2                  -  nU$ )Nr   rH   r  r  r  r   r   r+   r  Fr  r   )r   r  r   rP   r   r   r  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   re   r1   r   r   r   r^   r>   r=   r  s                         ru   r   AttnProcessor2_0.__call__
  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  r  r  r  r  rw   ru   rf   rf   
  sd    r 6:.2$(RR $R  3	R
 ,R "R 
R Rrw   rf   c                  P    \ rS rSrSrSSS jjr   S	           S
S jjrSrg)r}   i
  r
Processor for implementing scaled dot-product attention with pallas flash attention kernel if using `torch_xla`.
Nc                    [        [        S5      (       d  [        S5      e[        SS5      (       a  [        S5      e[	        5       (       a  [        SS5      (       a  [        S5      eXl        g )Nr1   zXXLAFlashAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.ry   rz   6XLA flash attention requires torch_xla version >= 2.3.r{   DSPMD support for XLA flash attention needs torch_xla version >= 2.4.)rd   re   r  r   r   r   )ri   r   s     ru   r3   !XLAFlashAttnProcessor2_0.__init__
  sa    q899j   U++VWW99-c599dee,rw   c           	        UnUR                   b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        S UUU4 5       5      (       a  Ubn  UR                  U
SSUR                  S   5      nUR!                  5       R#                  US:H  [!        S5      5      R#                  US:H  [!        S5      5      nUU-   nU[$        R&                  " UR                  S	   5      -  n[)        5       (       a  U R*                  OS n[-        UUUS
US9nO.[.        R1                  S5        [2        R4                  " UUUUSS
S9nUR	                  SS5      R7                  U
SUR                  U-  5      nUR9                  UR:                  5      nUR<                  S   " U5      nUR<                  S   " U5      nU	S:X  a$  UR	                  SS5      R7                  U
WWW5      nUR>                  (       a  X(-   nX!R@                  -  nU$ )Nr  r   r   r+   c              3  D   #    U  H  oR                   S    S:  v   M     g7f)r   i   N)r   ).0r   s     ru   	<genexpr>4XLAFlashAttnProcessor2_0.__call__.<locals>.<genexpr>0  s     I5H6||A$&5Hs    r   r  r  r   F)causalr   z[Unable to use the flash attention pallas kernel API call due to QKV sequence length < 4096.r  r   )!rP   r   r   r  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   allr  masked_fillr  r  r   r   r   r   r   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r  r  r  r5   r  r   s                         ru   r   !XLAFlashAttnProcessor2_0.__call__
  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C IeS%5HIII)!/!4!4ZA~G[G[\^G_!` #((* [1!4eFmD [1!4eCjA  N*TYYu{{1~..E4;IIT004N+E3e\jkMNNm ::sE^sV[M &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   )r   r;  )r   r8  r  r  r  r  rw   ru   r}   r}   
  sc    	- 6:.2$(aa $a  3	a
 ,a "a 
a arw   r}   c                  B    \ rS rSrSrS r  S         SS jjrSrg)	MochiVaeAttnProcessor2_0iZ  z(
Attention processor used in Mochi VAE.
c                D    [        [        S5      (       d  [        S5      eg r$  r  r  s    ru   r3   !MochiVaeAttnProcessor2_0.__init___  r&  rw   Nc           	     ^   UnUR                   S   S:H  nUc  UR                   OUR                   u  pxn	Ub<  UR                  XHU5      nUR                  XqR                  SUR                   S   5      nU(       a`  UR	                  U5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ UR                  U5      n
Uc  UnUR                  U5      nUR	                  U5      nUR                   S   nXR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " XXSUR                  S9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  U
R$                  5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ )Nr   r+   r   r   r  r  )r   r  r  rI   rX   r^   r>   r=   rV   rW   r  rQ   rR   re   r1   rE   r   r   r   )ri   r  r   r   r   r  is_single_framer   r  r   r  r  r  r5   r  s                  ru   r   !MochiVaeAttnProcessor2_0.__call__c  s{    !'--a0A5 $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN IIm4M !KKN=9M KKN=9M'' - 8),F,FFM  		-( ($1!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 663RVR`R`
 &//15==j"djj[cNcd%((5 A}5A}5##)4M%(B(BBrw   r  r=  r  r  r  rw   ru   r  r  Z  sV    r 6:.2II $I  3	I
 ,I 
I Irw   r  c                  ^    \ rS rSrSrS r      SS jr   S	           S
S jjrSrg)StableAudioAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Stable Audio model. It applies rotary embedding on query and key vector, and allows MHA, GQA or MQA.
c                D    [        [        S5      (       d  [        S5      eg )Nr1   z[StableAudioAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   $StableAudioAttnProcessor2_0.__init__  r  rw   c                    SSK Jn  US   R                  S   nUSS U24   USUS 24   peU" XRSSS9n[        R                  " Xv4SS	9nU$ )
Nr   r\  r   r+   .Tr   use_realuse_real_unbind_dimr   )r@  r  r   r   r  )	ri   r  	freqs_cisr  rot_dimx_to_rotatex_unrotated	x_rotatedouts	            ru   apply_partial_rotary_emb4StableAudioAttnProcessor2_0.apply_partial_rotary_emb  sa    
 	1A,$$R(#$S(7(]#3QsGH}5E[$[d`bc	ii0b9
rw   Nc           	     d   SSK Jn  UnUR                  nUS:X  a3  UR                  u  ppUR	                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XMU	5      nUR	                  XR                  SUR                  S   5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   UR                  -  nUR                  S   U-  nUR	                  U	SUR                  U5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUUR                  :w  a]  UR                  U-  n[        R                  " UUSUR                  S   U-  S9n[        R                  " UUSUR                  S   U-  S9nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUb  UR"                  nUR"                  nUR%                  [        R&                  5      nUR%                  [        R&                  5      nUS   R                  S   nUSS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR*                  (       d2  USS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR%                  U5      nUR%                  U5      n[,        R.                  " UUUUSSS9nUR                  SS5      R1                  U	SUR                  U-  5      nUR%                  UR"                  5      nUR2                  S   " U5      nUR2                  S   " U5      nUS:X  a$  UR                  SS
5      R1                  U	W
WW5      nUR4                  (       a  X'-   nX!R6                  -  nU$ )Nr   r\  r  r   r+   r  r   .Tr   r  r   r  Fr  )r@  r  r   r   r  r  r  rI   rV   rT   r!  rW   rX   r   r  rQ   rR   r   r   rq  r  r9   re   r1   r   r^   r>   r=   )ri   r  r   r   r   
rotary_embr  r  r  r   r  r  r  r  r   r  r  r  r  rj   heads_per_kv_headquery_dtype	key_dtyper  query_to_rotatequery_unrotatedquery_rotatedkey_to_rotatekey_unrotatedkey_rotateds                                 ru   r   $StableAudioAttnProcessor2_0.__call__  s    	1 "''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN		-( ($1!__$($C$CDY$Z!ii-.		/0;;r?djj099R=H,

:r4::x@JJ1aPhhz2x:DDQJ

:r8X>HHANtzz! $

h 6))#/@aUXU^U^_`UaduUuvC++(aU[[^N_=_E ;;"KK&E;;"++c"C !++K		IHHU]]+E&&'C m))"-G/4S(7(]/CU3PWPX=EY_O,_jSWmopMII}o>BGE**/23=/A3sGH}CU}.}jSWmopiim <"EHH[)E&&#C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  )r  r>  r  ztuple[torch.Tensor]r9  r>  r  )r  r   r   r>  r   r?  r   r?  r  r?  r9  r>  )	r   r@  rA  rB  rC  r3   r  r   rE  r  rw   ru   r  r    s    
 ' 
	& 6:.2*.gg $g  3	g
 ,g (g 
g grw   r  c                  N    \ rS rSrSrS r    S             SS jjrSrg)	HunyuanAttnProcessor2_0i4  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                D    [        [        S5      (       d  [        S5      eg r$  r  r  s    ru   r3    HunyuanAttnProcessor2_0.__init__:  r&  rw   Nc           	        SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUb#  U" UU5      nUR"                  (       d	  U" UU5      n[$        R&                  " UUUUSSS9nUR                  SS5      R)                  U
SUR                  U-  5      nUR+                  UR,                  5      nUR.                  S	   " U5      nUR.                  S   " U5      nU	S:X  a$  UR                  SS
5      R)                  U
WWW5      nUR0                  (       a  X(-   nX!R2                  -  nU$ Nr   r\  r  r   r+   r  Fr  r   r   )r@  r  rP   r   r   r  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r  r  r  r5   r  s                        ru   r    HunyuanAttnProcessor2_0.__call__>  s    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  rB  rC  r  r  rw   ru   r  r  4  sr    
r 6:.2$(04UU $U  3	U
 ,U "U .U 
U Urw   r  c                  N    \ rS rSrSrS r    S             SS jjrSrg)	FusedHunyuanAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0) with fused
projection layers. This is used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on
query and key vector.
c                D    [        [        S5      (       d  [        S5      eg )Nr1   z\FusedHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   %FusedHunyuanAttnProcessor2_0.__init__  s%    q899n  :rw   Nc           	     <   SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnnOoUR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR"                  b  UR#                  U5      nUR$                  b  UR%                  U5      nUb#  U" UU5      nUR&                  (       d	  U" UU5      n[(        R*                  " UUUUSS	S
9nUR                  SS5      R-                  U
SUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU	S:X  a$  UR                  SS5      R-                  U
WWW5      nUR4                  (       a  X(-   nX!R6                  -  nU$ )Nr   r\  r  r   r+   r   r   r  Fr  r   r   )r@  r  rP   r   r   r  r  r  rI   r.   r%  r   r(  rT   r!  rV   r'  rQ   rR   r9   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r)  r*  r  r  r  kvr5   r  s                           ru   r   %FusedHunyuanAttnProcessor2_0.__call__  s_    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM (++m,C2!+J %C DE3(,(G(GH](^%IIm,E12B"*JR<JCIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  rB  rC  r  r  rw   ru   r  r    sq     6:.2$(04WW $W  3	W
 ,W "W .W 
W Wrw   r  c                  N    \ rS rSrSrS r    S             SS jjrSrg)	PAGHunyuanAttnProcessor2_0i  O  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a normalization layer and rotary embedding on query and key vector. This
variant of the processor employs [Pertubed Attention Guidance](https://huggingface.co/papers/2403.17377).
c                D    [        [        S5      (       d  [        S5      eg )Nr1   zZPAGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   #PAGHunyuanAttnProcessor2_0.__init__  s%    q899l  :rw   Nc           	     f   SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S5      u  pUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      nUb#  U" UU5      nUR$                  (       d	  U" UU5      n[&        R(                  " UUUUSSS9nUR                  SS5      R+                  U
SUR                  U-  5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      n[2        R4                  " X/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ r  )r@  r  rP   r   r   r  r  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   r   r   r   r^   r   r  r>   r=   )ri   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r  r  r   r  r  r  r5   r  s                          ru   r   #PAGHunyuanAttnProcessor2_0.__call__
  s    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM 0=/B/B1/E, (='D##J_JeJe 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+, ($5!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o ??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrw   r  rB  rC  r  r  rw   ru   r  r    sq     6:.2$(04kk $k  3	k
 ,k "k .k 
k krw   r  c                  N    \ rS rSrSrS r    S             SS jjrSrg)	PAGCFGHunyuanAttnProcessor2_0ix  r  c                D    [        [        S5      (       d  [        S5      eg )Nr1   z]PAGCFGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   &PAGCFGHunyuanAttnProcessor2_0.__init__  s%    q899o  :rw   Nc           	        SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S5      u  pn[        R                  " X/5      nUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      nUb#  U" UU5      nUR(                  (       d	  U" UU5      n[*        R,                  " UUUUSSS	9nUR                  SS5      R/                  U
SUR                  U-  5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR#                  U5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      n[        R                  " UU/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ )Nr   r\  r  r   r   r+   r  Fr  r   r   )r@  r  rP   r   r   r  r  r  r   r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r  r  r  r   r  r  r  r5   r  s                           ru   r   &PAGCFGHunyuanAttnProcessor2_0.__call__  s"    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM FSEXEXYZE[B1B!II';&OP (='D##J_JeJe 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+, ($5!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o ??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#46G"HI##)4M%(B(BBrw   r  rB  rC  r  r  rw   ru   r  r  x  sq     6:.2$(04ll $l  3	l
 ,l "l .l 
l lrw   r  c                  R    \ rS rSrSrS r    S               SS jjrSrg)	LuminaAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the LuminaNextDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                D    [        [        S5      (       d  [        S5      eg r$  r  r  s    ru   r3   LuminaAttnProcessor2_0.__init__  r&  rw   Nc                   SSK Jn  UR                  n	U	S:X  a3  UR                  u  ppUR	                  XX-  5      R                  SS5      nUR                  u  pnUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUR                  S   nUUR                  -  nUR                  nUU-  nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUR	                  U
SUR                  U5      nUR	                  U
SUU5      nUR	                  U
SUU5      nUb  U" UUSS9nUb  U" UUSS9nUR                  U5      UR                  U5      nnUc  S nOGUb8  [        R                  " [        R                   " X5      5      UR"                  -  nOUR"                  nUR                  U-  nUS:  ah  UR%                  S5      R'                  SSSUS5      R)                  SS5      nUR%                  S5      R'                  SSSUS5      R)                  SS5      nUR+                  5       R	                  U
SSS5      nUR-                  SUR                  US5      nUR                  SS5      nUR                  SS5      nUR                  SS5      n[.        R0                  " UUUUUS	9nUR                  SS5      R                  U5      nU$ )
Nr   r\  r  r   r+   F)r  r   )r  rH   )r@  r  r   r   r  r  rV   rW   rX   rI   r   rQ   rR   r   r  r  logrH   r  r  r  r6  ro  re   r1   )ri   r  r   r   r   query_rotary_embkey_rotary_embbase_sequence_lengthr  r  r   r  r  r  r  r   r  r  r  r7   r5   r  r   rj   softmax_scalen_reps                             ru   r   LuminaAttnProcessor2_0.__call__  s    	1"''
?1>1D1D.J)..zFNS]]^_abcM)6)<)<&
Q 		-(ii-.		/0KKO	IIbM	

* ( ;;"KK&E;;"++c"C

:r4::x@hhz2x:

:r8X> '$U,<uME%"3GCXXe_cffUms ! M#/ $		$((?*Y Z]a]g]g g $

 

h&A:--"))!Q5!<DDQJCOOA&--aAua@HHANE (,,.33J1bI'..r4::PRS1%mmAq!1% 663}
 &//1588?rw   r  rB  )r  r   r   r>  r   r>  r   r?  r  r?  r  r?  r  r5  r9  r>  r  r  rw   ru   r  r    s}    
r /304.2+/SS $S  ,	S
 ,S .S ,S )S 
S Srw   r  c                  H    \ rS rSrSrS r   S           SS jjrSrg)	FusedAttnProcessor2_0iT  u  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). It uses
fused projection layers. For self-attention modules, all projection matrices (i.e., query, key, value) are fused.
For cross-attention modules, key and value projection matrices are fused.

> [!WARNING] > This API is currently 🧪 experimental in nature and can change in future.
c                D    [        [        S5      (       d  [        S5      eg )Nr1   z`FusedAttnProcessor2_0 requires at least PyTorch 2.0, to use it. Please upgrade PyTorch to > 2.0.r  r  s    ru   r3   FusedAttnProcessor2_0.__init__]  s%    q899r  :rw   Nc           	     F   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR
                  S   S	-  n[        R                  " UUSS
9u  nnnOoUR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR
                  S   S-  n[        R                  " UUSS
9u  nnUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      n[(        R*                  " UUUUSSS9nUR                  SS5      R-                  USUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU
S:X  a$  UR                  SS5      R-                  UWWW5      nUR4                  (       a  X)-   nX!R6                  -  nU$ )Nr   rH   r  r  r  r   r   r+   r   r   r  Fr  r   )r   r  r   rP   r   r   r  r  r  rI   r.   r%  r   r(  rT   r!  rV   r'  rQ   rR   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r  r  r  r   r  r  r  r  r   r)  r*  r  r  r  r  r5   r  s                            ru   r   FusedAttnProcessor2_0.__call__c  sb    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM (++m,C2!+J %C DE3(,(G(GH](^%IIm,E12B"*JR<JCIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  r  r  r  r  rw   ru   r  r  T  sc     6:.2$(TT $T  3	T
 ,T "T 
T Trw   r  c                     ^  \ rS rSrSr       S             SU 4S jjjr  S         S	S jjrSrU =r$ )
r   i  a.  
Processor for implementing memory efficient attention using xFormers for the Custom Diffusion method.

Args:
train_kv (`bool`, defaults to `True`):
    Whether to newly train the key and value matrices corresponding to the text features.
train_q_out (`bool`, defaults to `True`):
    Whether to newly train query matrices corresponding to the latent image features.
hidden_size (`int`, *optional*, defaults to `None`):
    The hidden size of the attention layer.
cross_attention_dim (`int`, *optional*, defaults to `None`):
    The number of channels in the `encoder_hidden_states`.
out_bias (`bool`, defaults to `True`):
    Whether to include the bias parameter in `train_q_out`.
dropout (`float`, *optional*, defaults to 0.0):
    The dropout probability to use.
attention_op (`Callable`, *optional*, defaults to `None`):
    The base
    [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to use
    as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best operator.
c                X  > [         TU ]  5         Xl        X l        X0l        X@l        Xpl        U R                  (       aH  [        R                  " U=(       d    UUSS9U l	        [        R                  " U=(       d    UUSS9U l
        U R                  (       a  [        R                  " X3SS9U l        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g r  )r2   r3   r   r   r   r:   r   r   rU   r   r  r  r]   r  r_   r`   )	ri   r   r   r   r:   rq   r?   r   rt   s	           ru   r3   -CustomDiffusionXFormersAttnProcessor.__init__  s     	 &&#6 ( ==)+3F3U+Wbin)oD&)+3F3U+Wbin)oD&)+;RW)XD&+-==+<D(((//		+Ya0bc((//

70CD	 rw   c           	        Uc  UR                   OUR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       a  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR%                  U5      R'                  5       nUR%                  U
5      R'                  5       n
UR%                  U5      R'                  5       n[(        R*                  R-                  XXU R.                  UR0                  S9nUR	                  UR                  5      nUR3                  U5      nU R                  (       a,  U R4                  S   " U5      nU R4                  S   " U5      nU$ UR6                  S   " U5      nUR6                  S   " U5      nU$ )NFTr   r  r4  r   )r   r  r   r  r   rV   r   r   rT   r!  r   r   r  rW   rX   r   r  r  r   r7  r   r   r   r   rH   r   r  r^   )ri   r  r   r   r   r   r  r   r  r  r  r  r  s                ru   r   -CustomDiffusionXFormersAttnProcessor.__call__  s    $9#@MF[FaFa 	'
Q 44^V`a..}=@@AQAQAWAWXEIIm..tyy/?/?/E/EFGE (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE&&u-88:$$S)446&&u-88: ??D<M<MUYU_U_ @ 
 &((5..}= 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrw   )	r   r:   r   r   r  r  r  r   r   )TFNNTr  N)r   r6  r   r6  r   r5  r:   r5  rq   r6  r?   r  r   r<  r=  r  r  rG  s   @ru   r   r     s    0 !"&*.(,EE E  	E
 (E E E &E E@ 6:.2>> $>  3	>
 ,> 
> >rw   r   c                  z   ^  \ rS rSrSr      S           SU 4S jjjr  S         S	S jjrSrU =r$ )
r   i.  uS  
Processor for implementing attention for the Custom Diffusion method using PyTorch 2.0’s memory-efficient scaled
dot-product attention.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
c                L  > [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g r  r  r  s          ru   r3   (CustomDiffusionAttnProcessor2_0.__init__B  r  rw   c           	        UR                   u  pVnUR                  XFU5      nU R                  (       a  U R                  U5      nOUR	                  U5      nUc  Sn	UnO$Sn	UR
                  (       a  UR                  U5      nU R                  (       a  U R                  UR                  U R                  R                  R                  5      5      n
U R                  UR                  U R                  R                  R                  5      5      nU
R                  UR                  R                  R                  5      n
UR                  UR                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR                   S   nXR$                  -  nUR'                  USUR$                  U5      R)                  SS5      nU
R'                  USUR$                  U5      R)                  SS5      n
UR'                  USUR$                  U5      R)                  SS5      n[*        R,                  " XXSSS9nUR)                  SS5      R/                  USUR$                  U-  5      nUR                  UR                  5      nU R                  (       a,  U R0                  S   " U5      nU R0                  S   " U5      nU$ UR2                  S   " U5      nUR2                  S   " U5      nU$ )	NFTr   r  r+   r   r  r   )r   r  r   r  rV   rT   r!  r   r   r   r   r   r  rW   rX   r   r  r  rI   r  r  re   r1   r   r  r^   )ri   r  r   r   r   r   r  r   r  r  r  r  r  r5   r  s                  ru   r   (CustomDiffusionAttnProcessor2_0.__call__\  s    *7)<)<&
Q44^V`a..}=EIIm,E (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E ))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE!''+	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP 663RW
 &//15==j"djj[cNcd%((5 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrw   r  r  r  r=  r  r  rG  s   @ru   r   r   .  s    *  "&*.EE E  	E
 (E E E E< 6:.2AA $A  3	A
 ,A 
A Arw   r   c                  F    \ rS rSrSrSS jr  S         S	S jjrSrg)
r   i  a  
Processor for implementing sliced attention.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
c                    Xl         g r;  r   ri   r   s     ru   r3   SlicedAttnProcessor.__init__      $rw   Nc                d   UnUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUc  UR                  OUR                  u  p{nUR	                  XKU5      nUR
                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  S   nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  u  nnn[        R                  " UUXR                  -  4UR                  UR                   S9n[#        US-
  U R$                  -  S-   5       Hj  nUU R$                  -  nUS-   U R$                  -  nUUU nUUU nUb  UUU OS nUR'                  UUU5      n[        R(                  " UUUU 5      nUUUU& Ml     UR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUS:X  a$  UR                  SS5      R/                  UWW	W
5      nUR0                  (       a  X%-   nX!R2                  -  nU$ )Nr  r   r   r+   r   r   r   )r   r   r  r  r  r.   rV   r   rT   r!  rW   rX   r   r  rI   r   r   r  r   r  r  r   r^   r   r>   r=   )ri   r  r   r   r   r  r  r   r  r  r  r  r   r  r-   r  r  batch_size_attentionrq  i	start_idxend_idxquery_slice	key_sliceattn_mask_slice
attn_slices                             ru   r   SlicedAttnProcessor.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q 44^V`a??& OOM,C,CAq,IJTTUVXYZM		-(kk"o&&u- ($1!__$($C$CDY$Z!ii-.		/0$$S)&&u-05-lA!<

1BCELL`e`k`k
 ,q0T__DqHIADOO+I1u/G	'2KIg.ICQC]nYw?cgO22;	?[J:uYw/GHJ/9M)G, J ..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r  )r   r4  r=  r  r  r  rw   ru   r   r     sU    % 6:.2GG $G  3	G
 ,G 
G Grw   r   c                  H    \ rS rSrSrS r   S           SS jjrSrg)	r   i  aP  
Processor for implementing sliced attention with extra learnable key and value matrices for the text encoder.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
c                    Xl         g r;  r  r  s     ru   r3   #SlicedAttnAddedKVProcessor.__init__  r  rw   Nc                (   UnUR                   b  UR                  X%5      nUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pxn	UR	                  XHU5      nUc  UnO"UR
                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n
U
R                  S   nUR                  U
5      n
UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                   " X/SS9n[        R                   " X/SS9nOUnUnU
R                  u  nnn	[        R"                  " UUXR$                  -  4U
R&                  U
R(                  S9n[+        US-
  U R,                  -  S-   5       Hj  nUU R,                  -  nUS-   U R,                  -  nU
UU nUUU nUb  UUU OS nUR/                  UUU5      n[        R0                  " UUUU 5      nUUUU& Ml     UR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nUR                  SS5      R7                  UR                  5      nX&-   nU$ )Nr   r   r+   r   r   r   r   )rP   r  r   r  r  rT   r!  r.   rV   r   rZ   r[   rL   rW   rX   r   r  r  rI   r   r   r  r   r  r  r   r^   r   )ri   r  r   r   r   r  r  r   r  r   r  r-   r  r  r  r  r   rq  r  r  r  r  r  r  r  s                            ru   r   #SlicedAttnAddedKVProcessor.__call__  s    !( --mBM%**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(kk"o&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E05-lA!<

1BCELL`e`k`k
 ,q0T__DqHIADOO+I1u/G	'2KIg.ICQC]nYw?cgO22;	?[J:uYw/GHJ/9M)G, J ..}= A}5A}5%//B7??O%0rw   r  r  )r  z'Attention'r   r>  r   r?  r   r?  r  r?  r9  r>  r  r  rw   ru   r   r     sc    % 6:.2$(LL $L  3	L
 ,L "L 
L Lrw   r   c                  B   ^  \ rS rSrSr    SU 4S jjrSS jrSrU =r$ )rO   iS  aU  
Spatially conditioned normalization as defined in https://huggingface.co/papers/2209.09002.

Args:
    f_channels (`int`):
        The number of channels for input to group normalization layer, and output of the spatial norm layer.
    zq_channels (`int`):
        The number of channels for the quantized vector as described in the paper.
c                   > [         TU ]  5         [        R                  " USSSS9U l        [        R
                  " X!SSSS9U l        [        R
                  " X!SSSS9U l        g )Nr3  ư>Tr   r   r   )rQ  strider  )r2   r3   r   rN   
norm_layerrL  conv_yconv_b)ri   r!   r"   rt   s      ru   r3   SpatialNorm.__init__^  sV    
 	,,J2SW`deiiQqZ[\iiQqZ[\rw   c                    UR                   SS  n[        R                  " X#SS9nU R                  U5      nX@R	                  U5      -  U R                  U5      -   nU$ )Nr   nearest)r  rn  )r   re   interpolater  r  r  )ri   fzqf_sizenorm_fnew_fs         ru   r   SpatialNorm.forwardh  sQ    ]]2;#R(4;;r?:rw   )r  r  r  )r!   r4  r"   r4  )r  r>  r  r>  r9  r>  )	r   r@  rA  rB  rC  r3   r   rE  rF  rG  s   @ru   rO   rO   S  s,    ]] ] rw   rO   c                  b   ^  \ rS rSrSrSU 4S jjr     S             SS jjrSrU =r$ )	r   ip  a  
Attention processor for Multiple IP-Adapters.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or list[`float`], defaults to 1.0):
        the weight scale of image prompt.
c                t  > [         TU ]  5         Xl        X l        [	        U[
        [        45      (       d  U/nX0l        [	        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l
        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        g s  snf s  snf NJ`scale` should be a list of integers with the same length as `num_tokens`.Fr0   )r2   r3   r   r:   r   tuplelistr   r   rM   rH   r   r]   r  rU   r   to_v_ipri   r   r:   r   rH   r   rt   s         ru   r3   IPAdapterAttnProcessor.__init__  s    &#6 *udm44$J$%&&Gc*o-Eu:Z(ijj
}}NSTWXbTcNdeNdRYY*eDNde
 }}NSTWXbTcNdeNdRYY*eDNde
 f fs   3D0D5c           
        UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUR                  UUU5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      nUR!                  U5      nUR!                  U5      nUR#                  UUU5      n[$        R&                  " UU5      nUR)                  U5      nUGb  [        U[*        5      (       d  [+        UR-                  S5      5      n[/        U5      [/        U R0                  5      s=:X  a  [/        W	5      :X  d<  O  [3        S
[/        U5       S[/        U R0                  5       S[/        W	5       S35      e[5        [7        XpR0                  U	5      5       H  u  nu  nnnUc  M  [        U[$        R8                  5      (       a  UR                  S:w  a  [3        S5      eUR                  S   UR                  S   :w  a.  [3        SUR                  S    SUR                  S    SU 35      e[        U[*        5      (       d  M  [/        U5      UR                  S   :X  a  M  [3        SUR                  S    S[/        U5       SU 35      e   OS /[/        U R0                  5      -  n[7        W	U R0                  U R:                  U R<                  U5       GH  u  nnnnnSn[        U[*        5      (       a  [?        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGbD  [        U[*        5      (       d  U/UR                  S   -  nUR                  S   n[A        U5       H  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      n#UR)                  U#5      n#[B        RD                  " US S 2US S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$RG                  URH                  URJ                  S9n$X&U   U#U$-  -  -   nM     GM  U" U5      n U" U5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      nUR)                  U5      nX&U-  -   nGM     URL                  S   " U5      nURL                  S   " U5      nUS:X  a$  UR                  SS5      RO                  UWWW5      nURP                  (       a  X(-   nX!RR                  -  nU$ )NYou have passed a tensor as `encoder_hidden_states`. This is deprecated and will be removed in a future release. Please make sure to update your script to pass `encoder_hidden_states` as a tuple to suppress this warning.!encoder_hidden_states not a tupler  Fstandard_warnr   r   r  r   "Length of ip_adapter_masks array ()) must match length of self.scale array (") and number of ip_hidden_states ()Each element of the ip_adapter_masks array should be a tensor with shape [1, num_images_for_ip_adapter, height, width]. Please use `IPAdapterMaskProcessor` to preprocess your maskNumber of masks (&) does not match number of ip images () at index #) does not match number of scales (c              3  *   #    U  H	  oS :H  v   M     g7fr   Nr  r  ss     ru   r  2IPAdapterAttnProcessor.__call__.<locals>.<genexpr>       -u!Avu   Tr   r+   r   )*r   r#  r   r   r   rP   r   r  r  r  r.   rV   rT   r!  rW   rX   r   r  r   r  r   r$  r  r   rH   rM   	enumeratezipTensorr   r%  r  r  r   
downsampler   r   r   r^   r   r>   r=   )%ri   r  r   r   r   r  rH   r   r  r   r  end_posr  r   r  r  r  r  r   r  r  r  r
  indexr  ip_statecurrent_ip_hidden_statesr   r%  skipcurrent_num_imagesr  ip_keyip_valueip_attention_probs_current_ip_hidden_statesmask_downsamples%                                        ru   r   IPAdapterAttnProcessor.__call__  s    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ 44^_V`a??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-$$S)&&u-33E3O		/59..}='.55#'(8(B(B1(E#F ()S_UDT@UU 8=M9N8O P336tzz?2C D,-.a1  7@DTV`V`br@s6t2E2D%| %dELL99TYY!^([ 
 zz!}q(99(/

1 ?44<NN14E3FkRWQXZ  "%..s5zTZZPQ]7R(/

1 ?114UKwP  7u( !%vDJJ7 HKdjj$,,FVH
C$eWgt D%&&-u---D!4#%eT22!&$**Q- 7)-A&"#56!()A!Q1*)M!N#*+CAq!QJ+O#P!%!7!7!?#'#9#9(#C-1-F-FufVZ-[*49II>PRZ4[1484J4JKd4e1*@*K*K Aq!,&5;;A>5;;A>	+ +:*<*<5;;W\WcWc*<*d(5aD]`oDo8p(p) 7, %%=>F&'?@H!33F;F#55h?H)-)B)B5&RV)W&/4yy9KX/V,/3/E/EF^/_,$1<T4T$TM_H
d A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   r:   r   r   rH   r   r%  Nr  r   NNNr   Nr  r   r   r>  r   r?  r   r?  r  r?  rH   r  r   r?  r  rG  s   @ru   r   r   p  su    
6 6:.2$(04__ $_  3	_
 ,_ "_ _ ._ _rw   r   c                  b   ^  \ rS rSrSrSU 4S jjr     S             SS jjrSrU =r$ )	r   i8  a  
Attention processor for IP-Adapter for PyTorch 2.0.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `list[float]`, defaults to 1.0):
        the weight scale of image prompt.
c                  > [         TU ]  5         [        [        S5      (       d"  [	        U R
                  R                   S35      eXl        X l        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        g s  snf s  snf )Nr1   z@ requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r"  Fr0   )r2   r3   rd   re   r  rt   r   r   r:   r   r#  r$  r   r   rM   rH   r   r]   r  rU   r   r%  r&  s         ru   r3   "IPAdapterAttnProcessor2_0.__init__G  s.   q899>>**++kl  '#6 *udm44$J$%&&Gc*o-Eu:Z(ijj
}}NSTWXbTcNdeNdRYY*eDNde
 }}NSTWXbTcNdeNdRYY*eDNde
 f fs   *E';E,c                >   UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb=  UR                  UUU5      nUR                  XR                  S
UR                  S
   5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S
   nUUR                  -  nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      n["        R$                  " UUUUSSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nUGb  [        U[,        5      (       d  [-        UR/                  S5      5      n[1        U5      [1        U R2                  5      s=:X  a  [1        W	5      :X  d<  O  [5        S[1        U5       S[1        U R2                  5       S[1        W	5       S35      e[7        [9        XpR2                  U	5      5       H  u  nu  nnnUc  M  [        U[:        R<                  5      (       a  UR                  S:w  a  [5        S5      eUR                  S   UR                  S   :w  a.  [5        SUR                  S    SUR                  S    SU 35      e[        U[,        5      (       d  M  [1        U5      UR                  S   :X  a  M  [5        SUR                  S    S[1        U5       SU 35      e   OS /[1        U R2                  5      -  n[9        W	U R2                  U R>                  U R@                  U5       GH  u  nnnnnSn[        U[,        5      (       a  [C        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGb  [        U[,        5      (       d  U/UR                  S   -  nUR                  S   n[E        U5       GH_  n U" US S 2U S S 2S S 24   5      n!U" US S 2U S S 2S S 24   5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9n#U#R                  SS	5      R'                  US
UR                  U-  5      n#U#R)                  UR*                  5      n#[F        RH                  " US S 2U S S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$R)                  UR*                  URJ                  S9n$X&U    U#U$-  -  -   nGMb     GM  U" U5      n!U" U5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nX&U-  -   nGM     URL                  S   " U5      nURL                  S   " U5      nUS:X  a$  UR                  S
S5      R'                  UWWW5      nURN                  (       a  X(-   nX!RP                  -  nU$ )Nr)  r*  r  Fr+  r   r   r  r   r+   r  r  r-  r.  r/  r0  r1  r2  r3  r4  r5  c              3  *   #    U  H	  oS :H  v   M     g7fr7  r  r8  s     ru   r  5IPAdapterAttnProcessor2_0.__call__.<locals>.<genexpr>  r;  r<  Tr   r   ))r   r#  r   r   r   rP   r   r  r  r  rI   r.   rV   rT   r!  rW   rX   re   r1   r   r   r   r$  r  r   rH   rM   r=  r>  r   r?  r   r%  r  r  r   r@  r   r^   r>   r=   )%ri   r  r   r   r   r  rH   r   r  r   r  rA  r  r   r  r  r  r  r   r  r  r  r5   r  rB  r  rC  rD  r   r%  rE  rF  r  rG  rH  rJ  rK  s%                                        ru   r   "IPAdapterAttnProcessor2_0.__call__c  s
    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP 6633RW
 &//15==j"djj[cNcd%((5'.55#'(8(B(B1(E#F ()S_UDT@UU 8=M9N8O P336tzz?2C D,-.a1  7@DTV`V`br@s6t2E2D%| %dELL99TYY!^([ 
 zz!}q(99(/

1 ?44<NN14E3FkRWQXZ  "%..s5zTZZPQ]7R(/

1 ?114UKwP  7u( !%vDJJ7 HKdjj$,,FVH
C$eWgt D%&&-u---D!4#%eT22!&$**Q- 7)-A&"#56!()A!Q1*)M!N#*+CAq!QJ+O#P!'ZTZZ!R!\!\]^`a!b#+==RX#V#`#`abde#f 564R4R!68ts^c51 5N4W4WXY[\4]4e4e&DJJ,A51 5N4P4PQVQ\Q\4]1*@*K*K Aq!,&5;;A>5;;A>	+ +:*<*<5;;W\WcWc*<*d(5aD]`oDo8p(p5 78 %%=>F&'?@H#[[RXNXXYZ\]^F'}}ZTZZR\\]^`abH 01/M/Mvx43Z_0, 0H/Q/QRSUV/W/_/_"B

X(=0, 0H/J/J5;;/W,$1<T4T$TMyH
~ A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   rM  rN  rP  rQ  r  rG  s   @ru   r   r   8  sv    
@ 6:.2$(04zz $z  3	z
 ,z "z z .z zrw   r   c                  p   ^  \ rS rSrSr    S SU 4S jjjr     S             S	S jjrSrU =r$ )
r   i   a  
Attention processor for IP-Adapter using xFormers.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `list[float]`, defaults to 1.0):
        the weight scale of image prompt.
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
c                  > [         TU ]  5         Xl        X l        XPl        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        g s  snf s  snf r!  )r2   r3   r   r:   r   r   r#  r$  r   r   rM   rH   r   r]   r  rU   r   r%  )ri   r   r:   r   rH   r   r   rt   s          ru   r3   'IPAdapterXFormersAttnProcessor.__init__4  s    	&#6 (*udm44$J$%&&Gc*o-Eu:Z(ijj
}}]bcfgqcr]st]sXYRYY*9k;US]st
 }}]bcfgqcr]st]sXYRYY*9k;US]st
 u us   9)E
)Ec           
        UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb6  UR                  UUU5      nUR                  u  nnnUR                  S
US
5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       n[&        R(                  R+                  UUUX@R,                  S9nUR/                  UR0                  5      nUR3                  U5      nW	(       Ga  UGb  [        U[4        5      (       d  [5        UR7                  S5      5      n[9        U5      [9        U R:                  5      s=:X  a  [9        U	5      :X  d<  O  [=        S[9        U5       S[9        U R:                  5       S[9        U	5       S35      e[?        [A        XpR:                  U	5      5       H  u  nu  nnnUc  M  [        U[B        RD                  5      (       a  UR                  S:w  a  [=        S5      eUR                  S   UR                  S   :w  a.  [=        SUR                  S    SUR                  S    SU 35      e[        U[4        5      (       d  M  [9        U5      UR                  S   :X  a  M  [=        SUR                  S    S[9        U5       SU 35      e   OS /[9        U R:                  5      -  n[A        XR:                  U RF                  U RH                  U5       GH  u  nnnnnSn[        U[4        5      (       a  [K        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGb  UR/                  [B        RL                  5      n[        U[4        5      (       d  U/UR                  S   -  nUR                  S   n[O        U5       GH3  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9n"U"R/                  UR0                  5      n"UR3                  U"5      n"[P        RR                  " US S 2US S 2S S 24   UU"R                  S   U"R                  S	   5      n#U#R/                  UR0                  URT                  S9n#X&U   U"U#-  -  -   nGM6     GM  U" U5      n U" U5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9nUR/                  UR0                  5      nUR3                  U5      nX&U-  -   nGM     URV                  S   " U5      nURV                  S   " U5      nUS:X  a$  UR                  S
S5      RY                  UWWW5      nURZ                  (       a  X(-   nX!R\                  -  nU$ )Nr)  r*  r  Fr+  r   r   r  r   r+   )r5  r6  r-  r.  r/  r0  r1  r2  r3  r4  r5  c              3  *   #    U  H	  oS :H  v   M     g7fr7  r  r8  s     ru   r  :IPAdapterXFormersAttnProcessor.__call__.<locals>.<genexpr>  s     15a65r<  T)r6  r   r   )/r   r#  r   r   r   rP   r   r  r  r  ro  r.   rV   rT   r!  rW   rX   r   r7  r   r   r   r   r   r   r   r$  r  r   rH   rM   r=  r>  r   r?  r   r%  r  r  r  r   r@  r   r^   r   r>   r=   )$ri   r  r   r   r   r  rH   r   r  r   r  rA  r  r   r  r  r  r  r   rq  r  r  r  rB  r  rC  rD  r   r%  rE  rF  r  rG  rH  rJ  rK  s$                                       ru   r   'IPAdapterXFormersAttnProcessor.__call__S  sk    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ %!88ZdeN "/!4!4A|Q+222|RHN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-88:$$S)446&&u-88: ??3<M<M @ 
 &((5..}=+!"2D99'+,<,F,Fq,I'J$,-TZZYCHXDYY$<SAQ=R<S T77:4::6G H 012!5  ;D,jj:JK;66eX  <$)$==a",!_# 
  ::a=HNN1,==","3DJJqM? C88@q8I7J+V[U\!^#  &eT223u:TU;V","3DJJqM? C558ZLE7!T# #;, %)6C

O#;  LO **dllDLLJZLG(%'4 eT**15111#aZDt'#wwu}}5)%66%*Gdjjm$;E-1ZZ]*!&'9!:A%,-EaAqj-Q%RF'./G1aQR
/S'TH%)%;%;F%C%N%N%PF'+'='=h'G'R'R'TH8@8_8_ %vxD<M<M 9` 95 9R8T8TUZU`U`8a58<8N8NOh8i5.D.O.O $Q1aZ 0 * 9 ? ? B 9 ? ? B	/O />.@.@u{{[`[g[g.@.hO,9!HHadsHs<t,tM+ ";. "))A!B#*+C#D!%!7!7!?!J!J!L#'#9#9(#C#N#N#P3;<<3Z3Z!688I8I 4[ 40 4L3N3Nu{{3[0373I3IJb3c0(5@X8X(XgLl A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrw   )r   r:   r   r   rH   r   r%  )NrO  r   Nr9  rP  )r  r   r   r  r   r  r   r  r  r  rH   r  r   r  r  rG  s   @ru   r   r      s    , !(,
 &
 
F ;?37)-59rr )r  8	r
 1r 'r r 3r rrw   r   c                  z   ^  \ rS rSrSr  S         SU 4S jjjr    S             S	S jjrSrU =r$ )
!SD3IPAdapterJointAttnProcessor2_0i  a"  
Attention processor for IP-Adapter used typically in processing the SD3-like self-attention projections, with
additional image-based information and timestep embeddings.

Args:
    hidden_size (`int`):
        The number of hidden channels.
    ip_hidden_states_dim (`int`):
        The image feature dimension.
    head_dim (`int`):
        The number of head channels.
    timesteps_emb_dim (`int`, defaults to 1280):
        The number of input channels for timestep embedding.
    scale (`float`, defaults to 0.5):
        IP-Adapter scale.
c                  > [         TU ]  5         SSKJnJn  U" XBS-  SSS9U l        [        R                  " X!SS9U l        [        R                  " X!SS9U l	        U" US5      U l
        U" US5      U l        U" US5      U l        XPl        g )Nr   )AdaLayerNormr   r   r  )
output_dimnorm_eps	chunk_dimFr0   )r2   r3   r4   rc  r   norm_ipr   rU   r   r%  rQ   rR   	norm_ip_krH   )	ri   r   ip_hidden_states_dimr  timesteps_emb_dimrH   rc  r   rt   s	           ru   r3   *SD3IPAdapterJointAttnProcessor2_0.__init__  s     	 	9#$5YZBZeiuvwyy!5Oyy!5Oh-h- 40
rw   c                	   UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nU	nU
nUnUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb>  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " U	U/SS9n	[        R                  " U
U/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS	9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  USS2SUR                   S   24   USS2UR                   S   S24   p2UR*                  (       d  UR-                  U5      nU R.                  S:w  Gae  UGba  U R1                  XVS
9nU R3                  U5      nU R5                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nU R                  U5      n	U R                  U5      nU R7                  U5      n[        R                  " UU/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS	9nUR                  SS5      R                  USUR                  U-  5      nUR'                  U	R(                  5      nX%U R.                  -  -   nUR8                  S   " U5      nUR8                  S   " U5      nUb  X#4$ U$ )a  
Perform the attention computation, integrating image features (if provided) and timestep embeddings.

If `ip_hidden_states` is `None`, this is equivalent to using JointAttnProcessor2_0.

Args:
    attn (`Attention`):
        Attention instance.
    hidden_states (`torch.FloatTensor`):
        Input `hidden_states`.
    encoder_hidden_states (`torch.FloatTensor`, *optional*):
        The encoder hidden states.
    attention_mask (`torch.FloatTensor`, *optional*):
        Attention mask.
    ip_hidden_states (`torch.FloatTensor`, *optional*):
        Image embeddings.
    temb (`torch.FloatTensor`, *optional*):
        Timestep embeddings.

Returns:
    `torch.FloatTensor`: Output hidden states.
r   r+   r   r   Nr   r  Fr  )r  )r   rV   rW   rX   rI   r  r  rQ   rR   r\   rZ   r[   rb   rc   r   r  re   r1   r   r   r   rC   ra   rH   rg  r   r%  rh  r^   )ri   r  r   r   r   r   r  r  r   r  r  r  r5   r  	img_queryimg_key	img_valuer  r  r  norm_ip_hidden_statesrG  rH  s                          ru   r   *SD3IPAdapterJointAttnProcessor2_0.__call__/  s   > !"((+
 		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP		;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/O/T/TB

H0i1o - .L-P-PB

H.i1o + 0P/T/TB

H0i1o -   ,373D3DEe3f0  ,151B1BCa1b.IIu&FGQOE))S"@AqICIIu&FGQOE66u5TWchi%//15==j"djj[cNcd%((5 , a!48>>!#4!445a!2!445 1 (((,8M(N% ::?/;$(LL1AL$M! \\"78F||$9:H [[RXFPPQRTUVF}}ZTZZJTTUVXYZH KK	*Ekk'*G^^F+F ))Wf-15CIIy(3;E ==e%[^jop/99!Q?DDZQSUYU_U_bjUjk/225;;?)tzz,IIM A}5A}5 , 77  rw   )rg  rh  rR   rQ   rH   r   r%  )i   g      ?)
r   r4  ri  r4  r  r4  rj  r4  rH   r  rB  )r  r   r   r  r   r  r   r  r   r  r  r  r9  r  r  rG  s   @ru   ra  ra    s    , "& " 	
   2 4837.2"&A!A! )A!  1	A!
 1A! ,A!  A! 
A! A!rw   ra  c                  H    \ rS rSrSrS r   S           SS jjrSrg)	PAGIdentitySelfAttnProcessor2_0i  
Processor for implementing PAG using scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
PAG reference: https://huggingface.co/papers/2403.17377
c                D    [        [        S5      (       d  [        S5      eg )Nr1   z_PAGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   (PAGIdentitySelfAttnProcessor2_0.__init__  s%    q899q  :rw   Nc           	     B   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUR                  S5      u  pUR                  u  pnUb<  UR                  XNU5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R                  USUR                  U-  5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      nUR                  u  pnUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      n[$        R&                  " X/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )
Nr  r   r   r+   r  Fr  r   r   )rP   r   r   r  r  r  r  rI   r.   rV   rW   rX   re   r1   r   r   r   r^   r   r  r>   r=   )ri   r  r   r   r   r  r  r  r   r  r  r  r  r  r  r   r  r  r  r5   r  s                        ru   r   (PAGIdentitySelfAttnProcessor2_0.__call__  s|    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM 0=/B/B1/E, *;)@)@&
Q%!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+,ii)*		+,IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o *;)@)@&
Q??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrw   r  r  r  r   r   r  r   r  r   r  r  r  r9  r>  r  r  rw   ru   rs  rs    sc    
 ;?37)-TT )T  8	T
 1T 'T 
T Trw   rs  c                  H    \ rS rSrSrS r   S           SS jjrSrg)	"PAGCFGIdentitySelfAttnProcessor2_0i  rt  c                D    [        [        S5      (       d  [        S5      eg )Nr1   zbPAGCFGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    ru   r3   +PAGCFGIdentitySelfAttnProcessor2_0.__init__  s%    q899t  :rw   Nc           	     v   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUR                  S5      u  pn[        R                  " X/5      nUR                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      nUR                  u  pnUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUnUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      n[        R                  " X/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )Nr  r   r   r   r+   r  Fr  r   r   )rP   r   r   r  r  r  r   r  r  rI   r.   rV   rW   rX   re   r1   r   r   r   r^   r>   r=   )ri   r  r   r   r   r  r  r  r   r  r  r  r  r  r  r  r   r  r  r  r5   r  s                         ru   r   +PAGCFGIdentitySelfAttnProcessor2_0.__call__"  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM FSEXEXYZE[B1B!II';&OP *;)@)@&
Q%!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+,ii)*		+,IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o *;)@)@&
Q??& $0A0K0KAq0Q R \ \]^`a b		+,!-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrw   r  r  ry  r  r  rw   ru   r{  r{    sc    
 ;?37)-XX )X  8	X
 1X 'X 
X Xrw   r{  c                  "    \ rS rSrSrSS jrSrg)re  i}  z<
Processor for implementing multiscale quadratic attention.
c                   UR                   SS  u  p4X4-  UR                  :  a  SnOSnUn[        UR                  5       5      u  pxp4UR                  n	UR                  SS5      nUR                  U5      n
UR                  U5      nUR                  U5      n[        R                  " XU/SS9nUR                  SS5      nU/nUR                   H  nUR                  U" U5      5        M     [        R                  " USS9nU(       a  UR                  [        R                  S9nUR                  USSUR                  -  X4-  5      nUR!                  SS	S9u  pnUR#                  U
5      n
UR#                  U5      nU(       a"  UR%                  XU5      nUR                  U	S9nOUR'                  XU5      n[        R                  " X'SX445      nUR)                  UR                  SS5      5      R                  SS5      nUR*                  S
:X  a2  UR-                  UR                  SS5      5      R                  SS5      nOUR-                  U5      nUR.                  (       a  X&-   nU$ )Nr   TFr   r+   r   r   ro  r   r(   )r   r_  r$  r  r   movedimrV   rW   rX   r   r  ra  r_   r   rq  r   r  rc  rt  ry  r^   r`  rd  r>   )ri   r  r   r  r  use_linear_attentionr  r   r   original_dtyper  r  r  multi_scale_qkvblocks                  ru   r   'SanaMultiscaleAttnProcessor2_0.__call__  sD   %++BC0>D333#' #(  '+M,>,>,@'A$
v&,,%--a4		-(ii&		-(		5u"51=%--b!4(/++E""5#78 , 		/q9),,5==,AM%--j"a$BYBY>Y[a[ij)//q/9E!!%($ 77EJM),,>,BM ::5uMMm"f5TUM$9$9!R$@AII"aP>>Z' MM-*?*?2*FGOOPRTUVM MM-8M##)4Mrw   r  N)r  r[  r   r>  r9  r>  r  r  rw   ru   re  re  }  s    4rw   re  c                      \ rS rSrSrS rSrg)LoRAAttnProcessori  z1
Processor for implementing attention with LoRA.
c                    g r;  r  r  s    ru   r3   LoRAAttnProcessor.__init__      rw   r  Nr   r@  rA  rB  rC  r3   rE  r  rw   ru   r  r        rw   r  c                      \ rS rSrSrS rSrg)LoRAAttnProcessor2_0i  zb
Processor for implementing attention with LoRA (enabled by default if you're using PyTorch 2.0).
c                    g r;  r  r  s    ru   r3   LoRAAttnProcessor2_0.__init__  r  rw   r  Nr  r  rw   ru   r  r    r  rw   r  c                      \ rS rSrSrS rSrg)LoRAXFormersAttnProcessori  z@
Processor for implementing attention with LoRA using xFormers.
c                    g r;  r  r  s    ru   r3   "LoRAXFormersAttnProcessor.__init__  r  rw   r  Nr  r  rw   ru   r  r    r  rw   r  c                      \ rS rSrSrS rSrg)LoRAAttnAddedKVProcessori  zr
Processor for implementing attention with LoRA with extra learnable key and value matrices for the text encoder.
c                    g r;  r  r  s    ru   r3   !LoRAAttnAddedKVProcessor.__init__  r  rw   r  Nr  r  rw   ru   r  r    r  rw   r  c                  <    \ rS rSrSr  S         SS jjrSrg)SanaLinearAttnProcessor2_0i  A
Processor for implementing scaled dot-product linear attention.
Nc                   UR                   nUc  UnUR                  U5      nUR                  U5      nUR                  U5      nUR                  b  UR	                  U5      nUR
                  b  UR                  U5      nUR                  SS5      R                  SUR                  S45      nUR                  SS5      R                  SUR                  S45      R                  SS5      nUR                  SS5      R                  SUR                  S45      n[        R                  " U5      n[        R                  " U5      nUR                  5       UR                  5       UR                  5       pn[        R                  " USSSS9n[        R                  " X5      n	[        R                  " X5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nr   r   r+   r   rk  rl  r   rm  r  r       )r   rV   rW   rX   rQ   rR   r  r  rI   re   relur  r  r   rp  r  r   r^   r  clip)
ri   r  r   r   r   r  r  r  r  rs  s
             ru   r   #SanaLinearAttnProcessor2_0.__call__  s    ',, ($1!		-(ii-.		/0;;"KK&E;;"++c"C1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)V3%aCRCi0M!Q)4Lu4TU%--a3==aC%((8A}5A}5U]]*)..vu=Mrw   r  r=  r  r  r  rw   ru   r  r    sI     6:.2,, $,  3	,
 ,, 
, ,rw   r  c                  <    \ rS rSrSr  S         SS jjrSrg) PAGCFGSanaLinearAttnProcessor2_0i  r  Nc                R   UR                   nUR                  S5      u  pgn[        R                  " Xg/5      nUR	                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SS5      R                  SUR                  S45      n	U
R                  SS5      R                  SUR                  S45      R                  SS5      n
UR                  SS5      R                  SUR                  S45      n[        R                  " U	5      n	[        R                  " U
5      n
U	R                  5       U
R                  5       UR                  5       pn	[        R                  " USSSS9n[        R                  " X5      n[        R                  " X5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nUR                  U5      R!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      n[        R                  " Xx/5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nr   r   r   r+   rk  rl  r   rm  r  r   r  r  )r   r  r   r  rV   rW   rX   r  r  rI   re   r  r  r  rp  r  r   r^   r  r  )ri   r  r   r   r   r  r  r  r  r  r  r  rs  s                ru   r   )PAGCFGSanaLinearAttnProcessor2_0.__call__  sS    ',,EREXEXYZE[B1B!II';&OP		+,ii)*		+,1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)!LL7-aCRCi8<MaQRTVTWi<X[`<`a-55a;EEaK-00@ KKN+<= KKN+<= !II&78;;NK KKN+<= KKN+<=		#4"HIU]]*)..vu=Mrw   r  r=  r  r  r  rw   ru   r  r    sI     6:.2// $/  3	/
 ,/ 
/ /rw   r  c                  <    \ rS rSrSr  S         SS jjrSrg)%PAGIdentitySanaLinearAttnProcessor2_0iH  r  Nc                   UR                   nUR                  S5      u  pgUR                  U5      nUR                  U5      n	UR	                  U5      n
UR                  SS5      R                  SUR                  S45      nU	R                  SS5      R                  SUR                  S45      R                  SS5      n	U
R                  SS5      R                  SUR                  S45      n
[        R                  " U5      n[        R                  " U	5      n	UR                  5       U	R                  5       U
R                  5       pn[        R                  " U
SSSS9n
[        R                  " X5      n[        R                  " X5      nUR                   [        R                  [        R                  4;   a  UR                  5       nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR!                  SS5      R                  SS5      nUR#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR	                  U5      R#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      n[        R&                  " Xg/5      nU[        R                  :X  a  UR)                  SS5      nU$ )Nr   r   r+   r   rk  rl  r   rm  r  r   r  r  )r   r  rV   rW   rX   r  r  rI   re   r  r  r  r   rp  r  r  r  r   r^   r  r  )ri   r  r   r   r   r  r  r  r  r  r  rs  s               ru   r   .PAGIdentitySanaLinearAttnProcessor2_0.__call__M  sf    ',,/</B/B1/E,		+,ii)*		+,1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)!LL7""u}}enn&EE 1 7 7 9-aCRCi8<MaQRTVTWi<X[`<`a-55a;EEaK-00@ KKN+<= KKN+<= !II&78;;NK KKN+<= KKN+<=		#4"HIU]]*)..vu=Mrw   r  r=  r  r  r  rw   ru   r  r  H  sI     6:.211 $1  3	1
 ,1 
1 1rw   r  c                      \ rS rSrS rSrg)FluxAttnProcessor2_0i  c                <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzq`FluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r  r  r   FluxAttnProcessorr   transformers.transformer_fluxr  clsr  r  r  r  s        ru   __new__FluxAttnProcessor2_0.__new__  s,     R('3FGD $1&11rw   r  Nr   r@  rA  rB  r  rE  r  rw   ru   r  r        2rw   r  c                      \ rS rSrSrS rSrg)FluxSingleAttnProcessor2_0i  r  c                <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nz|`FluxSingleAttnProcessor` is deprecated and will be removed in a future version. Please use `FluxAttnProcessorSDPA` instead.r  r  r   r  r  r  s        ru   r  "FluxSingleAttnProcessor2_0.__new__  s,     ].9LMD $1&11rw   r  Nr   r@  rA  rB  rC  r  rE  r  rw   ru   r  r    s    2rw   r  c                      \ rS rSrS rSrg)FusedFluxAttnProcessor2_0i  c                <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzv`FusedFluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r  r  r   r  r  r  s        ru   r  !FusedFluxAttnProcessor2_0.__new__  s,     W-w8KLD $1&11rw   r  Nr  r  rw   ru   r  r    r  rw   r  c                      \ rS rSrS rSrg)"FluxIPAdapterJointAttnProcessor2_0i  c                <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nz`FluxIPAdapterJointAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxIPAdapterAttnProcessor`r  r  r   )FluxIPAdapterAttnProcessor)r   r  r  )r  r  r  r  r  s        ru   r  *FluxIPAdapterJointAttnProcessor2_0.__new__  s,     i6ATUM)4:6::rw   r  Nr  r  rw   ru   r  r    s    ;rw   r  c                      \ rS rSrS rSrg)FluxAttnProcessor2_0_NPUi  c                J    Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	NzFluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.r  r  Fr+  r   r  _native_npur   r  r  _attention_backendr  r  r  r  r  rr   s         ru   r   FluxAttnProcessor2_0_NPU.__new__  s9    ^ 	 	,g7JZ_`D%'	'4	$rw   r  Nr  r  rw   ru   r  r        rw   r  c                      \ rS rSrS rSrg)FusedFluxAttnProcessor2_0_NPUi  c                J    Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	NzFusedFluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.r  r  Fr+  r   r  
_fused_npur  )ri   r  r  rr   s       ru   r  %FusedFluxAttnProcessor2_0_NPU.__new__  s9    ^ 	 	17<O_deD%'	'3	$rw   r  Nr  r  rw   ru   r  r    r  rw   r  c                      \ rS rSrSrS rSrg)r|   i  r  c                <   Sn[        SSUSS9  [        SS5      (       a  [        S5      e[        5       (       a  [        SS	5      (       a  [        S
5      eSSKJn  [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        U" U0 UD6nSUl        U$ )NzXLAFluxFlashAttnProcessor2_0 is deprecated and will be removed in diffusers 1.0.0. An alternative solution to using XLA Flash Attention will be provided in the future.r|   r  Fr+  ry   rz   r  r{   r  r   r  r   r   zpartition_spec was not used in the processor implementation when it was added. Passing it is a no-op and support for it will be removed._native_xla)	r   r   r  r   r  r  r   r  r  r  s         ru   r  $XLAFluxFlashAttnProcessor2_0.__new__  s    ` 	 	0';N^cdU++VWW99-c599deeDt9q=FJJ'7>JA   &1DE%t6v6	'4	$rw   r  Nr  r  rw   ru   r|   r|     s    rw   r|   )a
__future__r   r   r  typingr   r   torch.nn.functionalr   
functionalre   image_processorr   utilsr   r	   r
   utils.import_utilsr   r   r   utils.torch_utilsr   r   
get_loggerr   r   r  r   xformers.ops$torch_xla.experimental.custom_kernelr   torch_xla.runtimer   XLA_AVAILABLEr   r   rI  r[  r  r  rg   r   r   r   r   r  r  r"  r   r;  rE  rO  rU  ra  r   r   r   rf   r}   r  r  r  r  r  r  r  r  r   r   r   r   rO   r   r   r   ra  rs  r{  re  r  r  r  r  r  r  r  r  r  r  r  r  r  r|   ADDED_KV_ATTENTION_PROCESSORSCROSS_ATTENTION_PROCESSORSAttentionProcessorr  rw   ru   <module>r     s   #        4 > > d d F 
		H	%HC''H-MM Q&		 Q& Q&h		 4?3BII ?3D?
RYY ?
Df4 f4RE EPe299 eP@ @FI IXS! S!lY4 Y4xb4 b4JL4 L4^R! R!j[ [|Z! Z!z^! ^!BB4 B4JD4 D4ND DN[ [|o od[ [|q qhR RjB BJ_ _Dd dNx xvy yx] ]@c cLq299 qhobii odT TnY Yx")) :ERYY EPe ePeUXX__ ePh! h!V` `Fd dN9 9x    1 1h4 4n6 6r2 22 22 2; ;   @  	!  &  "#  	
  "" !!  	 $
$      $$     %!%" ###$ %& '( )* +, -. /0 "1"2 34 #5#6 !7!8 $9$: ;< => +?+@ &A &B C!D !E"!F !G#!H 'I$'J ,K%,L $M&$N %O'%P (Q((R S)T  U* V %W+%X (Y,(Z &[-&\ )].)^ _/` a0b  c1 d e2 rw   