
    
3jk                         S SK JrJr  S SKrS SKJr  S SKJs  Jr  SSK	J
r
  SSKJr  SSKJr  SSKJr   " S S	\R"                  5      r " S
 S\R"                  5      rg)    )OptionalTypeN   )maybe_add_mask)use_fused_attn)Mlp)trunc_normal_tf_c                   V  ^  \ rS rSr% Sr\R                  R                  \   \	S'   SSSSSSSS	SS
SS\
R                  SSS4S\S\S\S\S\\   S\S\S\S\S\S\S\S\\\
R"                        S\\\
R"                        S\4U 4S jjjrS rS!S\\R(                     4S jjrS rU =r$ )"AttentionPoolLatent   zAttention pooling w/ latent query

Setting out_features=0 disables the output projection, norm, and MLP layers (pre_logits mode).

fused_attnN   g      @TFr    tokeng        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layer	act_layerdropc                 p  > UUS.n[         TU ]  5         U=(       d    UnUc  UnX4-  S:X  d   eX@l        X4-  U l        XPl        U R                  S-  U l        Xl        [        5       U l        US:X  a6  Uc   e[        R                  " [        R                  " XQ40 UD65      U l        OS U l        U
=(       d    UU l        Xl        [        R                  " [        R                  " SU R                  U40 UD65      U l        [        R"                  " X34SU0UD6U l        [        R"                  " X3S-  4SU0UD6U l        U(       aJ  U=(       d    [        R(                  nU" U R                  40 UD6U l        U" U R                  40 UD6U l        O4[        R.                  " 5       U l        [        R.                  " 5       U l        US:  a}  [        R"                  " X240 UD6U l        [        R2                  " U5      U l        Ub	  U" U40 UD6O[        R.                  " 5       U l        [9        U[;        X&-  5      4X.S.UD6U l        OX[        R.                  " 5       U l        [        R2                  " U5      U l        [        R.                  " 5       U l        S U l        UnX l        U RA                  5         g )	Ndevicedtyper         absr   bias   )r   r   )!super__init__r   head_dimr   scalepoolr   r   nn	Parametertorchzerosr   r   r   latentLinearqkv	LayerNormq_normk_normIdentityprojDropout	proj_dropnormr   intmlpr   init_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r"   r#   ddqk_norm_layer	__class__s                       T/home/wildlama/miniconda3/lib/python3.13/site-packages/timm/layers/attention_pool.pyr)   AttentionPoolLatent.__init__   s+   ( /,	&L$)))"!."]]d*
	(*(((\\%++i*SPR*STDN!DN$1	$ll5;;q$//9#SPR#ST9EhE"E))I1}J8JrJ&6",,M'<<DK'<<DK++-DK++-DK!		)@R@DIZZ-DN:D:P
<626VXVaVaVcDI<\-E)F}Ua}z|}DHDIZZ-DNDIDH$L(    c                     U R                   b.  [        U R                   U R                   R                  S   S-  S9  [        U R                  U R                  S-  S9  g )Nr   r$   )std)r   r	   shaper1   r   )r@   s    rD   r?    AttentionPoolLatent.init_weightsY   sE    >>%T^^1E1Ea1HD1PQ$//T*ABrF   	attn_maskc                    UR                   u  p4nU R                  b6  XR                  R                  S5      R                  UR                  5      -   nU R
                  R                  USS5      nU R                  U5      R                  X0R                  U R                  U R                  5      R                  SS5      nU R                  U5      R                  X4SU R                  U R                  5      R                  SSSSS5      nUR                  S5      u  pU R!                  U5      U R#                  U	5      pU R$                  (       a  [&        R(                  " XyXS9nO@XpR*                  -  nXyR                  SS5      -  n[-        X5      nUR/                  SS	9nX-  nUR                  SS5      R                  X0R                  U5      nU R1                  U5      nU R3                  U5      nU R4                  b"  XR5                  U R7                  U5      5      -   nU R8                  S
:X  a  US S 2S4   nU$ U R8                  S:X  a  UR;                  S5      nU$ )Nr   r   r'         )rK   dimr   avg)rI   r   	unsqueezetor#   r1   expandr3   reshaper   r   r*   	transposer4   permuteunbindr6   r7   r   Fscaled_dot_product_attentionr+   r   softmaxr9   r;   r>   r<   r,   mean)r@   xrK   BNCq_latentr3   r4   kvattns               rD   forwardAttentionPoolLatent.forward^   s   ''a>>%NN,,Q/22177;;A;;%%aR0FF8$$QWaabcefgWWQZaGOOPQSTVWYZ\]^yy|{{1~t{{1~1??..qQLAJJA{{2r**D!$2D<<B<'DAKK1%%a!<IIaLNN188HHTYYq\**A 99!Q$A  YY%q	ArF   )r   r   r*   r7   r4   r1   r   r   r>   r<   r   r   r,   r   r9   r;   r3   r6   r+   )N)__name__
__module____qualname____firstlineno____doc__r/   jitFinalbool__annotations__r-   GELUr=   r   floatstrr   Moduler)   r?   Tensorrg   __static_attributes____classcell__rC   s   @rD   r   r      sA    		%%
 !%!'+"!!"$483577%CC C 	C
 C  }C C C C C C C C !bii1C  RYY0C  !C CJC
#HU\\$: # #rF   r   c                      ^  \ rS rSr% Sr\R                  R                  \   \	S'          SS\
S\
S\S\S\S	\\\R                        4U 4S
 jjjrS\R"                  S\R"                  4S jrSrU =r$ )AttentionPoolPrr   u1  Patch Representation Refinement (PRR) attention pool.

From "Locality-Attending Vision Transformer" (ICLR 2026).

Parameter-free multi-head self-attention that refines all patch representations
before pooling. No Q/K/V projections — input is reshaped directly into multi-head
format for self-attention.
r   rR   r   r   pre_norm	post_normr   c	                   > XxS.n	[         T
U ]  5         US;   d   SU S35       eX-  S:X  d   SU SU S35       eUc  U(       d  U(       a  [        R                  nX l        X-  U l        U R
                  S	-  U l        X0l        [        5       U l	        Xl
        U(       a	  U" U40 U	D6O[        R                  " 5       U l        U(       a  U" U40 U	D6U l        g [        R                  " 5       U l        g )
Nr!   )r   rS   z)pool_type must be 'token' or 'avg', got ''r   zdim (z") must be divisible by num_heads ()r$   )r(   r)   r-   r5   r   r*   r+   r,   r   r   r   r8   r}   r~   )r@   rR   r   r   r}   r~   r   r"   r#   rA   rC   s             rD   r)   AttentionPoolPrr.__init__   s     /,,f0YZcYdde.ff,!#`uSE1ST]S^^_%``#8yJ"(]]d*
	(*19
3-"-r{{}2;C.2.rF   r_   returnc                 L   UR                   u  p#nU R                  U5      nUR                  X#U R                  U R                  5      R                  SS5      nU R                  (       a  [        R                  " XUU5      nO4XPR                  -  UR                  SS5      -  nUR                  SS9nXe-  nUR                  SS5      R                  X#U5      nU R                  U5      nU R                  S:X  a  US S 2S4   nU$ U R                  S:X  a  UR                  S5      nU$ )	Nr   r'   rP   rM   rQ   r   r   rS   )rI   r}   rW   r   r*   rX   r   r[   r\   r+   r]   r~   r,   r^   )r@   r_   r`   ra   rb   qkvrf   s          rD   rg   AttentionPoolPrr.forward   s    ''aMM! iidnndmm<FFq!L??..s=A**$b"(==D<<B<'D
AKK1%%aA.NN1 99!Q$A  YY%q	ArF   )r   r*   r   r   r,   r~   r}   r+   )r   r   FFNNN)ri   rj   rk   rl   rm   r/   rn   ro   rp   rq   r=   rt   r   r   r-   ru   r)   rv   rg   rw   rx   ry   s   @rD   r{   r{      s     		%%
 $"#48OO O 	O
 O O !bii1O O: %,,  rF   r{   )typingr   r   r/   torch.nnr-   torch.nn.functional
functionalr[   	attentionr   configr   r>   r   weight_initr	   ru   r   r{    rF   rD   <module>r      sC    !     % "  )t")) tn?ryy ?rF   