
    
3j/2                         S r SSKJrJrJr  SSKrSSKJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJrJr  SS	KJr   " S
 S\R&                  5      r " S S\R&                  5      rg)a9  Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
    )OptionalUnionTupleN   )use_fused_attn)	to_2tuple)resample_abs_pos_embed)apply_rot_embed_catcreate_rope_embed)trunc_normal_c                   l  ^  \ rS rSr% Sr\R                  R                  \   \	S'                SS\
S\\
   S\\
\\
\
4   4   S\\
   S\\
   S	\\
   S
\S\S\S\S\S\4U 4S jjjrSS\4S jjrSS\\
   S\\   4S jjrS\R&                  S\
S\
S\R&                  4S jrSS\4S jjrSrU =r$ ) RotAttentionPool2d   as  Attention based 2D feature pooling w/ rotary (relative) pos embedding.
This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW

Setting out_features=0 disables the output projection (pre_logits mode).

fused_attnin_featuresout_featuresref_feat_size	embed_dimhead_dim	num_headsqkv_biasqkv_separate	pool_typeclass_token	drop_rate	rope_typec           
        > XS.n[         TU ]  5         U	S;   d   eU=(       d    U=U l        nXl        Uc  Xl        OUS:  a  X l        OX@l        [        U5      nUb  XF-  S:X  d   eXF-  nOXE-  S:X  d   eXE-  nX`l        XPl        U	R                  5       U l	        U R                  S-  U l
        [        5       U l        Xl        U
(       a2  [        R                  " [         R"                  " SU40 UD65      U l        OS U l        U(       ab  [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        S U l        O![        R&                  " XS-  4SU0UD6U l        [        R0                  " U5      U l        US:w  a!  [        R&                  " X@R                  40 UD6O[        R4                  " 5       U l        [9        S
UUUSUSS	.UD6U l        g )Ndevicedtype tokenr         r   bias   F)r   dimr   	in_pixelsref_feat_shaperotate_half )super__init__r   r   r   r   r   r   lowerr   scaler   r   r   nn	Parametertorchzeros	cls_tokenLinearqkvqkvDropoutdropIdentityprojr   	pos_embed)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r    dd	__class__s                   V/home/wildlama/miniconda3/lib/python3.13/site-packages/timm/layers/attention_pool2d.pyr-   RotAttentionPool2d.__init__$   s   " /M)))%.%=+=& +A , )!-0 (A--- -H'1,,,!-I" "*]]d*
(*"\\%++a*Ib*IJDN!DNYY{KHKKDFYY{KHKKDFYY{KHKKDFDHyy!mQ(QbQDHJJy)	EQUVEVBIIi):):AbA\^\g\g\i	* 
(
 
    zero_init_lastc                    U R                   Gc  U R                  R                  n[        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        g U R                   R                  n[        U R                   R                  US-  S9  [
        R                  R                  U R                   R                  5        g Nr$   )std)r9   r6   r   r   weightr0   initzeros_r%   r7   r8   r?   rE   r   s      rB   init_weightsRotAttentionPool2d.init_weightsg   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)rD   num_classesc                     Ub  US;   d   eX l         Ub[  US:  a!  [        R                  " U R                  U5      O[        R                  " 5       U l        US:  a  UOU R                  U l        g g Nr!   r   r   r0   r5   r   r<   r=   r   r?   rO   r   s      rB   resetRotAttentionPool2d.resetu   c     ---&N"BMPQ/		$..+>WYWbWbWdDI/:QDNND #rD   xHWreturnc                     U R                   S:X  a  US S 2S4   nU$ US S 2SS 24   R                  UR                  S   X#S5      R                  SSSS5      nU$ Nr#   r   r   r&      r   reshapeshapepermuter?   rW   rX   rY   s       rB   _poolRotAttentionPool2d._pool~   `    >>W$!Q$A  !QR%  Q26>>q!QJArD   
pre_logitsc                 `   UR                   u  p4pVXV-  nUR                  S5      R                  SS5      nU R                  c&  [        R
                  " UR                  SSS9U/SS9nO?[        R
                  " U R                  R                  UR                   S   SS5      U/SS9nU R                  c  U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      nU R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n	U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n
OaU R                  U5      R                  X7S-   SU R                  U R                  5      R                  SSSSS	5      nUR!                  S5      u  pn
U R"                  R%                  XV45      n['        U[(        5      (       a  [        R
                  " USS9n[        R
                  " US S 2S S 2S S2S S 24   [+        US S 2S S 2SS 2S S 24   U5      /SS9R-                  U
5      n[        R
                  " U	S S 2S S 2S S2S S 24   [+        U	S S 2S S 2SS 2S S 24   U5      /SS9R-                  U
5      n	U R.                  (       a!  [0        R2                  R5                  XU
5      nO5XR6                  -  nXR                  S
S5      -  nUR9                  SS9nX-  nUR                  SS5      R                  X7S-   S5      nU R;                  U5      nU(       a  U R=                  XU5      nU$ U R?                  U5      nU R=                  XU5      nU$ )Nr^   r   Tkeepdimr'   r   r]   r&      ) ra   flatten	transposer4   r2   catmeanexpandr9   r6   r`   r   r   r7   r8   rb   unbindr>   	get_embed
isinstancetupler
   type_asr   r0   
functionalscaled_dot_product_attentionr/   softmaxr;   rd   r=   )r?   rW   rg   B_rX   rY   Nr6   r7   r8   ropeattns                rB   forwardRotAttentionPool2d.forward   s   WW
aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA88q	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZA##A1uaOWWXY[\^_abdefAhhqkGA!~~''/dE""99Tr*DIIqArr1~':1Q12q[>4'PQWXYaabcdIIqArr1~':1Q12q[>4'PQWXYaabcd??::1CAJJA{{2r**D<<B<'DAKK1%%aQ3IIaL

1#AHIIaLJJqQrD   )r4   r;   r   r   r   r   r7   r   r   r   r>   r=   r6   r9   r   r/   r8   )N   N@   NTFr#   F        rp   NNFNN)__name__
__module____qualname____firstlineno____doc__r2   jitFinalbool__annotations__intr   r   r   strfloatr-   rM   rT   Tensorrd   r   __static_attributes____classcell__rA   s   @rB   r   r      s[   
 		%%
 +/9:'+&('+!!&$ %!"A
A
 #3-A
 !eCHo!56	A

  }A
 smA
  }A
 A
 A
 A
 A
 A
 A
 A
F*4 *S# S(3- Su||    %T % %rD   r   c                   f  ^  \ rS rSr% Sr\R                  R                  \   \	S'               SS\
S\\
\\
\
4   4   S\\
   S\\
   S\\
   S	\\
   S
\S\S\S\S\4U 4S jjjrSS\4S jjrSS\\
   S\\   4S jjrS\R&                  S\
S\
S\R&                  4S jrSS\4S jjrSrU =r$ )AttentionPool2d   a  Attention based 2D feature pooling w/ learned (absolute) pos embedding.
This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

It was based on impl in CLIP by OpenAI
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.

Setting out_features=0 disables the output projection (pre_logits mode).
r   r   	feat_sizer   r   r   r   r   r   r   r   r   c                   > XS.n[         TU ]  5         U	S;   d   eU=(       d    U=U l        nXl        Uc  Xl        OUS:  a  X0l        OX@l        Ub  XF-  S:X  d   eXF-  nOXE-  S:X  d   eXE-  n[        U5      U l        U R                  S   U R                  S   -  U l        X`l        XPl	        Xl
        U R                  S-  U l        [        5       U l        U
(       a2  [        R                  " [         R"                  " SU40 UD65      U l        OS U l        U(       ab  [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        [        R&                  " X4SU0UD6U l        S U l        O6S =U l        =U l        U l        [        R&                  " XS-  4SU0UD6U l        [        R0                  " U5      U l        US:w  a!  [        R&                  " X@R                  40 UD6O[        R4                  " 5       U l        [        R                  " [         R"                  " U R                  S-   U40 UD65      U l        U R;                  5         g )Nr   r!   r   r   r$   r%   r&   )r,   r-   r   r   r   r   r   seq_lenr   r   r   r/   r   r   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   rM   )r?   r   r   r   r   r   r   r   r   r   r   r   r   r    r@   rA   s                  rB   r-   AttentionPool2d.__init__   s     /M)))%.%=+=& +A , ) (A--- -H'1,,,!-I"9-~~a(4>>!+<<" "]]d*
(*\\%++a*Ib*IJDN!DNYY{KHKKDFYY{KHKKDFYY{KHKKDFDH'++DF+TVdfyy!mQ(QbQDHJJy)	EQUVEVBIIi):):AbA\^\g\g\i	ekk$,,2BK&VSU&VWrD   rE   c                 H   U R                   Gc  U R                  R                  n[        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        [        U R                  R                  US-  S9  [
        R                  R                  U R                  R                  5        OjU R                   R                  n[        U R                   R                  US-  S9  [
        R                  R                  U R                   R                  5        [        U R                  US-  S9  g rG   )r9   r6   r   r   rI   r0   rJ   rK   r%   r7   r8   r>   rL   s      rB   rM   AttentionPool2d.init_weights   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)dnn+*=>rD   rO   c                     Ub  US;   d   eX l         Ub[  US:  a!  [        R                  " U R                  U5      O[        R                  " 5       U l        US:  a  UOU R                  U l        g g rQ   rR   rS   s      rB   rT   AttentionPool2d.reset  rV   rD   rW   rX   rY   rZ   c                     U R                   S:X  a  US S 2S4   nU$ US S 2SS 24   R                  UR                  S   X#S5      R                  SSSS5      nU$ r\   r_   rc   s       rB   rd   AttentionPool2d._pool  rf   rD   rg   c                    UR                   u  p4pVXV-  nUR                  S5      R                  SS5      nU R                  c&  [        R
                  " UR                  SSS9U/SS9nO?[        R
                  " U R                  R                  UR                   S   SS5      U/SS9n[        U R                  R                  S5      XV4SS9nX-   nU R                  c  U R                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n	U R!                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      n
U R#                  U5      R                  X7S-   U R                  U R                  5      R                  SS5      nO_U R                  U5      R                  USS	U R                  U R                  5      R%                  SSS	SS
5      nUR'                  S5      u  pnU R(                  (       a!  [*        R,                  R/                  XU5      nO5XR0                  -  n	XR                  SS5      -  nUR3                  SS9nX-  nUR                  SS5      R                  X7S-   S5      nU R5                  U5      nU(       a  U R7                  XU5      nU$ U R9                  U5      nU R7                  XU5      nU$ )Nr^   r   Tri   rk   r   r]   )num_prefix_tokensr&   rl   rm   )ra   rn   ro   r4   r2   rp   rq   rr   r	   r>   	unsqueezer9   r6   r`   r   r   r7   r8   rb   rs   r   r0   rx   ry   r/   rz   r;   rd   r=   )r?   rW   rg   r{   r|   rX   rY   r}   r>   r6   r7   r8   r   s                rB   r   AttentionPool2d.forward  sm   WW
aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA*4>>+C+CA+Fbcd	M88q	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZAq	!!!UDNNDMMJTTUVXYZA##Ar1dnndmmLTTUVXY[\^_abcAhhqkGA!??::1CAJJA{{2r**D<<B<'DAKK1%%aQ3IIaL

1#AHIIaLJJqQrD   )r4   r;   r   r   r   r   r   r7   r   r   r   r>   r=   r6   r9   r/   r   r8   )r   NNr   NTFr#   Fr   NNr   r   )r   r   r   r   r   r2   r   r   r   r   r   r   r   r   r   r   r-   rM   rT   r   rd   r   r   r   r   s   @rB   r   r      s@   	 		%%
 67*.'+&('+!!&$ %!:: S%S/12: #3-	:
  }: sm:  }: : : : : : :x?4 ?S# S(3- Su||    !T ! !rD   r   )r   typingr   r   r   r2   torch.nnr0   configr   helpersr   r>   r	   pos_embed_sincosr
   r   weight_initr   Moduler   r   r+   rD   rB   <module>r      sL    * )   "  - D &U UpJbii JrD   