
    
3j                     \    S r SSKrSSKJs  Jr  SSKJrJr  SSK	r	SSK
Jr   " S S\5      rg)a  
SGDP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/sgdp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Spherical Cautious Optimizers: https://openreview.net/forum?id=OyT2CJ4fh7 
Copyright (c) 2020-present NAVER Corp.
MIT license
    N)	Optimizerrequired   )
projectionc            	       p   ^  \ rS rSr\SSSSSSSS4	U 4S jjr\R                  " 5       S	S j5       rSr	U =r
$ )
SGDP   r   Fg:0yE>g?c                 N   > [        UUUUUUUU	U
S9	n[        [        U ]  X5        g )N)	lrmomentum	dampeningweight_decaynesterovepsdeltawd_ratiocaution)dictsuperr   __init__)selfparamsr   r   r   r   r   r   r   r   r   defaults	__class__s               I/home/wildlama/miniconda3/lib/python3.13/site-packages/timm/optim/sgdp.pyr   SGDP.__init__   s;     %

 	dD"64    c                    S nUb%  [         R                  " 5          U" 5       nS S S 5        U R                   GH  nUS   nUS   nUS   nUS   nUR                  SS5      nUS    GHu  n	U	R                  c  M  U	R                  n
U R
                  U	   n[        U5      S:X  a  [         R                  " U	5      US'   US   nUR                  U5      R                  U
S	U-
  S
9  U(       a  XU-  -   nOUR                  5       nS	n[        U	R                  5      S:  a  [        XXS   US   US   U5      u  pOdU(       a]  X-  S:  R                  U
R                  5      nUR                  UR!                  5       R#                  SS95        UR                  U5        US:w  a&  U	R                  S	US   US   -  U-  SU-
  -  -
  5        U	R                  XS   * S
9  GMx     GM     U$ ! , (       d  f       GN= f)Nr   r   r   r   r   Fr   r   g      ?)alphar   r   r   r   gMbP?)minr   )torchenable_gradparam_groupsgetgradstatelen
zeros_likemul_add_cloneshaper   todtypediv_meanclamp_)r   closurelossgroupr   r   r   r   r   pr%   r&   bufd_pr   masks                   r   step	SGDP.step1   s   ""$y % &&E 0LZ(Hk*IZ(Hii	51G8_66>vv

1 u:?(-(8(8(;E*% J'"''BN'CC/C))+C qww<!#$.q7^US]M^`efk`lnu$vMCJN..tzz:DIIdiik00T0:;HHTN  1$FF2deN.C Ch NRST\R\ ]]^ s;,/A % 'R Y %$s   G''
G6 )N)__name__
__module____qualname____firstlineno__r   r   r!   no_gradr9   __static_attributes____classcell__)r   s   @r   r   r      s?     54 ]]_/ /r   r   )__doc__r!   torch.nn.functionalnn
functionalFtorch.optim.optimizerr   r   mathadampr   r   r;   r   r   <module>rK      s,       5  K9 Kr   