
    
3j                         S r SSKrSSKrSSKrSSKrSSKrSSKrSSKJrJ	r	  \R                  " \5      r " S S5      rg)zCheckpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
    N   )unwrap_modelget_state_dictc                   n    \ rS rSrSSSSSSSSS\S4S jrS	 rS
 rSS jrSS jr	SS jr
SS jrS rSrg)CheckpointSaver   N
checkpointrecovery F
   c                 ~   Xl         Xl        X l        X0l        X@l        XPl        / U l        S U l        S U l        SU l	        SU l
        SU l        Xl        Xl        X`l        Xpl        SU l        Xl        U
(       a  [$        R&                  O[$        R(                  U l        Xl        Xl        U R,                  S:  d   eg )Nr   Tz.pth.tarr   )modeltask	optimizerargs	model_ema
amp_scalercheckpoint_files
best_epochbest_metriccurr_recovery_fileprev_recovery_filecan_hardlinkcheckpoint_dirrecovery_dirsave_prefixrecovery_prefix	extension
decreasingoperatorltgtcmpmax_history	unwrap_fn)selfr   r   r   r   r   checkpoint_prefixr   r   r   r   r$   r%   r   s                 U/home/wildlama/miniconda3/lib/python3.13/site-packages/timm/utils/checkpoint_saver.py__init__CheckpointSaver.__init__   s    $ 
	"	"$ !#"$"$  -(,.#$",8;;(++&"1$$$    c                    U R                   (       a;   [        R                  R                  U5      (       a  [        R                  " U5        [        R                  " X5        g ! [
        [        4 a  nSU l          S nAN3S nAff = fNF)r   ospathexistsunlinkOSErrorNotImplementedErrorreplacer&   srcdstes       r(   _replaceCheckpointSaver._replaceD   s_    *77>>#&&IIcN 	

3 01 *$)!!*s   :A$ $B4B  Bc                 >   U R                   (       aR   [        R                  R                  U5      (       a  [        R                  " U5        [        R
                  " X5        g [        R                  " X5        g ! [        [        4 a  nSU l          S nAN3S nAff = fr-   )
r   r.   r/   r0   r1   linkr2   r3   shutilcopy2r5   s       r(   
_duplicateCheckpointSaver._duplicateM   sn    *77>>#&&IIcN! 	S 01 *$)!!*s   AA; ;BBBc                 B   U[        U R                  5      R                  R                  5       U R                  R                  5       SS.nU R                  b(  U R                  R                  US'   U R                  US'   U R                  b0  U R                  R                  5       X@R                  R                  '   U R                  bf  UR                  U R                  R                  U R                  S95        UR                  U R                  R                  SU R                  S95        OS[        U R                  U R                  5      US'   U R                  b#  [        U R                  U R                  5      US	'   Ub  X4S
'   [        R                   " XA5        g )N   )epocharchr   versionrD   r   )r%   T)emar%   
state_dictstate_dict_emametric)typer   __name__lowerr   rG   r   r   state_dict_keyr   updateget_checkpoint_stater%   r   r   torchsave)r&   	save_pathrC   rI   
save_states        r(   _saveCheckpointSaver._saveY   s=   $--335224	

 99 !%Jv!%Jv??&9=9S9S9UJ55699 dii<<t~~<VWdii<<QUQ_Q_<`a'5djj$..'QJ|$~~)/=dnndnn/]
+,#)x 

:)r+   c                    [        [        U R                  5      U5      nU R                  U-
  nUS:  d  [        U R                  5      U::  a  g U R                  US  nU HA  n [        R                  SR                  U5      5        [        R                  " US   5        MC     U R                  S U U l        g ! [         a/  n[        R                  SR                  U5      5         S nAM  S nAff = f)Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)minlenr   r$   _loggerdebugformatr.   remove	Exceptionerror)r&   trimdelete_index	to_deletedr8   s         r(   _cleanup_checkpoints$CheckpointSaver._cleanup_checkpointsp   s    3t,,-t4''$.!s4#8#89\I)),-8	AT7>>qAB		!A$  !% 5 5m| D  THOOPQRSSTs   $=B::
C3$C..C3c                    US:  d   e[         R                  R                  U R                  SU R                  -   5      n[         R                  R                  U R                  SU R                  -   5      nU R                  X1U5        U R                  X45        U R                  (       a  U R                  S   OS n[        U R                  5      U R                  :  d  Ub  U R                  X%S   5      (       Ga  [        U R                  5      U R                  :  a  U R                  S5        SR                  U R                  [        U5      /5      U R                  -   n[         R                  R                  U R                  U5      nU R                  XG5        U R                  R                  Xr45        [!        U R                  S U R"                  (       + S9U l        S	nU R                   H  n	US
R%                  U	5      -  nM     [&        R)                  U5        Ub  U R*                  b   U R                  X R*                  5      (       aT  Xl        X l        [         R                  R                  U R                  SU R                  -   5      n
U R                  XJ5        U R*                  c  S$ U R*                  U R,                  4$ )Nr   tmplastr   -c                     U S   $ )Nr    )xs    r(   <lambda>1CheckpointSaver.save_checkpoint.<locals>.<lambda>   s    adr+   )keyreversezCurrent checkpoints:
z {}

model_best)NN)r.   r/   joinr   r   rT   r9   r   rX   r$   r#   rc   r   strr?   appendsortedr   r[   rY   infor   r   )r&   rC   rI   tmp_save_pathlast_save_path
worst_filefilenamerR   checkpoints_strcbest_save_paths              r(   save_checkpointCheckpointSaver.save_checkpoint~   s&   zzT%8%8%$..:PQd&9&96DNN;RS

=0m4262G2GT**2.T
%%&)9)99~xx1..4(()T-=-==))!,xx!1!13u: >?$..PHT%8%8(CIOON6!!(())<=$*%%" OO+%D! 7O**7>>!#44 +LL)!t'7'7'?488FTdTdCeCe"'#) !#d.A.A<RVR`R`C`!a?#//7|`d>N>NPTP_P_=``r+   c                 F   US:  d   e[         R                  R                  U R                  SU R                  -   5      nU R                  X15        SR                  U R                  [        U5      [        U5      /5      U R                  -   n[         R                  R                  U R                  U5      nU R                  X55        [         R                  R                  U R                  5      (       aO   [        R                  SR                  U R                  5      5        [         R                  " U R                  5        U R"                  U l
        XPl        g ! [         a8  n[        R!                  SR                  X`R                  5      5         S nANUS nAff = f)Nr   recovery_tmpri   zCleaning recovery: {}z Exception '{}' while removing {})r.   r/   rr   r   r   rT   r   rs   r9   r0   r   rY   rZ   r[   r\   r]   r^   r   )r&   rC   	batch_idxrw   rz   rR   r8   s          r(   save_recoveryCheckpointSaver.save_recovery   s,   zzT%6%68WX

=(88T113u:s9~NORVR`R``GGLL!2!2H=	m/77>>$1122e5<<T=T=TUV		$112 #'"9"9"+  e@GGKbKbcddes   8AE 
F (.FF c                     [         R                  R                  U R                  U R                  5      n[
        R
                  " US-   U R                  -   5      n[        U5      n[        U5      (       a  US   $ S$ )N*r   r   )	r.   r/   rr   r   r   globr   ru   rX   )r&   recovery_pathfiless      r(   find_recoveryCheckpointSaver.find_recovery   s^    T%6%68L8LM		-#->?uu::uQx-2-r+   )r   r   r   r   r   r   r   r#   r   r   r   r$   r   r   r   r   r   r   r   r   r%   )N)r   )rK   
__module____qualname____firstlineno__r   r)   r9   r?   rT   rc   r~   r   r   __static_attributes__rk   r+   r(   r   r      sQ    
 *&"+%Z
*.E%aN,$.r+   r   )__doc__r   loggingr    r.   r=   rP   r   r   r   	getLoggerrK   rY   r   rk   r+   r(   <module>r      s?       	   / 

H
%e. e.r+   