
    
3jt                     f   S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKr	S SK
Jr  \R                  R                  5       rSrSS jrSS jr " S S\R$                  5      r " S	 S
\R$                  5      rSr/ SQrS rS rS rSS jrSS jrSS jrSS jrS rS rS r SS jr!SS jr"S r#g)    N)tqdm)iogGz?c           	      	   [        U R                  5      S:H  nU(       d  SOU R                  5       SS n	U=(       a    U	S:g  n
U R                  5       SS u  pU(       a'  U
(       a  U R                  SS9n OU R	                  5       n U R                  5       nUS:X  aN  [        R                  " U[        X,U5      US	9u  pnU[        R                  " U5      -  nUR                  5       nGO[        R                  R                  U5      u  pnS
U;   Gad  US:X  aB  [        R                  " U5      U-  n[        R                  " UU:  5      R                  5       nGOXUS:X  aa  [        R                   " US-  SS9n[        R                  " US-  5      nUU-  n[        R                  " UU:  5      R                  5       S-   nOUS:X  aV  [        R                   " USS9nU[        R                  " U5      -  n[        R                  " UU:  5      R                  5       nOUS:X  a  UR#                  S5      n[        [        R                  " U5      5      n[        R                   " USS9U-  n[%        [        R&                  " UUS-  5      5      S-   n[        S[        U[        U5      5      5      nO [        WU5      nUS:X  a  UR#                  S5      n[        R(                  " [        R                  " U5      5      n[        R(                  " [        R                  " USU 5      5      n[        UU-  5      n[*        R,                  " U SU SUS 35        O[*        R,                  " U SU 35        OUn[        SU5      n[        XU5      nUSS2SU24   nUSU nU[        R                  " U5      -  nUSU2SS24   nU(       a  [        R.                  " UR                  5       UR                  5       /5      nUR1                  5       S:  aQ  [        R2                  " UR1                  5       UR4                  S9SS n UU    n![        R6                  " U![8        5      n"O[        R6                  " U[8        5      n"U"* n#UR;                  U#U"5      nUR;                  U#U"5      nU(       a-  UR=                  UWSS5      nUR=                  UXS   U	S   5      nUU4$ )zB
Extracts LoRA weights from a weight difference tensor using SVD.
   N   )   r   r   r   )	start_dimsvd_lowrankqniteradaptiveadaptive_ratioadaptive_energydimadaptive_quantileadaptive_froz Extracted LoRA rank: z, Frobenius retained: .1%i )device)lenshapesizeflattensqueezefloattorchr
   mindiagtlinalgsvdmaxsumitemcumsumpowintsearchsortedsqrtlogginginfocatnumelrandpermr   quantileCLAMP_QUANTILEclampreshape)$diffkeyrank	algorithm	lora_typelowrank_itersadaptive_paramclamp_quantileconv2dkernel_size
conv2d_3x3out_dimin_dim
diff_floatUSVVhmin_s	lora_rankenergytotal_energy	thresholds_cummin_cum_sum	S_squaredS_fro_sqsum_S_squareds_fro	s_red_frofro_percentdistidxdist_samplehi_vallow_vals$                                       M/home/wildlama/comfy/ComfyUI/custom_nodes/ComfyUI-KJNodes/nodes/lora_nodes.pyextract_lorarY      s    $**o"F$$$))+a*:K1K61Jiik!A&OG<<!<,D<<>DJM!##J#dG2LTaba

1SSU <<##J/b",,		!~5!IIa%i0557	//ad2$yyA*\9	!IIfy&89>>@1D	11QA.,uyy|;!IIek&9:??A	n,EE!H	 9!56 %YA > I 2 2=.RSBS TUXYY	3y#a&#9:	 It,I N*EE!H	

599Y#78!JJuyy:I1F'GH	#I$56u$:9+E[\ghk[lmnu$:9+FGI9%	3	a)mjyM

1

Ayy!))+rzz|45::<'!..dkkB8GLCs)K^^K@F^^D.9F'GGGV$XXgv&IIgy!Q/ZZ	6q>;q>Jr7N    c                 D	   / nU R                   R                  5        H1  u  pUb  UR                  U5      (       d  M   UR                  U5        M3     U R                   R	                  5        H1  u  pUb  UR                  U5      (       d  M   UR                  U5        M3     [        U Vs/ s H;  oR                  S5      (       d!  U	(       d  M!  UR                  S5      (       d  M9  UPM=     sn5      n[        USUR                  S5       S3S9n[        R                  R                  U5      nU GH  nUR                  S5      (       GaC  U R                  USS9nUc$  UR                  S	5        UR                  S	5        MR  UR                  S
:X  a>  [        R                   " SU 35        AUR                  S	5        UR                  S	5        M  US:w  GaF  UR                  S:  ax  U	(       aK  UR#                  5       R%                  U5      R'                  5       USR)                  X?[        U5      S 5      '   AUR                  S	5        UR                  S	5        GM/   [+        UR%                  [,        5      XXeXzUS9nUS   R#                  5       R%                  U5      R'                  5       USR)                  X?[        U5      S 5      '   US	   R#                  5       R%                  U5      R'                  5       USR)                  X?[        U5      S 5      '   OKUR#                  5       R%                  U5      R'                  5       USR)                  X?[        U5      S 5      '   AUR                  S	5        UR                  S	5        GM^  U	(       d  GMh  UR                  S5      (       d  GM  U R                  USS9nUbL  UR#                  5       R%                  U5      R'                  5       USR)                  X?[        U5      S 5      '   AUR                  S	5        UR                  S	5        GM     UR3                  5         A [        R4                  R7                  5         U$ s  snf ! [.         a'  n[        R0                  " SU SU 35         S nAGN1S nAff = f)N.weightz.biaszExtracting LoRA (.))totaldescT)return_weightr      zSkipping 5D tensor for key fullr   z	{}{}.diffi)r9   r:   r;   r   z{}{}.lora_up.weightz{}{}.lora_down.weightz(Could not generate lora weights for key z, error z{}{}.diff_b)modelnamed_parameters
startswithappendnamed_buffersr   endswithr   stripcomfyutilsProgressBarpatch_weight_to_deviceupdatendimr+   r,   
contiguoustocpuformatrY   r   	Exceptionwarningclosemodel_managementsoft_empty_cache)
model_diffr6   prefix_modelprefix_lora	output_sdr8   r7   r9   	out_dtype	bias_diffr:   r;   sd_keysname_k
total_keysprogress_bar
comfy_pbarweight_diffouteweights                          rX   calc_lora_modelr   g   s[   G##4464??<#@#@NN4  7 ##1134??<#@#@NN4  4 iAJJy,A,AiaTUT^T^_fTgaijJj1B;CTCTUXCYBZZ[/\]L((4J ::i  $;;AT;RK"##A&!!!$1$:1#>?##A&!!!$F"##a' ^i^t^t^v^y^y  {D  _E  _I  _I  _K	+"4"4[CDUVXBY"Z[# ''*%%a(_&{~~f'=q	ly  jx  yCdghidjdududwdzdz  |E  eF  eJ  eJ  eLI3::;#lJ[\^H_`afijkflfwfwfyf|f|  ~G  gH  gL  gL  gNI5<<[CP\L]^`Jabc WbVlVlVnVqVqr{V|  WA  WA  WC	+,,[C<Mb:QRS"a Y1::g..66q6MF!X^XiXiXkXnXnoxXyX}X}X	-..{c,>OPR<STU"a Q T 	++-e j> ! _OO&NqcQYZ[Y\$]^^_s+   * Q)Q)&Q)!B<Q..
R8RRc                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LoraExtractKJ   c                 T   [         R                  " SSS[         R                  R                  S[         R                  [         R
                  /SS9[         R                  R                  S[         R                  [         R
                  /SS9[         R                  R                  S	S
S9[         R                  R                  SSSSSSS9[         R                  R                  S/ SQS9[         R                  R                  SSS/SSS9[         R                  R                  SSSSSSS9[         R                  R                  S/ SQS S!9[         R                  R                  S"SS9[         R                  R                  S#S$S%S&S'S(S9[         R                  R                  S)S*S9/S+9$ ),Nr   KJNodes/loraT	finetunedz1The finetuned model or clip to extract LoRA from.)tooltiporiginalz0The original base model or clip to diff against.filename_prefixzloras/ComfyUI_extracted_loradefaultr6   @   r      zNThe rank to use for standard LoRA, or maximum rank limit for adaptive methods.r   r   r#   stepr   r8   )standardrc   r   r   r   r   )optionsr7   
svd_linalgr
   z>SVD algorithm to use, svd_lowrank is faster but less accurate.r   r   r   r9      d   z<The number of subspace iterations for lowrank SVD algorithm.output_dtype)fp16bf16fp32r   )r   r   r   r:   g333333?              ?{Gz?zFor ratio mode, this is the ratio of the maximum singular value. For quantile mode, this is the quantile of the singular values. For fro mode, this is the Frobenius norm retention ratio.r;   F)node_idcategoryis_output_nodeinputs)r   Schema	MultiTypeInputModelClipStringIntComboBooleanFloatclss    rX   define_schemaLoraExtractKJ.define_schema   s   yy##"";2770C  NA"  B"":"''/BL~"		 1;YZVRQDq  Sc  d{  5S  T{\=4Q[h  ss  t_aQCa  ZX  Y~7OY_`

  d ;/3CVZ  ea  b

  !15 A	
 	
rZ   returnc                    US:X  a  US:w  a  [        S5      e[        R                  [        R                  [        R                  S.U   n[
        R                  " 5       n[
        R                  " X=5      u  pnnn0 n[        US5      nU(       a  UR                  5       nUR                  5       nUR                  5        VVs0 s H9  u  nnUR                  S5      (       a  M  UR                  S5      (       a  M6  UU_M;     nnnUR                  USS	5        [        UR                  US
SUXVX|XUS9nOEUR                  5       nUR                  S5      nUR                  USS	5        [        UUSSUXVX|XUS9nSU;   a	  U SU
S 3nOUnU SU SU SUS S3n[         R"                  R%                  UU5      n[&        R(                  R+                  UUS S9  [,        R.                  " 5       $ s  snnf )Nr
   r   zEsvd_lowrank algorithm is only supported for standard LoRA extraction.)r   r   r   patcherz.position_idsz.logit_scaleg      r    ztext_encoders.)r   r:   r;   zdiffusion_model.r   r   z.2f_rank_05_.safetensorsmetadata)
ValueErrorr   bfloat16float16float32folder_pathsget_output_directoryget_save_image_pathhasattrcloneget_key_patchesitemsrj   add_patchesr   r   ospathjoinrl   rm   save_torch_filer   
NodeOutput)r   r   r   r   r6   r8   r7   r9   r   r   r:   r;   dtype
output_dirfull_output_folderfilenamecounter	subfolderr~   is_clip	clip_diffkpr   vmrank_stroutput_checkpoints                              rX   executeLoraExtractKJ.execute   s    %)z*AdeeVWcd!668
LXLlLlm|  MJIgy/	)Y/!)I))+B#%88:t:41aQZZ5P$YZYcYcdrYs$!Q$:Bt!!"dC0'	(9(94EUW`bk  yF  Yb  R`  aI!A))*<=BMM"dC('41CEWYbdm  {H  [d  Tb  cI"#AnS%9:HH'jxj,qQSTabGGLL);=NO##I/@4#P}}# us   G.$G.<G. N
__name__
__module____qualname____firstlineno__classmethodr   r   r   r   __static_attributes__r   rZ   rX   r   r      sB    
 
* !  ce  cp  cp ! !rZ   r   c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LoraReduceRank   c                    [         R                  " SSSSSS[         R                  R                  S[        R
                  " S5      SS	9[         R                  R                  S
SSSSSS9[         R                  R                  S/ SQSSS9[         R                  R                  SSSSSSS9[         R                  R                  S/ SQSSS9[         R                  R                  SSS 9/S!9$ )"NLoraReduceRankKJr   r   zResize a LoRA model by reducing its rank. Based on kohya's sd-scripts: https://github.com/kohya-ss/sd-scripts/blob/main/networks/resize_lora.pyT	lora_nameloraszThe name of the LoRA.)r   r   new_rank   r   r   zLThe new rank to resize the LoRA. Acts as max rank when using dynamic_method.r   dynamic_method)disabledsv_ratiosv_cumulativesv_frosv_kneer   zyMethod to use for dynamically determining new alphas and dims. sv_knee finds the elbow point in the singular value curve.r   dynamic_paramg?r   g       @r   zParameter for dynamic methods. For sv_knee: sensitivity (1.0=standard knee, <1.0=more aggressive/lower rank, >1.0=more conservative).r   )match_originalr   r   r   r   zData type to save the LoRA as.verboser   )r   display_namer   descriptionr   is_experimentalr   )	r   r   r   r   r   get_filename_listr   r   r   r   s    rX   r   LoraReduceRank.define_schema   s    yy&)# j {L4R4RSZ4[e|}Zt!  Vd  e/9w  BL  VQ  R#TX  cj  k~7ak{  Ff  g

  D 9
 	
rZ   r   c           	        ^  [         R                  " SU5      n[        R                  R	                  USS9u  m nUS:X  a  [
        R                  n	OXUS:X  a  [
        R                  n	OAUS:X  a  [
        R                  n	O*US:X  a$  [        U 4S jT  5       5      n
T U
   R                  n	0 nT R                  5        H  u  pXUR                  S	S
5      '   M     @ [        R                  " S5        [        XW	[         X4U5      u  pnnUc  0 nUR#                  SS
5      nUS:X  a+  SU SU SU 3US'   [%        U5      US'   [%        U5      US'   OSU SU SU SU 3US'   SUS'   SUS'   ['        UR)                  5       5       Hj  nUU   n[+        U5      [
        R,                  :X  d  M'  UR                  R.                  (       d  MD  UR                  U	:w  d  MV  UR1                  U	5      UU'   Ml     [         R2                  " 5       nSU-   n[         R4                  " UU5      u  nnnnnUS:w  a  SU 3OS
n[%        [7        [8        R:                  " U5      5      5      nUS:X  a  UOSU 3nUR                  SS
5       SU SU U SUS S3	n[<        R>                  RA                  UU5      n[        R                  " SU 35        [        R                  RC                  UUUS 9  [D        RF                  " 5       $ )!Nr   T)return_metadatar   r   r   r   c              3      >#    U  HB  oR                  S 5      (       d  M  [        TU   [        R                  5      (       d  M>  Uv   MD     g7f)r\   N)rj   
isinstancer   Tensor).0r   lora_sds     rX   	<genexpr>)LoraReduceRank.execute.<locals>.<genexpr>  s6     #yw!**Y:OAT^_fgh_ikpkwkwTxAAws   A A	Az.defaultr   zResizing Lora...ss_training_commentr   zdimension is resized from z to z; ss_network_dimss_network_alphazDynamic resize with : z from Dynamiczloras/r   dynamic_z.safetensors_resized_from__to_r   r   zSaving resized LoRA to r   )$r   get_full_pathrl   rm   load_torch_filer   r   r   r   nextr   r   replacer+   r,   resize_lora_modelr   getstrlistkeystyper  is_floating_pointrs   r   r   r(   npmeanr   r   r   r   r   r   )!r   r   r   r   r   r   r   	lora_pathr   
save_dtypefirst_weight_keynew_lora_sdr   r   r~   old_dim	new_alpha	rank_listcommentr5   valuer   output_filename_prefixr   r   r   r   r   output_dtype_straverage_rankr   r   r	  s!                                   @rX   r   LoraReduceRank.execute   s-    ..w	B	!KK77	SW7X6!JV#JV#J--##yw#yy !1288JMMODA56		*b12 $'(3D[\fhnp~  PW  4X0	IyH,,4b9Z'0J7)SWX`Waacdkcl.mH*+),XH%&+.y>H'(0D^DTTVWdVeeklskttvw~v  /AH*+)2H%&+4H'(	()CcNEE{ell*u{{/L/L/LQVQ\Q\`jQj!&*!5	# *
 "668
!)I!5LXLlLl  nD  FP  MQIHgy/1=AQ1QQ|n-WY3rwwy123-;88L>AZ'//CDNSZR[[_`h_ijzi{{|  ~E  FH  }I  IV  WGGLL);=NO./@.ABC##I/@8#T}}rZ   r   Nr   r   rZ   rX   r   r      s6    
 
$ 4cecpcp 4 4rZ   r   gư>))	lora_downlora_up)lora_Alora_B)downupc           	          [        [        R                  " U 5      5      n[        R                  " U SS9U-  n[	        [        R
                  " X15      5      S-   n[        S[        U[        U 5      S-
  5      5      nU$ )Nr   r   r   )	r   r   r$   r&   r(   r)   r#   r   r   )rC   targetoriginal_sumcumulative_sumsindexs        rX   index_sv_cumulativer9  <  sc    1&Lll1!,|;O""?;<q@E3uc!fqj)*ELrZ   c           	      $   U R                  S5      n[        [        R                  " U5      5      n[        R                  " USS9U-  n[        [        R                  " XAS-  5      5      S-   n[        S[        U[        U 5      S-
  5      5      nU$ )Nr   r   r   r   )
r'   r   r   r$   r&   r(   r)   r#   r   r   )rC   r5  rM   rN   rO   r8  s         rX   index_sv_fror;  E  su    aIUYYy)*HLL2X=M""=!)<=AE3uc!fqj)*ELrZ   c           	          U S   nX!-  n[        [        R                  " X:  5      R                  5       5      n[	        S[        U[        U 5      S-
  5      5      nU$ )Nr   r   )r(   r   r$   r%   r#   r   r   )rC   r5  max_svmin_svr8  s        rX   index_sv_ratior?  O  sP    qTF_F		!*%**,-E3uc!fqj)*ELrZ   c                 b   [        U 5      nUS::  a  gU R                  5       R                  5       R                  5       n[        R
                  " SSU5      nX3S   -
  US   US   -
  S-   -  nSX-  -
  nXV-
  n[        [        R                  " U5      5      n[        S[        XS-
  5      5      nU$ )a  Find the knee/elbow point in the singular value curve.
Uses the Kneedle method: normalizes the curve to [0,1] on both axes,
then finds the point with maximum distance from a reference line.

sensitivity controls the aggressiveness:
  1.0 = standard knee point
  < 1.0 = more aggressive (lower rank), e.g. 0.5 roughly halves the knee rank
  > 1.0 = more conservative (higher rank)
The reference line tilts based on sensitivity: at lower values,
the line favors keeping fewer singular values.r   r   r   g|=r   )
r   rt   r   numpyr  linspacer(   argmaxr#   r   )	rC   sensitivitynS_npxyy_line	distancesr8  s	            rX   index_sv_kneerL  X  s     	AAAv557==?  "D 	Aq!A	RT!WtBx/%78A
 ;?"F 
I 		)$%E3u!e$%ELrZ   c                    U R                  U5      n U R                  [        R                  :w  a  U R	                  5       n U(       aU  US:w  aO  [        R
                  R                  U SS9u  pgn[        XqX#U5      n	U	S   nUSS2SU24   nUSU nUSU2SS24   nO9[        R                  " XSS9u  pgn
U
R                  5       nU[	        XQ-  5      S.n	A
[        R                  " [        R                  " U5      5      nXk-  nX-  nXhU	4$ )	zShared SVD extraction for both linear and conv weights.
Dynamic mode: single full SVD (need all singular values for rank selection).
Disabled mode: svd_lowrank only (rank is known, much faster for large matrices).r   F)full_matricesr   Nr   r   )r   r&  )rs   r   r   r   r   r!   r"   rank_resizer
   r    r   r*   )	weight_2drG   r   r   r   scalerB   rC   rE   
param_dictrD   sqrt_Ss               rX   _svd_extractrT  |  s    V$I%--'OO%	.J6 <<##IU#Cb ~eT
z*	a)mjyM

A ##I!DaSSU"+%@Q:RS
ZZ

1&F	
A	B*rZ   c                    U R                  5       u  pgp[        U R                  US5      XX4U5      u  pnUS   nUR                  XX5      R                  5       US'   U
R                  XaSS5      R                  5       US'   A
AA U$ NrA  r   r.  r   r/  r   rT  r3   rt   )r   rG   r   r   r   rQ  out_sizein_sizer=   r   rB   rE   rR  s                rX   extract_convrZ    s    (.%H{$x$iX]A: :&I jj[VZZ\J{IIh1a@DDFJy	2vrZ   c                     U R                  5       u  pg[        XX#XE5      u  pn
U
S   nU	R                  X5      R                  5       U
S'   UR                  Xa5      R                  5       U
S'   AA	A U
$ )Nr   r.  r/  rW  )r   rG   r   r   r   rQ  rX  rY  rB   rE   rR  s              rX   extract_linearr\    sw    H$>&A: :&I jj<@@BJ{IIh:>>@Jy	2vrZ   c           
      6   U R                   u  p4pVUR                   u  px  n	X8:X  a  XV:X  d   SU SU SU SU S3	5       eU R                  U5      n UR                  U5      nUR                  US5      U R                  US5      -  n
U
R                  XtXU5      nAA U$ )Nrank  z or kernel 	 mismatchrA  r   rs   r3   )r.  r/  r   in_rankrY  r=   k_rX  out_rankr   mergedr   s               rX   
merge_convrf    s    (1%Gk&}}H1;#4xgYazQ\]h\iijkmjnnw6xx4V$Ijj G__Xr*Y->->w-KKF^^H{HFMrZ   c                     U R                   u  p4UR                   u  pVX6:X  d   SU SU S35       eU R                  U5      n UR                  U5      nX-  nAA U$ )Nr^  r_  r`  )r   rs   )r.  r/  r   rb  rY  rX  rd  r   s           rX   merge_linearrh    sh     G HE%y(9 EEV$Ijj G FMrZ   c                 &   U R                   u  p4pVnUR                   u  p    n
X9:X  d   SU SU	 S35       eU R                  U5      n UR                  U5      nUR                  US5      U R                  US5      -  nUR                  XXVU5      nAA U$ )Nr^  r_  r`  rA  ra  )r.  r/  r   rb  rY  kDkHkWrX  rd  r   re  r   s                rX   merge_conv3drm    s    #,?? Gbb")--H1aE%y(9 EEV$Ijj G__Xr*Y->->w-KKF^^Hrr:FMrZ   c                    U R                  5       u  pgpn
[        U R                  US5      XX4U5      u  pnUS   nUR                  XXU
5      R                  5       US'   UR                  XaSSS5      R                  5       US'   AAA U$ rV  rW  )r   rG   r   r   r   rQ  rX  rY  rj  rk  rl  rB   rE   rR  s                 rX   extract_conv3dro    s    $*KKM!Hrr$x$iX]A: :&I jjRRHLLNJ{IIh1aCGGIJy	2vrZ   c                 x   0 nUS:X  a  [        X5      S-   n[        XF-  5      nOuUS:X  a  [        X5      S-   n[        XF-  5      nOSUS:X  a  [        X5      S-   n[        XF-  5      nO1US:X  a  [	        X5      S-   n[        XF-  5      nOUn[        XF-  5      nU S   [
        ::  a  Sn[        XF-  5      nOXa:  a  Un[        XF-  5      n[        R                  " [        R                  " U 5      5      n[        R                  " [        R                  " U S U 5      5      n	U R                  S5      n
[        R                  " [        R                  " U
5      5      n[        R                  " [        R                  " U
S U 5      5      n[        X-  5      nXeS'   XuS	'   X-  US
'   XS'   U S   XS-
     -  US'   U$ )Nr   r   r   r   r   r   r   r   r&  sum_retainedfro_retained	max_ratio)r?  r   r9  r;  rL  MIN_SVr   r$   absr'   r*   )rC   r6   r   r   rQ  rR  r   r&  s_sums_rankrM   rP   rQ   rR   s                 rX   rO  rO    s   J#!!3a7%*+		?	*&q81<%*+		8	#1A5%*+		9	$ 2Q6%*+	%*+	tv~%*+		%*+	 IIeiil#EYYuyy9H./FaIJJuyy+,E

599Yy%9:;I	)*K%z'{"(!1J~!,~dQ!|_4J{rZ   c                 F
   S nS nSn	/ n
/ nU(       a  [         R                  " SU SU SU 35        S nS nU R                  5       nS nS n[        U  Vs/ s H  nUR	                  S5      (       d  M  UPM     sn5      n[
        R                  R                  U5      n[        U R                  5       SSS9 GH  u  nnUR                  S	5      nS n[         H  n[        U5      S
:  a:  US   US   :X  a.  S	R                  US S 5      nS	US   -   nS	US   -   nS	US   -   n  OI[        U5      S:  d  M]  US   US   :X  d  Mk  S	R                  US S 5      nS	US   -   nS	US   -   nSn  O   Uc  M  UnUnU R                  UW-   W-   S 5      nU R                  US-   S 5      nUS L=(       a    US LnU(       Ga   [        UR                  5       5      S:H  n[        UR                  5       5      S:H  nUR                  5       S   n[        U=(       d    SU5      nU(       a.  UR                   u  n n!n"n#UR                   u  n$n%  n#U$U!-  U"-  U"-  n&O]U(       a3  UR                   u  n n!n'n(n)UR                   u  n$n%    n#U$U!-  U'-  U(-  U)-  n&O#UR                   u  n n!UR                   u  n$n%U$U!-  n&U&S:  aP  [         R"                  " SU SU&S S35        [        R$                  " SU SU&S S35        UR'                  S5        GMJ  Uc  Sn*OUU-  n*U(       a  [)        XU5      n+[+        U+XXSU*5      n,O<U(       a  [-        XU5      n+[/        U+XXSU*5      n,O[1        XU5      n+[3        U+XXSU*5      n,U(       a|  SU,;   av  U,S   n-U,S   n.U,S   n/[4        R6                  " U/5      (       d  U
R9                  [;        U/5      5        US SU.S  S!U/S  S"U-S# S$U,S%    3	n0[        R$                  " U05        U	U0-  n	U(       a  U(       a  U	S&U,S%    S'U,S(    S3-  n	OU	S-  n	U,S(   nU,S)   R=                  U5      R?                  5       XW-   U-   '   U,S*   R=                  U5      R?                  5       UUU-   U-   '   [@        RB                  " U,S(   5      R=                  U5      XS-   '   S nS nS nS nS+nUR9                  U,S%   5        A,UR'                  S5        GM     U(       aF  [         R                  " S,[4        RD                  " U
5      S- S.[4        RF                  " U
5      S/ 35        XX4$ s  snf )0N
z6Dynamically determining new alphas and dims based off r  z, max rank is r\   TzResizing LoRA weights)leaver`   r]   r   r   r   rA  r   z.alphar   rb   i z	Skipping z: merged weight too large (,z
 elements)z	SKIPPED: z - too large (r   rr  rs  rq  75z | sum(S) retained: r   z, fro retained: z, max(S) ratio: z0.1fz, new dim: r   z, dynamic | dim: z	, alpha: r&  r.  r/  Fz"Average Frobenius norm retention: z.2%z | std: z0.3f)$r+   r,   copyr   rj   rl   rm   rn   r   r   splitLORA_DOWN_UP_FORMATSr   r  r   r#   r   rw   writerp   rf  rZ  rm  ro  rh  r\  r  isnanrh   r   rs   rr   r   tensorr   std)1r	  r   r"  r   r   r   r   max_old_rankr&  verbose_strfro_listr'  lora_down_weightlora_up_weight	o_lora_sdblock_down_nameblock_up_namer   r   pbarr5   r)  	key_parts_formatlora_down_namelora_up_nameweight_name
lora_alphaweights_loadedr<   conv3dold_rankrb  rY  r=   r   rX  rd  merged_sizerj  rk  rl  rQ  full_weight_matrixrR  rs  rq  rr  log_strs1                                                    rX   r  r  $  s   LIKHIMnM]]_`m_nn|  ~F  }G  H  	INIOMBAAJJy,AaBCJ;;"":.D7==?$=TU
UIIcN	+G 9~"wqzYr]'B"%((9Sb>":!$wqz!1"WQZ/!IbM19~"wqzYr]'B"%((9Sb>":!$wqz!1"WQZ/  ,  " (  ]\%AK%OQUV[[8!;TB
)5T.PT:T)..01Q6F)..01Q6F',,.q1H|0q(;L 3C3I3I0+q+9+?+?((Aq&0;>L/?/E/E,"b".<.B.B+(Aq!&025:R?#3#9#9 %3%9%9"(&0[()O+<<WXcdeWffp qr

Y&7~kRS_T^_`A!"X-%/0@RX%Y")*<hXeotu
%12BTZ%["+,>Zgqvw
%12BTZ%["+,>Zgqvw
>Z7&{3	).9).9xx--OOE,$78,R00D\RUDVVfgstwfx  yI  JS  TX  IY  Yd  eo  pz  e{  d|  }

7#w&>!2:j3I2J)T^_jTkSllnoot#";/IHRS^H_HbHbcmHnHyHyH{I6DEDNyDYD\D\]gDhDsDsDuIml2[@A49LLKAX4Y4\4\]g4hI01"O M#!N"NZ
34AK VN 9"''(:KC9PPXY[Y_Y_`hYijnXopqI88Y Cs   T0T)r   r   T)Fr   T)r   )r   )$r   comfy.model_managementrl   comfy.utilsr   r   r+   r   rB  r  comfy_api.latestr   ry   get_torch_devicer   r1   rY   r   	ComfyNoder   r   rt  r  r9  r;  r?  rL  rT  rZ  r\  rf  rh  rm  ro  rO  r  r   rZ   rX   <module>r     s        	    				0	0	2Up<~9BLL 9vIR\\ Id 
  H>
"0f}9rZ   