
    +jO                     d   S SK r S SKrS SKrS SKrS SKJr  S SKJrJr  SSK	J
r
  SSKJr  SSKJr  SSKJr  SS	KJr  S
 r " S S\ R(                  R*                  5      r " S S5      r " S S\5      r " S S5      r " S S5      r " S S\5      r " S S5      r " S S5      r " S S5      rg)    N)CheckpointLoaderSimple)DFloat11Modelcompress_model   )DFloat11ModelPatcher)decompress_state_dict_func_map)DFloat11FluxDiffusersModel)!convert_diffusers_to_comfyui_flux)MODEL_TO_PATTERN_DICTc                 <   U R                  5        VVs0 s Hy  u  p[        UR                  S5      UR                  S5      UR                  S5      UR                  S5      UR                  S5      UR                  S5      45      (       a  Mw  X_M{     snn$ s  snnf )Nsign_mantissaencoded_exponentlutsgapsoutput_positionssplit_positions)itemsanyendswith)
state_dictkeytensors      \/home/wildlama/comfy/ComfyUI/custom_nodes/ComfyUI-DFloat11-Extended/dfloat11_model_loader.pyfilter_df11_keysr      s    +5+;+;+=  |+=KCS#,,WfJgiliuiu  wI  jJ  LO  LX  LX  Y_  L`  be  bn  bn  ou  bv  x{  xD  xD  EW  xX  Z]  Zf  Zf  gx  Zy  Jz  F{KCK+=  |  |  |s   A6BBc                   f    \ rS rSr " S S\R
                  R                  R                  5      rSrg)disable_weight_init_df11   c                   ,   ^  \ rS rSrSU 4S jjrSrU =r$ )disable_weight_init_df11.Linear   c                 l   > [         [        R                  R                  R                  U ]  XX4U5        g N)supercomfyopsdisable_weight_initLinear__init__)selfin_featuresout_featuresbiasdevicedtype	__class__s         r   r(   (disable_weight_init_df11.Linear.__init__   s+    %))//66F{bfpuv     )TNN)__name__
__module____qualname____firstlineno__r(   __static_attributes____classcell__)r/   s   @r   r'   r      s    	 	r1   r'   r2   N)	r3   r4   r5   r6   r$   r%   r&   r'   r7   r2   r1   r   r   r      s    ..55 r1   r   c                   <    \ rS rSrSr \S 5       rSrSrSr	S r
Srg	)
DFloat11ModelLoaderAdvanced   
A custom node to load a DFloat11 diffusion model from the `diffusion_models` directory.

DFloat11 models are >30% smaller than their float16 counterparts, yet produce bit-for-bit identical outputs.
c           
      r    S[         R                  " S5      4SSSS.4SSSS	S
SS.4SSSS.4SS/SSS.4S.0$ )Nrequireddiffusion_modelsBOOLEANFzWhether to offload to CPU RAM)defaulttooltipINTr   i  r   z4If set to 0, all blocks will be offloaded to CPU RAM)rA   minmaxsteprB   TzWhether to lock/pin the weights to CPU RAM. Enabling this option increases RAM usage (which might cause OOM), but should increase speed
custom_opsload_state_dictzStrategy for compatibility with `dynamic_vram`, if it is enabled. `custom_ops` is better since it causes a smaller spike in RAM usage)dfloat11_model_namecpu_offloadcpu_offload_blocks
pin_memorydynamic_vram_compatibilityfolder_pathsget_filename_listclss    r   INPUT_TYPES'DFloat11ModelLoaderAdvanced.INPUT_TYPES$   s     (4(F(FGY(Z'\ )uIh+ij',!Ac[\  j`  /a  'b(d  HQ  +R  S0<>O/P^j  x  S@  /A
 	
r1   MODELload_dfloat11_model_advancedDFloat11c           
         U(       d  SnSn[         R                  " SU5      n[        R                  R	                  U5      n[        S UR                  5        5       5      (       d  [        SU S35      e[        R                  R                  5       n[        R                  R                  5       n	0 n
SU;   a  SU;   d  S	U;   a  S U
S
'   SU;   a  SU;  a  S U
S'   SU;   a"  SU;   a  US   R                  5       S:X  a  S U
S'   SU;   a  US   R                  5       S:X  a  S U
S'   SU;   Ga&  SU;   Ga  US   R                  5       S:X  ai  [        R                  " SS/SS9U
S'   [        R                  " SS/SS9U
S'   [        R                  " SS/SS9U
S'   [        R                  " SS/SS9U
S'   OUS   R                  5       S :X  ai  [        R                  " S!S"/SS9U
S'   [        R                  " S#S"/SS9U
S'   [        R                  " SS/SS9U
S'   [        R                  " SS/SS9U
S'   O[        S$US   R                  5        35      e[        R                  R                   R#                  Xz-  S%5      nUcH  0 n
S&U;   a  S U
S''   S U
S('   [        R                  R                   R#                  Xz-  S%5      nUc   S)5       e[%        U5      R&                  nUS*:X  aD  UR(                  R+                  S+S,5      (       a#  UR(                  R+                  S-S,5      (       a  S.n[        R,                  R.                  (       a  US/:X  a  [0        Ul        UR5                  [        R6                  [        R6                  5        UR9                  US%5      n[        R,                  R.                  (       d  UR;                  U	5      n[        R,                  R.                  (       a*  US0:X  a$  UR<                  R?                  [A        U5      S,S,S19  [B        RD                  " U[F        U   UR<                  U	UUS:  a  UOS US29  [I        XU	S394$ )4Nr   Tr?   c              3   B   #    U  H  oR                  S 5      v   M     g7fr   Nr   .0ks     r   	<genexpr>KDFloat11ModelLoaderAdvanced.load_dfloat11_model_advanced.<locals>.<genexpr><        M;La::011;L   The model '' is not a DFloat11 model.z&double_blocks.0.img_mlp.gate_proj.biasztxt_norm.scaleztxt_norm.weightz(double_blocks.0.img_mlp.gate_proj.weightz.double_stream_modulation_img.lin.sign_mantissaz'double_stream_modulation_img.lin.weightz adaLN_modulation.1.sign_mantissaztime_embedding.sign_mantissazlayers.0.sign_mantissai   zlayers.0.mlp.linear_fc2.weightz-transformer_blocks.0.attn.norm_added_q.weight@   z&transformer_blocks.0.img_mlp.w1.weightz5encoder.lyric_encoder.layers.0.input_layernorm.weightzdecoder.layers.0.sign_mantissai  i   i   metar-   z%decoder.layers.0.mlp.gate_proj.weightz(decoder.layers.0.self_attn.q_proj.weightz6encoder.lyric_encoder.layers.0.self_attn.q_proj.weightz3encoder.lyric_encoder.layers.0.mlp.gate_proj.weighti  i &  i 
     z1Detected Ace-Step-v1.5 model, but unsure of size  z"llm_adapter.blocks.0.sign_mantissaz-llm_adapter.blocks.0.cross_attn.q_proj.weightzblocks.0.mlp.layer1.weightzUnable to detect model typeFluxSchnellyak_mlpFtxt_norm	OvisImagerG   rH   )strictassign)pattern_dictbfloat16_modelr-   rJ   rK   rL   load_deviceoffload_device)%rO   get_full_path_or_raiser$   utilsload_torch_filer   keys
ValueErrormodel_managementget_torch_deviceunet_offload_devicenumeltorchempty	Exceptionsdmodel_detectionmodel_config_from_unettyper3   unet_configgetmemory_managementaimdo_enabledr   custom_operationsset_inference_dtypebfloat16	get_modeltodiffusion_modelrH   r   r   from_single_filer   r   )r)   rI   rJ   rK   rL   rM   dfloat11_model_pathr   rt   ru   missing_keysmodel_config	df11_typemodels                 r   rW   8DFloat11ModelLoaderAdvanced.load_dfloat11_model_advanced4   s   !"J*AABTVij[[001DE
M:??;LMMM{+>*??YZ[[,,==?//CCE 3zAGW[eGeiz  I  jIGKLCD;zINw  @J  OJFJLBC-;@^bl@lq{  }U  rV  r\  r\  r^  bk  rk=AL9::jHZ  YH  NI  NO  NO  NQ  UW  NWEILABBjPUu  zD  VD:;AACxOHMUY[_T`ioHpDEKP;;X\^bWclrKsGHY^YdYdfjlpeq  {A  ZBUVV[VaVacgimbnw}V~RS<=CCERHMUY[_T`ioHpDEKP;;X\^bWclrKsGHY^YdYdfjlpeq  {A  ZBUVV[VaVacgimbnw}V~RS  "ST^_  UA  UG  UG  UI  TJ  !K  L  L xx//FFzG`bdeL3zAPTLM :>L56 8833JJ:KdfhiL+J-JJ+&//	%,*B*B*F*FyRW*X*X]i]u]u]y]y  {E  GL  ^M  ^M#I""005OS_5_-EL*((H&&z26&&44HH^,E""005OSd5d!!112B:2NW\ej1k&&.y9 00!#5G!5K1QU!	
 !P^_
 	
r1   r2   N)r3   r4   r5   r6   __doc__classmethodrS   RETURN_TYPESFUNCTIONCATEGORYrW   r7   r2   r1   r   r:   r:      s7     	
 	
 L-HH^
r1   r:   c                   2    \ rS rSrSr\S 5       rSrS rSr	g)DFloat11ModelLoader   r<   c                 8    SS[         R                  " S5      400$ )Nr>   rI   r?   rN   rQ   s    r   rS   DFloat11ModelLoader.INPUT_TYPES   s*     %(F(FGY(Z'\
 	
r1   load_dfloat11_modelc                 (    U R                  USSSSS9$ )NFr   TrG   )rJ   rK   rL   rM   )rW   )r)   rI   s     r   r   'DFloat11ModelLoader.load_dfloat11_model   s1    001DTYpq  AE  dp0  q  	qr1   r2   N)
r3   r4   r5   r6   r   r   rS   r   r   r7   r2   r1   r   r   r      s(     
 
 %Hqr1   r   c                   6    \ rS rSr\S 5       rSrSrSrS r	Sr
g)	DFloat11Decompressor   c                 H    S[         R                  " S5      4SS/4/ SQ4S.0$ )Nr>   r?   zFlux.2-Klein-4BzFlux.2-Klein-9B)rA   
fp8_e4m3fnfp8_e4m3fn_fastfp8_e5m2)rI   decompress_recipeweight_dtyperN   rQ   s    r   rS    DFloat11Decompressor.INPUT_TYPES   s:     (4(F(FGY(Z'\'8:K&L%N!Y [
 	
r1   rU   decompress_dfloat11_modelrX   c                    [         R                  " SU5      n[        R                  R	                  U5      n[        S UR                  5        5       5      (       d  [        SU S35      e[        U   " U5      n0 nUS:X  a  [        R                  US'   O8US:X  a  [        R                  US'   SUS	'   OUS
:X  a  [        R                  US'   [        R                  R                  XgS9nU4$ )Nr?   c              3   B   #    U  H  oR                  S 5      v   M     g7fr[   r\   r]   s     r   r`   ADFloat11Decompressor.decompress_dfloat11_model.<locals>.<genexpr>   s     R;Qa::011;Qrc   rd   re   r   r.   r   Tfp8_optimizationsr   )model_options)rO   rv   r$   rw   rx   r   ry   rz   r   r   float8_e4m3fnfloat8_e5m2r   load_diffusion_model_state_dict)	r)   rI   r   r   r   df11_state_dictreconstructed_state_dictr   model_patchers	            r   r   .DFloat11Decompressor.decompress_dfloat11_model   s    *AABTVij++556IJR?;O;O;QRRR{+>*??YZ[[#ABS#TUd#e <'%*%8%8M'"..%*%8%8M'"15M-.Z'%*%6%6M'"@@AY@wr1   r2   N)r3   r4   r5   r6   r   rS   r   r   r   r   r7   r2   r1   r   r   r      s*    
 
 L*HH r1   r   c                   6    \ rS rSr\S 5       rSrSrSrS r	Sr
g)	DFloat11LoadingPatch   c                 2    SSSS04/ SQ4SSSS	S
S
SS.4S.0$ )Nr>   rV   rB   z$The model to display information for)v1zv1.5v2FLOATg      ?g        g      $@g{Gz?z9The multiplier to scale ComfyUI's memory usage estimation)rA   rD   rE   rF   roundrB   )r   load_versionmemory_usage_factor_scaler2   rQ   s    r   rS    DFloat11LoadingPatch.INPUT_TYPES   sM     ")I7]+^!_!5 7.53s[_hlw{  ID  8E  .F
 	
r1   rU   patch_loading_methodsrX   c                     UR                  5       nUR                  U5        UR                  R                  =R                  U-  sl        UR                  R                  R                  UR                  l        U4$ r"   )cloner   r   r   memory_usage_factor)r)   r   r   r   new_model_patchers        r   r   *DFloat11LoadingPatch.patch_loading_methods   sg    )//1//=,,@@D]]@6G6M6M6Z6Z6n6n3!##r1   r2   N)r3   r4   r5   r6   r   rS   r   r   r   r   r7   r2   r1   r   r   r      s*    
 
 L&HH	$r1   r   c                   6    \ rS rSr\S 5       rSrSrSrS r	Sr
g)	CheckpointLoaderWithDFloat11   c                 p    S[         R                  " S5      SS04[         R                  " S5      SS04S.0$ )Nr>   checkpointsrB   +The name of the checkpoint (model) to load.r?   z#The diffusion model in DF11 format.)	ckpt_namerI   rN   )ss    r   rS   (CheckpointLoaderWithDFloat11.INPUT_TYPES   s`     *<<]Ki  ZG  NH  I(4(F(FGY(Z]f  iN  ]O  (P
 	
r1   load_checkpoint_with_df11rX   z;Loads a diffusion model checkpoint, along with a DF11 unet.c                    [         R                  " SU5      n[        R                  R	                  USS[         R
                  " S5      S9nUtpVpx[         R                  " SU5      n	[        R                  R                  U	5      n
[        S U
R                  5        5       5      (       d  [        SU S35      e[        R                  R                  5       n[        R                  R                  5       n[        UR                  5      R                   n[#        UR                  UUS	9nA[$        R&                  " U	[(        U   UR                  R*                  US
9  XU4$ )Nr   T
embeddings)
output_vaeoutput_clipembedding_directoryr?   c              3   B   #    U  H  oR                  S 5      v   M     g7fr[   r\   r]   s     r   r`   ICheckpointLoaderWithDFloat11.load_checkpoint_with_df11.<locals>.<genexpr>   rb   rc   rd   re   rs   rq   rr   r-   )rO   rv   r$   r   load_checkpoint_guess_configget_folder_pathsrw   rx   r   ry   rz   r{   r|   r}   r   r   r3   r   r   r   r   r   )r)   r   rI   	ckpt_pathoutr   clipvae_r   r   rt   ru   r   df11_model_patchers                  r   r   6CheckpointLoaderWithDFloat11.load_checkpoint_with_df11   sN    77yQ	hh33I$\`  wC  wT  wT  Ua  wb3  c'*$S*AABTVij[[001DE
M:??;LMMM{+>*??YZ[[,,==?//CCE,,-66	1#)
 &&.y9-33CC!		
 ##..r1   r2   N)r3   r4   r5   r6   r   rS   r   r   DESCRIPTIONr   r7   r2   r1   r   r   r      s*    
 
 +HHOK/r1   r   c                   :    \ rS rSrSr\S 5       rSrSrSr	S r
Srg	)
DFloat11DiffusersModelLoaderi  z
A custom node to load a diffusers-native DFloat11 diffusion model from the `diffusion_models` directory.

DFloat11 models are >30% smaller than their float16 counterparts, yet produce bit-for-bit identical outputs.
c                 >    S[         R                  " S5      4S/4S.0$ )Nr>   r?   Flux)rI   
model_typerN   rQ   s    r   rS   (DFloat11DiffusersModelLoader.INPUT_TYPES"  s/     (4(F(FGY(Z'\ &yl
 	
r1   rU   r   rX   c                    [         R                  " SU5      n[        R                  R	                  U5      n[        S UR                  5        5       5      (       d  [        SU S35      e[        R                  R                  5       n[        R                  R                  5       n0 SS_S/ SQ_S	S
_SS_SS_SS_SS_S/ _SS_SS_SS_SS_SS_SS_SS _S!S_S"S#_S$S#0EnS%U;   US!'   [        R                  R                  U5      nUR                  [        R                  [        R                  5        UR!                  US&5      n	U	R#                  U5      n	[$        R&                  " U[(        U   U	R*                  US'9  [-        XUS(94$ ))Nr?   c              3   B   #    U  H  oR                  S 5      v   M     g7fr[   r\   r]   s     r   r`   CDFloat11DiffusersModelLoader.load_dfloat11_model.<locals>.<genexpr>5  rb   rc   rd   re   image_modelfluxaxes_dim)   8   r   	num_heads   	mlp_ratiog      @thetai'  out_channelsr   qkv_biasTtxt_ids_dimsin_channelshidden_sizei   context_in_dimri   
patch_size   
vec_in_dimi   depth   depth_single_blocks&   guidance_embedrl   Frm   z1time_text_embed.guidance_embedder.linear_1.weightrj   r   rs   )rO   rv   r$   rw   rx   r   ry   rz   r{   r|   r}   supported_modelsr   r   r   r   r   r   r	   r   r   r   r   )
r)   rI   r   r   r   rt   ru   r   r   r   s
             r   r   0DFloat11DiffusersModelLoader.load_dfloat11_model/  s   *AABTVij [[001DE
M:??;LMMM{+>*??YZ[[,,==?//CCE
6

 
 	

 U
 B
 
 B
 2
 4
 d
 !
 #
 R
 "2
  d!
" u#
$ %
* )\_i(i$%--22;?((H&&z26("33.z: 00!		
 !P^_
 	
r1   r2   N)r3   r4   r5   r6   r   r   rS   r   r   r   r   r7   r2   r1   r   r   r     s1     
 
 L$HH1
r1   r   c                   :    \ rS rSrSr\S 5       rSrSrSr	S r
Srg	)
DFloat11ModelCompressorib  z
A custom node to compress a DFloat11 diffusion model from the `diffusion_models` directory.

DFloat11 models are >30% smaller than their float16 counterparts, yet produce bit-for-bit identical outputs.
c                 t    S[         R                  " S5      4[        [        R                  " 5       5      4S.0$ )Nr>   r?   )bfloat16_model_namer   rO   rP   listr   ry   rQ   s    r   rS   #DFloat11ModelCompressor.INPUT_TYPESi  s=     (4(F(FGY(Z'\#$9$>$>$@AC
 	
r1   STRINGload_bfloat16_modelrX   c           	      2   [         R                  " SU5      n[        R                  R	                  US[
        R                  05      n[        R                  R                  U5      S    S3n[        UR                  R                  [        U   USSSS9  U4$ )Nr?   r.   r   -DF11Tr   i  r   rq   	save_pathsave_single_filecheck_correctnessblock_range)rO   rv   r$   r   load_diffusion_modelr   r   ospathsplitextr   r   r   r   )r)   r  r   bfloat16_model_pathr   r  s         r   r  +DFloat11ModelCompressor.load_bfloat16_modelv  s    *AABTVij--.AGennC]^ww''(;<Q?@F	++--.z: "#!	
 |r1   r2   N)r3   r4   r5   r6   r   r   rS   r   r   r   r  r7   r2   r1   r   r  r  b  s1     
 
 L$HHr1   r  c                   :    \ rS rSrSr\S 5       rSrSrSr	S r
Srg	)
DFloat11CheckpointCompressori  z
A custom node to compress a DFloat11 diffusion model (unet only) from the `checkpoints` directory.

DFloat11 models are >30% smaller than their float16 counterparts, yet produce bit-for-bit identical outputs.
c                 z    S[         R                  " S5      SS04[        [        R                  " 5       5      4S.0$ )Nr>   r   rB   r   )r   r   r  rQ   s    r   rS   (DFloat11CheckpointCompressor.INPUT_TYPES  sO     *<<]Ki  ZG  NH  I#$9$>$>$@AC
 	
r1   r  load_bfloat16_checkpointrX   c           	         [         R                  " SU5      n[        R                  R	                  USS9u  pE[        R
                  R                  U5      n[        R                  R                  XF5      n[        R                  R                  XF5      n[        R                  R                  5       n	[        R                  R                  5       n
[        R
                  R                  XFUS9nUR                  X5        [        UR                  5      n[        R                  R!                  X|US9n[        R                  R#                  X}5      nUR%                  XFUS9nUR'                  XF5        [        R(                  R+                  XU
S9nUR,                  R.                  n[0        R2                  R5                  U5      S    S	3n[7        U[8        U   USSS
S9  U4$ )Nr   T)return_metadata)metadata)model_paramssupported_dtypesr   rh   rs   r   r  r  r  )rO   rv   r$   rw   rx   r   unet_prefix_from_state_dictcalculate_parametersr   r{   r|   r}   r   r   r  supported_inference_dtypes
unet_dtypeunet_inital_load_devicer   load_model_weightsr   ModelPatcherr   r   r  r  r  r   r   )r)   r   r   r   r   r)  diffusion_model_prefix
parametersr   rt   ru   r   unet_weight_dtyper/  inital_load_devicer   r   r   r  s                      r   r&  5DFloat11CheckpointCompressor.load_bfloat16_checkpoint  s    77yQ	${{::9VZ:[
!&!6!6!R!RS]!^[[55jY
{{//
S,,==?//CCE,,CCJqyCz((D !H!HI++66J  BN6  O
"33KKJc&&zRd&e  D++88hv8w'--==ww''	2156e<	!.z:!" 	
 |r1   r2   N)r3   r4   r5   r6   r   r   rS   r   r   r   r&  r7   r2   r1   r   r#  r#    s1     
 
 L)HH&r1   r#  )r$   rO   r   r  nodesr   dfloat11r   r   dfloat11_customr   dfloat11_decompressr   dfloat11_diffusersr	   convert_fixed_tensorsr
   rq   r   r   r%   r&   r   r:   r   r   r   r   r   r  r#  r2   r1   r   <module>r>     s       	 ( 2 1 ? : D /|uyy<< {
 {
zq5 q*#  # N$ $6,/#9 ,/`E
 E
N# #L: :r1   