
    +ju                     6   d dl Z d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Zd dl
Zd dlmZmZ d dlmZ d dlmZ  G d de j        j                  Z G d d	e j        j                  Z G d
 de j        j                  Zd Z G d d          Z G d d          Z G d d          Z G d d          Z G d de          Z G d d          Z G d d          Z G d de j        j                  Z G d dej                  Z eeeee dZ!dS )     N)nn)WanMultiTalkAttentionBlockMultiTalkAudioProjModel)io)
SUPIRPatchc                   ,     e Zd Zddef fdZd Z xZS )BlockWiseControlBlock   Ndimc                 j   t                                                       |                    |d          | _        |                    |d          | _        |                    ||          | _        t          j        	                                | _
        |                    ||          | _        d S )Ngư>)eps)super__init__RMSNormx_rmsy_rmsLinear
input_projtorchr   GELUactoutput_proj)selfr   devicedtype
operations	__class__s        >/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_model_patch.pyr   zBlockWiseControlBlock.__init__   s    '''66
'''66
$++C558==??%,,S#66    c                     |                      |          |                     |          }}|                     ||z             }|                     |          }|                     |          }|S N)r   r   r   r   r   )r   xys      r   forwardzBlockWiseControlBlock.forward   s[    zz!}}djjmm1OOAE""HHQKKQr   )r
   NNN)__name__
__module____qualname__intr   r$   __classcell__r   s   @r   r	   r	      sX        7 7C 7 7 7 7 7 7      r   r	   c            	       H     e Zd Z	 	 	 	 	 ddededed	ef fd
Zd Zd Z xZS )QwenImageBlockWiseControlNet<   @   r   r
   N
num_layersin_dimadditional_in_dimr   c                     t                                                       || _                            ||z             | _        t
          j                            fdt          |          D                       | _	        d S )Nr   r   c                 6    g | ]}t                     S r   r   r   )r	   ).0_r   r   r   r   s     r   
<listcomp>z9QwenImageBlockWiseControlNet.__init__.<locals>.<listcomp>/   s;        &c&R\]]]  r   )
r   r   r1   r   img_inr   r   
ModuleListrangecontrolnet_blocks)	r   r/   r0   r1   r   r   r   r   r   s	       ````r   r   z%QwenImageBlockWiseControlNet.__init__#   s     	!2 ''1B(BCPV^c'dd!&!4!4      z**  "
 "
r   c                 4   t           j                                                            |d d d df                   |d d d df<   d}t           j        j                            |d||f          }|j        }|                    |d         |d         |d         dz  d|d         dz  d          }|	                    dddddd	          }|
                    |d         |d         dz  |d         dz  z  |d         dz            }|                     |          S )
N         r            )comfylatent_formatsWan21
process_inldm
common_ditpad_to_patch_sizeshapeviewpermutereshaper:   )r   latent_image
patch_sizehidden_states
orig_shapes        r   process_input_latent_imagez7QwenImageBlockWiseControlNet.process_input_latent_image5   s/   $399;;FF|TUTUTUWZXZWZTZG[\\QQQV
	,>>|aQ[]gMhii"(
%**:a=*Q-TV[\I\^_aklnaostatvwxx%--aAq!Q??%--jmjnPQ>QV`acVdhiVi=jlvwxly|}l}~~{{=)))r   c                 0     | j         |         ||          S r!   )r=   )r   imgcontrolnet_conditioningblock_ids       r   control_blockz*QwenImageBlockWiseControlNet.control_block?   s    /t%h/5LMMMr   )r-   r.   r   r
   NNN)r%   r&   r'   r(   r   rV   r[   r)   r*   s   @r   r,   r,   "   s         !",0
 

 
 	

 
 
 
 
 
 
$* * *N N N N N N Nr   r,   c                        e Zd ZdZ	 	 	 	 	 	 dded	ed
ededef
 fdZd Zdej	        de
j        de
j        de
j        dej        dej	        fdZ xZS )SigLIPMultiFeatProjModela  
    SigLIP Multi-Feature Projection Model for processing style features from different layers
    and projecting them into a unified hidden space.

    Args:
        siglip_token_nums (int): Number of SigLIP tokens, default 257
        style_token_nums (int): Number of style tokens, default 256
        siglip_token_dims (int): Dimension of SigLIP tokens, default 1536
        hidden_size (int): Hidden layer size, default 3072
        context_layer_norm (bool): Whether to use context layer normalization, default False
      r.     r
   TNsiglip_token_numsstyle_token_numssiglip_token_dimshidden_sizecontext_layer_normc	                    t                                                       t          j        |                    ||          t          j                              | _        |r|                    |          nt          j                    | _	        |                    ||d          | _
        t          j        |                    ||          t          j                              | _        |r|                    |          nt          j                    | _        |                    ||d          | _        t          j        |                    ||          t          j                              | _        |r|                    |          nt          j                    | _        |                    ||d          | _        d S )NT)bias)r   r   r   
Sequentialr   SiLUhigh_embedding_linear	LayerNormIdentityhigh_layer_normhigh_projectionmid_embedding_linearmid_layer_normmid_projectionlow_embedding_linearlow_layer_normlow_projection)
r   r`   ra   rb   rc   rd   r   r   r   r   s
            r   r   z!SigLIPMultiFeatProjModel.__init__P   s    	 &(]/1ABBGII&
 &
"
 8J\J  !2333r{}} 	  *001BKVZ0[[ %'M/1ABBGII%
 %
!
 8J\J  !2333r{}} 	 )//0A;UY/ZZ %'M/1ABBGII%
 %
!
 8J\J  !2333r{}} 	 )//0A;UY/ZZr   c                    t          | j                                                  j        }|                     |d         | j        | j        | j        |          }|                     |d         | j        | j        | j	        |          }|                     |d         | j
        | j        | j        |          }t          j        |||fd          S )a  
        Forward pass function

        Args:
            siglip_outputs: Output from SigLIP model, containing hidden_states

        Returns:
            torch.Tensor: Concatenated multi-layer features with shape [bs, 3*style_token_nums, hidden_size]
        r@   rA   r   r   )nextri   
parametersr   _process_layer_featuresrl   rm   rn   ro   rp   rq   rr   rs   r   cat)r   siglip_outputsr   high_embeddingmid_embeddinglow_embeddings         r   r$   z SigLIPMultiFeatProjModel.forwardy   s     T/::<<==C 551&  
 
 441%
 
 441%
 
 y.-GQOOOOr   rT   embedding_linear
layer_norm
projectionr   returnc                      ||                     |                              dd                                        dd          } ||          } ||          }|S )a  
        Helper function to process features from a single layer

        Args:
            hidden_states: Input hidden states [bs, seq_len, dim]
            embedding_linear: Embedding linear layer
            layer_norm: Layer normalization
            projection: Projection layer
            dtype: Target data type

        Returns:
            torch.Tensor: Processed features [bs, style_token_nums, hidden_size]
        rA   r@   )to	transpose)r   rT   r~   r   r   r   	embeddings          r   rx   z0SigLIPMultiFeatProjModel._process_layer_features   so    , %$U##--a33
 

)Aq// 	
 Jy))	 Jy))	r   )r^   r.   r_   r
   TNNN)r%   r&   r'   __doc__r(   boolr   r$   r   Tensorr   Moduler   rx   r)   r*   s   @r   r]   r]   C   s
       
 
 "% "!%#',0'[ '['[ '[ 	'[
 '[ !'[ '[ '[ '[ '[ '[R(P (P (PT |  )  I	 
 I  {  
               r   r]   c                    ddddd}i }t          |                                           D ]}| |         }|}|                    d          r|g}%|                    d          r|g|z   }A|                    d          r2||gz   }t          j        |d	
          }|                    dd          }|                                D ]\  }}|                    ||          }|||<   |S )Nz.attention.out.biasz.attention.k_norm.weightz.attention.q_norm.weightz.attention.out.weight)z.attention.to_out.0.biasz.attention.norm_k.weightz.attention.norm_q.weightz.attention.to_out.0.weightz.attention.to_k.weightz.attention.to_q.weightz.attention.to_v.weightr   ru   z.attention.qkv.weight)sortedkeysendswithr   ry   replaceitems)	sdreplace_keysout_sdkwk_outccrrrs	            r   z_image_convertr      s   0E0J0J2I L FBGGII  qE>>233 	B>>233 	rB>>233 	UqcB	"!$$$AMM":<STTE!'')) 	) 	)EArMM!R((EEuMr   c                   :    e Zd Zed             ZdZdZdZdZd Z	dS )ModelPatchLoaderc                 4    ddt          j        d          fiiS )Nrequirednamemodel_patches)folder_pathsget_filename_listss    r   INPUT_TYPESzModelPatchLoader.INPUT_TYPES   s&    f|'Eo'V'V&Y   ! 	!r   MODEL_PATCHload_model_patchTzadvanced/loadersc           
         t          j        d|          }t          j                            |d          }t          j                            |          }d|v rV|d         j        d         dz
  }t          |t          j        	                                |t          j
        j                  }nd	|v rct          j                            |d
did          }t          t          j        	                                |t          j
        j                  }nbd|v rt          |          }i }d|vrd|d<   d|d<   d|d<   d|v rDd|d<   d|d<   d|d<   |                    dd           }|t!          j        |          dk    rd|d<   t          j        j        j        j        d+t          j        	                                |t          j
        j        d|}nd|v rwt-          ddd|d         j        d         |d         j        d         |d         j        d         t          j        	                                t          j
        j                   }n&d!|v sd"|v ri }	d!|v rd#|	d$<   d%|	d&<   n
d#|	d#<   d%|	d%<   d'}
i }t/          |                                          D ]D}|                    |
          r-|                    |          ||t7          |
          d          <   Et          j                            ||	d          }|                    d(d            t          j        j        j                            t          j        	                                |t          j
        j                  }|r||_        t          j         !                    |t          j        "                                t          j        	                                )          }|#                    ||$                                *           |fS ),Nr   T)	safe_loadz controlnet_blocks.0.y_rms.weightzimg_in.weightrA   r.   )r1   r   r   r   z$feature_embedder.mid_layer_norm.biaszfeature_embedder. )filter_keysr6   z!control_all_x_embedder.2-1.weightz*control_layers.4.adaLN_modulation.0.weightrE   n_control_layers   r1   refiner_controlz+control_layers.14.adaLN_modulation.0.weight   z)control_noise_refiner.0.after_proj.weightr   brokenzaudio_proj.proj1.weightrF       rD   z%blocks.0.audio_cross_attn.proj.weightzaudio_proj.norm.weight)audio_windowcontext_tokens	vae_scaler0   intermediate_dimout_dimr   r   z-model.control_model.input_hint_block.0.weightz'control_model.input_hint_block.0.weightzcontrol_model.zmodel.control_model.zproject_modules.z&model.diffusion_model.project_modules.z"first_stage_model.denoise_encoder.zcontrol_model.mask_LQ)load_deviceoffload_device)assign )%r   get_full_path_or_raiserG   utilsload_torch_fileweight_dtyperN   r,   model_managementunet_offload_deviceopsmanual_caststate_dict_prefix_replacer]   r   getr   count_nonzerorK   lumina
controlnetZImage_ControlMultiTalkModelPatchlistr   
startswithpoplensupirsupir_modulesSUPIRdenoise_encoder_sdmodel_patcherCoreModelPatcherget_torch_deviceload_state_dict
is_dynamic)r   r   model_patch_pathr   r   r1   modelconfig
ref_weightprefix_replace	de_prefixr   r   r   s                 r   r   z!ModelPatchLoader.load_model_patch   s   '>PTUU[(()9T(JJ((,,-33 "? 3 9! <r A0CT]b]s  ^H  ^H  ^J  ^J  RW  di  dm  dy  z  z  zEE3r9966r<OQS;Tbf6ggB,E4J4^4^4`4`hmz  {D  {P  Q  Q  QEE0B66 $$BF;2EE-.)*.0*+,0()<BB-/)*.0*+,0()VV$OQUVV
)*:66!;;+/x(I$/>  meF\FpFpFrFrz  MR  MV  Mb  m  m  fl  m  mEE&",,'!"2EFLQO%'(A%B%H%K78>qA 1EEGG$y46 6 6EE =BBFosuFuFuN>"DD9I56K]GHH3C/05G12 =I!#"''))__ G G<<	** G=?VVAYY&qY'9:66r>W[6\\BFF*D111IO177u?U?i?i?k?ksx  FK  FO  F[7  \  \E! >+=(+<<UPUPfPwPwPyPy  KP  Ka  Ku  Ku  Kw  Kw<  x  xb)A)A)C)CDDDr   N)
r%   r&   r'   classmethodr   RETURN_TYPESFUNCTIONEXPERIMENTALCATEGORYr   r   r   r   r   r      sR        ! ! [! $L!HL!H;  ;  ;  ;  ; r   r   c                   .    e Zd ZddZd Zd Zd Zd ZdS )DiffSynthCnetPatchNc                     || _         || _        || _        || _        || _        |j                            |                     |                    | _        |j	        d         |j	        d         f| _
        d S )NrA   r@   )model_patchvaeimagestrengthmaskr   rV   encode_latent_condencoded_imagerN   encoded_image_size)r   r   r   r   r   r   s         r   r   zDiffSynthCnetPatch.__init__,  sm    &
 	(.II$JaJabgJhJhii#(;q>5;q>"Br   c                    | j                             |          }| j        j        j        dk    r| j        3t          j        |          d d d | j        j        j        dz  f         }nSt          j	        
                    | j                            dd          |j        d         |j        d         dd	          }t          j        ||gd
          S |S )Nr   rD   rA   Tr   keepdimrC   rB   bilinearnoneru   )r   encoder   r   r1   r   r   	ones_likerG   r   common_upscalemeanrN   ry   )r   r   rR   mask_s       r   r   z%DiffSynthCnetPatch.encode_latent_cond5  s    xu--!3a77y 55aaa9g$:J:P:bfg:g9g6gh2249>>aQU>3V3VXdXjkmXnp|  qC  DF  qG  IS  U[  \  \9lE2::::r   c           
      0   |                     d          }|                     d          }|                     d          }| j                                        }| j        *| j        |j        d         |z  |j        d         |z  fk    rt          j                            | j	        
                    dd          |j        d         |z  |j        d         |z  dd          }t          j                            d	
          }| j        j                            |                     |
                    dd                              | _        |j        d         |j        d         f| _        t          j                            |           |d d d | j        j        d         fxx         | j        j                            |d d d | j        j        d         f         | j                            |j                  |          | j        z  z  cc<   ||d<   |S )Nr"   rX   block_indexrB   rC   rA   areacenterTonly_currently_used)r   r   spacial_compression_encoder   r   rN   rG   r   r   r   movedimr   loaded_modelsr   r   rV   r   load_models_gpur[   r   r   r   )r   kwargsr"   rX   r   spacial_compressionimage_scaledr   s           r   __call__zDiffSynthCnetPatch.__call__A  s{   JJsOOjjjj//"hAACC%)@QWR[SfEfhihoprhs  wJ  iJ  EK  *K  *K ;55dj6H6HQ6O6OQRQXY[Q\_rQrtut{|~t  CV  uV  X^  `h  i  iL!2@@UY@ZZM!%!1!7!R!RSWSjSjkwkk  AB  DF  lG  lG  TH  TH  "I  "ID'3'9"'=|?QRT?U&VD#"22=AAAAAA+"(+++,,,1A1G1U1UVYZ[Z[Z[]y^b^p^vwx^y]yZyVz  }A  }O  }R  }R  SV  S\  }]  }]  _j  2k  2k  nr  n{  2{  	|,,,ur   c                 x    t          |t          j                  r| j                            |          | _        | S r!   
isinstancer   r   r   r   r   device_or_dtypes     r   r   zDiffSynthCnetPatch.toQ  5    ou|44 	H!%!3!6!6!G!GDr   c                     | j         gS r!   r   r   s    r   modelszDiffSynthCnetPatch.modelsV       !!r   r!   r%   r&   r'   r   r   r  r   r  r   r   r   r   r   +  sh        C C C C
  
  
      
" " " " "r   r   c                   0    e Zd ZddZddZd Zd Zd ZdS )ZImageControlPatchNc                    || _         || _        || _        || _        || _        || _        | j         j        j        dk    | _        d}| j        #| j        | j        j	        | j        j	        k    rd}|rd | _
        n|                     | j        | j                  | _
        | j        *| j        j	        d         | j        j	        d         f| _        n)| j        j	        d         | j        j	        d         f| _        d | _        d S )Nr   FTrA   r@   )r   r   r   inpaint_imager   r   r   r1   
is_inpaintrN   r   r   r   	temp_data)r   r   r   r   r   r  r   skip_encodings           r   r   zZImageControlPatch.__init__Z  s    &
*	 *0BQF:!d&8&Dz4#5#;;; $ 	U!%D!%!8!8TEW!X!XDz!+/+=+CA+FHZH`abHc*d''+/:+;A+>
@PQR@S*T'r   c           	         d }|It           j                                                            | j                            |                    }| j        r|t          j        |          dz  }| j	        t           j
                            | j	                            | j	        j        d         d| j	        j        d         | j	        j        d                                       dd          |j        d         |j        d         d	d
          }|dz
  |                    dd                                          z  dz   }t           j                                                            | j                            |                    }| j	        !t          j        |          d d d df         }nt           j
                            | j	                            | j	        j        d         d| j	        j        d         | j	        j        d                                       dd                              |j                  |j        d         |j        d         dd
          }|^t           j                                                            | j                            t          j        |          dz                      }t          j        |||gd          S |S )Ng      ?r   rC   rB   rA   Tr   r   r   )r   nearestru   )rG   rH   FluxrJ   r   r   r  r   r   r   r   r   rO   rN   r   r   round
zeros_liker   r   ry   )r   control_imager  rR   mask_inpaintinpaint_image_latentr   s          r   r   z%ZImageControlPatch.encode_latent_condr  s>   $ /4466AA$(//R_B`B`aaL? 	 $ % > > Dy$${99$)..YZI[]_aeajapqsatvzv  wF  GI  wJ  ;K  ;K  ;P  ;P  UV  `d  ;P  ;e  ;e  gt  gz  {}  g~  @M  @S  TV  @W  Yc  em   n   n"/#"59M9MaQS9T9T9Z9Z9\9\!\`c c#(#7#<#<#>#>#I#I$(//ZgJhJh#i#i y ()=>>qqq"1"uE2249>>$)/RSBTVXZ^ZcZijlZmosoxo~  @B  pC  4D  4D  4I  4I  NO  Y]  4I  4^  4^  4a  4a  i}  iD  4a  4E  4E  G[  Ga  bd  Ge  g{  gA  BD  gE  GP  RZ  [  [#$388::EEdhooV[VefsVtVtwzVzF{F{||9lE3GHaPPPPr   c           
      *   |                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     dd	          }	| j                                        }
| j        *| j        |j        d
         |
z  |j        d         |
z  fk    rd }| j        t          j        	                    | j        
                    dd          |j        d         |
z  |j        d
         |
z  dd          
                    dd          }|j        d         |j        d
         f| _        d }| j        t          j        	                    | j        
                    dd          |j        d         |
z  |j        d
         |
z  dd          
                    dd          }|j        d         |j        d
         f| _        t          j                            d          }|                     ||          | _        t          j                            |           | j        j        j        }t'          d|z            }||z  }||z  }|                    d           |                    d           ||dz
  k    r	d | _        |S | j        | j        d         |k    r|	dk    rDdd | j                            || j                            |j                  ||          ff| _        nCdd | j                            || j                            |j                  ||          ff| _        |	dk    r| j        d         dz   }|| j        j                            || j        d         d         |d d d | j        d         d         j        d         f         d ||          f| _        | j        d         d         K|d d d | j        d         d         j        d         fxx         | j        d         d         | j        z  z  cc<   n5| j        d         |k     r| j        d         dz   |k     r| j        d         dz   }|| j        j                            || j        d         d         |d d d | j        d         d         j        d         f         d ||          f| _        | j        d         |k     r| j        d         dz   |k     || j        d         k    rf|d d d | j        d         d         j        d         fxx         | j        d         d         | j        z  z  cc<   || j        d         dz   k    rd | _        |S )Nr"   rX   	img_inputtxtpevecr   
block_typer   rB   rC   rA   r   r   r  Tr      r   noise_refiner)r   r   r   r   r   rN   r   rG   r   r   r   r  r   r   r   r   r   r   r   r  r   r  r   r   forward_noise_refiner_blockr   forward_control_block)r   r  r"   rX   r"  r#  r$  r%  r   r&  r  r  inpaint_scaledr   cnet_blocksdiv
cnet_indexcnet_index_float
next_layers                      r   r  zZImageControlPatch.__call__  s   JJsOOjjJJ{++	jjZZjjjj//ZZb11
"hAACC%)@QWR[SfEfhihoprhs  wJ  iJ  EK  *K  *KLz%${99$*:L:LRQR:S:SUVU\]_U`cvUvxyx  AC  yD  GZ  yZ  \b  dl   m   m   u   u  vw  y{   |   |+7+=b+A<CUVXCY*Z'!N!-!&!;!;D<N<V<VWY[\<]<]_`_fgi_j  nA  `A  CD  CJ  KM  CN  Qd  Cd  fl  nv  "w  "w  "  "  @A  CE  "F  "F+9+?+C^EYZ\E]*^'!2@@UY@ZZM!%!8!8~!V!VD"22=AAA&,=B$%%!S(
'#-

5

5{Q//!DNM>!T^A%6%C%C_,,"$tT-=-C-CCI[I^I^_b_hIiIikmor-s-s&t!u"$tT-=-C-CCI[I^I^_b_hIiIikmor-s-s&t!u((*Q.J($*:*@*\*\]hjnjxyzj{|}j~  AJ  KL  KL  KL  Nl  OS  O]  ^_  O`  ab  Oc  Oi  jk  Ol  Nl  Kl  Am  os  uw  y|  +}  +}  ~DN~a #/AAA5q)!,215556664>!;LQ;ORVR_;_`666.#j00dnQ6G!6K{5Z5Z!^A.2
",d.>.D.Z.Z[egkguvwgxyzg{  ~G  HI  HI  HI  Ki  LP  LZ  [\  L]  ^_  L`  Lf  gh  Li  Ki  Hi  ~j  lp  rt  vy  /z  /z  "{ .#j00dnQ6G!6K{5Z5Z  4>!#444AAA5q)!,215556664>!;LQ;ORVR_;_`666$."3a"777%)DNr   c                     t          |t          j                  r-| j        | j                            |          | _        d | _        | S r!   )r  r   r   r   r   r  r  s     r   r   zZImageControlPatch.to  sD    ou|44 	"!-%)%7%:%:?%K%K"!DNr   c                     | j         gS r!   r  r  s    r   r  zZImageControlPatch.models  r  r   )NNr  r   r   r   r  r  Y  sj           0       6; ; ;z  " " " " "r   r  c                   <    e Zd Zed             ZdZdZdZdZd	dZ	dS )
QwenImageDiffsynthControlnetc           
      *    ddddddddd	d
fdddidS )NMODELr   VAEIMAGEFLOAT      ?      $      $@{Gz?defaultminmaxstep)r   r   r   r   r   r   MASKr   optionalr   r   s    r   r   z(QwenImageDiffsynthControlnet.INPUT_TYPES  sG    '1-=%-'1+2EZ^hl4m4m*n	    $Y/1 1 	1r   r6  diffsynth_controlnetTzadvanced/loaders/qwenNr=  c           	      >   |                                 }||d d d d d d d df         }||d d d d d d d df         }|E|j        dk    r|                    d          }|j        dk    r|                    d          }d|z
  }t          |j        t
          j        j        j        j	                  r@t          ||||||          }	|                    |	           |                    |	           n&|                    t          |||||                     |fS )NrE   rA   rD   r@   r=  )r  r   )clonendim	unsqueezer  r   rG   rK   r   r   r   r  set_model_noise_refiner_patchset_model_double_block_patchr   )
r   r   r   r   r   r   r  r   model_patchedpatchs
             r   rJ  z1QwenImageDiffsynthControlnet.diffsynth_controlnet  s@   !!!QQQ2A2+&E$)!!!QQQ2A2+6MyA~~~~a((yA~~~~a((:Dk')9)D)STT 	t&{CXelpqqqE77>>>66u====667I+WZ\ackmq7r7rsssr   )Nr=  NN)
r%   r&   r'   r   r   r   r   r   r   rJ  r   r   r   r4  r4    sW        1 1 [1 L%HL&H           r   r4  c                   (    e Zd Zed             ZdZdS )ZImageFunControlnetc           	      ,    ddddddddd	fd
dddddS )Nr6  r   r8  r<  r=  r>  r?  r@  rA  )r   r   r   r   r:  rF  )r   r  r   rH  r   r   s    r   r   zZImageFunControlnet.INPUT_TYPES  sM    '1-=%-+2EZ^hl4m4m*n   
 '1:W`aac c 	cr   zadvanced/loaders/zimageN)r%   r&   r'   r   r   r   r   r   r   rT  rT    s1        c c [c )HHHr   rT  c                   &    e Zd Zd Zd Zd Zd ZdS )UsoStyleProjectorPatchc                 "    || _         || _        d S r!   )r   r   )r   r   r   s      r   r   zUsoStyleProjectorPatch.__init__  s    &*r   c           	         |                     d          }|                     d          }| j                            | j                            |j                                                |j                  }t          j        ||gd          }||d<   t          j        t          j        |j	        d         |j	        d         d|j        |j
                  |gd          |d<   |S )Ntxt_idsr#  rA   ru   r   rE   )r   r   )r   r   r   r   r   r   r   ry   zerosrN   r   )r   r  rZ  r#  siglip_embeddings        r   r  zUsoStyleProjectorPatch.__call__
  s    **Y''jj+11$2D2G2G	2R2RSSVVWZW`aai)3/Q777u!Iu{3C3I!3LN^NdefNgijryr  IP  IW  (X  (X  (X  Za  'b  hi  j  j  jyr   c                 x    t          |t          j                  r| j                            |          | _        | S r!   r  r  s     r   r   zUsoStyleProjectorPatch.to  r
  r   c                     | j         gS r!   r  r  s    r   r  zUsoStyleProjectorPatch.models  r  r   N)r%   r&   r'   r   r  r   r  r   r   r   rW  rW    sP        + + +    
" " " " "r   rW  c                   :    e Zd Zed             ZdZdZdZdZd Z	dS )USOStyleReferencec                     dddddiS )Nr   r6  r   )CLIP_VISION_OUTPUT)r   r   clip_vision_outputr   r   s    r   r   zUSOStyleReference.INPUT_TYPES  s!    j,<3K    ! 	!r   r6  apply_patchTzmodel/patch/fluxc                     t          j        |j        d d df         |j        d d df         |j        f          }|                                }|                    t          ||                     |fS )Nii)r   stackall_hidden_statespenultimate_hidden_statesrL  set_model_post_input_patchrW  )r   r   r   rc  r   rQ  s         r   rd  zUSOStyleReference.apply_patch)  s    %7%I!!!S&%QSeSwxyxyxy{~x~S  BT  Bn  %o  p  p001GUb1c1cdddr   N)
r%   r&   r'   r   r   r   r   r   r   rd  r   r   r   r`  r`    sR        ! ! [!
 LHL!H         r   r`  c                   N     e Zd Z	 	 	 	 	 	 	 	 dd	ed
edededededef fdZ xZS )r   rF            r   rD   (   Nr   r   r0   r   r   r   r/   c           
        	
 t                                                       t          |||z   dz
  ||	
          | _        t          j                            	
fdt          |          D                       | _        d S )NrA   )seq_len
seq_len_vfr   r   r   r   r   r   c           	      8    g | ]}t                     S r5   )r   )r7   r8   r   r   r0   r   r   s     r   r9   z0MultiTalkModelPatch.__init__.<locals>.<listcomp>I  s>        +676QVcmnnn  r   )	r   r   r   
audio_projr   r   r;   r<   blocks)r   r   r   r0   r   r   r   r/   r   r   r   r   s      ``   ```r   r   zMultiTalkModelPatch.__init__1  s     	1$'	1!3!1-%	
 	
 	
 h))       z**  
 
r   )
rF   rk  rl  rm  r   rD   rn  NNN)r%   r&   r'   r(   r   r)   r*   s   @r   r   r   0  s          # ,0
 

 
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
r   r   c                       e Zd Zedej        fd            Zed             Zedej        j	        dej
        j	        dej        j	        dej        j	        ded	ed
ededej        fd            ZdS )
SUPIRApplyr   c                    t          j        dddt           j                            d          t           j                            d          t           j                            d          t           j                            d          t           j                            dd	d
ddd          t           j                            dd	d
ddd          t           j                            ddd
dddd          t           j                            ddd
d	ddd          gt           j                                        g          S )Nrv  zmodel/patch/supirTr   r   r   r   strength_startr=          r?  r@  z7Control strength at the start of sampling (high sigma).)rB  rC  rD  rE  tooltipstrength_endzVControl strength at the end of sampling (low sigma). Linearly interpolated from start.restore_cfgg      @g      4@g?zaPulls denoised output toward the input latent. Higher = stronger fidelity to input. 0 to disable.)rB  rC  rD  rE  advancedrz  restore_cfg_s_tming?z4Sigma threshold below which restore_cfg is disabled.)node_idcategoryis_experimentalinputsoutputs)	r   SchemaModelInput
ModelPatchVaeImageFloatOutput)clss    r   define_schemazSUPIRApply.define_schemaQ  s/   y ( w''##M22U##w''/#4VZ'`  b b~sTX'  A A}csSVae (K  L L3TsPSZ^im']  _ _ X__&&'%
 
 
 	
r   c                 d   t          |j        dd          }|s|                    |          S |j        }|                                |_        d |                                D             }|j                            |dd           	 |                    |          ||_        S # ||_        w xY w)zHEncode using denoise_encoder weights from SUPIR checkpoint if available.r   Nc                 "    i | ]\  }}d | |fS )zencoder.r   )r7   r   vs      r   
<dictcomp>z;SUPIRApply._encode_with_denoise_encoder.<locals>.<dictcomp>r  s&    GGGDAq>a>>A4GGGr   r=  ry  )strength_patchstrength_model)getattrr   r   patcherrL  r   add_patches)r  r   r   r   
denoise_sdorig_patcherpatchess          r   _encode_with_denoise_encoderz'SUPIRApply._encode_with_denoise_encoderh  s     [.0DdKK
 	%::e$$$ {"((**GGJ4D4D4F4FGGGCPPP	'::e$$&CKK,CK&&&&s   
B& &	B/r   r   r   r   rx  r{  r|  r~  c                   |                                 }	|                    d                              |                     |||d d d d d d d df                             }
t	          ||j        j        |
||          }|                    |	           dk    r|                    d          }|                    |	                    |
                    }|                    |
                    |d d d d d d d df                             dfd}|	                    |           t          j        |	          S )Nlatent_formatrE   ry  gB>٬:-@c                    | d         }| d         }|                                 dk    r|d                                         }n|                                }|	k    r                    |j        |j                  }|j        d         }|j        d         |k    rh|j        d         dk    r|                    |ddd          n>|                    ||j        d         z   dz
  |j        d         z  ddd          d |         }|                                 dk    r|                    dddd          n|}||z
  }|||
z  z  z  z
  }|S )Ndenoisedsigmar   r3   rA   rC   )	r   itemr   r   r   rN   expandrepeatrO   )argsr  r  r   refb	sigma_vald_centerr|  r~  	sigma_maxx_centers           r   restore_cfg_functionz0SUPIRApply.execute.<locals>.restore_cfg_function  s   
+W99;;??aAA

A)))"++X_HN+SSC q)Ay|q((;>9Q<1;L;LcjjBB777RUR\R\^_bebklmbn^nqr^rwz  xA  BC  xD  ^D  FG  IJ  LM  SN  SN  OQ  PQ  OQ  SR;@99;;??

2q!Q 7 7 7PUI'#~H'(y97LQ\6\*]]Hr   )rL  get_model_objectrJ   r  r   r   project_modulesregisterdecodeprocess_outr   #set_model_sampler_post_cfg_functionr   
NodeOutput)r  r   r   r   r   rx  r{  r|  r~  rQ  hint_latentrR  r  decodedr  r  r  s          ``      @@r   executezSUPIRApply.executey  sh    ,,_==HH,,S+uQQQ111bqb[?QRRT T;(9(I;Xfhtuu}%%%!22?CCMjj!:!:;!G!GHHG$//

7111aaaBQB;;O0P0PQQHI               " ==>RSSS}]+++r   N)r%   r&   r'   r   r   r  r  r  r  Typer  r  r  floatr  r  r   r   r   rv  rv  P  s        
bi 
 
 
 [
, ' ' ['  ",rx} ",2=;M ",TVTZT_ ",hjhphu ", %",5:",IN",di",npn{", ", ", [", ", ",r   rv  )r   r4  rT  r`  rv  )"r   r   r   comfy.utilsrG   	comfy.opscomfy.model_managementcomfy.ldm.common_ditcomfy.latent_formatscomfy.ldm.lumina.controlnetcomfy.ldm.supir.supir_modulescomfy.ldm.wan.model_multitalkr   r   comfy_api.latestr   comfy.ldm.supir.supir_patchr   r   r	   r,   r]   r   r   r   r  r4  rT  rW  r`  r   	ComfyNoderv  NODE_CLASS_MAPPINGSr   r   r   <module>r     s                                  " " " " $ $ $ $ ] ] ] ] ] ] ] ]       2 2 2 2 2 2    EHO   $N N N N N58? N N NB@ @ @ @ @ux @ @ @D  :F  F  F  F  F  F  F  F R," ," ," ," ," ," ," ,"\y" y" y" y" y" y" y" y"v#  #  #  #  #  #  #  # J
) 
) 
) 
) 
)6 
) 
) 
)" " " " " " " ".               (
 
 
 
 
%(/ 
 
 
@L, L, L, L, L, L, L, L,` )$@.*    r   