
    +jN                     .   d dl Z d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlZd dlZ G d de	j                  Z G d d	e	j                  Z G d
 de          Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  ZdZ G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de	j                  Z G d de          Zdefd ZdS )!    N)override)ComfyExtensionio)HunyuanVideo15SRModel)LatentUpsamplerc                   N    e Zd Zed             Zedej        fd            ZeZdS )CLIPTextEncodeHunyuanDiTc                 ,   t          j        ddt           j                            d          t           j                            ddd          t           j                            ddd          gt           j                                        g          S )	Nr	   advanced/conditioningclipbertT	multilinedynamic_promptsmt5xlnode_idcategoryinputsoutputs)r   SchemaClipInputStringConditioningOutputclss    :/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_hunyuan.pydefine_schemaz&CLIPTextEncodeHunyuanDiT.define_schema   s}    y.,f%%	$MM	4NN &&((
 
 
 	
    returnc                     |                     |          }|                     |          d         |d<   t          j        |                    |                    S )Nr   )tokenizer   
NodeOutputencode_from_tokens_scheduled)r   r   r   r   tokenss        r   executez CLIPTextEncodeHunyuanDiT.execute   sK    t$$--..w7w}T>>vFFGGGr!   N	__name__
__module____qualname__classmethodr    r   r%   r(   encode r!   r   r	   r	      s\        
 
 [
 H2= H H H [H FFFr!   r	   c                   P    e Zd Zed             Zeddej        fd            ZeZdS )EmptyHunyuanLatentVideoc                    t          j        dddt           j                            dddt          j        d          t           j                            dd	dt          j        d          t           j                            d
ddt          j        d          t           j                            dddd          gt           j                                        g          S )Nr1   zEmpty HunyuanVideo 1.0 Latentzmodel/latent/videowidthP     defaultminmaxstepheight  length         
batch_size   r7   r8   r9   r   display_namer   r   r   r   r   Intr   nodesMAX_RESOLUTIONLatentr   r   s    r   r    z%EmptyHunyuanLatentVideo.define_schema'   s    y-8)Wcru?SZ\]]Xs@T[]^^Xrqe>RYZ[[\1!FF	 	  ""
 
 
 	
r!   r?   r"   c                     t          j        |d|dz
  dz  dz   |dz  |dz  gt          j                                                  }t          j        |dd          S )Nr5   r?   r@      devicesamplesdownscale_ratio_spacialtorchzeroscomfymodel_managementintermediate_devicer   r%   r   r3   r;   r=   rA   latents         r   r(   zEmptyHunyuanLatentVideo.execute8   s|    j"
q/@A.EvQR{TY]^T^_hmh~  iS  iS  iU  iU  V  V  V}ANNOOOr!   Nr?   	r*   r+   r,   r-   r    r   r%   r(   generater/   r!   r   r1   r1   &   sb        
 
 [
  P PR] P P P [P HHHr!   r1   c                   V     e Zd Ze fd            Zeddej        fd            Z xZS )EmptyHunyuanVideo15Latentc                 d    t                                                      }d|_        d|_        |S )Nr^   zEmpty HunyuanVideo 1.5 Latent)superr    r   rE   )r   schema	__class__s     r   r    z'EmptyHunyuanVideo15Latent.define_schemaA   s,    &&((4=r!   r?   r"   c                     t          j        |d|dz
  dz  dz   |dz  |dz  gt          j                                                  }t          j        |dd          S )N    r?   r@   r5   rM   rO   rR   rX   s         r   r(   z!EmptyHunyuanVideo15Latent.executeH   s     j"
q/@A.EvQS|UZ^`U`ajo  kA  kU  kU  kW  kW  X  X  X}BOOPPPr!   rZ   )	r*   r+   r,   r-   r    r   r%   r(   __classcell__)rb   s   @r   r^   r^   @   sz            [ Q QR] Q Q Q [Q Q Q Q Qr!   r^   c                   L    e Zd Zed             Zeddej        fd            ZdS )HunyuanVideo15ImageToVideoc                 r   t          j        ddt           j                            d          t           j                            d          t           j                            d          t           j                            dddt          j        d	          t           j                            d
ddt          j        d	          t           j                            dddt          j        d	          t           j                            dddd          t           j                            dd          t           j	                            dd          g	t           j        
                    d          t           j        
                    d          t           j        
                    d          g          S )Nrg   model/conditioning/video_modelspositivenegativevaer3   r4   r5   r6   r;   r<   r=   !   r?   r@   rA   rB   rC   start_imageToptionalclip_vision_outputrE   rY   r   )r   r   r   r   VaerG   rH   rI   ImageClipVisionOutputr   rJ   r   s    r   r    z(HunyuanVideo15ImageToVideo.define_schemaP   sK   y06%%j11%%j11U##Wcru?SZ\]]Xs@T[]^^Xrqe>RYZ[[\1!FF}t<<#))*>)NN
 &&J&??&&J&??	  h 77
 
 
 	
r!   Nr"   c
                 X   t          j        |d|dz
  dz  dz   |dz  |dz  gt          j                                                  }
|t          j                            |d |                             dd          ||dd                              dd          }|                    |d d d d d d d d	f                   }t          j        |
j	        d
         d|
j	        d         |
j	        d	         |
j	        d         ft          j                                                  }||d d d d d |j	        d         d d d d f<   t          j
        dd|
j	        d         |j	        d         |j	        d         f|j        |j                  }d|d d d d d |j	        d
         dz
  dz  dz   f<   t          j        |||d          }t          j        |||d          }|	.t          j        |d|	i          }t          j        |d|	i          }i }|
|d<   t          j        |||          S )Nrd   r?   r@   r5   rM   bilinearcenter   r      rN   dtype        concat_latent_imageconcat_maskrq   rP   )rS   rT   rU   rV   rW   utilscommon_upscalemovedimr.   shapeonesrN   r~   node_helpersconditioning_set_valuesr   r%   )r   rj   rk   rl   r3   r;   r=   rA   rn   rq   rY   encodedr   mask
out_latents                  r   r(   z"HunyuanVideo15ImageToVideo.executeg   s   j"
q/@A.EvQS|UZ^`U`ajo  kA  kU  kU  kW  kW  X  X  X"+44[&5I5Q5QRTVW5X5XZ_agisu}~~  G  G  HI  KM  N  NKjjQQQ111bqb[!9::G"'+v|AFLQROU[UabcUdflfrstfu.v  @E  @V  @j  @j  @l  @l  #m  #m  #mAH111&7w}Q'7&7AAA =>:q!V\!_6I6OPR6SUhUnoqUrs  }H  }O  Wb  Wh  i  i  iDADDAAA=++A.2q8A===>#;H^q  CG  GH  GH  I  IH#;H^q  CG  GH  GH  I  IH)#;HG[]oFpqqH#;HG[]oFpqqH
 &
9}Xx<<<r!   )NNr*   r+   r,   r-   r    r   r%   r(   r/   r!   r   rg   rg   O   se        
 
 [
, = =  @B  @M = = = [= = =r!   rg   c                   L    e Zd Zed             Zeddej        fd            ZdS )HunyuanVideo15SuperResolutionc                    t          j        dddt           j                            d          t           j                            d          t           j                            dd          t           j                            d	d          t           j                            d
d          t           j                            d          t           j                            dddddd          gt           j        	                    d          t           j        	                    d          t           j        	                    d          g          S )Nr   z"Hunyuan Video 1.5 Super Resolutionri   rj   rk   rl   Tro   rn   rq   rY   noise_augmentationgffffff?r         ?{Gz?r7   r8   r9   r:   advancedrr   rD   )
r   r   r   r   rs   rt   ru   rJ   Floatr   r   s    r   r    z+HunyuanVideo15SuperResolution.define_schema   s   y3=6%%j11%%j11UT22}t<<#))*>)NN	))3TsPSZ^imnn	 &&J&??&&J&??	  h 77
 
 
 	
r!   Nr"   c                    |d         }|j         d         }	t          j        |j         d         |	dz  dz   |j         d         |j         d         |j         d         gt          j                                                  }
||
d d |	dz   d|	z  dz   f<   d|
d d d|	z  dz   f<   |t          j                            |                    dd          |j         d         d	z  |j         d         d	z  d
d                              dd          }|	                    |d d d d d d d df                   }||
d d d |	d |j         d         d d d d f<   d|
d d |	dz   df<   t          j        ||
|d          }t          j        ||
|d          }|.t          j        |d|i          }t          j        |d|i          }t          j        |||          S )NrP   r?   r   r{   r|   rw   rM   r5   rx   ry   rz   r   r   rq   )r   rS   rT   rU   rV   rW   r   r   r   r.   r   r   r   r%   )r   rj   rk   rY   r   rl   rn   rq   	in_latentin_channelscond_latentr   s               r   r(   z%HunyuanVideo15SuperResolution.execute   s   9%	oa(k9?1#5{Q7JIO\^L_ajapqsatv  wF  GI  wJ  #K  TY  Tj  T~  T~  T@  T@  A  A  A@IAAA{Q[1)<<<=./AAAq;**+"+44[5H5HQ5O5OQZQ`acQdgiQiktkz{}k~  BD  lD  FP  RZ  [  [  c  c  de  gi  j  jKjjQQQ111bqb[!9::GDKK<K<):'-*:):AAAqqq@A12K;?A-.7Ze  ~P  CQ  CQ  R  R7Ze  ~P  CQ  CQ  R  R)#;HG[]oFpqqH#;HG[]oFpqqH}Xx888r!   )NNNr   r/   r!   r   r   r      sa        
 
 [
, 9 9}  ~K 9 9 9 [9 9 9r!   r   c                   J    e Zd Zed             Zedej        fd            ZdS )LatentUpscaleModelLoaderc                     t          j        dddt           j                            dt	          j        d                    gt           j                                        g          S )Nr   zLoad Latent Upscale Modelzmodel/loaders
model_namelatent_upscale_modelsoptionsrD   )r   r   Combor   folder_pathsget_filename_listLatentUpscaleModelr   r   s    r   r    z&LatentUpscaleModelLoader.define_schema   sc    y.4$|\5STk5l5lmm %,,..

 

 

 
	
r!   r"   c                 b   t          j        d|          }t          j                            |dd          \  }dv rd         j        d         d         j        d         d         j        d         t          d	                                 D                       d
d}d}t          ||          }|	                               nPdv rd 
                                D             d         j        d         d         j        d         t          fdt          t          d                                 D                                 D                       d}d}t          ||          }|	                               ndv rt          j        |d                   }t          j        |                              t          j                            t(          j        t(          j        g                    }|                               t1          j        |          S )Nr   T)	safe_loadreturn_metadatazblocks.0.block.0.conv.weightzin_conv.conv.weightr?   zout_conv.conv.weightr   c                 f    g | ].}|                     d           |                    d          ,|/S )zblocks.z.block.0.conv.weight
startswithendswith.0ks     r   
<listcomp>z4LatentUpscaleModelLoader.execute.<locals>.<listcomp>   s>    "z"z"z1<<	;R;R"zWXWaWabxWyWy"z1"z"z"zr!   F)r   out_channelshidden_channels
num_blocksglobal_residual720pzup.0.block.0.conv1.conv.weightc                 D    i | ]\  }}|                     d dd          |S )nin_shortcutznin_shortcut.convr?   )replace)r   keyvalues      r   
<dictcomp>z4LatentUpscaleModelLoader.execute.<locals>.<dictcomp>   s2    jjjQ[QTV[#++n.A1EEujjjr!   zconv_in.conv.weightzconv_out.conv.weightc              3   F   K   | ]}d | d         j         d         V  dS )up..block.0.conv1.conv.weightr   N)r   )r   isds     r   	<genexpr>z3LatentUpscaleModelLoader.execute.<locals>.<genexpr>   sV        ,N  ,NabB/RQ/R/R/R,S,YZ[,\  ,N  ,N  ,N  ,N  ,N  ,Nr!   c                 f    g | ].}|                     d           |                    d          ,|/S )r   r   r   r   s     r   r   z4LatentUpscaleModelLoader.execute.<locals>.<listcomp>   s      qK  qK  qKwx  JK  JV  JV  W\  J]  J]  qK  bc  bl  bl  mI  bJ  bJ  qKqr  qK  qK  qKr!   )
z_channelsr   block_out_channels1080pz%post_upsample_res_blocks.0.conv2.biasconfig)allowed_dtypes)r~   )r   get_full_path_or_raiserU   r   load_torch_filer   lenkeysr   load_sditemstuplerangejsonloadsr   from_configtorV   	vae_dtyperS   bfloat16float32load_state_dictr   r%   )r   r   
model_pathmetadatar   
model_typemodelr   s          @r   r(   z LatentUpscaleModelLoader.execute   s   !89PR\]]
{22:_c2ddH)R//!"78>qA "#9 : @ C#%&;#<#B1#E!"z"zbggii"z"z"z{{#( F  J)*f==EMM"-33jj_a_g_g_i_ijjjB !67=a@ "#9 : @ C&+  ,N  ,N  ,N  ,Nfklo  qK  qK|~  }D  }D  }F  }F  qK  qK  qK  mL  mL  gM  gM  ,N  ,N  ,N  'N  'N F
 !J)*f==EMM"4::Z 233F#/77::AWAaAarw  sA  CH  CP  rQAa  BR  BR:  S  SE!!"%%%}U###r!   Nr   r/   r!   r   r   r      sV        
 
 [
 $BM $ $ $ [$ $ $r!   r   c                   J    e Zd Zed             Zedej        fd            ZdS )$HunyuanVideo15LatentUpscaleWithModelc                    t          j        dddt           j                            d          t           j                            d          t           j                            dg dd	          t           j                            d
dddd          t           j                            ddddd          t           j                            dddg          gt           j                                        g          S )Nr   z*Hunyuan Video 15 Latent Upscale With Modelmodel/latentr   rP   upscale_method)znearest-exactrx   areabicubicbislerprx   )r   r7   r3   i   r   i @  rL   r6   r;   i  cropdisabledry   r   rD   )r   r   r   r   rJ   r   rG   r   r   s    r   r    z2HunyuanVideo15LatentUpscaleWithModel.define_schema   s    y:E#%++G44		**/9t9t9t  I  J  JWdu1MMXsu1MMv
H/EFF 	  ""
 
 
 	
r!   r"   c                    |dk    r|dk    rt          j        |          S |dk    rVt          d|          }t          dt          |d         j        d         |z  |d         j        d         z                      }n||dk    rVt          d|          }t          dt          |d         j        d         |z  |d         j        d         z                      }n t          d|          }t          d|          }t
          j                            |d         |dz  |dz  ||          }|                    |          }t          j        d|	                                
                                i          S )Nr   @   rP   rw   r|   r5   )r   r%   r9   roundr   rU   r   r   resample_latentcpufloat)r   r   rP   r   r3   r;   r   ss           r   r(   z,HunyuanVideo15LatentUpscaleWithModel.execute   sL   A::&A++=)))zzRBgi&8&>r&BV&KgV_N`NfgiNj&j k kll1BRwy'9'?'Ce'KgV_N`NfgiNj'j!k!kllBR**79+=u{FVXLZhjnooA%%a((A=)QUUWW]]__!=>>>r!   Nr   r/   r!   r   r   r      sV        
 
 [
$ ?R] ? ? ? [? ? ?r!   r   aZ  <|start_header_id|>system<|end_header_id|>

<image>
Describe the video by detailing the following aspects according to the reference image: 1. The main content and theme of the video.2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.4. background environment, light, style and atmosphere.5. camera angles, movements, and transitions used in the video:<|eot_id|>

<|start_header_id|>user<|end_header_id|>

{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>

c                   N    e Zd Zed             Zedej        fd            ZeZdS )#TextEncodeHunyuanVideo_ImageToVideoc                 n   t          j        ddt           j                            d          t           j                            d          t           j                            ddd          t           j                            dd	d
ddd          gt           j                                        g          S )Nr   r   r   rq   promptTr   image_interleaver{   r?   i   zqHow much the image influences things vs the text prompt. Higher number means more influence from the text prompt.)r7   r8   r9   tooltipr   r   )	r   r   r   r   ru   r   rG   r   r   r   s    r   r    z1TextEncodeHunyuanVideo_ImageToVideo.define_schema  s    y9,f%%#))*>??	D$OO& P!   	 &&((!
 
 
 	
r!   r"   c                     |                     |t          |j        |          }t          j        |                    |                    S )N)llama_templateimage_embedsr   )r$    PROMPT_TEMPLATE_ENCODE_VIDEO_I2Vmm_projectedr   r%   r&   )r   r   rq   r   r   r'   s         r   r(   z+TextEncodeHunyuanVideo_ImageToVideo.execute*  sM    v6Vew  fE  Xh  i  i}T>>vFFGGGr!   Nr)   r/   r!   r   r   r     s\        
 
 [
, HBM H H H [H FFFr!   r   c                   P    e Zd Zed             Zeddej        fd            ZeZdS )HunyuanImageToVideoc                    t          j        ddt           j                            d          t           j                            d          t           j                            dddt          j        d          t           j                            d	d
dt          j        d          t           j                            dddt          j        d          t           j                            dddd          t           j                            dg dd          t           j	                            dd          gt           j        
                    d          t           j        
                    d          g          S )Nr   ri   rj   rl   r3   r4   r5   r6   r;   r<   r=   5   r?   r@   rA   rB   rC   guidance_type)v1 (concat)v2 (replace)customT)r   r   rn   ro   rr   rY   r   )r   r   r   r   rs   rG   rH   rI   r   rt   r   rJ   r   s    r   r    z!HunyuanImageToVideo.define_schema3  s'   y)6%%j11U##Wcru?SZ\]]Xs@T[]^^Xrqe>RYZ[[\1!FF8a8a8alpqq}t<<	 &&J&??	  h 77
 
 
 	
r!   Nr"   c	                 *   t          j        |d|dz
  dz  dz   |dz  |dz  gt          j                                                  }	i }
|/t          j                            |d |d d d d d df                             dd          ||dd	                              dd          }|                    |          }t          j	        dd|	j
        d
         |j
        d         |j
        d         f|j        |j                  }d|d d d d d |j
        d         dz
  dz  dz   f<   |dk    r||d}n3|dk    r#ddi}||	d d d d d |j
        d
         f<   ||
d<   n
|dk    rd|i}t          j        ||          }|	|
d<   t          j        ||
          S )Nr5   r?   r@   rL   rM   rz   rw   rx   ry   r{   r|   r}   r   r   r   r   r   guiding_frame_index
noise_maskr  
ref_latentrP   )rS   rT   rU   rV   rW   r   r   r   r.   r   r   rN   r~   r   r   r   r%   )r   rj   rl   r3   r;   r=   rA   r   rn   rY   r   r   r   conds                 r   r(   zHunyuanImageToVideo.executeH  s@   j"
q/@A.EvQR{TY]^T^_hmh~  iS  iS  iU  iU  V  V  V
"+44[&!!!QQQPRQRPRAR5S5[5[\^`a5b5bdikqs}  @H  I  I  Q  Q  RS  UW  X  XK"%**["9"9:q!V\!_6I6OPR6SUhUnoqUrs  }H  }O  Wb  Wh  i  i  iDADDAAA=++A.2q8A===>--/BSWXX.00-q1>Qqqq!!!:17:::;+/
<(((**$&9:#;HdKKH &
9}Xz222r!   Nr)   r/   r!   r   r   r   2  s^        
 
 [
( 3 3kmkx 3 3 3 [32 FFFr!   r   c                   P    e Zd Zed             Zeddej        fd            ZeZdS )EmptyHunyuanImageLatentc                 d   t          j        ddt           j                            dddt          j        d          t           j                            dddt          j        d          t           j                            d	d
d
d          gt           j                                        g          S )Nr	  r   r3   i   r   rd   r6   r;   rA   r?   rB   rC   r   rF   r   s    r   r    z%EmptyHunyuanImageLatent.define_schemaf  s    y-#Wd@T[]^^XtAU\^__\1!FF 	  ""
 
 
 	
r!   r?   r"   c                     t          j        |d|dz  |dz  gt          j                                                  }t          j        d|i          S )Nr   rd   rM   rP   rR   )r   r3   r;   rA   rY   s        r   r(   zEmptyHunyuanImageLatent.executeu  sL    j"flERKHQVQgQ{Q{Q}Q}~~~}i/000r!   NrZ   r[   r/   r!   r   r	  r	  e  s\        
 
 [
 1 1R] 1 1 1 [1 HHHr!   r	  c                   J    e Zd Zed             Zedej        fd            ZdS )HunyuanRefinerLatentc                    t          j        dddt           j                            d          t           j                            d          t           j                            d          t           j                            ddd	d
dd          gt           j                            d          t           j                            d          t           j                            d          g          S )Nr  zHunyuan Latent Refinerri   rj   rk   rY   r   g?r   r   r   Tr   rr   rD   )r   r   r   r   rJ   r   r   r   s    r   r    z"HunyuanRefinerLatent.define_schema~  s    y*16%%j11%%j11	))3TsPSZ^imnn	 &&J&??&&J&??	  h 77
 
 
 	
r!   r"   c                 n   |d         }t          j        |||d          }t          j        |||d          }i }t          j        |j        d         d|j        d         |j        d         |j        d         gt
          j                                                  |d<   t          j	        |||          S )	NrP   r   r   rd   r   r|   rw   rM   )
r   r   rS   rT   r   rU   rV   rW   r   r%   )r   rj   rk   rY   r   r   s         r   r(   zHunyuanRefinerLatent.execute  s    	"7Z`  yK  CL  CL  M  M7Z`  yK  CL  CL  M  M
 %V\!_b&,rBRTZT`acTdflfrsufv,w  AF  AW  Ak  Ak  Am  Am  !n  !n  !n
9}Xx<<<r!   Nr   r/   r!   r   r  r  }  sV        
 
 [
& = = = = [= = =r!   r  c                   L    e Zd Zedeeej                          fd            ZdS )HunyuanExtensionr"   c                    K   t           t          t          t          t          t
          t          t          t          t          t          gS r  )r	   r   r1   r^   rg   r   r   r   r   r	  r  )selfs    r   get_node_listzHunyuanExtension.get_node_list  s3       %/#%&)0$# 
 	
r!   N)	r*   r+   r,   r   listtyper   	ComfyNoder  r/   r!   r   r  r    sE        
T$r|*<%= 
 
 
 X
 
 
r!   r  r"   c                  "   K   t                      S r  )r  r/   r!   r   comfy_entrypointr    s      r!   )rH   r   rS   comfy.model_managementrU   typing_extensionsr   comfy_api.latestr   r   !comfy.ldm.hunyuan_video.upsamplerr   %comfy.ldm.lightricks.latent_upsamplerr   r   r   r  r	   r1   r^   rg   r   r   r   r   r   r   r	  r  r  r  r/   r!   r   <module>r     s             & & & & & & / / / / / / / / C C C C C C A A A A A A         r|   4    bl   4Q Q Q Q Q 7 Q Q Q/= /= /= /= /= /= /= /=d+9 +9 +9 +9 +9BL +9 +9 +9\.$ .$ .$ .$ .$r| .$ .$ .$b$? $? $? $? $?2< $? $? $?P8 !    ",   @0 0 0 0 0", 0 0 0f    bl   0= = = = =2< = = =>
 
 
 
 
~ 
 
 
$ 0      r!   