
    
3j^.                         S SK Jr  S SKrS SKrS SKJr  S SKJr  SSK	J
r
  SSKJrJr  SS	KJrJr   " S
 S\R"                  R$                  5      r\
" SS9 " S S5      5       rg)    )ListN)ModelPatcher)CallbacksMP   )
comfy_node)STGBlockWrapperSTGFlag   )GuiderParametersModalityc                   H  ^  \ rS rSrS\S\\\4   S\\	   4U 4S jjr
SS jr\S\S\4S	 j5       r\S\4S
 j5       rS rS\S\S\S\4S jrS\R*                  4S jrS\R*                  S\R*                  4S jr0 S4S\R*                  S\R*                  S\4S jjrSrU =r$ )MultimodalGuider   model
parametersskip_blocksc                 $  > UR                  5       nSU l        S U l        S U l        UR	                  [
        R                  SU R                  5        [        TU ]%  U5        [        SUS9U l        U R                  XR                  5        X l        g )Nr   mm_guider_on_pre_runF)do_skipskip_layers)clonecurrent_steplast_denoised_vlast_denoised_aadd_callback_with_keyr   
ON_PRE_RUNreset_current_stepsuper__init__r	   stg_flagpatch_modelr   )selfr   r   r   	__class__s       W/home/wildlama/comfy/ComfyUI/custom_nodes/ComfyUI-LTXVideo/guiders/multimodal_guider.pyr   MultimodalGuider.__init__   s     ####""$:D<S<S	
 	#

 	.$    Nc                 .    SU l         S U l        S U l        g )Nr   )r   r   r   )r"   model_patchers     r$   r   #MultimodalGuider.reset_current_step&   s    ##r&   r    c                     U R                  U5      n[        U5       H#  u  pEUR                  [        XRU5      SSU5        M%     g )Nditdouble_block)get_transformer_blocks	enumerateset_model_patch_replacer   )clsr   r    transformer_blocksiblocks         r$   r!   MultimodalGuider.patch_model+   sC     77>!"45HA))3UNA 6r&   c                     U R                  S5      nSnUR                  R                  S:X  a  SnU R                  U5      $ )Ndiffusion_modelz"diffusion_model.transformer_blocksLTXVTransformer3Dz.diffusion_model.transformer.transformer_blocks)get_model_objectr#   __name__)r   r6   keys      r$   r-   'MultimodalGuider.get_transformer_blocks4   sC    001BC2$$--1DDBC%%c**r&   c                 *    U R                  XS.5        g )N)positivenegative)inner_set_conds)r"   r=   r>   s      r$   	set_condsMultimodalGuider.set_conds<   s    (IJr&   run_vxrun_ax	audio_ptb	video_ptbc                     [        5       nSnU(       a  US-  nU(       a  US-  nSnUS:X  a  SOSnU(       a  UR                  U5        U(       a  UR                  U5        [        U5      $ )Nr   r
   r   )setaddlist)	r"   rB   rC   rD   rE   stg_indexesnum_self_attnsvideo_attn_idxaudio_attn_idxs	            r$   calc_stg_indexes!MultimodalGuider.calc_stg_indexes?   sf     eaNaN,1qOON+OON+K  r&   xc                     U R                   R                  S0 5      S   R                  S0 5      R                  SS 5      R                  n[        R                  R                  X5      u  p4X44$ )Nr=   r   model_condslatent_shapes)condsgetcondcomfyutilsunpack_latents)r"   rP   rS   vxaxs        r$   rY   MultimodalGuider.unpack_latentsW   sZ    JJNN:r*1-S#S$'T	 	 ++A=vr&   rZ   r[   c                 L    [         R                  R                  X/5      u  p4X44$ N)rW   rX   pack_latents)r"   rZ   r[   rP   rS   s        r$   r_   MultimodalGuider.pack_latentsa   s$     ;;33RH=r&   timestepmodel_optionsc                    U R                   nUS-   U l         U R                  R                  SS 5      nU R                  R                  SS 5      nU R                  R                  [        R
                  R                  [        5       5      nU R                  R                  [        R                  R                  [        5       5      n	U	R                  U5      (       + n
UR                  U5      (       + nU	R                  U5      nUR                  U5      nU R                  U5      u  pU
(       d  U R                  nU(       d  U R                  nU R                  X5      u  nnU
(       d	  U(       d  U$  XS   S'   XS   S'   XS   S'   XS   S'   [        R                   R#                  U R$                  U/UUU5      S	   nUS   S	 US   S	 US   S	 US   S	 U R                  U5      u  nnS
u  nnS
u  nnS
u  nn['        S X4 5       5      (       az   XS   S'   XS   S'   XS   S'   XS   S'   [        R                   R#                  U R$                  U/UUU5      S	   nU R                  U5      u  nnUS   S	 US   S	 US   S	 US   S	 ['        S X4 5       5      (       a   U R)                  U
U=(       a    UR+                  5       S	:  UR,                  U	R,                  5      nXS   S'   XS   S'   XS   S'   XS   S'   S	US   S'   UUS   S'   SU R.                  l	        [        R                   R#                  U R$                  U/UUU5      S	   nU R                  U5      u  nnSU R.                  l	        US   S	 US   S	 US   S	 US   S	 US   S	 US   S	 ['        S X4 5       5      (       a|   XS   S'   XS   S'   SUS   S'   SUS   S'   [        R                   R#                  U R$                  U/UUU5      S	   nU R                  U5      u  nnUS   S	 US   S	 US   S	 US   S	 U
(       a  U	R1                  UUUU5      nOU R                  nU(       a  UR1                  UUUU5      nOU R                  nU R                  X5      u  nnUR                  S/ 5       H#  nUUUU R$                  WUUUUUWS.nU" U5      nM%     U R                  U5      u  U l        U l        U$ ! US   S	 US   S	 US   S	 US   S	 f = f! US   S	 US   S	 US   S	 US   S	 f = f! SU R.                  l	        US   S	 US   S	 US   S	 US   S	 US   S	 US   S	 f = f! US   S	 US   S	 US   S	 US   S	 f = f)Nr
   r=   r>   transformer_optionsrB   rC   a2v_cross_attnv2a_cross_attnr   )r   r   c              3   @   #    U  H  oR                  5       v   M     g 7fr^   )	do_uncond.0paramss     r$   	<genexpr>1MultimodalGuider.predict_noise.<locals>.<genexpr>   s     M0Lf!!0L   c              3   @   #    U  H  oR                  5       v   M     g 7fr^   )do_perturbedri   s     r$   rl   rm      s     P3O""$$3Orn   	ptb_indexrJ   TFc              3   @   #    U  H  oR                  5       v   M     g 7fr^   )do_modalityri   s     r$   rl   rm      s     O2N!!##2Nrn   sampler_post_cfg_function)denoisedrV   uncondr   uncond_denoisedcond_denoisedsigmarb   inputperturbed_condperturbed_cond_denoised)r   rT   rU   r   r   AUDIOvaluer   VIDEOr   do_cross_attnrY   r   r   r_   rW   samplerscalc_cond_batchinner_modelanyrN   numelperturb_attnr    	calculate) r"   rP   ra   rb   seedr   positive_condnegative_condaudio_paramsvideo_paramsrB   rC   run_a2vrun_v2arZ   r[   _noise_pred_posv_noise_pred_posa_noise_pred_posa_noise_pred_negv_noise_pred_nega_noise_pred_perturbedv_noise_pred_perturbeda_noise_pred_modalityv_noise_pred_modalitynoise_pred_negrJ   noise_pred_perturbednoise_pred_modalityfnargss                                    r$   predict_noiseMultimodalGuider.predict_noisee   sM    (((1,

z48

z48)-)<)<NN  "2"4*
 *.)<)<NN  "2"4*
 ")),77!)),77,,\:,,\:$$Q'%%B%%B  (1fH	G=C/0:=C/0:EL/01ABEL/01AB"^^;;   N 34X>34X>345EF345EF-1-@-@-P**-1**9=6 67;44M0LMMMKAG34X>AG34X>IP345EFIP345EF!&!?!?$$"O!" " 6:5H5H5X2 "2!"78B!"78B!"789IJ!"789IJPL3OPPPH"33-rxxzA~ -- --	 BH34X>AG34X>IP345EFIP345EFDE34[AFQ34]C(,%',~~'E'E$$"O!( ($ BFATAT(B>&(> ).%!"78E!"78B!"78B!"789IJ!"789IJ!"78GO<2NOOOCAG34X>AG34X>IN345EFIN345EF&+nn&D&D$$"O!' '# @D?R?R'@<%'< ""789IJ!"789IJ!"78B!"78B''  &%	B %%B''  &%	B %%B  (1  ##$?DB%'))#1!/!!."/+?D 4A E" 6:5H5H5K2d2E 34X>34X>345EF345EF, ""78B!"78B!"789IJ!"789IJ8 ).%!"78E!"78B!"78B!"789IJ!"789IJ!"78G& ""789IJ!"789IJ!"78B!"78Bs4   3AS0 A!T &CT* 8A#U$ 0T
T'*7U!$U>)r   r   r   r   r    r^   )r9   
__module____qualname____firstlineno__r   dictstrr   r   intr   r   classmethodr	   r!   staticmethodr-   r@   boolrN   torchTensorrY   r_   r   __static_attributes____classcell__)r#   s   @r$   r   r      s   %% ../% #Y	%0$
     +l + +K!! ! 	!
 !0  u||     !t<<t ,,t 	t tr&   r   )namec                   6    \ rS rSr\S 5       rSrSrSrS r	Sr
g)	MultimodalGuiderNodei  c           	           SSSSSSSSS.4S	.0$ )
Nrequired)MODEL)CONDITIONING)GUIDER_PARAMETERSSTRING T)default	multiline)r   r=   r>   r   r    )ss    r$   INPUT_TYPES MultimodalGuiderNode.INPUT_TYPES  s2     #--4 "6 	
 	
r&   )GUIDER
get_guiderzlightricks/LTXVc                     UR                  S5       Vs/ s H2  ofR                  5       (       d  M  [        UR                  5       5      PM4     nn[        XU5      nUR	                  X#5        X#4Ul        U4$ s  snf )N,)splitstripr   r   r@   	raw_conds)r"   r   r=   r>   r   r   nguiders           r$   r   MultimodalGuiderNode.get_guider2  sh    /:/@/@/ES/E!~s1779~/ES!%[A,$/y	 Ts
   A7A7r   N)r9   r   r   r   r   r   RETURN_TYPESFUNCTIONCATEGORYr   r   r   r&   r$   r   r     s*    
 
 LH Hr&   r   )typingr   comfy.samplersrW   r   comfy.model_patcherr   comfy.patcher_extensionr   nodes_registryr   stgr   r	   r   r   r   r   	CFGGuiderr   r   r   r&   r$   <module>r      sU       , / ' * 2Lu~~// L^ #$  %r&   