
    
3jֲ             	          S SK JrJrJr  S SKrS SKrS SKJr  S SKrS SK	J
r
  S SKrS SKrS SKrS SKJr  S SKJs  Js  Jr  S SKJr   S SKJrJr  \R>                  " 5       r S SK!r! " S S\5      r" " S	 S
\5      r# " S S\
RH                  5      r%S0 S4S jr&S0 S4S jr'S r(S0 4S jr) " S S5      r* " S S\
RH                  5      r+S r, " S S5      r- " S S\
RH                  5      r.S SK/r/S SK0J1r1  S SK2J3s  J4r5  S SK6r6S SK7r7S SK8J9r9  S SK
J:r:  \/Rv                  Rx                  r= " S S5      r>S r?S  r@SXS" jrA " S# S$5      rB " S% S&\
RH                  5      rC " S' S(5      rD " S) S*\
RH                  5      rE " S+ S,\
RH                  5      rF S SKGrGS SKHJIrJ  S-rK\K(       a  \GR                  S.\JR                  S/\JR                  S0\JR                  S1\JR                  4S2 j5       rN\GR                  S0\JR                  S3\JR                  4S4 j5       rO\GR                  S.\JR                  S/\JR                  S0\JR                  S1\JR                  4S5 j5       rPSYS6 jrQSZS7 jrRSYS8 jrSS9 rT    S[S:\\R                  \R                  4   S;\\R                  \R                  4   4S< jjrV " S= S>5      rW " S? S@\
RH                  5      rX " SA SB\
RH                  5      rYSC rZS!r[Sr\ S SDK]J^r^J_r_J`r`JaraJbrbJcrc  \b" 5       r\ S SEK]Jere  \Z" 5       rf\g" \eSF5      =(       a    \fSG:  r[ S SIK]Jiri   S SKK]Jkrk  S SKJr   S SMKmJnro   S SNKpJqrrJsrt  SO ruSSSS0 4SP jrv0 4SQ jrw0 4SR jrx0 4SS jry " ST SU\
RH                  5      rzS SK{r{ " SV SW\
RH                  5      r|g! \ a    SrSr GNPf = f! \ a    S!rK GNf = f! \d a     Nf = f! \ a     S SHK]Jhre   N! \ a    Sre  Nf = ff = f! \ a     S SJK]Jjri   N! \ a    Sri  Nf = ff = f! \ a     S SLK]Jlrk   N! \ a    Srk  Nf = ff = f! \ a    Sro Nf = f! \ a    S=rrrt Nf = f)\    )get_noise_maskLTXVAddGuide_append_guide_attention_entryN)Tuple)io)apply_rotary_emb)GuideAttentionMask_attention_with_guide_maskc                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LTXVAddGuideMulti   c                    / n[        SS5       H  n/ n[        SUS-   5       H  nUR                  [        R                  R	                  SU 35      [        R
                  R	                  SU 3SSSSU S	3S
9[        R                  R	                  SU 3SSSSSU S	3S9/5        M     UR                  [        R                  R                  [        U5      US95        M     [        R                  " SSS[        R                  R	                  SSS9[        R                  R	                  SSS9[        R                  R	                  SSS9[        R                  R	                  SSS9[        R                  R	                  SUS S!S"9/[        R                  R                  SS#9[        R                  R                  SS#9[        R                  R                  SS$S%9/S&9$ )'N   r   image_
frame_idx_r   '  zFrame index for guide .)defaultminmaxtooltip	strength_      ?              $@{Gz?zStrength for guide r   r   r   stepr   keyinputsr   KJNodes/ltxvzAdd multiple guide images at specified frame indices with strengths, uses DynamicCombo which requires ComfyUI 0.8.1 and frontend 1.33.4 or later.positivez@Positive conditioning to which guide keyframe info will be addedr   negativez@Negative conditioning to which guide keyframe info will be addedvaez)Video VAE used to encode the guide imageslatentz8Video latent, guides are added to the end of this latent
num_guideszNumber of Guidesz#Select how many guide images to useoptionsdisplay_namer   r,   zVideo latent with added guidesr,   r   node_idcategorydescriptionr"   outputs)rangeextendr   ImageInputIntFloatappendDynamicComboOptionstrSchemaConditioningVaeLatentOutput)clsr+   r)   guide_inputsis        M/home/wildlama/comfy/ComfyUI/custom_nodes/ComfyUI-KJNodes/nodes/ltxv_nodes.pydefine_schemaLTXVAddGuideMulti.define_schema   s   2,JL1j1n-##HHNNVA3<0FFLL$QC( !! "81 = !  HHNNYqc?CSdY]i|}~|  @A  hBN  C
% 
 . NN2??11
O# 2   '& yy'# l%%j:|%}%%j:|%}U,WX		2lm%% #!3A	 &  &&J&?&&J&?		  h@` a!
 	
    returnc                 
   UR                   nUS   n[        U5      nUR                  u    pp[        UR	                  5        Vs/ s H  oR                  S5      (       d  M  UPM     sn5      nU H  nUR                  S5      S   nUSU 3   nUSU 3   nUSU 3   nU R                  X<UUU5      u  nnU R                  X[        U5      UU5      u  nnUUR                  S   -   U
::  d   S5       eU R                  UUUUUUUU5      u  ppxUR                  S   UR                  S	   -  UR                  S
   -  n[        UR                  SS  5      n[        XUUUS9u  pM     [        R                  " XXxS.5      $ s  snf )Nsamplesr   _r   r   r      z=Conditioning frames exceed the length of the latent sequence.      strengthrL   
noise_mask)downscale_index_formular   shapesortedkeys
startswithsplitencodeget_latent_indexlenappend_keyframelistr   r   
NodeOutput)rC   r$   r&   r'   r(   r)   scale_factorslatent_imagerT   rM   latent_lengthlatent_heightlatent_widthk
image_keysimg_keyrE   imgf_idxrR   image_1t	frame_idx
latent_idxpre_filter_countguide_latent_shapes                             rF   executeLTXVAddGuideMulti.executeD   s   33i(#F+
;G;M;M81]
 
(9T(91\\(=SQ(9TU
!Gc"1%AvaS\*CA3/0E!IaS/2HC}c=YJGQ$'$8$8RUV]R^`egt$u!Iz
*m;|=||;;>;N;N	<8H  !wwqzAGGAJ6C!%aggabk!2!>xScew  CK  "LHh5 "8 }}X\1dee= Us   F !F  N
__name__
__module____qualname____firstlineno__classmethodrG   r   r`   rq   __static_attributes__rs   rI   rF   r   r      s7    *
 *
X (fR]] (f (frI   r   c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LTXVAddGuidesFromBatcho   c                 F   [         R                  " SSS[         R                  R                  S5      [         R                  R                  S5      [         R                  R                  S5      [         R
                  R                  S5      [         R                  R                  SS	S
9[         R                  R                  SSSSSSS9/[         R                  R                  SS9[         R                  R                  SS9[         R
                  R                  SS9/S9$ )Nr|   zconditioning/ltxvzAdds multiple guide images from a batch to the latent at corresponding frame indices. Non-black images in the batch are used as guides.r$   r&   r'   r(   imagesz9Batch of images - non-black images will be used as guidesr%   rR   r   r   r   r   zStrength for all guides.r   r-   r/   )	r   r>   r?   r7   r@   rA   r6   r9   rB   rC   s    rF   rG   $LTXVAddGuidesFromBatch.define_schemaq   s    yy,( b%%j1%%j1U#		)x1lmz3CTPT^xy &&J&?&&J&?		  h 7
 	
rI   rJ   c                    UR                   nUS   n[        U5      n	UR                  u    ppUR                  S   n[        U5       H  nX_US-    nUR	                  5       S:  d  M   UnU R                  X=UUU5      u  nnU R                  X[        U5      UU5      u  nnUUR                  S   -   U::  ar  U R                  UUUUU	UUU5      u  ppUR                  S   UR                  S   -  UR                  S   -  n[        UR                  SS  5      n[        XUUUS9u  pM  [        S	U S
35        M     [        R                  " XXS.5      $ )NrL   r   r   MbP?rN   rO   rP   rQ   z!Warning: Skipping guide at index z4 - conditioning frames exceed latent sequence lengthrS   )rU   r   rV   r4   r   r[   r\   r]   r^   r_   r   printr   r`   )rC   r$   r&   r'   r(   r   rR   ra   rb   rT   rM   rc   rd   re   
batch_sizerE   ri   rj   rk   rl   rm   rn   ro   rp   s                           rF   rq   LTXVAddGuidesFromBatch.execute   sy   33i(#F+
;G;M;M81] \\!_
z"A1Q3-C wwy5  ZZ=#}]
(+(<(<XVYZaVbdikx(y%	:
*m;CFCVCV  !$" %	D@H ()wwqzAGGAJ'>'K$)-aggabk):&)Fx[km  KS  *T&Hh=aS@tuv9 #< }}X\1deerI   rs   Nrt   rs   rI   rF   r|   r|   o   s6    
 
( (f2== (f (frI   r|   c                   T    \ rS rSr\S 5       r\SS\R                  4S jj5       rSr	g)LTXVAudioVideoMask   c                    [         R                  " SSS[         R                  R                  SSS9[         R                  R                  SSS9[         R                  R                  SS	S
SSS9[         R                  R                  SS
S
SSSS9[         R                  R                  SSS
SSSS9[         R                  R                  SS
S
SSSS9[         R                  R                  SSS
SSSS9[         R
                  R                  S/ SQSSS9[         R
                  R                  S/ SQSS S!S"9/	[         R                  R                  SS#9[         R                  R                  SS#9/S$9$ )%Nr   r#   zCreates noise masks for video and audio latents based on specified time ranges. New content is generated within these masked regionsvideo_latentToptionalaudio_latent	video_fps   r         Y@r   )r   r   r   r   video_start_timeg     @z)Start time in seconds for the video mask.r   video_end_timeg      @z'End time in seconds for the video mask.audio_start_timez)Start time in seconds for the audio mask.audio_end_timez'End time in seconds for the audio mask.
max_length)truncatepadpartialr   zz'truncate': cut latent to end_time length. 'pad': extend latent to end_time. 'partial': mask range within existing latent.)r+   r   r   existing_mask_mode)addsubtract	overwriter   zHow to combine with existing noise masks if present. 'add' will take the max of existing and new mask, 'overwrite' will replace with new mask. 'subtract' will set the masked region to 0 instead of 1, effectively unmasking it.)r+   r   r   r   r-   r/   )r   r>   rA   r7   r9   ComborB   r   s    rF   rG    LTXVAudioVideoMask.define_schema   s|   yy(# _		>		>{BCUQUV13CW[_  jU  V/#7Y]  hQ  R13CW[_  jU  V/#7Y]  hQ  R :& Y	   3=]hlv{  Fi  j" 		  n =		  n =)
 	
rI   NrJ   c
           
      8   Sn
SnSnSnX-  U-  nUGbM  US   R                   S   n[        [        X!-  5      5      n[        [        X1-  5      5      nUS-
  U
-  S-   nUS:X  a  UU:  a  UU-
  n[        R                  " US   R                   S	   US   R                   S   UUS   R                   S
   US   R                   S   US   R
                  US   R                  S9n[        R                  " US   U/SS9nUR                   S   nO,US:X  a!  US   S S 2S S 2S U24   nUR                   S   nOUS   nUS-
  U
-  S-   n[        R                  " S	/[        [        SUU
-   U
5      5      -   5      n[        R                  " UUSS9n[        R                  " UUSS9S-
  n[        S	U5      n[        UU5      nSU;   Ga  US   b  US:w  a  US   R                  5       nUS:X  a  UR                   S   US   R                   S   :  a  [        R                  " UR                   S	   UR                   S   UR                   S   UR                   S   -
  UR                   S
   UR                   S   UR
                  UR                  S9n[        R                  " UU/SS9nOAUS:X  a  US S 2S S 2S UR                   S   24   nO[        R                   " U5      S S 2S S24   nUS:w  a  SOSUS S 2S S 2UU24'   US:X  aI  UR                   S   US   R                   S   :  a&  US:w  a  SOSUS S 2S S 2US   R                   S   S 24'   UR#                  5       nUUS'   UUS'   U	Gb  U	S   R                   S   n[        [        XN-  5      5      n[        [        X^-  5      5      S-   nUS:X  a  UU:  a  UU-
  n[        R                  " U	S   R                   S	   U	S   R                   S   UU	S   R                   S
   U	S   R
                  U	S   R                  S9n[        R                  " U	S   U/SS9nUR                   S   nO,US:X  a!  U	S   S S 2S S 2S U24   nUR                   S   nOU	S   n[        S	U5      n[        UU5      nSU	;   a  U	S   b  US:w  a  U	S   R                  5       n US:X  a  UR                   S   U	S   R                   S   :  a  [        R                  " U R                   S	   U R                   S   UR                   S   U R                   S   -
  U R                   S
   U R
                  U R                  S9n[        R                  " U U/SS9n O8US:X  a  U S S 2S S 2S UR                   S   24   n O[        R                   " U5      n SU S S 2S S 2UU24'   US:X  aA  UR                   S   U	S   R                   S   :  a  SU S S 2S S 2U	S   R                   S   S 24'   U	R#                  5       n	UU	S'   U U	S'   [$        R&                  " X5      $ )N      i>  rP   rL   rN   r   r   r   rO   dtypedevicedimr   left)siderightrT   r   r   r   r   )rV   introundtorchzerosr   r   catnparrayr_   r4   searchsortedr   r   clone
zeros_likecopyr   r`   )!rC   r   r   r   r   r   r   r   r   r   time_scale_factormel_hop_lengthsampling_ratelatent_downsample_factoraudio_latents_per_secondvideo_latent_frame_countvideo_pixel_frame_start_rawvideo_pixel_frame_end_rawrequired_latent_frames
pad_framespaddingvideo_samplesvideo_pixel_frame_countxpvideo_latent_frame_index_startvideo_latent_frame_index_end
video_maskmask_paddingaudio_latent_frame_countaudio_latent_frame_index_startaudio_latent_frame_index_endaudio_samples
audio_masks!                                    rF   rq   LTXVAudioVideoMask.execute   s    #$ $1$BE]$] #'3I'>'D'DQ'G$*-e4D4P.Q*R'(+E.2L,M(N% '@!&CHY%Y\]%]" U"'=@X'X36NN
++ +11!4 +11!4 +11!4 +11!4&y177'	299 !&		<	+BG*LRS T+8+>+>q+A(z) ,Y 71>U?U>U8U V+8+>+>q+A( ,Y 7 (@!'CGX&X[\&\#1#U1.EHY.Y[l%m nnoB .0__RA\ci-j*+-??2?X_f+gjk+k(-04R-S*+./KMe+f( |+\0J0V[mq|[|),7==?
&=+>+>q+ALQZD[DaDabcDd+d#(;;"((+"((+%++A.1A1A!1DD"((+"((+(..)00$L "'J+E1!MJ:-!+Aq2I=3F3Fq3I2I,I!JJ"--m<QUC
cu  zD  dD\_  JMJq!;<XXXYU"}':':1'=Y@W@]@]^_@`'`M_cmMmcsv
1ai!8!>!>q!A!BBC',,.L&3L#)3L&#'3I'>'D'DQ'G$-07G7b1c-d*+.u^5^/_+`cd+d( U"'CF^'^9<TT
++ +11!4 +11!4 +11!4&y177'	299 !&		<	+BG*LRS T+8+>+>q+A(z) ,Y 71>[?[>[8[ \+8+>+>q+A( ,Y 7-04R-S*+./KMe+f( |+\0J0V[mq|[|),7==?
&=+>+>q+ALQZD[DaDabcDd+d#(;;"((+"((+%++A.1A1A!1DD"((+(..)00$L "'J+E1!MJ:-!+Aq2I=3F3Fq3I2I,I!JJ"--m<
\_Jq!;<XXXYU"}':':1'=Y@W@]@]^_@`'`FI
1ai!8!>!>q!A!BBC',,.L&3L#)3L&}}\88rI   rs   )r   r   NNrt   rs   rI   rF   r   r      sE    
 
6 L9  @B  @M  @M L9 L9rI   r   c                    U R                  U R                  U5      5      R                  UR                  5      nU R	                  U5      R                  UR                  5      nUcP  [
        R                  R                  R                  R                  XXpR                  X4S9R                  S5      nAAU$ [
        R                  R                  R                  R                  XXpR                  XSSUS9R                  S5      nAAU$ )u   Compute attention and return the result. Cleans up intermediate tensors.

When `mask` is non-None, bypass wrap_attn and call attention_pytorch directly —
sage and similar backends drop arbitrary masks.
headsattn_precisiontransformer_optionsrN   T)r   maskr   _inside_attn_wrapperr   )k_normto_ktor   to_vcomfyldmmodules	attentionoptimized_attentionr   flattenattention_pytorch)	selfquerycontextr   r   r   rf   vxs	            rF   _compute_attentionr   _  s    	DIIg&'**5;;7A		'ekk*A|II'';;Eazzjx;  c  k  k  lm  n 	
1H II''99%AZZ^b  Y]  sF9  G  O  O  PQ  R	1HrI   c           	      8    [        XX$XVS9n[        XX4U5      nXx4$ )Nr   )r   )	r   r   context_positivenag_contextr   r   r   
x_positive
x_negatives	            rF   nag_attentionr   n  s*    #D1ASfrJ#DNabJ!!rI   c                 @   U R                   (       aE  UR                  U R                  S-
  5      R                  5       R	                  XR                  S9nAO/X R                  S-
  -  nAXR                  -  R                  U5      n[        R                  " USSSS9n[        R                  " USSSS9nXT-  n[        R                  " USS9  X`R                  :  nAX@R                  -  US-   -  nAAUR                  [        R                  " XxS	5      5        AAU R                   (       a9  UR                  U5      R                  U R                  5      R	                  U5      $ UR                  U R                  5        UR	                  USU R                  -
  -  5      $ )
Nr   alphaT)pr   keepdimr   )nangHz>r   )inplacemul_	nag_scaleneg_add_sub_r   normnan_to_num_nag_tauwhere	nag_alpha)	r   r   r   nag_guidancenorm_positivenorm_guidancescaler   
adjustments	            rF   normalized_attention_guidancer  t  sV   ||!t~~'9:??AFFzYgYgFh!^^a%78"^^399,GJJzQBEMJJ|qb$GM)E	e&<<D,,.=43GHJ}ekk$C89j||  ,11$..AFFzRR$..)  q4>>/A!BCCrI   c           
         UcN  UR                  S5      nUb:  U" UR                  S   UR                  S   UR                  UR                  U5      nUR                  S   S:X  a  XpSu  pO0[        R
                  " USSS9u  py[        R
                  " USSS9u  pU R                  U R                  U5      5      nA[        XXR                  U R                  XCS9u  pAA[        XU5      nAAU	b  U
b  U R                  U R                  U	5      5      nU R                  U R                  U
5      5      nU R                  U
5      n[        R                   R"                  R$                  R'                  UUUU R(                  U R                  US9n[        R*                  " UU/SS9nOUnU R,                  b  U R-                  U5      nUR                  u  nnnUR/                  UUU R(                  U R0                  5      nS	[        R2                  " U5      -  nUUR5                  S
5      -  nUR/                  UUU R(                  U R0                  -  5      nU R7                  U5      $ )Npromptrelay_mask_fnr   r   NNrN   r   )r   r   r   r          @r   )getrV   r   r   r   chunkq_normto_qr   r   r   r  r   r   r   r   r   r   r   r   r   r   to_gate_logitsviewdim_headsigmoid	unsqueezeto_out)r   r   r   r   r   kwargsmask_providerx_poscontext_posx_negcontext_negq_posr   r   	x_pos_outq_negk_negv_neg	x_neg_outoutgate_logitsbrl   rM   gatess                            rF   ltxv_crossattn_forward_nagr#    s9   |+//0EF$ W]]1-=qwwRefD }}Q1{'{{{1aQ/#(;;wq#A  KK		%()E*4EUEUfjfyfy  Pc  oJU-d
KIJ [4DIIe,-DIIk23		+&II%%//CCE5RW_c_i_iz~  {N  {N  dwC  x	iiI.A6&))!,))1ahhq!TZZ7emmK00EOOB''hhq!TZZ$--78;;srI   c                   (    \ rS rSrSS jrSS jrSrg)LTXVCrossAttentionPatchi  c                 @    Xl         X l        X0l        X@l        XPl        g N)r   r   r   r   r   )r   r   r   r   r   r   s         rF   __init__ LTXVCrossAttentionPatch.__init__  s    """rI   Nc                 <   ^  U 4S jn[         R                  " X15      $ )Nc                    > TR                   U l         TR                  U l        TR                  U l        TR                  U l        TR                  U l        [        U /UQ70 UD6$ r'  )r   r   r   r   r   r#  self_moduleargsr  r   s      rF   wrapped_attention:LTXVCrossAttentionPatch.__get__.<locals>.wrapped_attention  sV    &*&6&6K#$(NNK!$(NNK!"&,,K"&,,K-kKDKFKKrI   types
MethodType)r   objobjtyper/  s   `   rF   __get__LTXVCrossAttentionPatch.__get__  s    	L  177rI   )r   r   r   r   r   Tr'  ru   rv   rw   rx   r(  r6  rz   rs   rI   rF   r%  r%    s    
8rI   r%  c                   T    \ rS rSr\S 5       r\SS\R                  4S jj5       rSr	g)LTX2_NAGi  c                 "   [         R                  " SSSSS[         R                  R                  S5      [         R                  R                  SSS	S
SSS9[         R                  R                  SSS	SSSS9[         R                  R                  SSS	SSSS9[         R
                  R                  SSS9[         R
                  R                  SSS9[         R                  R                  SSSSS9/[         R                  R                  SS9/S9$ )Nr;  zLTX2 NAGr#   z;https://github.com/ChenDarYen/Normalized-Attention-GuidanceTmodelr   g      &@r   r   r   z$Strength of negative guidance effectr   r   g      ?r   zMixing coefficient in that controls the balance between the normalized guided representation and the original positive representation.r   g      @r   zgClipping threshold that controls how much the guided attention can deviate from the positive attention.nag_cond_videor   nag_cond_audior   z}If true, modifies tensors in place to save memory. Leads to different numerical results which may change the output slightly.)r   r   r   r-   r0   r,   r1   r2   is_experimentalr"   r3   )r   r>   Modelr7   r9   r?   BooleanrB   r   s    rF   rG   LTX2_NAG.define_schema  s   yy##U w'{DcuSX  cI  J{DcsQV  ai  jy#3Du  _H  I%%&6%F%%&6%F

  D4  RQ   R W5
 	
rI   NrJ   c           	         US:X  a  [         R                  " U5      $ [        R                  " 5       n[        R                  " 5       n	UR
                  R                  n
U
c   UR
                  R                  R                  n
UR                  5       nUR                  S5      nUR                  nUR                  nS =nnUGb  US   S   R                  X5      n[        USS 5      nUb.  UR                  S   UUR                   -   :X  a  US S 2S S 2S U24   nUR"                  (       aX  UR$                  (       aG  UR&                  R                  U5        UR'                  U5      nUR&                  R                  U	5        [)        US5      (       aJ  UR*                  R                  U5        UR+                  U5      S   nUR*                  R                  U	5        UR-                  SSU5      n[/        UR0                  5       HK  u  nn[3        XX4US9R5                  UR6                  UR8                  5      nUR;                  SU S	3U5        MM     UGb  UR<                  Gb  US   S   R                  X5      n[        USS 5      nUb.  UR                  S   UUR                   -   :X  a  US S 2S S 2US 24   nUR"                  (       aX  UR$                  (       aG  UR<                  R                  U5        UR=                  U5      nUR<                  R                  U	5        [)        US
5      (       aJ  UR>                  R                  U5        UR?                  U5      S   nUR>                  R                  U	5        UR-                  SSU5      n[/        UR0                  5       HK  u  nn[3        UX#XGS9R5                  UR@                  UR8                  5      nUR;                  SU S3U5        MM     [         R                  " U5      $ )Nr   diffusion_modelcross_attention_dimr   video_embeddings_connectorr   )r   #diffusion_model.transformer_blocks.z.attn2.forwardaudio_embeddings_connectorz.audio_attn2.forward)!r   r`   mmget_torch_deviceunet_offload_devicer=  manual_cast_dtyperF  r   r   get_model_object	inner_dimaudio_inner_dimr   getattrrV   audio_cross_attention_dimcaption_proj_before_connectorcaption_projection_first_linearcaption_projectionhasattrrH  r  	enumeratetransformer_blocksr%  r6  attn2	__class__add_object_patchaudio_caption_projectionrJ  audio_attn2)rC   r=  r   r   r   r>  r?  r   r   offload_devicer   model_clonerF  img_dim	audio_dimcontext_videocontext_audio	vid_splitidxblockpatched_attn2patched_audio_attn2s                         rF   rq   LTX2_NAG.execute  s   >==''$$&//1--=KK//55Ekkm%667HI!++#33	(,,%*1-a033FBM1FMI$)<)<R)@IP_PyPyDy)y -aJYJ.> ?<<ApAp2255f= / B B= Q2255nE(DEE::==fE / J J= YZ[ \::==nM)..q"g>M'(J(JK
U 7R[nu v ~ ~  @E  @K  @K  MR  M\  M\  !],,/RSVRWWe-fhuv L %/*R*R*^*1-a033FBM1FMI$)<)<R)@IP_PyPyDy)y -aIJ.> ?<<ApAp88;;FC / H H W88;;NK(DEE::==fE / J J= YZ[ \::==nM)..q"i@M'(J(JK
U&=mYcj&|  'E  'E  FK  FW  FW  Y^  Yh  Yh  'i#,,/RSVRWWk-l  oB  C L }}[))rI   rs   )NNTrt   rs   rI   rF   r;  r;    s>    
 
* 6*vx  wD  wD 6* 6*rI   r;  c                 n   UR                   S   U R                  :  a  UR                   S   U R                  -  n[        U R                  5       HQ  nX2-  nX0R                  S-
  :  a  US-   U-  OUR                   S   nU R	                  US S 2XE24   5      US S 2XE24'   MS     U$ U R	                  U5      $ )Nr   )rV   dim_threshold
num_chunksr4   net)r   r   
chunk_sizerE   	start_idxend_idxs         rF   ffn_chunked_forwardrr  '  s    wwqzD&&&WWQZ4??2
t'AI.///A2E.Eq1u
*177ST:G&*hhqI4E1E/F&GAa""# ( xx{rI   c                   (    \ rS rSrSS jrSS jrSrg)LTXVffnChunkPatchi2  c                     Xl         X l        g r'  )rm  rl  )r   rm  rl  s      rF   r(  LTXVffnChunkPatch.__init__3  s    $*rI   Nc                 <   ^  U 4S jn[         R                  " X15      $ )Nc                 f   > TR                   U l         TR                  U l        [        U /UQ70 UD6$ r'  )rm  rl  rr  r,  s      rF   wrapped_forward2LTXVffnChunkPatch.__get__.<locals>.wrapped_forward8  s2    %)__K"(,(:(:K%&{DTDVDDrI   r1  r   r4  r5  ry  s   `   rF   r6  LTXVffnChunkPatch.__get__7  s    	E 55rI   )rl  rm  )   r'  r9  rs   rI   rF   rt  rt  2  s    +6rI   rt  c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LTXVChunkFeedForwardi>  c                 .   [         R                  " SSSSS[         R                  R                  S5      [         R                  R                  SSS	S
S	SS9[         R                  R                  SSSSSSS9/[         R                  R                  SS9/S9$ )Nr  zLTXV Chunk FeedForwardr#   zhEXPERIMENTAL AND MAY CHANGE THE MODEL OUTPUT!! Chunks feedforward activations to reduce peak VRAM usage.Tr=  chunksrN   r   d   zUNumber of chunks to split the feedforward activations into to reduce peak VRAM usage.r   rl  r}  r   i @     z2Dimension threshold above which to apply chunking.r-   r@  )r   r>   rB  r7   r8   rB   r   s    rF   rG   "LTXVChunkFeedForward.define_schema@  s    yy*1# C w'XqaSq  Sj  k_duSV  aU  V W5
 	
rI   rJ   c                 n   US:X  a  [         R                  " U5      $ UR                  5       nUR                  S5      n[	        UR
                  5       HJ  u  pg[        X#5      R                  UR                  UR                  5      nUR                  SU S3U5        ML     [         R                  " U5      $ )Nr   rF  rI  z.ff.forward)r   r`   r   rO  rX  rY  rt  r6  ffr[  r\  )	rC   r=  r  rl  r`  rF  rf  rg  rh  s	            rF   rq   LTXVChunkFeedForward.executeR  s    Q;==''kkm%667HI#O$F$FGJC-fDLLUXXW\WfWfgM((+NseS^)_ano H }}[))rI   rs   Nrt   rs   rI   rF   r  r  >  s3    
 
" *bmm * *rI   r  )Thread)r6   )BytesIOc                   0    \ rS rSrSS jrS rS rS rSrg)	WrappedPrevieweril  Nc                     SU l         X@l        SU l        SU l        X0l        [
        R                  " USS9R                  SS5      U l        Ub  [
        R                  " USS9U l	        g S U l	        g )NTr   cpu)r   r   )
first_previewtaeltx	last_timec_indexrater   tensor	transposelatent_rgb_factorslatent_rgb_factors_bias)r   r  r  r  r  s        rF   r(  WrappedPreviewer.__init__m  so    !	"',,/A%"P"Z"Z[\^_"`^u  _Bu||4KTY'Z$  HL$rI   c                 d   UR                   S:X  a3  UR                  SS5      nUR                  SUR                  SS  -   5      nUR	                  S5      n[
        R
                  " 5       n[        X@R                  -
  U R                  -  5      nU R                  XPR                  -  -   U l        XS:  a  UnOUS::  a  g U R                  (       aO  SU l	        [        R                  SX0R                  [        R                  S	.5        USU R                  -  -   U l        U R                  U-   U:  a!  UR                  U R                  * S5      S U nOX R                  U R                  U-    n[        U R                   X R                  U4S
9R#                  5         U R                  U-   U-  U l        g )N   rN   r   )r   r   FVHS_latentpreview)lengthr  id)targetr.  )ndimmovedimreshaperV   sizetimer   r  r  r  serv	send_synclast_node_idr  rollr  process_previewsrun)r   preview_formatx0
num_imagesnew_timenum_previewss         rF   decode_latent_to_preview_image/WrappedPreviewer.decode_latent_to_preview_imagev  s_   77a<AaBE"((23-/0BWWQZ
99;H~~5BC,yy*@@$%LQ!&DNN.*ii_c_p_p0qr%$))3DN<<,&3$,,*=L9BLL!<=Bd++2||3=3? 	@@C|3zArI   c           	         SnSnU R                  U5      nUR                  S5      U:  d  UR                  S5      U:  a6  [        R                  " UR	                  SS5      SSS	9R	                  SS5      nUR                  S5      U:  d  UR                  S5      U:  a  UR	                  SS5      nUR                  S5      UR                  S
5      :  a=  XAR                  S5      -  UR                  S
5      -  n[        R                  " XU4SS9nO<XAR                  S
5      -  UR                  S5      -  n[        R                  " XU4SS9nUR	                  SS5      nUR                  SS5      R                  S5      R                  S[        R                  S9nU GHE  n	[        R                  " U	R                  5       5      n
[        5       nUR                  SR                  SSS9S-  5        UR                  UR                  SSS95        UR                  [         R"                  " S[$        R&                  =(       d    SR)                  S5      5      5        U
R+                  USSSS9  [$        R-                  [.        R0                  R2                  UR5                  5       [$        R6                  5        U R8                  b  US-   US-
  S-  S-   -  nGM=  US-   U-  nGMH     g )Ni   r  r   rN   r   r   rP   nearest)scale_factormoderO   )r     r  r   r   big)r  	byteorder16p asciiJPEG_   )formatqualitycompress_levelr   )decode_latent_to_previewr  Finterpolater  clampmulr   r   uint8r6   	fromarraynumpyr  writeto_bytesstructpackr  r  r[   saver  serverBinaryEventTypesPREVIEW_IMAGEgetvalue	client_idr  )r   image_tensorindlengmax_sizemin_sizeheightwidthpreviews_ubytepreviewrE   messages               rF   r  !WrappedPreviewer.process_previews  s}   44\BQ(*l.?.?.BX.M==)=)=b)CRSZcdllmnoqrLQ(*l.?.?.BX.M'//15L  #l&7&7&::"%6%6q%99l>O>OPQ>RR }}\8;LS\]!$5$5a$88\=N=Nq=QQ }}\e;LS\]'//"5L&,,Q266t<@@UZU`U`@a &G0AiGMM1,,a5,A!CDMM#,,a5,ABMM&++ed.?.?.E2-M-Mg-VWXFF762aFHNN622@@"++-t~~?{{&QwDFa<!#34Qw$& &rI   c                    U R                   b  UR                  S5      R                  U R                   R                  R                  S   R
                  R                  [        S9nU R                   R                  R                  U5      S   R                  SSSS5      nU$ U R                  R                  UR                  UR                  S9U l
        U R                  b3  U R                  R                  UR                  UR                  S9U l        [        R                  " UR                  SS5      U R                  U R                  S9n[        R                   " U5      nU$ )Nr   r   r   rN   rO   r   )bias)r  r  r   first_stage_modeldecoderweightr   r   decodepermuter  r  r  linearr  r   r  )r   r  x_samplerb   s       rF   r  )WrappedPreviewer.decode_latent_to_preview  s'   ;;"a##$++*G*G*O*OPQ*R*Y*Y*_*_hn#oB{{44;;B?BJJ1aQRTUVHO&*&=&=&@&@rxxXZXaXa&@&bD#++7/3/K/K/N/NUWU]U]fhfofo/N/p,88BJJq"$5t7N7N)-)E)EGL !==6LrI   )r  r  r  r  r  r  r  )r   N)	ru   rv   rw   rx   r(  r  r  r  rz   rs   rI   rF   r  r  l  s    L2"'H rI   r  c                     [        U 5      $ )zReturn (latent_rgb_factors, latent_rgb_factors_bias) for LTX (is_23=False) or
LTX2/LTXAV (is_23=True). Exposed so other nodes can build a WrappedPreviewer with
the right LTX factors without duplicating the 128-row tables.)_get_ltx_rgb_factors_impl)is_23s    rF   get_ltx_rgb_factorsr    s     %U++rI   c                 <   U (       Gd	  / / SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ S	QP/ S
QP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ S QP/ S!QP/ S"QP/ S#QP/ S$QP/ S%QP/ S&QP/ S'QP/ S(QP/ S)QP/ S*QP/ S+QP/ S,QP/ S-QP/ S.QP/ S/QP/ S0QP/ S1QP/ S2QP/ S3QP/ S4QP/ S5QP/ S6QP/ S7QP/ S8QP/ S9QP/ S:QP/ S;QP/ S<QP/ S=QP/ S>QP/ S?QP/ S@QP/ SAQP/ SBQP/ SCQP/ SDQP/ SEQP/ SFQP/ SGQP/ SHQP/ SIQP/ SJQP/ SKQP/ SLQP/ SMQP/ SNQP/ SOQP/ SPQP/ SQQP/ SRQP/ SSQP/ STQP/ SUQP/ SVQP/ SWQP/ SXQP/ SYQP/ SZQP/ S[QP/ S\QP/ S]QP/ S^QP/ S_QP/ S`QP/ SaQP/ SbQP/ ScQP/ SdQP/ SeQP/ SfQP/ SgQP/ ShQP/ SiQP/ SjQP/ SkQP/ SlQP/ SmQP/ SnQP/ SoQP/ SpQP/ SqQP/ SrQP/ SsQP/ StQP/ SuQP/ SvQP/ SwQP/ SxQP/ SyQP/ SzQP/ S{QP/ S|QP/ S}QP/ S~QP/ SQP/ SQPn/ SQnX4$ / / SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ SQP/ GS QP/ GSQPn/ GSQnX4$ (  N)gQ?gH?gF%u?)g{Gzd?gJ4aa2U0*3)gQI?Ǻf?gMb`?)gy&1?g+eXgMO)Έ<,Ԛ}?HPsׂ?)_vOf?gj+݃?gMSt$)glxz,C|-C6jga2U0*c)gaTR'glgڊe)g5;Nё?g	^)ː?ggs?)j+݃gHPx?gy&1l?)g9vgJ4qga2U0*S)gZӼㄿ-C6*g&S){Gzd o_y??ܵ|?)ga2U0*3?g_vOn?g_vO?)ŏ1w-!?	^)ˀ?gQI?)g~:?Dioɔ?gq?)gHPsׂg$ۗgtF_)g#~jgMSt$gHPsb)gUN@?gjMS?g/n?)?"uqg~jt)g=U?gR!u?g䃞ͪϕ?)ga+ei?gHPsrH}])g+e⨿g@߾g%u)ZӼ}g_Qڛg(0)gMb?g46<R?gj+?)g(?g/n?)gn~jtgAǘ)gUN@?r   J4a?)g??r  gL7A`?)g9vzg 	g'酿)gd`T?+eX?ŏ1w-!o?)r  ~jth_Le)r  gŏ1w-!?g)<,Ԋg0*xg rh)I+v?gF%u{?ŏ1w-!o) 	?r  g;On?)gΈ?gQ?UN@?)gSt$gׁg'饿)g@ǈ?~jtX?K=U)0*?g{Gzt?gŏ1w-!_?)g?gݓ?g&?)gHPsr?g9v?gǘ?)g6?皙?r  )g_vOv燧W2ġg{򰠿)gw-!l<,Ԛg+eX)gjM󎓿g-C6geX)gˡEgwJ4q?)gMO?gJY8?geX?)g@ǘrg<,Ԛ捿)r  gͪՖ?gJ4?)r  gMbgu)gŏ1w-!_g"uq?-C6J?)g~jtxg]K=gHP)ga2U0*sg
ףp=
go_)gz6>W?]K=?r  )gAǘ?go_?r  )r  ga+eyg_LU?)g=U給y):gvq-)Mbp?ǘr  )g-C6zr  /nR)gZӼ㔿g&gŏ1w-!)g/nR?r  gV-?)gB>٬gݓg9v)r  r  r  )gTN?ggs?gD?)r  g{?r  )r  r	  gd`T)r  gQIg%u)r  瀷@ǘ?g8d`?)g{Pkg~jt?r  )g 	gH}m?g-C6?)r  gH}Mr  )gS!uq{r  gZd;O)/nbgr  )r  gqor  )Mb@g
F%ur  )gQIg2U0*gL7A`吿)r  g	cgS㥛)r  g
F%u?gy):?)r  r  gQ?)ggs_Q{g-C6j?)g'长r  r  )g_Lg]K=g:pΈ)r  ga2U0*s?r  )gDl?gJ4?g0*?)g)\(gI++)r  gݵ|г?gǺ?)g-C6Z?g_vOnUN@)g=U?gׁsF?r  )r  r  g!u?)ga2U0*S?_LugJ4)g+eXw?gǺfg{򰐿)gHPgCl竿gȘ)gŏ1w-!r  gǺV)gV}bgZӼg'Wʢ)g_Qڋ?	^)p?gS㥋?)gǺF?g_Qk?ggsu)r  g46<Rgg?)ga2U0*C?g0*Dr   )g2%䃎r  r  )g/$gqo?g_vO?)r  g{Gz?g;On?)gz6>W[r  r  )g_L?g?W[?g~jt?)r  gy):?UN@s?)g2U0*?r  r  )g o_Ή?gZӼ?gMbp)r  r  g?߾?)g=yX?g37?g46<R?)g-C6gF%u{gŏ1w-!)g7[ Ag5;Nсr  )r!  gMb?g\ Ac̝?)gvq-?r  g9v?)g@ǘgvOjMgd`T)gUN@ga+ey?gy&1l)r  r"  gSt$)g;Onr?gntggsu?)g"u?gZӼ}?r  )r  gqga2U0*)gAfgF%ugS㥋)g_vO~g_Q[?gׁsF?)r
  gy&1|?gǺV?)g_LUg_Q[r  )g~jth?/$?r$  )r  gJ4gMb)g_vOr
  r#  )g2%䃞r  g46<R)g0*gݓZgPkw)gۊe?g:#J{/?gUN@s)g   `Jb?g   g    g,)g    ]lg   䅉g   ʆ)g   y?g   @"?g   {	?)g   `:?g   @	E?g   E?)g   `tg   ^g   ?)g   `w?g   `?g   ఛ?)g   `g    ෿g   @٤)g   `Wg    u.1?g   Y(?)g   T?g    ҋ?g   b?)g    W?g   m?g   `:w)g   `~g   ,ψg    mȐ)g    ?)?g   :?g   `F}?)g    ֹ?g    ɹ?g   @aܩ?)g    ?g   ?g    $w?)g   %g    Ng   &V)g    {ڛg   Ǚg   -)g   @?g   ?g   F?)g   ߭g   }g   z)g   Q?g   o?g   ؂?)g    LW?g   @֊?g    ͆?)g   Hi?g   ݋?g   ^+)g   Ӌg   Lg   ҂)g   @k?g   ?g   @[?)g   @+g   `g   @]?)g   `?g   @?g    wX?)g   e{~g   H?g   9?)g   ig   g   w)g    Esg   -?g   ?)g   W?g   ?g    HҊ?)g    b?g   `XZSg   PE3?)g   =x<g   `<<?g   `?)g   dg   ?g   @M)g    )ig    Ƒg    Ҁ)g   ˆg   g   `vh)g   =[?g   ?g   k?)g   tg    upg    )g   çhg   Fg   `~)g   `b?g    5n?g   `ᗒ?)g   @?g   Ҡ?g   8u?)g   ۉ?g    V?g   )g   )og   @xg    )g   g    ug    +K)g    lg    ȃ?g   `Ր?)g   @'g   ~d?g   ߃)g    ng   Mg   w)g   ?g   z?g   m?)g   b?g   $?g   o?)g   U)g    #g   G)g   %g    }'ag   G8?)g   yg   "r{g    ~)g   t?g    M?g    v?)g    ?g   ?g    S?)g    $g   `g    4Ր)g   kg   No?g   `)d)g   9]g   O?g   )g   4g    Sg   ?)g   @<6?g   ?g   ̐?)g   ؾ?g   @?g   @-?)g   1?g   m?g   Yc?)g    }g    !z?g    Sj?)g    lF~?g   %~g     )g   ?g   `?g    b?)g   Wg   ng   `q)g   p!?g   ?g   ]?)g    g   -g   _?)g   3g    ⮿g   o)g   <g   `otg    է)g   `ig   ?'g   )g    @vg    U<g    	)g   ?g    R?g   @Ü?)g   @?g    2t?g   4lz)g   3?g    Sb?g    V?)g   9g   `g    綿)g   iʎ?g   ?g    AR?)g   ҋg    ?g    S۔?)g   `#ƿg   @! ſg   @Zſ)g    M+?g   @qX?g   [)g   @Jw?g   o?g   @>?)g   ލ?g   g   `S)g   `?g   m7?g   ௦?)g   ?g    ʦ?g    cʖ?)g    θ?g   @V?g    ר?)g    ?g   ?g    ?)g   g   `g   9)g    g   {g    d)g   kه?g   f?g   `n?)g    @Ӻ?g   @5?g    Ȱ?)g    ?g   hg   )g    !?g   !b?g   )c?)g   |g   Ng   ȕ)g   `-e?g   @y?g    @Y?)g   A?g   ug    唿)g    ?g   ?g   Q?)g   @e?g   `&~g   @p)g    ?g   ?g   E~)g    5Ig    >g   A)g    u?g   @?g   ?)g   1Sg   @}Ag   )g   ?g    !ʟ?g   ?)g   MPrg   w?g    u?)g   @tg   ѐg   `ylQ)g    ?g   e}?g   w)g   %?g   @vg    s?)g   @wS?g   ?g    ?)g    nug   kg   `iNP)g    母?g   m?g   ``?)g    6ug    g    L)g   @vg   ?g   ߀)g    zbg   Qg    婿)g    "?g   ~?g    ?)g    Ʋ^?g   %g   )g   @Yߔg   @Lug   @)g   g   N(g   䀯)g    Ug   !?g    \?)g   ̀?g   @y%q?g   f?)g    E4?g   ?g   <?)g    ,dÿg    
ÿg   `	V¿)g   .v?g    ߄?g    gr)g   Stg   xvg   `x)g   n?g   s?g   SUl?)g   `a.?g   ӹ?g    |?)g   Pr?g   K'?g   @?)g   j{g    f^g    LD)g   >wg    g   @)g   𺡿g   ͙g   񼑿)g   @Sg   og   8)g   `ܑg   g   )g   @ѣg    tg   q)g   `Cg   Hg    ʲrs   )r  r  r  s      rF   r  r    s   A+A+A ,A ,	A
 ,A ,A ,A ,A ,A ,A ,A ,A ,A ,A ,A  ,!A" ,#A$ ,%A& ,'A( ,)A* ,+A, ,-A. ,/A0 ,1A2 ,3A4 ,5A6 ,7A8 ,9A: ,;A< ,=A> ,?A@ ,AAB ,CAD ,EAF ,GAH ,IAJ ,KAL ,MAN ,OAP ,QAR ,SAT ,UAV ,WAX ,YAZ ,[A\ ,]A^ ,_A` ,aAb ,cAd ,eAf ,gAh ,iAj ,kAl ,mAn ,oAp ,qAr ,sAt ,uAv ,wAx ,yAz ,{A| ,}A~ ,A@ ,AAB ,CAD ,EAF ,GAH ,IAJ ,KAL ,MAN ,OAP ,QAR ,SAT ,UAV ,WAX ,YAZ ,[A\ ,]A^ ,_A` ,aAb ,cAd ,eAf ,gAh ,iAj ,kAl ,mAn ,oAp ,qAr ,sAt ,uAv ,wAx ,yAz ,{A| ,}A~ ,A@ ,AAB ,CAD ,EAF ,GAH ,IAJ ,KAL ,MAN ,OAP ,QAR ,SAT ,UAV ,WAX ,YAZ ,[A\ ,]A^ ,_A` ,aAb ,cAd ,eAf ,gAh ,iAj ,kAl ,mAn ,oAp ,qAr ,sAt ,uAv ,wAx ,yAz ,{A| ,}A~ ,A@ ,AAD #< 66 @Ia  @I  dh  @I  jl  @I  nn  @I  ps  @I  ut  @I  vx  @I  z{  @I  }~	  @I  @
@  @I  BG  @I  IK  @I  MK  @I  MN  @I  PT  @I  VZ  @I  \\  @I  ^a  @I  ce  @I  gh  @I  jl  @I  nr  @I  tt  @I  vy  @I  {y  @I  {}  @I  A  @I  CE  @I  GH  @I  JP   @I  R V!  @I  X!]"  @I  _"d#  @I  f#i$  @I  k$l%  @I  n%r&  @I  t&x'  @I  z'|(  @I  ~(~)  @I  @*D+  @I  F+K,  @I  M,Q-  @I  S-U.  @I  W.[/  @I  ]/`0  @I  b0_1  @I  a1c2  @I  e2h3  @I  j3o4  @I  q4x5  @I  z5|6  @I  ~6}7  @I  7D9  @I  F9K:  @I  M:R;  @I  T;W<  @I  Y<Y=  @I  [=Y>  @I  [>^?  @I  `?c@  @I  e@iA  @I  kAlB  @I  nBsC  @I  uCuD  @I  wDyE  @I  {E}F  @I  FBH  @I  DHHI  @I  JIPJ  @I  RJTK  @I  VKYL  @I  [L]M  @I  _MbN  @I  dNeO  @I  gOjP  @I  lPkQ  @I  mQnR  @I  pRrS  @I  tSxT  @I  zTzU  @I  |U|V  @I  ~V}W  @I  WAY  @I  CYGZ  @I  IZN[  @I  P[R\  @I  T\S]  @I  U]Y^  @I  [^]_  @I  __b`  @I  d`ga  @I  iamb  @I  oboc  @I  qcud  @I  wdxe  @I  ze|f  @I  ~f~g  @I  @hCi  @I  EiFj  @I  HjJk  @I  LkQl  @I  SlUm  @I  Wm[n  @I  ]n[o  @I  ]ocp  @I  epeq  @I  gqmr  @I  orrs  @I  tsvt  @I  xtwu  @I  yu{v  @I  }vAx  @I  CxFy  @I  HyKz  @I  MzN{  @I  P{O|  @I  Q|R}  @I  T}U~  @I  W~[  @I  ]`@  @I  b@`A  @I  bAaB  @I  cBgC  @I  iClD  @I  nDrE  @I  tEwF  @I  yF{G  @I  }GH  @I"a66rI   Fc
                    ^^^^^^^ [        U	5      u  pSmTS;  a  Sm[        XXg(       a  TOS S9m[        R                  R	                  U5      mUUUUUUU4S jnU$ )Nr  )r  PNG)r  r  c                 &  > UbV  TbS  [         R                  " TSS  5      nUS S 2S S 2S U24   R                  UR                  S   /[	        T5      SS  -   5      nTS:  a  US S 2S S 2S T* 24   nTbo  TR
                  R                  R                  U5      nT" UR                  [        R                  5      5      nTR
                  R                  R                  U5      nS nT
(       a  T
R                  T	U5      nTR                  U S-   X55        g )Nr   r   )mathprodr  rV   r_   r  per_channel_statisticsun_normalizer   r   bfloat16	normalizer  update_absolute)r   r  r   total_stepscutpreview_byteslatent_upscale_modelnum_keyframespbarr  	previewerrV   r'   s         rF   callback"prepare_callback.<locals>.callbacka  s    >e/))E!"I&CAq$3$J''!UAB(GHB1Aq/M>/)*B+&&==JJ2NB&ruuU^^'<=B&&==GGKB%DD^UWXMTAX{BrI   )r  r  r   utilsProgressBar)r=  stepsx0_output_dictrV   r2  r'   r  r  r3  r  r  r  r6  r4  r  r5  s      ```  `    @@@rF   prepare_callbackr<  X  sa    2Ee2L/N_, !3SWgm`cswxI;;""5)DC C" OrI   c                   $    \ rS rSrSS jrS rSrg)OuterSampleCallbackWrapperit  Nc                 B    Xl         X l        X0l        X@l        0 U l        g r'  )r2  r'   preview_rater  	x0_output)r   r2  r'   r@  r  s        rF   r(  #OuterSampleCallbackWrapper.__init__u  s    $8!(rI   c                   ^^ UR                   nUR                  R                  R                  nUR                  (       + nUmU R
                  b  U R
                  R                  [        5        U R                  b:  U R                  (       a)  U R                  R                  R                  [        5        SnSUR                  ;   ah  [        UR                  S   5      S:  aL  UR                  S   S   R                  S5      nUb(  [        [        R                  " USSS S 2S4   5      5      n[!        UR                  [        U5      S-
  [        U
5      S:  a  U
S   OS U R"                  U R
                  U R                  U R$                  U R                  XS9
mUU4S jnU" X#XEUUXU
S9	nU R
                  b.  U R
                  R                  [&        R(                  " 5       5        U$ )Nr   r$   keyframe_idxsr   )rV   r;  r2  r'   r  r  r3  r  c                 2   > T" XX#5        Tb
  T" XX#5        g g r'  rs   )r   r  r   r/  new_callbackoriginal_callbacks       rF   combined_callback>OuterSampleCallbackWrapper.__call__.<locals>.combined_callback  s"    12 ,!$A; -rI   latent_shapes)	class_objmodel_patcherr=  rF  rU  r2  r   r   r'   r  r  condsr]   r	  r   uniquer<  rA  r@  rK  rM  )r   executornoiserb   samplersigmasdenoise_maskr6  disable_pbarseedrK  guiderrF  r  r3  rD  rH  r  rF  rG  s                     @@rF   __call__#OuterSampleCallbackWrapper.__call__|  s   ## ..44DD#CCC$$$0%%((088DKKHH&&))&1%#fll:.F*G!*K"LL4Q7;;OLM( #ELLq!Qz1J$K L'(<(<c&k1nhklyhz}~h~TabcTd  EI7;~~\`\u\u{  |D  |D  KO  K\  K\  ei  ep  ep  @M[	< uG\K\^j  AN  O$$0%%(()?)?)AB
rI   )r2  r@  r  r'   rA  )NNr   Fru   rv   rw   rx   r(  rX  rz   rs   rI   rF   r>  r>  t  s    rI   r>  c                   p    \ rS rSr\S\R                  4S j5       r\SS\R                  4S jj5       r	Sr
g)LTX2SamplingPreviewOverridei  rJ   c                 b   [         R                  " SSSSS[         R                  R                  SSS9[         R                  R                  S	S
SSSSS9[         R
                  R                  SSSS9[         R                  R                  SSSS9/[         R                  R                  SS9/S9$ )Nr\  zLTX2 Sampling Preview OverridezhOverrides the LTX2 preview sampling preview function, temporary measure until previews are in comfy corer#   Tr=  %The model to add preview override to.r%   r@  r   r   <   zPreview frame rate.r   r2  z=Optional upscale model to use for higher resolution previews.r   r   r'   z?VAE model to use normalizing the latents for the upscale model.z)The model with Sampling Preview Override.r0   r,   r2   r1   rA  r"   r3   )r   r>   rB  r7   r8   LatentUpscaleModelr@   rB   r   s    rF   rG   )LTX2SamplingPreviewOverride.define_schema  s    yy19 C# w0WX^QA2AWlm%%++,BT  \[+  \UT;|}	 (ST
 	
rI   Nc           
      &   UR                  5       nSnUb(  UR                  R                  R                  S:X  a  SnS nUR	                  [
        R                  R                  R                  S[        X4X%5      5        [        R                  " U5      $ )NFTAEHVTsampling_preview)r   r  r[  ru   add_wrapper_with_keyr   patcher_extension
WrappersMPOUTER_SAMPLEr>  r   r`   )rC   r=  r@  r2  r'   r  s         rF   rq   #LTX2SamplingPreviewOverride.execute  s    ?$$..777B%)$""5#:#:#E#E#R#RTf  iC  DX  _k  it  	u}}U##rI   rs   r  ru   rv   rw   rx   ry   r   r>   rG   r`   rq   rz   rs   rI   rF   r\  r\    sA    
bii 
 
$ $RTR_R_ $ $rI   r\  c                        \ rS rSrS rS rSrg)$OuterSampleAudioNormalizationWrapperi  c                     Xl         g r'  )audio_normalization_factors)r   rp  s     rF   r(  -OuterSampleAudioNormalizationWrapper.__init__  s    +F(rI   c                   ^ ^^^ UR                   nUR                  R                  R                  n0 mUR                  S   S-
  T l        [        R                  R                  T R
                  5      mST l	        [        R                  " UR                  R                  UR                  R                  R                  5      mUUU U4S jnUnT R                  R                  5       R!                  S5      nU Vs/ s H  n[#        U5      PM     nnT R
                  n[%        U5      U:  a3  [%        U5      S:  a$  UR'                  US   /U[%        U5      -
  -  5        [)        U5       VVs/ s H  u  nnUS:w  d  M  US-   PM     nnnS nU" UU5      nSnU GH4  nU[%        U5      S-
  -  nU" X#UUXgXU
S9	nS	T;   a(  UR                  R                  R+                  TS	   5      nUS-
  [%        U5      :  d  Mb  UR-                  [        R                  R/                  X:5      S 5      u  nnUbJ  UR-                  [        R                  R/                  Xj5      S 5      S   nUU-  UUS-
     -  USU-
  -  -   nOUUUS-
     -  n[        R                  R1                  UR3                  UU5      5      S   n[5        S
UUUS-
     4-  5        GM7     U$ s  snf s  snnf )Nr   r   r   c                    > Tb  UTS'   S nT(       a  TR                  SU5      nT=R                  S-  sl        TR                  TR                  TR                  U5        g )Nr  r  r   )r  	full_stepr.  r/  )	r   r  r   r/  r1  r4  r5  r   rA  s	        rF   custom_callbackFOuterSampleAudioNormalizationWrapper.__call__.<locals>.custom_callback  sW    $"$	$ M ) H HQS TNNaN  1A1A=QrI   ,r   c                     U(       d  U /$ [        [        U5      5      n/ nSnU H   nXE:  a  UR                  XUS-    5        UnM"     U[        U 5      :  a  UR                  XS 5        U$ )z
Splits arr into chunks according to indices (split points).
Indices are treated as starting a new chunk at each index in the list.
r   r   N)rW   setr:   r]   )arrindicessplit_pointsr  prevrf  s         rF   split_by_indicesGOuterSampleAudioNormalizationWrapper.__call__.<locals>.split_by_indices  sq    
 u!#g,/LFD#:MM#S1W"56 $ c#hc%j)MrI   rJ  r  z5After %d steps, the audio latent was normalized by %f)rL  rM  r=  rF  rV   r/  r   r8  r9  rt  latent_previewget_previewerload_devicelatent_formatrp  striprZ   floatr]   r5   rX  process_latent_out separate_audio_and_video_latentsunpack_latentspack_latents!recombine_audio_and_video_latentsr   )r   rP  rQ  rb   rR  rS  rT  r6  rU  rV  rK  rW  ltxavru  rp  factor
sigmas_lenrE   asampling_split_indicesr~  sigmas_chunkssigmas_chunkvxaxr   r4  r5  rA  s   `                         @@@rF   rX  -OuterSampleAudioNormalizationWrapper.__call__  s   ##$$**::	!<<+a/{{&&t'7'78"001E1E1Q1QSYSgSgSmSmS{S{|		R 	R #&*&F&F&L&L&N&T&TUX&Y#C^&_C^uV}C^#&_ %%
*+j8SA\=]`a=a'..0KB0O/PT^ad  fA  bB  UB  0C  D 5>>Y4Z!g4ZDAq^_cf^f%!a%4Z!g	$ )1GH)L\"Q&&A#E,P\ht  KX  YLy %3399LLYW[_]1us677??@Z@Z[g@wy}~B+!&!G!GHbHbcoH  BF  "G  HI  "JJj+Fq1u+MMPRVWZdVdPeeB9!a%@@B${{778_8_`bdf8ghijkMQRTopqtupuTvPwwx! *$ e '` "hs   ;K.K?	K)rp  rt  r/  NrZ  rs   rI   rF   rn  rn    s    GIrI   rn  c                   l    \ rS rSr\S\R                  4S j5       r\S\R                  4S j5       r	Sr
g)"LTX2AudioLatentNormalizingSamplingi
  rJ   c                     [         R                  " SSSSS[         R                  R                  SSS9[         R                  R                  S	S
SS9/[         R                  R                  SS9/S9$ )Nr  z&LTX2 Audio Latent Normalizing Samplingz_Improves LTX2 generated audio quality by normalizing audio latents at specified sampling steps.r#   Tr=  r^  r%   rp  z1,1,0.25,1,1,0.25,1,1zComma-separated list of audio normalization factors to apply at each sampling step. For example, '1,1,0.25,1,1,0.25,1,1' will apply a factor of 0.25 at the 3rd and 6th steps.r   r   z1The model with Audio Latent Normalizing Sampling.ra  )r   r>   rB  r7   StringrB   r   s    rF   rG   0LTX2AudioLatentNormalizingSampling.define_schema  sv    yy8Ay# w0WX		 =G^  iY  Z
 ([\
 	
rI   c                     UR                  5       nUR                  [        R                  R                  R
                  S[        U5      5        [        R                  " U5      $ )Nltx2_audio_normalization)	r   rg  r   rh  ri  rj  rn  r   r`   )rC   r=  rp  s      rF   rq   *LTX2AudioLatentNormalizingSampling.execute  sW    ""5#:#:#E#E#R#RTn  qU  Vq  qr  	s}}U##rI   rs   Nrl  rs   rI   rF   r  r  
  s>    
bii 
 
  $BMM $ $rI   r  c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LTXVImgToVideoInplaceKJi#  c                    / n[        SS5       H  n/ n[        SUS-   5       H  nUR                  [        R                  R	                  SU 3SSU S3S9[        R
                  R	                  SU 3S	S
SSSU S3SS9[        R                  R	                  SU 3SSSSSU S3S9/5        M     UR                  [        R                  R                  [        U5      US95        M     [        R                  " SSS[        R                  R	                  SSS9[        R                  R	                  SSS9[        R                  R	                  SUS S!S"9/[        R                  R                  SS#S$9/S%9$ )&Nr   r   r   TzImage z! to insert into the video latent.r`  index_r   r   r   zFrame index for image z (in pixel space).)r   r   r   r   r   r   r   r   r   r   zStrength for image r   r   r    r  r#   zReplaces video latent frames with the encoded input images, uses DynamicCombo which requires ComfyUI 0.8.1 and frontend 1.33.4 or later.r'   z#Video VAE used to encode the imagesr%   r(   z"Video latent to insert images intor  zNumber of Imagesz Select how many images to insertr*   zHThe video latent with the images inserted and latent noise mask updated.r.   r/   )r4   r5   r   r6   r7   r8   r9   r:   r;   r<   r=   r>   r@   rA   rB   )rC   r+   r  image_inputsrE   s        rF   rG   %LTXVImgToVideoInplaceKJ.define_schema$  s   2,JL1j1n-##HHNNVA3<$&QRPSStHuNvFFLL  !! "8;M N!% !  HHNNYqc?CScX\h{|}{~~  gAN  B%  . NN2??11
O# 2  ! '* yy-# cU,QR		2VW%% #!3>	 & 	 		  h  AK   L
 	
rI   rJ   c           	         US   R                  5       nUR                  nUu  pgnUR                  u  ppnX-  nX-  nSU;   a  US   R                  5       nO3[        R                  " U	SU
SS4[        R
                  UR                  S9n[        UR                  5        Vs/ s H  nUR                  S5      (       d  M  UPM     sn5      nU GHV  nUR                  S5      S   nUSU 3   nUc  M%  UR                  SU 35      nUc  M>  USU 3   nUR                  S   U:w  d  UR                  S	   U:w  aC  [        R                  R                  UR                  S
S5      XSS5      R                  SS
5      nOUnUS S 2S S 2S S 2S S24   nUR!                  U5      nUS   nU
S-
  U-  S-   nUS:  a  UU-   nUU-  n[#        S[%        UU
S-
  5      5      n[%        UUR                  S	   -   U
5      nUS S 2S S 2S UU-
  24   US S 2S S 2UU24'   SU-
  US S 2S S 2UU24'   GMY     [&        R(                  " XOS.5      $ s  snf )NrL   rT   r   r   r   rM   r  r   rN   r   bilinearcenterrO   r   r   rS   )r   rU   rV   r   onesfloat32r   rW   rX   rY   rZ   r	  r   r8  common_upscaler  r[   r   r   r   r`   )rC   r'   r(   r  rL   ra   rM   height_scale_factorwidth_scale_factorbatchlatent_framesrd   re   r  r  conditioning_latent_frames_maskrf   rg   rh   rE   imageindexrR   pixelsencode_pixelsrl   r   pixel_frame_countrn   	end_indexs                                 rF   rq   LTXVImgToVideoInplaceKJ.executeO  s    #))+335B2 2?F}}<-14 6!.4\.B.H.H.J+.3jj=!Q/mm~~/+ 
(9T(91Q\\(=SQ(9TU
!Gc"1%As|,E}NNVA3<0E}!IaS/2H{{1~'5;;q>U+B33EMM"a4H%Ycemnvvwxz|}"1aBQB;/M

=)A !.a 0 "/!26G G! Kqy)E1 "33J QJ0A BCJ J3]CI 34Aq:Q9z;Q:Q4Q2RGAq*Y../ KNPX.+Aq*Y2F,FGO "R }}`aaW Us   !I?Irs   Nrt   rs   rI   rF   r  r  #  s:    (
 (
T Bb Bb BbrI   r  TepsBLOCK_NIS_BF16HAS_BROADCASTc
                 \   [         R                  " S5      n
[         R                  " SU5      nX:  nX-  U-   n[         R                  " X-   USS9R	                  [         R
                  5      n[         R                  " X-  SS9U-  n[         R                  R                  X-   5      nUU-  nU	(       a  X-  nUU-  U-   nOUn[         R                  " UU-   USS9R	                  [         R
                  5      n[         R                  " UU-   USS9R	                  [         R
                  5      nUSU-   -  U-   nU(       a6  [         R                  " X-   UR	                  [         R                  5      US9  g [         R                  " X-   UR	                  [         R                  5      US9  g )Nr   r   r   otheraxisr   r   tl
program_idarangeloadr   r  sumr(  rsqrtstorer,  float16)X_ptrOut_ptr	Scale_ptr	Shift_ptrNscale_n_rowsr  r  r  r  rowcolsr   offr   mean_sqrrmsx_norm	scale_row	scale_offr  shiftr  s                          rF   _rms_norm_scale_shift_kernelr    sK    mmAyyG$xgnGGEKd#699"**E&&Q'!+ww}}W]+T *I!A,II	I-DDGG

S	I-DDGG

Se$u,HHW]CFF2;;$7dCHHW]CFF2::$6TBrI   BLOCK_HDc
                    [         R                  " S5      n
X-  nX-  U-  nX-  nUS-  n[         R                  " SU	5      nX:  nX-  U-   XF-  -  X-  -   nX-  U-  X-  -   U-   U-  n[         R                  " U U-   U-   USS9R	                  [         R
                  5      n[         R                  " U U-   U-   U-   USS9R	                  [         R
                  5      n[         R                  " UU-   U-   USS9R	                  [         R
                  5      n[         R                  " UU-   U-   U-   USS9R	                  [         R
                  5      n[         R                  " UU-   U-   USS9R	                  [         R
                  5      n[         R                  " UU-   U-   USS9R	                  [         R
                  5      nUU-  UU-  -
  nUU-  UU-  -   nUU-  UU-  -
  nUU-  UU-  -   nU(       a  [         R                  " U U-   U-   UR	                  [         R                  5      US9  [         R                  " U U-   U-   U-   UR	                  [         R                  5      US9  [         R                  " UU-   U-   UR	                  [         R                  5      US9  [         R                  " UU-   U-   U-   UR	                  [         R                  5      US9  g [         R                  " U U-   U-   UR	                  [         R                  5      US9  [         R                  " U U-   U-   U-   UR	                  [         R                  5      US9  [         R                  " UU-   U-   UR	                  [         R                  5      US9  [         R                  " UU-   U-   U-   UR	                  [         R                  5      US9  g )Nr   rN   r   r  r   r   )	r  r  r  r  r   r  r  r,  r  )Q_ptrK_ptrCos_ptrSin_ptrHTDTHr  r  bhtrl   hr!  D_halfr  r   qk_basecs_baseq_xq_yk_xk_ycossinq_oxq_oyk_oxk_oys                                rF   _rope_qk_split_kernelr    s7    a Q	a1h' EAI!%(150EAI%)V3 ggego5DLOOPRPZPZ[ggego.$6TMPPQSQ[Q[\ggego5DLOOPRPZPZ[ggego.$6TMPPQSQ[Q[\ggg'$6TMPPQSQ[Q[\ggg'$6TMPPQSQ[Q[\ Sy39$Sy39$Sy39$Sy39$HHUW_4dggbkk6JQUVHHUW_v-5twwr{{7KRVWHHUW_4dggbkk6JQUVHHUW_v-5twwr{{7KRVWHHUW_4dggbjj6IPTUHHUW_v-5twwrzz7JQUVHHUW_4dggbjj6IPTUHHUW_v-5twwrzz7JQUVrI   c                 (   [         R                  " S5      n[         R                  " SU
5      nX:  nX-  U-   n[         R                  " U U-   USS9R	                  [         R
                  5      n[         R                  " UU-  SS9U-  nU[         R                  R                  UU	-   5      -  nU(       a  X-  nUU-  U-   nOUn[         R                  " UU-   USS9R	                  [         R
                  5      n[         R                  " UU-   USS9R	                  [         R
                  5      n[         R                  " UU-   USS9R	                  [         R
                  5      n[         R                  " UU-   USS9R	                  [         R
                  5      nUSU-   -  U-   nUSU-   -  U-   nU(       am  [         R                  " UU-   UR	                  [         R                  5      US9  [         R                  " UU-   UR	                  [         R                  5      US9  g[         R                  " UU-   UR	                  [         R                  5      US9  [         R                  " UU-   UR	                  [         R                  5      US9  g)z?Compute rms_norm(x) once, write two differently-scaled outputs.r   r   r  r  r   r   Nr  )r  Out1_ptrOut2_ptr
Scale1_ptr
Shift1_ptr
Scale2_ptr
Shift2_ptrr  r  r  r  r  r  r  r  r   r  r   r  r  r  s_offscale1shift1scale2shift2out1out2s                               rF   !_rms_norm_dual_scale_shift_kernelr     s    mmAyyG$xgnGGECKd#699"**E&&QQ'!+RWW]]7S=11*IMD(EEe+$cBEEbjjQe+$cBEEbjjQe+$cBEEbjjQe+$cBEEbjjQv&/v&/HHX^TWWR[[%9EHHX^TWWR[[%9EHHX^TWWRZZ%8tDHHX^TWWRZZ%8tDrI   c                    U(       Ga>  [         (       Ga2  U R                  (       Ga   U R                  nU R                  S   nU R                  5       R	                  SU5      nUR                  S   nUR                  5       R	                  SU5      n	UR                  5       R	                  SU5      n
U	R                  S   n[
        R                  " U5      n[        R                  " U5      n[        [        US-  S5      S5      n[        U4   " X|XXkX=U R                  [
        R                  :H  X:  US9  UR                  U5      $ [        R                   R"                  R%                  XS9SU-   -  U-   $ )Nr   r   r  r      r  r  r  r  	num_warpsr  )
HAS_TRITONis_cudarV   
contiguousr  r   
empty_liketritonnext_power_of_2r   r   r  r   r,  r  r   r   
common_ditrms_norm)r   r  r  r  
use_triton
orig_shapehiddenx_2dn_rowsscale_2dshift_2dr  r  r  r  s                  rF   fused_norm_scale_shiftr    s7   jjQYYYWW
||~%%b&1A##%--b&9##%--b&9~~a(t$((0GsNA.3	$fY/xWW.'0	
 xx
##yy##,,Q,8AIFNNrI   c                 "   U(       a  [         (       a  U R                  (       d  [        X5      [        X5      4$ US   US   pT[        U5      S:  a  US   OSnU(       a   UR                  S:w  d  U R                  S:w  a  [        X5      [        X5      4$ UR
                  u  pxpU
S-  nU R
                  XyX-  4:w  d  UR
                  XyX-  4:w  a  [        X5      [        X5      4$ U R                  5       n UR                  5       nUR                  5       nUR                  5       n[        R                  " U
5      n[        [        US-  S5      S5      n[        Xx-  U	-  4   " XXXUX-  U R                  [        R                  :H  UUS	9  X4$ )
zApply split RoPE to q and k in one fused kernel pass.
q, k: [B, T, H*D] contiguous
freqs_cis: (cos, sin, split_pe) where cos/sin: [B, H, T, D//2]
Falls back to original apply_rotary_emb if preconditions not met.
r   r   rN   FrP   rO       r   )r  r  r  )r  r  _apply_roper]   r  rV   r  r
  r  r   r   r  r   r   r,  )qrf   	freqs_cisr  r  r  split_peB_cosr  T_cosr  r  cos_csin_cr  r  s                   rF   fused_rope_qkr   5  sb    ::!))1(+a*CCC|Yq\"9~1y|uHsxx1}!1(+a*CCC!iiEe
Aww5''177uQU6K+K1(+a*CCC	A	ANNENNE%%f-HCB*A.I59u,./	e	!	ENN* 4KrI   c                    U(       Ga  [         (       Ga  U R                  (       Ga  U R                  nU R                  S   nU R                  5       R	                  SU5      n	U	R                  S   n
UR                  5       R	                  SU5      nUR                  5       R	                  SU5      nUR                  5       R	                  SU5      nUR                  5       R	                  SU5      nUR                  S   n[
        R                  " U	5      n[
        R                  " U	5      n[        R                  " U5      n[        [        US-  S5      S5      n[        U
4   " U	UUXXXUUU R                  [
        R                  :H  X:  US9  UR                  U5      UR                  U5      4$ [        R                   R"                  R%                  XS9nUSU-   -  U-   USU-   -  U-   4$ )zQRMS-norm x once, return two scaled outputs: (x_norm*(1+s1)+b1, x_norm*(1+s2)+b2).r   r   r  r   r  r  r  )r  r  rV   r  r  r   r	  r
  r  r   r   r   r   r,  r  r   r   r  r  )r   r  r  r  r  r  r  r  r  r  r  	scale1_2d	shift1_2d	scale2_2d	shift2_2dr  r  r  r  r  r  s                        rF   fused_norm_dual_scale_shiftr&  \  s   jjQYYYWW
<<>))"f5::a=%%'//F;	%%'//F;	%%'//F;	%%'//F;	 q)%%**62GsNA.3	)6)4$)WWW.'0	
 yy$dii
&;;;%%..q.:V$v-vV/Dv/MMMrI   c
           
         U R                   (       a  U R                  XAR                  S   U[        SS5      5      u  pnUR                  S   nUS   R	                  UR
                  UR                  S9UR                  XR                  S   SS5      -   R                  SS	9u  p[        XU
[        U S
S5      S9nA
AUSU-   -  U-   nAAU" UUXS9U-  $ U" [        R                  R                  R                  U5      UXS9$ )zDrop-in replacement for _apply_text_cross_attention with fused norm+scale+shift.
Patched onto the block instance so self._apply_text_cross_attention resolves here.r      	   r  r  r   rN   r   r   use_triton_kernelsTr  )r   r   r   )cross_attention_adalnget_ada_valuesrV   slicer   r   r   r  unbindr  rR  r   r   r  r  )r   r   r   attnscale_shift_tableprompt_scale_shift_tabletimestepprompt_timestepattention_maskr   shift_qscale_qgater   shift_kvscale_kv
attn_inputencoder_hidden_statess                     rF   #_apply_text_cross_attention_patchedr=  {  s$    !!!%!4!45FPQ
T\^cdegh^i!j$WWQZ
$Z0331881773S%%j2G2G2JArRS
&Q&- 	 ,A7>tEY[_7`b
W '1x< 88 ChJ(='RTXY 	Y		%%a(' rI   r   rJ   c                    UR                  SS5      nUR                  SS5      n[        U SS5      n[        U SS5      n[        U SS5      n[        U SS5      n[        U S	S5      nUu  nnU=(       a    UR                  5       S
:  =(       a    US:g  nU=(       a8    UR                  SS5      =(       a    UR                  5       S
:  =(       a    US:g  nU=(       a    UR                  SS5      =(       a    US:g  nU(       Ga  U R                  U R                  UR
                  S
   U[        S
S5      5      u  nn [        UU UUS9n!AA U(       Ga  U R                  n"U"R                  U!5      n#U"R                  U!5      n$U"R                  U!5      n%U"R                  U#5      n#U"R                  U$5      n$Ub  [        U#U$USS9u  n#n$Uc.  [        R                   " U#U$U%U"R"                  U"R$                  US9n&On[&        b9  [)        U[&        5      (       a$  [+        U#U$U%U"R"                  UU"R$                  US9n&O.[        R,                  " U#U$U%U"R"                  UU"R$                  US9n&A#A$A%U"R.                  b  U"R/                  U!5      n'U&R
                  u  n(n)n*U&R1                  U(U)U"R"                  U"R2                  5      n&U&R5                  S[6        R8                  " U'5      -  R;                  S5      5        U&R1                  U(U)U"R"                  U"R2                  -  5      n&A'U"R=                  U&5      n+A&A!OU R                  U!UUUS9n+A!U R                  U R                  UR
                  S
   U[        SS5      5      S
   n,UR?                  U+U,US9  A,A+URA                  U RC                  UX RD                  U R                  [        U SS 5      UUXO5	      US9  U(       a  U R                  U RF                  UR
                  S
   U[        S
S5      5      u  n-n.[        UU.U-US9n/A-A.U RI                  U/XS9n+A/U R                  U RF                  UR
                  S
   U[        SS5      5      S
   n0UR?                  U+U0US9  A0A+URA                  U RC                  UX0RJ                  U RF                  [        U SS 5      UUXO5	      US9  U(       d  U(       Ga  U(       a  U(       a  U R                  U RL                  S S2S S 24   UR
                  S
   U5      n1U R                  U RN                  S S2S S 24   UR
                  S
   U5      n2[Q        UU1S
   U1S   U1S   U1S   US9u  n3n4[Q        UU2S
   U2S   U2S   U2S   US9u  n5n6A1A2GO$U(       a  U R                  U RL                  S S2S S 24   UR
                  S
   U5      S S u  n7n8U R                  U RN                  S S2S S 24   UR
                  S
   U5      S S u  n9n:[        UU7U8US9n3[        UU9U:US9n5A7A8A9A:OU R                  U RL                  S S2S S 24   UR
                  S
   U5      SS u  n7n8U R                  U RN                  S S2S S 24   UR
                  S
   U5      SS u  n9n:[        UU7U8US9n4[        UU9U:US9n6A7A8A9A:U(       a]  U RS                  W3W5XUS9n;A3A5U R                  U RL                  SS 2S S 24   UR
                  S
   U5      S
   n<UR?                  U;U<US9  A<A;U(       a]  U RU                  W6W4XUS9n=A6A4U R                  U RN                  SS 2S S 24   UR
                  S
   U5      S
   n>UR?                  U=U>US9  A>A=U(       a  U R                  U R                  UR
                  S
   U[        SS5      5      u  n?n@[        UU@U?US9nAA?A@U RW                  UA5      nBAAU R                  U R                  UR
                  S
   U[        SS5      5      S
   nCUR?                  UBUCUS9  ACABU(       a  U R                  U RF                  UR
                  S
   U[        SS5      5      u  nDnE[        UUEUDUS9nFADAEU RY                  UF5      nBAFU R                  U RF                  UR
                  S
   U[        SS5      5      S
   nGUR?                  UBUGUS9  AGABUU4$ )Nrun_vxTrun_axr*  video_scaler   audio_scaleaudio_to_video_scalevideo_to_audio_scaler   r   a2v_cross_attnv2a_cross_attnrN   r+  r   r   r  r   )per   r   rO   )valuer2  r   )rH  r   audio_prompt_scale_shift_tablerP   r   )r   rH  k_per   r  r(  )-r	  rR  numelr-  r1  rV   r.  r  attn1r  r   r   r  r   r   _comfy_attnr   r   r   _GuideAttentionMask
isinstance_ltx_attn_with_guide_maskoptimized_attention_maskedr  r  r  r   r   r  r  r  addcmul_r   _apply_text_cross_attentionrZ  audio_scale_shift_tableaudio_attn1r^  scale_shift_table_a2v_ca_videoscale_shift_table_a2v_ca_audior&  audio_to_video_attnvideo_to_audio_attnr  audio_ff)Hr   r   	v_context	a_contextr5  
v_timestep
a_timestepv_pea_pe
v_cross_pe
a_cross_pev_cross_scale_shift_timestepa_cross_scale_shift_timestepv_cross_gate_timestepa_cross_gate_timestepr   self_attention_maskv_prompt_timestepa_prompt_timestepr  r?  r@  r  rA  rB  rC  rD  r  r  run_a2vrun_v2a
vshift_msa
vscale_msanorm_vx_a1r  rf   r   _sa_out_gate_b_trM   	attn1_out	vgate_msa
ashift_msa
ascale_msanorm_ax	agate_msa	v_ca_vals	a_ca_valsvx_a2vvx_v2aax_a2vax_v2asvbvsabaa2v_outgate_out_a2vv2a_outgate_out_v2a
vshift_mlp
vscale_mlp	vx_scaledff_out	vgate_mlp
ashift_mlp
ascale_mlp	ax_scaled	agate_mlpsH                                                                           rF   ltx2_forwardr    s
    %((48$((48T#7>
dM37dM37&t-CSI&t-CSIBABHHJNA{c/A0445EtLQSQYQYQ[^_Q_dx|dl0445EtLlQeilQl&*&9&9$:P:PRTRZRZ[\R]_ikpqrtukv&w"J
,RZT^_GJjjHHW%HHW%HHW%JJqMJJqM#(AtEDAq&.)==aAsyyadasas  J]  ^G(4DWYl9m9m71aL_ps  qC  qC  Yl  mG)DDQ1ciiYl  ~A  ~P  ~P  fy  zGq!%%1..w7E 'IBA%ll2r399cllKGLL#e(<"<!G!G!KL%ll2r399s||3KLGJJw/	W JJw4>QgzJ{	++D,B,BBHHQKQ[]bcdfg]hijklIKK	9KK@9GGD44Izz4+A+A8$?-~U "	   &*&9&9$:V:VXZX`X`abXceoqvwxz{q|&}"J
,RZT^_GJ((T(cI++D,H,H"((ST+WachijlmcnopqrIKK	9KK@9GGD44I//1M1M>E-~U "	   g7 !//0S0STVUVTVXYTY0Z\^\d\def\g  jF  G	 //0S0STVUVTVXYTY0Z\^\d\def\g  jF  G	!<R1yYZ|]fgh]iktuvkw  EO  "P!<R1yYZ|]fgh]iktuvkw  EO  "Py,,T-P-PQSRSQSUVQV-WY[YaYabcYd  gC  D  EG  FG  HB,,T-P-PQSRSQSUVQV-WY[YaYabcYd  gC  D  EG  FG  HB/BzR/BzRB,,T-P-PQSRSQSUVQV-WY[YaYabcYd  gC  D  EF  GH  IB,,T-P-PQSRSQSUVQV-WY[YaYabcYd  gC  D  EF  GH  IB/BzR/BzRB 2266j  @S2  TF#2243V3VWXWY[\W\3]_a_g_ghi_j  mB   C  DE   FG\9MN ' 2266j  @S2  TF#2243V3VWXWY[\W\3]_a_g_ghi_j  mB   C  DE   FG\9MN ' %)%8%89O9OQSQYQYZ[Q\^hjopqstju%v"J
.r:zV`aIJWWY'F++D,B,BBHHQKQ[]bcdfg]hijklIKK	K=6 %)%8%89U9UWYW_W_`aWbdnpuvwyzp{%|"J
.r:zV`aIJ]]9-F++D,H,H"((ST+WachijlmcnopqrIKK	K=62vrI   c                   (    \ rS rSrSS jrSS jrSrg)LTX2ForwardPatchi.  c                 @    Xl         X l        X@l        X0l        XPl        g r'  )rA  rB  rD  rC  r*  )r   videoaudioaudio_to_videovideo_to_audior  s         rF   r(  LTX2ForwardPatch.__init__/  s       $2!$2!",rI   Nc                 <   ^  U 4S jn[         R                  " X15      $ )Nc                   > TR                   U l         TR                  U l        TR                  U l        TR                  U l        TR                  U l        [
        R                  " [        U 5      U l        [        U /UQ70 UD6$ r'  )
rA  rB  rD  rC  r*  r2  r3  r=  rT  r  r,  s      rF   ry  1LTX2ForwardPatch.__get__.<locals>.wrapped_forward6  sw    &*&6&6K#&*&6&6K#/3/H/HK,/3/H/HK,-1-D-DK*6;6F6F3[7BK3=d=f==rI   r1  r{  s   `   rF   r6  LTX2ForwardPatch.__get__5  s    	> 55rI   )rB  rC  r*  rA  rD  r8  r'  r9  rs   rI   rF   r  r  .  s    -
6rI   r  c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)LTX2AttentionTunerPatchiA  c                 *   [         R                  " SSSSS[         R                  R                  S5      [         R                  R                  SSS	S
9[         R
                  R                  SSSSSSS9[         R
                  R                  SSSSSSS9[         R
                  R                  SSSSSSS9[         R
                  R                  SSSSSSS9[         R                  R                  SSSS
9/[         R                  R                  SS9/S9$ )Nr  zLTX2 Attention Tuner Patchr#   zqEXPERIMENTAL! Custom LTX2 forward pass with attention scaling factors per modality, also reduces peak VRAM usage.Tr=  blocksr  zlComma separated list of transformer block indices to apply the patch to. Leave empty to apply to all blocks.r  rA  r   r   r  r   z#Scaling factor for video attention.r   rB  z#Scaling factor for audio attention.rC  rD  triton_kernelszkUse Triton fused kernels for norm+scale+shift and rope application operations, can be very slightly faster.r-   r@  )r   r>   rB  r7   r  r9   rC  rB   r   s    rF   rG   %LTX2AttentionTunerPatch.define_schemaC  s*   yy-5# L w'		"  ?m  n}csRV  aF  G}csRV  aF  G5sQT[_  jO  P5sQT[_  jO  P

  !14  Jw   x W5
 	
rI   rJ   c           	         UR                  5       nUR                  S5      n	UR                  5       S:X  a(  [        [	        [        U	R                  5      5      5      n
O/[        S UR                  5       R                  S5       5       5      n
[        R                  " S[        U
5       SU 35        [	        [        U	R                  5      5       Hx  nU	R                  U   nX;   a%  [        X4XVUS9R                  XR                  5      nO&[        SSSSUS9R                  XR                  5      nUR                  S	U S
3U5        Mz     [        R                   " U5      $ )NrF  r  c              3   8   #    U  H  n[        U5      v   M     g 7fr'  )r   ).0rf  s     rF   	<genexpr>2LTX2AttentionTunerPatch.execute.<locals>.<genexpr>b  s     !P6Os#c((6Os   rw  zBApplying LTX2 Attention Tuner Patch with custom scales to blocks: z, triton_kernels=r+  r   rI  z.forward)r   rO  r  ry  r4   r]   rY  rZ   logginginforW   r  r6  r[  r\  r   r`   )rC   r=  r  rA  rB  rC  rD  r  r`  rF  selected_blocksrf  rg  patched_forwards                 rF   rq   LTX2AttentionTunerPatch.executeY  sn   kkm%667HI <<>R!%O,N,N(O"PQO!!Pflln6J6J36O!PPOYZ`apZqYr  sD  ES  DT  U  	V _??@AC#66s;E%"2;Ma  ES  #T  #\  #\  ]b  ds  ds  #t"23S#R`"a"i"ijo  rA  rA  #B((+NseS[)\^mn B }}[))rI   rs   Nrt   rs   rI   rF   r  r  A  s>    
 
* *}  ~K  ~K * *rI   r  c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g)%LTX2MemoryEfficientSageAttentionPatchiq  c                     [         R                  " SSSSS[         R                  R                  S5      [         R                  R                  SSSS	9/[         R                  R                  SS
9/S9$ )Nr  z!LTX2 Mem Eff Sage Attention Patchr#   zEXPERIMENTAL! Activates custom sageattention to reduce peak VRAM usage, overrides the attention mode. Requires latest sageattention version.Tr=  r  zHUse Triton fused RoPE kernel on the self-attention Q/K. Requires Triton.r  r-   r@  )r   r>   rB  r7   rC  rB   r   s    rF   rG   3LTX2MemoryEfficientSageAttentionPatch.define_schemas  su    yy;<# g w'

  !14  JT   U
 W5
 	
rI   rJ   c                    [         c  [        S5      eUR                  5       nUR                  S5      n[        R
                  " SU 35        [        UR                  5       H]  u  pVX&R                  l	        UR                  SU S3[        R                  UR                  UR                  R                  5      5        M_     [        R                  " U5      $ )Nzsageattention is not new enough version or could not determine CUDA architecture, cannot apply LTX2 Memory Efficient Sage Attention Patch.rF  z^Applying LTX2 Memory Efficient Sage Attention Patch to all transformer blocks, triton_kernels=rI  z.attn1.forward)_cuda_archsRuntimeErrorr   rO  r  r  rX  rY  rM  r*  r\  ltx2_sageattn_forwardr6  r[  r   r`   )rC   r=  r  r`  rF  rf  rg  s          rF   rq   -LTX2MemoryEfficientSageAttentionPatch.execute  s       l  m  mkkm%667HIu  wE  vF  G  	H#O$F$FGJC-;KK*((+NseSa)bdy  eB  eB  CH  CN  CN  PU  P[  P[  Pe  Pe  ef  g H }}[))rI   rs   Nrt   rs   rI   rF   r  r  q  s3    
 
  *r}} * *rI   r  c                       [         R                  R                  n U b)  U R                  S5      u  p[	        U5      [	        U5      4$ g! [
         a     gf = f)Nr   )r   r   )r   versioncudarZ   r   	Exception)r  majorminors      rF   get_cuda_versionr    sT    --$$"==-LEu:s5z)) s   AA	 	
AA)per_thread_int8_tritonper_warp_int8_cudaper_block_int8_tritonper_channel_fp8get_cuda_arch_versions
attn_false)_qattn_sm892qk_int8_sv_f8_accum_f16_fuse_v_scale_attn_inst_buf)   r   )sm89_compile)_qattn_sm80)sm80_compile)_qattn_sm90)sm90_compile)
apply_rope)WanT2VCrossAttentionWanI2VCrossAttentionc                    U u  p#nU R                  5         UR                  S   nSnSnSnSn	Sn
US-  nSn[        S   S;   a  UR                  UR	                  SS	S
95        [        X#USSSSS9u  pnnAA[        R                  " UR                  5       XR                  S9nUR                  [        R                  5      nA[        R                  " XUUUUXxXU
5        GO[        S   S:X  aC  UR                  UR	                  SS	S
95        [        X#XS9u  pnnAA[        XXNUXaS SS9	u  nnAGOD[        S   S:X  a  [         (       d  SnOSnSnUR                  UR	                  SS	S
95        [        X#USSSSS9u  pnnAA[#        XFUSS9u  nnnA[        R                  " UR                  5       XR                  S9nUS:X  a  [$        R&                  " XUUUUUXxXU
5        O$US:X  a  [$        R(                  " XUUUUUXxXU
5        AAGOh[        S   S:X  a  UR                  UR	                  SS	S
95        [        X#USSSSS9u  pnnAA[#        XFSS9u  nnnA[        R                  " UR                  5       XR                  S9n[*        R(                  " XUUUUUXxXU
5        AAO[        S   S:X  a  [         (       d  SnOSnSnSn	[-        X#UR	                  SS	S
9USSSS9u  pnnAA[#        XFUSS9u  nnnA[        R                  " UR                  5       XR                  S9nUS:X  a  [$        R.                  " XUUUUUXxXU
5        O$US:X  a  [$        R&                  " XUUUUUXxXU
5        AAAAAAW$ ) Nr   NHDr   rO   g      g      |@>   sm80sm86r   T)r   r      r  @   )tensor_layoutBLKQWARPQBLKKWARPKr   sm75)sm_scaler  F)r  output_dtype	attn_mask
return_lsesm89z	fp32+fp32z	fp32+fp16g      @)r  	scale_maxsmooth_vsm90r  )r  r  sm120fp32rN   )kmr  r  r  r  )clearrV   r  r   meanr  r   emptyr  r   r   r  r  qk_int8_sv_f16_accum_f32_attnr  r  sageplus_sm89_availabler  r  r  2qk_int8_sv_f8_accum_f32_fuse_v_scale_attn_inst_bufr  r  )qk_int8_sv_f8_accum_f32_fuse_v_scale_attn)qkvr   r  rf   r   head_dim_ogr  _tensor_layout	_is_caual_qk_quant_gran_return_lser  quant_v_scale_maxq_int8q_scalek_int8k_scaleov_fp16rM   pv_accum_dtypev_fp8v_scales                          rF   _sageattn_int8_fp8_nhdr    sf    GA!IIK''"+KMNINKD H1~))	qvv!Tv*++A!Vcjmuw  A  IK  ,L(qKKU==Iemm$11&&!WV]_m  {I  U`  	a	Q6	!	qvv!Tv*++@PX+v(q&!g]z~  LQ  R1	Q6	!&&(N(N $	qvv!Tv*++A!Vcjmuw  A  IK  ,L(q+AVgrwxwKKU==I[(JJ6[`bcelnuw~  AO  \j  vA  B{*JJ6[`bcelnuw~  AO  \j  vA  B7	Q6	!	qvv!Tv*++A!Vcjltv  ~A  IL  ,M(q+AUZ[wKKU==IFFvW\^_ahjqsz  }K  Xf  r}  	~7	Q7	"&&#N(N $+=aqvvRS]avGbr  GJ  RT  []  ,^(q+AVgrwxwKKU==IV#AA&RWYZ\celnu  xF  Sa  mx  y{*JJ6[`bcelnuw~  AO  \j  vA  B7HrI   c           
         UR                   nUc  UOUnU R                  U5      nU R                  U5      nU R                  U5      n	U R	                  U	5      n	Ub3  [        U SS5      n
Uc  [        XXJS9u  pO[        X5      n[        X5      n	U R                  U5      nUGb.  [        b6  [        U[        5      (       a!  [        XXR                  X0R                  US9nO+[        R                  " XXR                  X0R                  US9nU R                   b  U R!                  U5      nUR"                  u  pnUR%                  XU R                  U R&                  5      nUR)                  S[*        R,                  " U5      -  R/                  S5      5        UR%                  XU R                  U R&                  -  5      nAU R1                  U5      $ UR"                  u  nnnU R&                  nUR%                  UUU R                  U5      nU	R%                  UU	R"                  S   U R                  U5      n	UR%                  UUR"                  S   U R                  U5      nXU/nAA	A[3        UU5      nU R                   bI  U R!                  U5      nUR)                  S[*        R,                  " U5      -  R/                  S5      5        AU R1                  UR%                  UUS5      5      $ )Nr*  Fr+  rG  r  r   r   )r   r  r  r   r   rR  r   r   r   rO  rP  rQ  r   r   rN  rR  r  rV   r  r  r   r   r  r  r  r  )r   r   r   r   rH  rK  r   r   r  rf   r  r   r  r   rs  rt  rM   r   seq_lenr  r  s                        rF   r  r    s   GGE?aG 			!AAA		'AAA	~T#7?
< rADAq 'A )A		'A *z$@S/T/T)!::tTgTg  ~Q  RA66qQ

Datat  K^  _A*--a0KIBArtzz4==9AFFC%--44??CDrtzzDMM9:A{{1~ WWJ--K	z7DJJ<A	z1771:tzz;?A	z1771:tzz;?A)C	1asE*A &))!,	emmK00;;B?@;;qvvj'2677rI   c                    UR                   n/ UR                  SS QU R                  PU R                  P7u  pVpxU R	                  U R                  U5      5      R                  XVXx5      n	U R                  U R                  U5      5      R                  XVXx5      n
[        XU5      u  pU R                  U5      R                  XVXx5      nXU/nA	A
A[        X5      nU R                  UR                  XVXx-  5      5      $ )z|
Args:
    x(Tensor): Shape [B, L, num_heads, C / num_heads]
    freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2]
NrN   )r   rV   	num_headshead_dimnorm_qr  r  norm_krf   _wan_apply_roper   r  r  )r   r   freqsr   r   r!  sndr  rf   r   r  r  s                 rF   wan_sageattn_forwardr  R  s     GGE<!''"1+<t~~<t}}<JA!DFF1I##A!/ADFF1I##A!/A1'DAq	qQ"A)C	1as*A66!&&qu%&&rI   c                    UR                   n/ UR                  S S QU R                  PU R                  P7u  pgpU R	                  U R                  U5      5      R                  XgX5      n
U R                  U R                  U5      5      R                  USX5      nU R                  U5      R                  USX5      nXU/nA
AA[        X5      nU R                  UR                  XgX-  5      5      $ NrN   r   )r   rV   r  r  r  r  r  r  rf   r   r  r  )r   r   r   r   r  r   r!  r  r	  r
  r  rf   r   r  r  s                  rF   wan_t2v_cross_sageattn_forwardr  g  s    GGE<!''"1+<t~~<t}}<JA!DFF1I##A!/ADFF7O$))!R6AwQA)A)C	1as*A66!&&qu%&&rI   c                     UR                   n/ UR                  S S QU R                  PU R                  P7u  pgpUS S 2S U24   n
US S 2US 24   nU R	                  U R                  U5      5      R                  XgX5      nU R                  U R                  U
5      5      R                  USX5      nU R                  U
5      R                  USX5      nXU/nAA[        X5      nU R                  U R                  U5      5      R                  USX5      nU R                  U5      R                  USX5      nUUU/nAAA[        UU5      nUR                  U5        AU R                  UR                  XgX-  5      5      $ r  )r   rV   r  r  r  r  r  
norm_k_imgk_imgv_imgr  r  rf   r   r   r  )r   r   r   context_img_lenr   r   r!  r  r	  r
  context_imgr  r  r  qkv_imgimg_orf   r   r  r  s                       rF   wan_i2v_cross_sageattn_forwardr  v  sd   GGE<!''"1+<t~~<t}}<JA!!-o--.Ka))*GDFF1I##A!/AOODJJ{3499!RFEJJ{#((B5EGu"72EDFF7O$))!R6AwQA)Aa)C	1asE*AFF5M66!&&qu%&&rI   c                   P    \ rS rSr\S 5       r\S\R                  4S j5       rSr	g))WanVideoMemoryEfficientSageAttentionPatchi  c                     [         R                  " SSSSS[         R                  R                  S5      /[         R                  R	                  SS9/S9$ )	Nr  z%WanVideo Mem Eff Sage Attention PatchzKJNodes/wanzEXPERIMENTAL! Activates custom sageattention on the WanVideo self-attention to reduce peak VRAM usage, overrides the attention mode. Requires latest sageattention version.Tr=  r-   r@  )r   r>   rB  r7   rB   r   s    rF   rG   7WanVideoMemoryEfficientSageAttentionPatch.define_schema  sS    yy?@" F w' W5
 	
rI   rJ   c                    [         c  [        S5      e[        c  [        S5      eUR                  5       nUR	                  S5      n[
        R                  " S5        [        UR                  5       H  u  pEUR                  SU S3[        R                  UR                  UR                  R                  5      5        [        USS 5      nUbG  [        U5      [         L a5  UR                  SU S3["        R                  XfR                  5      5        M  Uc  M  [        U5      [$        L d  M  UR                  SU S3[&        R                  XfR                  5      5        M     [(        R*                  " U5      $ )	Nzsageattention is not new enough version or could not determine CUDA architecture, cannot apply WanVideo Memory Efficient Sage Attention Patch.zrCould not import apply_rope from comfy.ldm.flux.math, cannot apply WanVideo Memory Efficient Sage Attention Patch.rF  zQApplying WanVideo Memory Efficient Sage Attention Patch to all transformer blockszdiffusion_model.blocks.z.self_attn.forward
cross_attnz.cross_attn.forward)r  r  r  r   rO  r  r  rX  r  r\  r  r6  	self_attnr[  rR  type_WanI2VCrossAttentionr  _WanT2VCrossAttentionr  r   r`   )rC   r=  r`  rF  rf  rg  r  s          rF   rq   1WanVideoMemoryEfficientSageAttentionPatch.execute  s      p  q  q"   T  U  Ukkm%667HIhi#O$:$:;JC((+B3%GY)Z\p\x\xy~  zI  zI  KP  KZ  KZ  Kd  Kd  ]e  f d;J%$z*:>S*S,,/FseK^-_a  bH  bH  IS  Ui  Ui  bj  k'D,<@U,U,,/FseK^-_a  bH  bH  IS  Ui  Ui  bj  k < }}[))rI   rs   Nrt   rs   rI   rF   r  r    s3    
 
 *r}} * *rI   r  c                   T    \ rS rSr\S 5       r\SS\R                  4S jj5       rSr	g)LTX2LoraLoaderAdvancedi  c                    [         R                  " SSSSS[         R                  R                  S[        R
                  " S5      SS	9[         R                  R                  S
SS9[         R                  R                  SSSSSSS9[         R                  R                  SSSSS9[         R                  " S5      R                  SSSS9[         R                  R                  SSSSSSS9[         R                  R                  SSSSSSS9[         R                  R                  S SSSSS!S9[         R                  R                  S"SSSSS#S9[         R                  R                  S$SSSSS%S9/
[         R                  R                  S
S&S'9[         R                  R                  S(S)S'9[         R                  R                  S*S+S'9/S,9$ )-Nr$  zLTX2 LoRA Loader Advancedr#   zDAdvanced LoRA loader with per-block strength control for LTX2 modelsT	lora_nameloraszThe name of the LoRA.)r+   r   r=  z0The diffusion model the LoRA will be applied to.r%   strength_modelr   g      Yr   r   zGHow strongly to modify the diffusion model. This value can be negative.r   opt_lora_pathzAbsolute path of the LoRA.)r   force_inputr   SELECTEDDITBLOCKSr  z!Selected DiT blocks configurationr`  r  r   z$Strength for video attention layers.r  z3Strength for video to audio cross-attention layers.r  z$Strength for audio attention layers.r  z3Strength for audio to video cross-attention layers.r  z6Strength for layers not caught by other layer filters.zThe modified diffusion model.r.   rankzPossible rank of the LoRA.loaded_keys_infoz+List of loaded keys and their alpha values.r@  )r   r>   r   r7   folder_pathsget_filename_listrB  r9   r  CustomrB   r   s    rF   rG   $LTX2LoraLoaderAdvanced.define_schema  s   yy,4#^ {L4R4RSZ4[e|}w0bc/&eZ^  ir  s		$DYuv		-.44XVy4zw#D  [A  B/#3UY  dY  Zw#D  [A  B/#3UY  dY  Zw#D  [S  T W>]^		  f>Z [		  .@Jw x%
 	
rI   NrJ   c                 	   SSK Jn  SS KnU	(       a  U	nO[        R                  " SU5      nU" USS9nSn[        S UR                  5        5       S 5      nU(       a:  [        SU S	UU   R                   35        [        UU   R                  S   5      nO[        S
5        Sn0 nUb&  UR                  R                  UR                  U5      nUR                  R                  UU5      n/ nU
b  U
 H  n[        UR                  5       5       H  nSn[        U[        5      (       a	  UU;   a  SnO([        U[         5      (       a  U H  nUU;   d  M  Sn  O   U(       d  MT  U
U   nUS:X  a  UR#                  U5        Mr  UU   n[%        US5      (       d  M  [        UR&                  5      nUUS'   [!        U5      UU   l        M     M     [        UR                  5       5       H  nUU;   a  M  [        U[        5      (       a  UO$[        U[         5      (       a  US   O
[        U5      nS nSU;   a  UnO)SU;   a  UnO SU;   d  SU;   a  UnOSU;   d  SU;   a  UnOUnUc  M  US:X  a  UR#                  U5        M  US:w  d  M  UU   n[%        US5      (       d  M  [        UR&                  5      nUS   b  US   OSnUU-  US'   [!        U5      UU   l        M     U H  nUU;   d  M  UU	 M     / nUR)                  5        H  u  nn[%        US5      (       a[  [        U[        5      (       a  UO
[        U5      nUR&                  S   b  UR&                  S   OSnUR#                  U SU 35        Mr  [        U[        5      (       a  UO
[        U5      nUR#                  U S[+        U5      R,                   35        M     Ub"  UR/                  5       n U R1                  UU5      n[3        W5      n/ n!U HA  n"U"U;  d  M  [        U"[        5      (       a  U"O
[        U"5      nU!R#                  SU 35        MC     U!(       a  UR5                  U!5        SR7                  U5      n#[8        R:                  " W UU#5      $ )Nr   )load_torch_filer'  T)	safe_loadunknownc              3   T   #    U  H  oR                  S 5      (       d  M  Uv   M      g7f)r  N)endswith)r  r!   s     rF   r  1LTX2LoraLoaderAdvanced.execute.<locals>.<genexpr>  s     P+3h9O33+s   (	(z!Shape of the first 'weight' key (z): z"No key ending with 'weight' found.zCouldn't find rankFweightsrN   rZ  rY  
audio_attnzaudio_ff.netr0  zff.netr   Nonez: alpha=z: type=zNOT LOADED: 
)comfy.utilsr3  
comfy.lorar.  get_full_pathnextrX   r   rV   r=   loramodel_lora_keys_unetr=  	load_lorar_   rP  tupler:   rW  r9  itemsr  ru   r   add_patchesry  r5   joinr   r`   )$rC   r=  r&  r(  r  r  r  r  r  r)  r  r3  r   	lora_pathrA  r,  
weight_keykey_maploadedkeys_to_deleterg  r!   matchrf   ratiorI  weights_listkey_strstrength_multipliercurrent_alphaloaded_keys_listr   new_modelpatcher
not_loadedr   r-  s$                                       rF   rq   LTX2LoraLoaderAdvanced.execute  sI   /%I$227IFIyD9 P$))+PRVW
5j\T*EUE[E[D\]^tJ'--a01D67'Djj55ekk7KG%%dG4 .C!E!#s++ $#C//!$A$z(, % "%
 u &u A:*11#6 %+3KE&ui88/3EMM/B27Q6;L6Is 3) /  0 &Cn$'S11c*SRWBXBXA^abe^fG #' %/&4#&'1&4#(Ng,E&+#7"h'&9&+# ',# #.&!+"))#.(C/"3KEui00'+EMM':;G?;VQ\_*7:M*MQ.3L.As+K 'N "Cf}3K "
  ,,.JCui((!+C!5!5#3s8,1MM!,<,Ha(f ''7)8E7(CD!+C!5!5#3s8 ''7)74;;O;O:P(QR ) ${{} ,,V^DA F
Az)!S11!s1v!!L	":; 
 ##J/99%56}}-t5EFFrI   rs   r  rt   rs   rI   rF   r$  r$    sH    
 
4 G  QS  Q^  Q^ G GrI   r$  )NNNNr   Fr   F)gư>Tr8  )NNNNNNNNNNNNNNNNN)}comfy_extras.nodes_ltr   r   r   r2  r(  typingr   r   comfy_api.latestr   r  r   r   r  comfy.model_managementmodel_managementrK  comfy.ldm.modules.attentionr   r   r   rN  comfy.ldm.lightricks.modelr   r  r	   rO  r
   rQ  ImportErrorrL  r   r  r   r|   	ComfyNoder   r   r   r  r#  r%  r;  rr  rt  r  r  	threadingr  torch.nn.functionalnn
functionalr  r  r  PILr6   r  PromptServerinstancer  r  r  r  r<  r>  r\  rn  r  r  r
  triton.languagelanguager  r  jit	constexprr  r  r   r  r   r&  r=  Tensorr  r  r  r  r  r  r  sageattention.corer  r  r  r  r  r  r  r  cuda_versionrW  r  r  r  r  r  comfy.ldm.flux.mathr  r  comfy.ldm.wan.modelr  r!  r  r   r  r  r  r  r  r  r.  r$  rs   rI   rF   <module>rp     s
   ] ]         # 1 1 F% N 
			 Xf Xft@f\ @fFk9 k9Z =AVX_c  NRgipt "D< 7;PR ,^8 8(O*r|| O*d	
6 
6 *2<<  *J        ##W  W t,H7V8# #J$",, $BM M`$ $2obbll obb J ZZ C \\ C 	 C
  C || C  CD ZZ/W /W ,,/W /Wb ZZ&E \\	&E
 &E &E ||&E &ERO4$NN>6 FJ AEnr26	Wu||U\\12W
 
u||U\\)	*Wr6 6&.*bll .*` *BLL  *F	   	 Z  Z(*K.#%L%k3gh  Emy  ~E  nE./ 8A9 A
M` ,0dt$df 68r >@ '* JL ' [] '4&* &*R ]GR\\ ]GY=  % $%R$  JX  		  B   B   B   O  948819s   N N$ 'N3 ?"N? "O! )P 6P% =P3 
N! N!$N0/N03N<;N<?OOOOOO!P (O00O<8P ;O<<P P"
PPP"PP"%P0/P03
Q ?Q 