
    
3j<X                        S SK r S SKJr  S SKJr  S SKJrJrJr  S SKr	S SK
r	S SKr	S SKrS SKrS SKJr  S SKJrJrJrJrJr  S SKJr  S SKJr  S	S
KJr  S	SKJr  \ R<                  " \5      r S\!S\!4S jr"\"" S5      r#\"" S5      r$S\RJ                  S\R                  4S jr& " S S5      r' " S S\RP                  RR                  5      r*S<S jr+S=S jr,S\!S\!S\!4S jr-\" SSS9 " S S5      5       r.\!R_                  S S!5      r0S" r1\" S#S$S9 " S% S#5      5       r2   S>S&\S'\S(\3S)\\R                     S*\4S+\4S\!4S, jjr5  S?S&\S'\S-\!S.\!S*\4S+\4S\!4S/ jjr6  S?S&\S'\S-\!S)\R                  S.\!S*\4S+\4S\!4S0 jjr7S\RJ                  S1\4S2\4\8-  S\RJ                  4S3 jr9S@S4\44S5 jjr:S6 r; SAS7\RP                  RR                  S8\!S9\\!   S\\\RP                  RR                  \!\RP                  Rx                  \!4      4S: jjr=S; r>g)B    N)glob)Path)ListOptionalTuple)Image)AutoImageProcessorAutoTokenizerGemma3ConfigGemma3ForConditionalGenerationGemma3Processor)ROPE_INIT_FUNCTIONS)MODEL_MAPPING_NAMES   )
comfy_node)load_text_embeddings_pipelinefilenamereturnc                     [        [        5      R                  S-  U -  nUR                  5       (       a  UR	                  SS9R                  5       $  g! [         a%  n[        R                  SU  SU 35         SnAgSnAff = f)z-Load system prompt from file at module level.system_promptszutf-8)encodingzCould not load : N )	r   __file__parentexists	read_textstrip	Exceptionloggerwarning)r   prompt_pathes      K/home/wildlama/comfy/ComfyUI/custom_nodes/ComfyUI-LTXVideo/gemma_encoder.py_load_system_promptr%      s    :8n++.>>I(('(:@@BB     :
"QC899:s   AA 
BA??Bzgemma_t2v_system_prompt.txtzgemma_i2v_system_prompt.txttensorc                     U R                  5       S:X  a  U S   n U R                  5       R                  5       S-  R                  S5      n[        R
                  " U5      $ )z*Convert ComfyUI image tensor to PIL Image.   r      uint8)dimcpunumpyastyper   	fromarray)r&   numpy_images     r$   tensor_to_pilr1   +   sL    zz|q::<%%'#-55g>K??;''    c                   @    \ rS rSrS
S\S\4S jjrSS\S\4S jjrSr	g	)LTXVGemmaTokenizer3   tokenizer_path
max_lengthc                     [         R                  " USUS9U l        SU R                  l        U R                  R                  c%  U R                  R
                  U R                  l        X l        g )NT)local_files_onlymodel_max_lengthleft)r
   from_pretrained	tokenizerpadding_side	pad_token	eos_tokenr7   )selfr6   r7   s      r$   __init__LTXVGemmaTokenizer.__init__4   sT    &66TJ
 '-#>>##+'+~~'?'?DNN$$r2   textreturn_word_idsc                    UR                  5       nU R                  USU R                  SSS9nUR                  nUR                  n[        [        US   US   5      5       VVVs/ s H  u  nu  pxXxU4PM     n	nnnSU	0n
U(       dC  U
R                  5        VVVVVs0 s H   u  pX VVVs/ s H	  u  poU4PM     snnn_M"     n
nnnnnU
$ s  snnnf s  snnnf s  snnnnnf )Nr7   Tpt)paddingr7   
truncationreturn_tensorsr   gemma)r   r=   r7   	input_idsattention_mask	enumeratezipitems)rA   rD   rE   encodedrL   rM   itoken_idattntuplesoutkvtw_s                   r$   tokenize_with_weights(LTXVGemmaTokenizer.tokenize_with_weights?   s    zz|..  ! 
 %%	 // (1Yq\>RSCT1U'V
'V##H Q'V 	 
 <?IIKHKDA1Q/Q'!1vQ//KCH

 0Hs   %CC+C;	CC)r7   r=   N)   )F)
__name__
__module____qualname____firstlineno__strintrB   boolr\   __static_attributes__ r2   r$   r4   r4   3   s-    	%s 	% 	%#   r2   r4   c                      ^  \ rS rSrS\R
                  S4S\S\S-  4U 4S jjjrS r	S r
SS	 jrS
 rS rS rSrU =r$ )LTXVGemmaTextEncoderModelV   Nr,   model	processorc                   > [         TU ]  5         Xl        X@l        UR	                  US9U l        UR	                  US9U l        [        U/5      U l        [        R                  R                  U R                  5      S-   U l        g )Ndtypei   )superrB   rk   rl   tofeature_extractorembeddings_processorsetdtypescomfymodel_managementmodule_size_model_memory_required)rA   rk   rr   rs   rl   ro   device	__class__s          r$   rB   "LTXVGemmaTextEncoderModel.__init__W   s{     	
"!2!5!5E!5!B$8$;$;%$;$H!5'l ""..tzz:=NN 	#r2   c                     g Nrg   )rA   optionss     r$   set_clip_options*LTXVGemmaTextEncoderModel.set_clip_optionsl       r2   c                     g r~   rg   )rA   s    r$   reset_clip_options,LTXVGemmaTextEncoderModel.reset_clip_optionso   r   r2   c                     U R                  UUSS9n[        R                  " UR                  SS9nU R	                  XRU5      nU$ )NT)rL   rM   output_hidden_statesr+   )rk   torchstackhidden_statesrr   )rA   rL   rM   r>   outputsall_layer_hiddensfeaturess          r$   forward!LTXVGemmaTextEncoderModel.forwardr   sU    **)!%  

 "KK(=(=2F ))|
 r2   c                    US   n[         R                  " U Vs/ s H  o3S   PM	     sn/U R                  R                  S9n[         R                  " U Vs/ s H  oUS   PM	     sn/U R                  R                  S9nU R	                  U R                  R                  5        U " XFSS9n[        [        UR                  5       5      5      R                  nUS-
  R	                  U5      R                  UR                  S   SSUR                  S   45      [         R                  " U5      R                  -  n	U R                  R                  Xy5      u  pU
S SU04$ s  snf s  snf )	NrK   r   rz   r   r;   )r>   r   rM   )r   r&   rk   rz   rq   nextitervaluesro   reshapeshapefinfomaxrs   create_embeddings)rA   token_weight_pairstoken_pairsrY   rL   rZ   rM   r   encoded_input_dtypeconnector_attention_maskrQ   masks               r$   encode_token_weights.LTXVGemmaTextEncoderModel.encode_token_weights   sK   (1LL'(KqdK()$**2C2C
	 '(KqdK()$**2C2C
 	

!!"	G #4(9#:;AA$2Q$6#:#:;N#O#W#W!!!$a^-A-A"-EF$
KK+,00$1 
 11CC
 /666+ ) )s   EE#c                 6    U R                   R                  USS9$ )NF)strict)rk   load_state_dict)rA   sds     r$   load_sd!LTXVGemmaTextEncoderModel.load_sd   s    zz))"U);;r2   c                     U R                   $ r~   )ry   )rA   input_shapes     r$   memory_required)LTXVGemmaTextEncoderModel.memory_required   s    ***r2   )ry   ru   rs   rr   rk   rl   )right)r_   r`   ra   rb   r   bfloat16r   r   rB   r   r   r   r   r   r   rf   __classcell__)r{   s   @r$   ri   ri   V   sV     -1nn
-

 #T)
 
*74<+ +r2   ri   c                 0   ^ ^  " UU 4S jS[         5      nU$ )Nc                   6   >^  \ rS rSrS0 4U UU4S jjrSrU =r$ )1ltxv_gemma_tokenizer.<locals>._LTXVGemmaTokenizer   Nc                 "   > [         TU ]  TTS9  g )Nr7   )rp   rB   )rA   embedding_directorytokenizer_datar{   r7   r6   s      r$   rB   :ltxv_gemma_tokenizer.<locals>._LTXVGemmaTokenizer.__init__   s    G^
Cr2   rg   r_   r`   ra   rb   rB   rf   r   )r{   r7   r6   s   @r$   _LTXVGemmaTokenizerr      s    /3B 	D 	Dr2   r   )r4   )r6   r7   r   s   `` r$   ltxv_gemma_tokenizerr      s     D D0 D r2   c                 8   ^ ^^^  " UU UU4S jS[         5      nU$ )Nc                   <   >^  \ rS rSrS Y0 4U UUU4S jjrSrU =r$ )3ltxv_gemma_clip.<locals>._LTXVGemmaTextEncoderModel   r,   c           	         > [         R                  n[        R                  " TSUS9n[	        T	UTS-  S9u  pV[
        TU ]  UUUT
UUS9  g )NT)r9   torch_dtypezproj_linear.safetensors)ro   fallback_proj_path)rk   rr   rs   rl   ro   rz   )r   r   r   r<   r   rp   rB   )rA   rz   ro   model_optionsgemma_modelrr   rs   r{   encoder_path	ltxv_pathrl   s          r$   rB   <ltxv_gemma_clip.<locals>._LTXVGemmaTextEncoderModel.__init__   sk    NNE8HH!%!K 7T#/2K#K73 G!"3%9#  r2   rg   r   )r{   ro   r   r   rl   s   @r$   _LTXVGemmaTextEncoderModelr      s    "'uB 	 	 	r2   r   )ri   )r   r   rl   ro   r   s   ```` r$   ltxv_gemma_clipr      s     %> 2 &%r2   	root_pathpatternc                     [        U  S3SS9 Vs/ s H/  n[        U5      R                  U5      (       d  M$  [        U5      PM1     nnU(       d  [        SU SU  35      e[	        US   R
                  5      $ s  snf )zj
Recursively search for files matching a glob pattern and return the parent directory of the first match.
z/**T)	recursivezNo files matching pattern 'z' found under r   )r   r   matchFileNotFoundErrorrc   r   )r   r   pmatchess       r$   find_matching_dirr      s     3'488A7==! 	Q8  
 )'.L
 	
 wqz  !!s
   #A8A8LTXVGemmaCLIPModelLoaderzGemma 3 Model Loader)namedescriptionc                   R    \ rS rSr\S 5       rSrSrSrSr	Sr
SrS	\S
\S\4S jrSrg)r      c           	          S[         R                  " S5      SS04[         R                  " S5      SS04SSS	S
SS.4S.0$ )Nrequiredtext_encoderstooltipz+The name of the text encoder model to load.checkpointsz#The name of the ltxv model to load.INTr^      i      defaultminr   step)
gemma_pathr   r7   )folder_pathsget_filename_listss    r$   INPUT_TYPES$LTXVGemmaCLIPModelLoader.INPUT_TYPES   sf      22?C MN
 !22=A EF
  $RJ
 	
r2   CLIP)clip
load_modellightricks/LTXVzLTXV Gemma CLIP LoaderFr   r   r7   c           
         [        [        R                  " SU5      5      nUR                  S   n[        [	        US5      5      n[        [	        US5      5      n[        [	        US5      5      n[        XcS9n	S n
 [        R                  " [        U5      SS9n[        UU	" 5       R                  S	9n
[        R                  S
U S35        [        R                   n[        R                  " SU5      n["        R$                  R'                  U	[)        X~XS9S9n["        R*                  R-                  U5      4$ ! [         a%  n[        R                  SU SU 35         S nANS nAff = f)Nr   r   ztokenizer.modelzmodel*.safetensorszpreprocessor_config.jsonr   Tr9   image_processorr=   zLoaded processor from z - enhancement enabledzCould not load processor from r   r   )rl   ro   )r=   r   )r   r   get_full_pathparentsr   r   r	   r<   rc   r   r=   r    infor   r!   r   r   rv   supported_models_base
ClipTargetr   r   r   )rA   r   r   r7   path
model_rootr6   gemma_model_pathprocessor_pathtokenizer_classrl   r   r#   
clip_dtypeltxv_full_pathclip_targets                   r$   r   #LTXVGemmaCLIPModelLoader.load_model   sU   L..
KL\\!_
/
<MNO 1*>R ST/
<VWX.~U		O0@@N#!%O ( /)+55I KK0<RST ^^
%33M9M11<<%  I = 
 k*,,  	ONN;J<r!MNN	Os   ;AD8 8
E'E""E'rg   N)r_   r`   ra   rb   classmethodr   RETURN_TYPESRETURN_NAMESFUNCTIONCATEGORYTITLEOUTPUT_NODErc   rd   r   rf   rg   r2   r$   r   r      sN    
 
$ LLH H$EK-S -S -c -r2   u   ‘’“”—– ′−z	''""-- '-c                     U R                  [        5      n [        U 5       H   u  pUR                  5       (       d  M  XS  s  $    U $ r~   )	translate_UNICODE_REPLACEMENTSrN   isalpha)rD   rR   chars      r$   clean_responser    s>    >>/0D T?<<>>8O # Kr2   LTXVGemmaEnhancePromptzGemma 3 Prompt Enhancerc                       \ rS rSrSr\S 5       rSrSrSr	Sr
SrS	rS
r  SS\S\S\S\S\\R&                     S\4S jjrSrg)r  i%  z@Enhance prompts using Gemma 3 model. Supports T2V and I2V modes.c           	      \    SSSSS.4SS[         S.4SSSS	S
S.4SSS04S.SSSSSS.4S.S.$ )Nr   STRINGTr   )	multiliner   r          r^   r   r   BOOLEANr   F)r   promptsystem_prompt
max_tokens
bypass_i2v)IMAGE*   r   l    )r   r   r   )imageseed)r   optional)DEFAULT_T2V_SYSTEM_PROMPTr   s    r$   r   "LTXVGemmaEnhancePrompt.INPUT_TYPES)  sw     "#4B%GH%)#<"  #BtRH  )9e*<=" $ "1Z@#
 	
r2   )r  )enhanced_promptenhancer   zLTXV Gemma Enhance PromptTzFEnhance text prompts using Gemma 3 VLLM for improved video generation.Nr  r  r  r  r  r  c           
         [        U[        5      (       d  SnUR                  5         UR                  n[	        US5      (       a  UR
                  c*  [	        US5      (       a  [        U5      u  pO8[        S5      e[        U[        5      (       a  UR                  n	UR
                  n
US L=(       a    U(       + nU(       aA  UR                  5       [        R                  5       :X  a  [        n[        R                  S5        U(       a  UR                  5       (       d  [        S5      eU(       a  [        U5      n[!        W
W	UUUUUS9nO[#        W
W	UUUUS9n[%        U5      nU4$ )	Nr  rl   
gemma3_12bzProcessor not loaded - enhancement not available. Ensure your model directory has chat_template.json, processor_config.json, and preprocessor_config.json files.z7Auto-selected I2V system prompt for image-to-video modez@system_prompt is required and cannot be empty or whitespace-only)rl   rk   r  r  r  max_new_tokensr  )rl   rk   r  r  r&  r  )
isinstancerd   r   cond_stage_modelhasattrrl    transformers_gemma3_from_encoder
ValueErrorri   rk   r   r   DEFAULT_I2V_SYSTEM_PROMPTr    r   r1   enhance_i2venhance_t2vr  )rA   r   r  r  r  r  r  r  encoderrk   rl   use_i2v	pil_imager"  s                 r$   r#  LTXVGemmaEnhancePrompt.enhanceO  sY    $$$D''w,,0A0A0Iw--#CG#L y : 
 !:;;MME))It#6J }**,0I0O0O0QQ5MKKQRM$7$7$9$9R  %e,I)#+)O *#+)O )9!!r2   rg   )Nr  )r_   r`   ra   rb   __doc__r  r   r  r  r  r  r  r  DESCRIPTIONrc   rd   re   r   r   Tensorr#  rf   rg   r2   r$   r  r  %  s    J
 
6 L'LH H'EKP  )-@" @" 	@"
 @" @" %@" @" @"r2   rl   rk   messagesr  r&  r  c                 X   U c  [        S5      eU R                  R                  USSS9nU " UUSS9R                  UR                  5      nU R                  R
                  b  U R                  R
                  OSn[        Xx5      n[        R                  " 5          [        R                  R                  UR                  /S	9   [        R                  " UR                  R                  UR                  S
9   [        R                  " U5        UR                  " S0 UDUSSS.D6n	U	S   [!        UR"                  S   5      S n
U R                  R%                  U
SS9nSSS5        SSS5        SSS5        W$ ! , (       d  f       N = f! , (       d  f       N)= f! , (       d  f       W$ = f)z4Common enhancement logic for both T2V and I2V modes.Nz0Processor not loaded - enhancement not availableFT)tokenizeadd_generation_promptrG   )rD   imagesrJ   r   )devices)device_typero   gffffff?)r&  	do_sampletemperature)skip_special_tokensrg   )r+  r=   apply_chat_templaterq   rz   pad_token_id#_pad_inputs_for_attention_alignmentr   inference_moderandomfork_rngautocasttypero   manual_seedgeneratelenrL   decode)rl   rk   r6  r  r&  r  rD   model_inputsrA  r   generated_idsr"  s               r$   _enhancerN    s    KLL225 3 D  	b	  ++7 	(( 
 7|RL 	u||n55<<#4#4EKKH$.. 

)	
  
3|'='=a'@#A#CD#--44t 5 
 	I 	6 	   	IH 	65 	  s=   )F4F	7A'E8F	&F8
FF		
F	F
F)r  r  c                 4    SUS.SSU S3S./n[        XXdUS9$ )z)Enhance a text prompt for T2V generation.systemrolecontentuserUser Raw Input Prompt: .)r&  r  rN  )rl   rk   r  r  r&  r  r6  s          r$   r.  r.    s;     m4&=fXQ$GHH ( r2   c           	      H    SUS.SSS0SSU S3S	./S./n[        U UUUUUS
9$ )zAEnhance a text prompt for I2V generation using a reference image.rP  rQ  rT  rG  r  rD   rU  rV  )rG  rD   )r  r&  r  rW  )rl   rk   r  r  r  r&  r  r6  s           r$   r-  r-    s[     m4!+B6(!)LM	
	H % r2   padding_lengthvaluec           
          [         R                  " U [         R                  " SU4UU R                  U R                  S9/SS9$ )z>Concatenate a tensor with a padding tensor of the given value.r   ro   rz   r   )r   catfullro   rz   )r&   rY  rZ  s      r$   _cat_with_paddingr_    sG     99JJN#ll}}		
  r2   	alignmentc                 
   U R                   R                  S   nX2-   S-
  U-  U-  nXC-
  nUS:  aR  [        U R                   XQ5      U S'   [        U R                  US5      U S'   SU ;   a  U S   b  [        U S   US5      U S'   U $ )a  Pad sequence length to multiple of alignment for Flash Attention compatibility.

Flash Attention within SDPA requires sequence lengths aligned to 8 bytes.
This pads input_ids, attention_mask, and token_type_ids (if present) to prevent
'p.attn_bias_ptr is not correctly aligned' errors.
r   r   rL   rM   token_type_ids)rL   r   r_  rM   )rL  rA  r`  seq_len
padded_lenrY  s         r$   rB  rB    s     $$**1-G&*y8IEJ)N$5""N%
[! *;''*
%&
 ,-.:->-..L)* r2   c                     [         R                  " US 5      nUc  g U R                  5        H!  nUR                  R                  U:X  d  M  Us  $    g r~   )r   getmodulesr{   r_   )super_model
model_name
class_namemodules       r$   _locate_model_within_modelrl  &  sN    $((T:J%%'$$
2M ( r2   rootleaf_param_namemust_have_in_pathc                 Z   [        U R                  5       5      n/ nU R                  SS9 Hq  u  pVUR                  S5      nUS   nX:w  a  M"  Ub  X';  a  M,  SR	                  US S 5      n	UR                  XS:X  a  U OS 5      n
U
c  M^  UR                  XXe45        Ms     U(       d  g US   $ )NT)recurserV  r   r   r   )dictnamed_modulesnamed_parameterssplitjoinrf  append)rm  rn  ro  rg  
candidates	full_namer   partsleaf
owner_pathowners              r$   &_locate_unique_parameter_owner_by_leafr~  0  s     4%%'(GMOJ--d-;	$Ry"(->-KXXeCRj)
Jb0@dK=556 < a=r2   c           	         [        [        5      R                  S-  n[        R                  " US-  5      n[
        R                  " S5         [        U5      nS S S 5        UR                  R                  n[        WU5      nUc  [        S5      eUR                  U R                  R                  R                  R!                  5       SSS9  UR"                  R                  n[        X65      nUc  [        S5      eUR$                  nUR                  U R                  R                  R$                  R!                  5       SSS9  UR&                  R                  U R                  R                  R&                  R!                  5       SSS9  UR                  n[)        US	UR*                  UR,                  -  5      n	UR.                  n
U R                  n[1        UR2                  R4                  S
   5      n[
        R6                  " U[
        R8                  SS9R;                  S
5      nUR2                  R=                  SU5        [
        R>                  " UR*                  S-  US9nUR@                  R=                  SU5        SU
[
        R6                  " S
U	S[
        RB                  S9RE                  U[
        RF                  S9U	-  -  -  nURH                  R=                  SU5        [J        URL                  S      " X+5      u  nnURN                  R=                  SU5        Sn[Q        USSS9nUc  [        S5      eUu  nn  nUR@                  RR                  n[U        UU[
        RV                  RY                  UUS95        URE                  U5        [[        USS9n[\        R^                  " [a        U5      SS9n[c        UU" 5       Rd                  S9nUU4$ ! , (       d  f       GN= f) Ngemma_configszgemma3cfg.jsonmetazWCan't locate text model while converting text encoder to Gemma3ForConditionalGenerationTF)assignr   zYCan't locate vision model while converting text encoder to Gemma3ForConditionalGenerationhead_dimr   r,   r\  position_idsg      ?r   embed_scaleg      ?   rn   )rz   ro   inv_freq	rope_typeweightlm_head)rn  ro  zTCan't locate lm_head while converting text encoder to Gemma3ForConditionalGeneration)requires_gradr^   r   r   r   )3r   r   r   r   from_json_filer   rz   r   text_config
model_typerl  r+  r   r%  transformerrk   
state_dictvision_configvision_modelmulti_modal_projectorgetattrhidden_sizenum_attention_headsrope_local_base_freqrJ  
embeddingsr  arangelong	unsqueezeregister_bufferr&   embed_tokensint64rq   floatrotary_emb_localr   rope_scaling
rotary_embr~  r  setattrnn	Parameterr   r	   r<   rc   r   r=   )r/  
jsons_pathconfig	metamodelt_model_namet_modelv_tower_namev_towerv_modelr+   baserz   positions_lengthr  r  local_rope_freqs
rope_freqsr[   lm_head_requires_grad
loc_resultlm_head_ownerlm_head_attrreal_wr   r   rl   s                             r$   r*  r*  L  s   h&&8J((6F)FGF	f	26:	 
%%00L(LAGe
 	
 &&,,779$u   ''22L(AGg
 	
 ""G&&33>>@  
 ##33&&<<GGI 4 
 F
&*f&8&8F<V<V&V
WC&&D^^F7--::1=><<

5il  &&~|D,,v1136vFK((DLLC%++699U[[ :  	
 ,,Z9IJ'(;(;K(HIMJ &&z:>!78yJ b
 	
 )3%M<A!!((F61FG
 LL*:$GO(88JO  '!#--I ia 
	s   O
O*)   )NN)Nr  r  )r  r  )r   r~   )?loggingr   pathlibr   typingr   r   r   comfy.model_managementrv   comfy.sdcomfy.supported_models_baser   r   PILr   transformersr	   r
   r   r   r    transformers.modeling_rope_utilsr   &transformers.models.auto.modeling_autor   nodes_registryr   text_embeddings_connectorsr   	getLoggerr_   r    rc   r%   r   r,  r5  r1   r4   r  Moduleri   r   r   r   r   	maketransr  r  r  listrd   rN  r.  r-  r  r_  rB  rl  r  r~  r*  rg   r2   r$   <module>r     s      ( (   "     A F & E			8	$# #  00MN /0MN (%,, (5;; (   FJ+ J+Z&:" "s "s "  +9OP:- :- Q:-z <m 
 )7PQi" i" Ri"` $(..). . EKK 	.
 . . 	.l )  	
   	0 )  ;;	
    	<LL ; \\	(s > (,
((//  } eEHHOOS%((*<*<cABC	8S r2   