
    
3j              
          S SK r S SKJr  S SKrS SKJr  S SKJs  Jr  SSK	J
r
Jr  SSKJrJr  SSKJrJr  SSKJr  SS	KJrJr  SS
KJrJrJr  SSKJr  SSKJr  SSKJ r J!r!J"r"  SSK#J$r$  SSK%J&r&  SSK'J(r(  \RR                  " \*5      r+S r,S r-S\R\                  S\R\                  4S jr/SSS\R\                  S\R\                  4S jr0 " S S\Rb                  5      r2 " S S5      r3 " S S\R
                  Rb                  \5      r4 " S S \Rb                  5      r5 " S! S"\Rb                  5      r6\ " S# S$\Rb                  5      5       r7 " S% S&\&\
\\\\5      r8g)'    N)Any   )ConfigMixinregister_to_config)FromOriginalModelMixinPeftAdapterMixin)apply_lora_scalelogging)maybe_allow_in_graph   )ContextParallelInputContextParallelOutput)AttentionMixinAttentionModuleMixinFeedForward)dispatch_attention_fn)
CacheMixin)PixArtAlphaTextProjectionTimestepEmbedding	Timesteps)Transformer2DModelOutput)
ModelMixin)FP32LayerNormc           	          U R                   u  p#pEnUu  pxn	XtU-  -
  U-  n
XU-  -
  U-  nXU	-  -
  U	-  n[        R                  R                  R	                  U SUSUSU
4SS9$ )Nr   	replicate)mode)shapetorchnn
functionalpad)xkernel_sizebcthwptphpwpad_tpad_hpad_ws                j/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/transformers/transformer_helios.pypad_for_3d_convr0   '   sy    GGMA!JBBr6]b Er6]b Er6]b E88""1q%E1e&D;"WW    c                 R    [         R                  R                  R                  XUS9$ )N)stride)r   r   r    
avg_pool3d)r"   r#   s     r/   center_down_sample_3dr5   0   s"    88))!)MMr1   hidden_states	freqs_cisc                 \   U R                  SS5      R                  S5      u  p#UR                  S5      R                  SSS9u  pE[        R
                  " U 5      nX$SSS S24   -  X5SSS S24   -  -
  USSS S24'   X%SSS S24   -  X4SSS S24   -  -   USSS S24'   UR                  U 5      $ )	N)r9   r   r   dim.r      )	unflattenunbind	unsqueezechunkr   
empty_liketype_as)r6   r7   x_1x_2cossinouts          r/   apply_rotary_emb_transposedrI   4   s     &&r73::2>HC""2&,,QB,7HC


=
)CsADqDy>)Cc14a4i.,@@CQTT	NsADqDy>)Cc14a4i.,@@CQTT	N;;}%%r1   attnHeliosAttentionencoder_hidden_statesc                 j   Uc  UnU R                   (       ag  U R                  (       d#  U R                  U5      R                  SSS9u  p4nOfU R	                  U5      nU R                  U5      R                  SSS9u  pEO3U R	                  U5      nU R                  U5      nU R                  U5      nX4U4$ )Nr   r9   r;   r   )fused_projectionsis_cross_attentionto_qkvrA   to_qto_kvto_kto_v)rJ   r6   rL   querykeyvalues         r/   _get_qkv_projectionsrX   @   s    $ -&& $M : @ @ @ KE IIm,E$9:@@@KJC		-(ii-.		/0ur1   c                   z   ^  \ rS rSrSS\S\S\4U 4S jjjrS\R                  S\R                  S\4S	 jr
S
rU =r$ )HeliosOutputNormT   r<   epselementwise_affinec                    > [         TU ]  5         [        R                  " [        R
                  " SSU5      US-  -  5      U l        [        XSS9U l        g )Nr=   r         ?Fr]   )	super__init__r   	Parameterr   randnscale_shift_tabler   norm)selfr<   r\   r]   	__class__s       r/   rb   HeliosOutputNorm.__init__U   sC    !#ekk!Q.DsCx.O!P!#uE	r1   r6   temboriginal_context_lengthc                    US S 2U* S 2S S 24   nU R                   R                  S5      R                  UR                  5      UR                  S5      -   R	                  SSS9u  pEUR                  S5      R                  UR                  5      UR                  S5      R                  UR                  5      pTUS S 2U* S 2S S 24   nU R                  UR                  5       5      SU-   -  U-   R                  U5      nU$ )Nr   r   r;   r=   )	re   r@   todevicerA   squeezerf   floatrC   )rg   r6   rj   rk   shiftscales         r/   forwardHeliosOutputNorm.forwardZ   s    A//0!34..88;>>t{{Kdnn]^N__ffghnofp}}Q'**=+?+?@%--PQBRBUBUVcVjVjBku%a*A)A)BA&EF=#6#6#89QYG%OXXYfgr1   )rf   re   )ư>F)__name__
__module____qualname____firstlineno__intrp   boolrb   r   Tensorrs   __static_attributes____classcell__rh   s   @r/   rZ   rZ   T   sO    FC Fe F F F
U\\  `c  r1   rZ   c                       \ rS rSrSrSrS r    SSSS\R                  S\R                  S-  S\R                  S-  S	\	\R                  \R                  4   S-  S
\
S\R                  4S jjrSrg)HeliosAttnProcessorc   Nc                 D    [        [        S5      (       d  [        S5      eg )Nscaled_dot_product_attentionzeHeliosAttnProcessor requires PyTorch 2.0. To use it, please upgrade PyTorch to version 2.0 or higher.)hasattrFImportErrorrg   s    r/   rb   HeliosAttnProcessor.__init__g   s%    q899w  :r1   rJ   rK   r6   rL   attention_mask
rotary_embrk   returnc                    [        XU5      u  pxn	UR                  U5      nUR                  U5      nUR                  SUR                  S45      nUR                  SUR                  S45      nU	R                  SUR                  S45      n	Ub  [        Xu5      n[        X5      nUR                  (       d  UR                  (       a  UR                  S   U-
  n
U
S:  a  S[        R                  " UR                  5      UR                  S-
  -  -   nUR                  S:X  a  UR                  SSSS5      n[        R                  " US S 2S U
24   U-  US S 2U
S 24   /SS9n[!        UUU	USS	U R"                  Uc  U R$                  OS S
9nUR'                  SS5      nUR)                  U5      nUR*                  S   " U5      nUR*                  S   " U5      nU$ )Nr   r9   r=   r         ?per_headr;           F)	attn_mask	dropout_p	is_causalbackendparallel_configr   )rX   norm_qnorm_kr>   headsrI   rO   is_amplify_historyr   r   sigmoidhistory_key_scale	max_scalehistory_scale_modeviewcatr   _attention_backend_parallel_configflattenrC   to_out)rg   rJ   r6   rL   r   r   rk   rU   rV   rW   history_seq_len	scale_keys               r/   __call__HeliosAttnProcessor.__call__m   s    1F[\EE"kk#DJJ#34mmA

B/0DJJ#34!/BE-c>C&&4+B+B+11!47NNO"%--0F0F"G4>>\_K_"``	**j8 )q!R ;IiiQ(8(8%8!9I!Es1oN^K^G_ `fgh-$++6K6ST22Y]

 &--a3%--e4A}5A}5r1    NNNN)rv   rw   rx   ry   r   r   rb   r   r|   tuplerz   r   r}   r   r1   r/   r   r   c   s     6:.2?C'+// ||/  %||d2	/
 t+/ %,,45</ "%/ 
/ /r1   r   c                   l  ^  \ rS rSr\r\/r          SS\S\S\S\S\S\S-  S	\S-  4U 4S
 jjjr	S r
\R                  " 5       S 5       r    SS\R                  S\R                  S-  S\R                  S-  S\\R                  \R                  4   S-  S\S\R                  4S jjrSrU =r$ )rK      Nr<   r   dim_headr\   dropoutadded_kv_proj_dimcross_attention_dim_headc                   > [         TU ]  5         X2-  U l        X l        X`l        Xpl        Uc  U R                  OXr-  U l        [        R                  R                  XR                  SS9U l
        [        R                  R                  XR                  SS9U l        [        R                  R                  XR                  SS9U l        [        R                  R                  [        R                  R                  U R                  USS9[        R                  R                  U5      /5      U l        [        R                  R!                  X2-  USS9U l        [        R                  R!                  X2-  USS9U l        S =U l        U l        Ub  [        R                  R                  X`R                  SS9U l        [        R                  R                  X`R                  SS9U l        [        R                  R!                  X2-  US9U l        U	b  Xl        O	US LU l        U R/                  U5        Xl        U
(       a  US:X  a0  [        R2                  " [        R4                  " S5      5      U l        ODUS:X  a0  [        R2                  " [        R4                  " U5      5      U l        O[9        SU 35      eXl        S	U l        g g )
NTbias)r\   r]   )r\   scalarr=   r   zUnknown history_scale_mode: g      $@)ra   rb   	inner_dimr   r   r   kv_inner_dimr   r   LinearrQ   rS   rT   
ModuleListDropoutr   RMSNormr   r   
add_k_proj
add_v_projnorm_added_krO   set_processorr   rc   onesr   
ValueErrorr   r   )rg   r<   r   r   r\   r   r   r   	processorrO   r   r   rh   s               r/   rb   HeliosAttention.__init__   s.    	!)
!2(@%.F.NDNNTlTtHHOOCdOC	HHOOC):):OF	HHOOC):):OF	hh))$?  )
 hh&&x'7SUY&Zhh&&x'7SUY&Z,00$/(#hhoo.?VZo[DO#hhoo.?VZo[DO % 0 01As 0 KD)&8#&>d&JD#9%"4!X-)+ejjm)D&#z1)+ejj6G)H& #?@R?S!TUU&8#!DN r1   c                    [        U SS5      (       a  g U R                  (       GdL  [        R                  " U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  /5      n[        R                  " U R                  R                  R                  U R                  R                  R                  U R                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R                  R                  XS.SSS9  GO[        R                  " U R                  R
                  R                  U R                  R
                  R                  /5      n[        R                  " U R                  R                  R                  U R                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R                   R                  XS.SSS9  U R"                  Gb  [        R                  " U R$                  R
                  R                  U R&                  R
                  R                  /5      n[        R                  " U R$                  R                  R                  U R&                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R(                  R                  XS.SSS9  SU l        g ! , (       d  f       GN\= f! , (       d  f       GN`= f! , (       d  f       NW= f)NrN   FmetaTr   )weightr   )strictassign)getattrrO   r   r   rQ   r   datarS   rT   r   r   rn   r   r   rP   load_state_dictrR   r   r   r   to_added_kvrN   )rg   concatenated_weightsconcatenated_biasout_featuresin_featuress        r/   fuse_projections HeliosAttention.fuse_projections   s   4,e44&&&#(99dii.>.>.C.CTYYEUEUEZEZ\`\e\e\l\l\q\q-r#s  %		499>>+>+>		@S@SUYU^U^UcUcUhUh*i j(<(B(B%Lf% iiM &KK''/KTXae (  $)99dii.>.>.C.CTYYEUEUEZEZ-[#\  %		499>>+>+>		@S@S*T U(<(B(B%Lf%YY{tL
 &JJ&&/KTXae '  !!-#(99doo.D.D.I.I4??KaKaKfKf-g#h  %		4??+?+?+D+DdooFZFZF_F_*` a(<(B(B%Lf%#%99[T#R  &,,/KTXae -  "&3 &% &% &%s$   0N >N2O 
N/2
O
Oc                     [        U SS5      (       d  g [        U S5      (       a  [        U S5        [        U S5      (       a  [        U S5        [        U S5      (       a  [        U S5        SU l        g )NrN   FrP   rR   r   )r   r   delattrrN   r   s    r/   unfuse_projections"HeliosAttention.unfuse_projections   sb    t0%884""D(#4!!D'"4''D-(!&r1   r6   rL   r   r   rk   r   c                 2    U R                   " U UUUUU40 UD6$ N)r   )rg   r6   rL   r   r   rk   kwargss          r/   rs   HeliosAttention.forward  s1     ~~!#
 
 	
r1   )r   r   r   r   rN   r   r   r   r   r   rO   r   r   r   r   r   r   rS   rR   r   rQ   rP   rT   )
   @   gh㈵>r   NNNNFr   r   )rv   rw   rx   ry   r   _default_processor_cls_available_processorsrz   rp   rb   r   r   no_gradr   r|   r   rs   r}   r~   r   s   @r/   rK   rK      s,   001
 (,/3 %8"8" 8" 	8"
 8" 8" :8" #&*8" 8"t!&F ]]_' '  6:.2?C'+
||
  %||d2
 t+	

 %,,45<
 "%
 

 
r1   c                      ^  \ rS rSrS\S\S\S\4U 4S jjr  SS\R                  S	\R                  S-  S
\4S jjr	Sr
U =r$ )HeliosTimeTextEmbeddingi"  r<   time_freq_dimtime_proj_dimtext_embed_dimc                    > [         TU ]  5         [        USSS9U l        [	        X!S9U l        [        R                  " 5       U l        [        R                  " X5      U l
        [        XASS9U l        g )NTr   )num_channelsflip_sin_to_cosdownscale_freq_shift)in_channelstime_embed_dim	gelu_tanh)act_fn)ra   rb   r   timesteps_projr   time_embedderr   SiLUr   r   	time_projr   text_embedder)rg   r<   r   r   r   rh   s        r/   rb    HeliosTimeTextEmbedding.__init__#  s]     	']TXopq.=]ggi366~S^_r1   NtimesteprL   is_return_encoder_hidden_statesc                    U R                  U5      n[        [        U R                  R	                  5       5      5      R
                  nUR
                  U:w  a%  U[        R                  :w  a  UR                  U5      nU R                  U5      R                  U5      nU R                  U R                  U5      5      nUb  U(       a  U R                  U5      nXVU4$ r   )r   nextiterr   
parametersdtyper   int8rm   rC   r   r   r   )rg   r   rL   r   time_embedder_dtyperj   timestep_projs          r/   rs   HeliosTimeTextEmbedding.forward2  s     &&x0"4(:(:(E(E(G#HIOO>>005HEJJ5V{{#67H!!(+334IJt{{4'89 ,1P$($6$67L$M!$999r1   )r   r   r   r   r   )NT)rv   rw   rx   ry   rz   rb   r   r|   r{   rs   r}   r~   r   s   @r/   r   r   "  sn    `` ` 	`
 `$ 6:04	:,,:  %||d2: *.	: :r1   r   c                      ^  \ rS rSrU 4S jrS r\R                  " 5       S 5       r\R                  " 5       S 5       r	\R                  " 5       S 5       r
SrU =r$ )HeliosRotaryPosEmbediF  c                 T  > [         TU ]  5         Uu  U l        U l        U l        X l        U R                  SU R                  U R                  5      SS9  U R                  SU R                  U R                  5      SS9  U R                  SU R                  U R                  5      SS9  g )Nfreqs_base_tF)
persistentfreqs_base_yfreqs_base_x)ra   rb   DTDYDXthetaregister_buffer_get_freqs_base)rg   rope_dimr  rh   s      r/   rb   HeliosRotaryPosEmbed.__init__G  s    $,!$'
^T-A-A$''-JW\]^T-A-A$''-JW\]^T-A-A$''-JW\]r1   c           	      ~    SU R                   [        R                  " SUS[        R                  S9S US-   U-  -  -  $ )Nr   r   r   )r   )r  r   arangefloat32)rg   r<   s     r/   r	  $HeliosRotaryPosEmbed._get_freqs_baseO  s>    djjU\\!S!5==%QR^UX\]U]%_be%efggr1   c                     [         R                  " SX5      nUR                  SSS9nUR                  5       UR	                  5       4$ )Nzd,bthw->dbthwr   r   r;   )r   einsumrepeat_interleaverF   rG   )rg   
freqs_baseposfreqss       r/   get_frequency_batched*HeliosRotaryPosEmbed.get_frequency_batchedR  s>    _j>''q'1yy{EIIK''r1   c                     [         R                  " U5      n[         R                  " X[         R                  S9n[         R                  " X$[         R                  S9n[         R                  " XVSS9u  pxXx4$ )Nrn   r   ij)indexing)r   rn   r  r  meshgrid)	rg   heightwidth
device_strrn   grid_y_coordsgrid_x_coordsgrid_ygrid_xs	            r/   _get_spatial_meshgrid*HeliosRotaryPosEmbed._get_spatial_meshgridX  sR    j)V%--PUOtT~r1   c                 x   UR                   S   nUR                   S   nUR                  U[        R                  S9nU R	                  X#[        U5      5      u  pxUS S 2S S 2S S 4   R                  XVX#5      n	US S S S 2S S 24   R                  XVSS5      n
US S S S 2S S 24   R                  XVSS5      nU R                  U R                  U	5      u  pU R                  U R                  U
5      u  pU R                  U R                  U5      u  nn[        R                  " XUXU/SS9nUR                  SSSSS5      $ )	Nr   r=   r  r9   r;   r   r      )r   rm   r   r  r$  strexpandr  r   r  r  r   permute)rg   frame_indicesr  r  rn   
batch_size
num_framesr"  r#  grid_tgrid_y_batchgrid_x_batchfreqs_cos_tfreqs_sin_tfreqs_cos_yfreqs_sin_yfreqs_cos_xfreqs_sin_xresults                      r/   rs   HeliosRotaryPosEmbed.forward`  sB   "((+
"((+
%((emm(L33F3v;Oq!T4/077
PV^dD!Q./66zrSUVdD!Q./66zrSUV#'#=#=d>O>OQW#X #'#=#=d>O>OQ]#^ #'#=#=d>O>OQ]#^ [Kk;]hiopq~~aAq!,,r1   )r  r  r  r  )rv   rw   rx   ry   rb   r	  r   r   r  r$  rs   r}   r~   r   s   @r/   r   r   F  s]    ^h ]]_( (
 ]]_  ]]_- -r1   r   c                      ^  \ rS rSr       SS\S\S\S\S\S\S	\S-  S
\S\S\4U 4S jjjr SS\	R                  S\	R                  S\	R                  S\	R                  S\S\	R                  4S jjrSrU =r$ )HeliosTransformerBlockiu  Nr<   ffn_dim	num_headsqk_normcross_attn_normr\   r   guidance_cross_attnr   r   c                   > [         TU ]  5         [        XSS9U l        [	        UUX-  US [        5       U	U
S9U l        [	        UUX-  UUX-  [        5       S9U l        U(       a
  [        XSS9O[        R                  " 5       U l
        [        XSS9U l        [        XSS9U l        [        R                  " [        R                   " SS	U5      US
-  -  5      U l        Xl        g )NFr`   )r<   r   r   r\   r   r   r   r   )r<   r   r   r\   r   r   r   Tzgelu-approximate)r   activation_fnr=      r_   )ra   rb   r   norm1rK   r   attn1attn2r   Identitynorm2r   ffnnorm3rc   r   rd   re   r?  )rg   r<   r;  r<  r=  r>  r\   r   r?  r   r   rh   s              r/   rb   HeliosTransformerBlock.__init__w  s     	 #3F
$%%))+11	

 %%/%(%5)+

 JY]3E^`^i^i^k
 sEWX"3F
!#ekk!Q.DsCx.O!P $7 r1   r6   rL   rj   r   rk   r   c                 f   UR                   S:X  a  U R                  R                  S5      UR                  5       -   R	                  SSS9u  pgppUR                  S5      nUR                  S5      nUR                  S5      nU	R                  S5      n	U
R                  S5      n
UR                  S5      nO/U R                  UR                  5       -   R	                  SSS9u  pgppU R                  UR                  5       5      SU-   -  U-   R                  U5      nU R                  US S UU5      nUR                  5       X-  -   R                  U5      nU R                  (       a  UR                  S   U-
  n[        R                  " XU/SS9u  pU R                  UR                  5       5      R                  U5      nU R                  UUS S U5      nX-   n[        R                  " X/SS9nOGU R                  UR                  5       5      R                  U5      nU R                  UUS S U5      nX-   nU R!                  UR                  5       5      SU
-   -  U	-   R                  U5      nU R#                  U5      nUR                  5       UR                  5       U-  -   R                  U5      nU$ )Nr'  r   rB  r   r;   r=   )ndimre   r@   rp   rA   ro   rC  rC   rD  r?  r   r   splitrG  rE  r   rI  rH  )rg   r6   rL   rj   r   rk   	shift_msa	scale_msagate_msac_shift_msac_scale_msa
c_gate_msanorm_hidden_statesattn_outputr   history_hidden_states	ff_outputs                    r/   rs   HeliosTransformerBlock.forward  s    99>&&003djjlBeA1eo QI( "))!,I!))!,I''*H%--a0K%--a0K#++A.J &&5eA1eo QI(
 #jj)<)<)>?1y=QT]]ffgtujj#
 ',,.1GGPPQ^_ ##+11!47NNO38;;1HIq40! "&M,?,?,A!B!J!J=!Y**"%'K *7M!II'<&LRSTM!%M,?,?,A!B!J!J=!Y**"%'K *7M #jj)<)<)>?1{?SVaajj
 HH/0	&,,.1BZ1OOXXYfgr1   )rD  rE  rH  r?  rC  rG  rI  re   )rms_norm_across_headsFru   NFFr   r   )rv   rw   rx   ry   rz   r(  r{   rp   rb   r   r|   rs   r}   r~   r   s   @r/   r:  r:  u  s     / %(,$)#(",/7/7 /7 	/7
 /7 /7 /7 :/7 "/7 !/7  /7 /7n (,F||F  %||F ll	F
 LLF "%F 
F Fr1   r:  c            ,       D  ^ \ rS rSrSrSr/ SQrSS/r/ SQrS/r	S/r
\" S	S
SS9\" S	S
SS9S.S\" S	S
SS90S\" S	S
SS90S.\" S5       V VVs0 s H  nSU S3[        S	S
S9_M     snnn E\" S5       V VVs0 s H  nSU S3[        S	S
S9_M     snnn E\" S5       V VVs0 s H  nSU S3[        S	S
S9_M     snnn Er\                    S4S\\S4   S\S\S\S\S\S\S\S \S!\S"\S-  S#\S$\S-  S%\\S4   S&\S'\S(\S)\S*\S+\S,S4*U4S- jjj5       r\" S.5               S5S\R0                  S/\R2                  S0\R0                  S1\S.\\\4   S-  S,\R0                  \\\R0                  4   -  4S2 jj5       rS3rU=r$ s  snnn f s  snnn f s  snnn f )6HeliosTransformer3DModeli  a?  
A Transformer model for video-like data used in the Helios model.

Args:
    patch_size (`tuple[int]`, defaults to `(1, 2, 2)`):
        3D patch dimensions for video embedding (t_patch, h_patch, w_patch).
    num_attention_heads (`int`, defaults to `40`):
        Fixed length for text embeddings.
    attention_head_dim (`int`, defaults to `128`):
        The number of channels in each head.
    in_channels (`int`, defaults to `16`):
        The number of channels in the input.
    out_channels (`int`, defaults to `16`):
        The number of channels in the output.
    text_dim (`int`, defaults to `512`):
        Input dimension for text embeddings.
    freq_dim (`int`, defaults to `256`):
        Dimension for sinusoidal time embeddings.
    ffn_dim (`int`, defaults to `13824`):
        Intermediate dimension in feed-forward network.
    num_layers (`int`, defaults to `40`):
        The number of layers of transformer blocks to use.
    window_size (`tuple[int]`, defaults to `(-1, -1)`):
        Window size for local attention (-1 indicates global attention).
    cross_attn_norm (`bool`, defaults to `True`):
        Enable cross-attention normalization.
    qk_norm (`bool`, defaults to `True`):
        Enable query/key normalization.
    eps (`float`, defaults to `1e-6`):
        Epsilon value for normalization layers.
    add_img_emb (`bool`, defaults to `False`):
        Whether to use img_emb.
    added_kv_proj_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the added key and value projections. If `None`, no projection is used.
T)patch_embeddingpatch_short	patch_mid
patch_longcondition_embedderrf   r:  rZ   )r   re   rC  rG  rI  r   norm_added_qr=   r   F)	split_dimexpected_dimssplit_output)r6   r   r6   )zblocks.*.attn1zblocks.*.attn2zblocks.*.ffn(   zblocks.z.attn1)
gather_dimrc  z.attn2z.ffnN
patch_size.num_attention_headsattention_head_dimr   out_channelstext_dimfreq_dimr;  
num_layersr>  r=  r\   r   r
  
rope_thetar?  zero_history_timestephas_multi_term_memory_patchr   r   r   c                 $  > [         TU ]  5         X#-  nU=(       d    Un[        XS9U l        [        R
                  " UUXS9U l        UU l        U(       a  [        R
                  " UUXS9U l        [        R
                  " UU[        S U 5       5      [        S U 5       5      S9U l
        [        R
                  " UU[        S U 5       5      [        S U 5       5      S9U l        [        UUUS-  US9U l        [        R                  " [        U	5       Vs/ s H  n[!        UUUUU
UUUUUS	9
PM     sn5      U l        [%        UUS
S9U l        [        R(                  " UU[*        R,                  " U5      -  5      U l        S
U l        g s  snf )N)r
  r  )r#   r3   c              3   ,   #    U  H
  nS U-  v   M     g7fr   Nr   .0ps     r/   	<genexpr>4HeliosTransformer3DModel.__init__.<locals>.<genexpr>f       !<A!a%   c              3   ,   #    U  H
  nS U-  v   M     g7frs  r   rt  s     r/   rw  rx  g       7JqQUJrz  c              3   ,   #    U  H
  nS U-  v   M     g7fr'  Nr   rt  s     r/   rw  rx  l  ry  rz  c              3   ,   #    U  H
  nS U-  v   M     g7fr~  r   rt  s     r/   rw  rx  m  r|  rz  rB  )r<   r   r   r   )r?  r   r   Fr`   )ra   rb   r   roper   Conv3dr\  ro  r]  r   r^  r_  r   r`  r   ranger:  blocksrZ   norm_outr   mathprodproj_outgradient_checkpointing)rg   rg  rh  ri  r   rj  rk  rl  r;  rm  r>  r=  r\   r   r
  rn  r?  ro  rp  r   r   r   _rh   s                          r/   rb   !HeliosTransformer3DModel.__init__>  s   0 	'<	#2{ )(M	!yyiZk &;"&!yyiZkDYY!!<!<<7J77	DN !ii!!<!<<7J77	DO #:"#a-#	#
 mm z* +A ''#%(;'9'9 +
& )CER		)\DIIj<Q-QR&+#+s   Fattention_kwargsr   rL   return_dictc           
      P   UR                   S   nU R                  R                  u  pnU R                  U5      nUR                   u    nnnnUc6  [        R
                  " SU5      R                  S5      R                  US5      nUR                  S5      R                  SS5      nU R                  UUUUR                  S9nUR                  S5      R                  SS5      nUR                   S   nUb  Ub  U R                  U5      nUR                   u      nnnUR                  S5      R                  SS5      nU R                  UUUUR                  S9nUR                  S5      R                  SS5      n[        R                  " X/SS9n[        R                  " UU/SS9nU	b  Ub  [        U	S5      n	U R                  U	5      n	U	R                  S5      R                  SS5      n	U R                  UWWU	R                  S9n[        US5      n[!        US5      nUR                  S5      R                  SS5      n[        R                  " X/SS9n[        R                  " UU/SS9nU
b  Ub  [        U
S	5      n
U R#                  U
5      n
U
R                  S5      R                  SS5      n
U R                  UWWU
R                  S9n[        US
5      n[!        US
5      nUR                  S5      R                  SS5      n[        R                  " X/SS9n[        R                  " UU/SS9nUR                   S   U-
  nUb  U R$                  (       a  [        R&                  " SUR(                  UR                  S9nU R+                  UUSS9u  nnnUR                  S5      R                  UUS5      nUR-                  SS5      R/                  SSSS5      R                  USUS5      nU R+                  X#5      u  n n!nU!R-                  SS5      n!Ub!  U R$                  (       d  UR                   S   n"OUn"U R/                  USS5      R                  UU"S5      n U!R/                  USSS5      R                  USU"S5      n!Ub?  U R$                  (       a.  [        R                  " WU /SS9n [        R                  " WU!/SS9n!U!R0                  S:X  a  U!R3                  SSSS5      n!UR5                  5       nUR5                  5       nUR5                  5       n[        R6                  " 5       (       a;  U R8                  (       a*  U R:                   H  n#U R=                  U#UUU!UU5      nM     OU R:                   H  n#U#" UUU!UU5      nM     U R?                  UU U5      nU RA                  U5      nURC                  UUUUXUS5      nUR3                  SSSSSSSS5      nUR                  SS5      R                  SS5      R                  SS5      n$U(       d  U$4$ [E        U$S9$ )Nr   r9   r   r=   )r+  r  r  rn   r;   )r   r'  r'  )r   r   r   )r'  r   r   )r'  r'  r'  )r   rn   F)r   )rB  r9   rB  r'  r         )sample)#r   configrg  r\  r   r  r@   r)  r   	transposer  rn   r]  r   r0   r^  r5   r_  ro  zerosr   r`  r>   r   rL  r*  
contiguousis_grad_enabledr  r  _gradient_checkpointing_funcr  r  reshaper   )%rg   r6   r   rL   indices_hidden_statesindices_latents_history_shortindices_latents_history_midindices_latents_history_longlatents_history_shortlatents_history_midlatents_history_longr  r  r,  p_tp_hp_wr  post_patch_num_framespost_patch_heightpost_patch_widthr   rk   H1W1rotary_emb_history_shortrotary_emb_history_midrotary_emb_history_longhistory_context_lengthtimestep_t0temb_t0timestep_proj_t0rj   r   main_repeat_sizeblockoutputs%                                        r/   rs    HeliosTransformer3DModel.forward  sv   $ #((+
..# ,,];KXK^K^H1#%68H ($)LL4I$J$T$TUV$W$^$^_ikm$n!%--a0::1a@YY/$" ''	  

  ''*44Q:
"/"5"5a"8 !,1N1Z$($4$45J$K!399OAq!R$9$A$A!$D$N$NqRS$T!'+yy;,33	 (1 ($ (@'G'G'J'T'TUVXY'Z$!II'<&LRSTM$<j#IqQJ */J/V"12Ey"Q"&..1D"E"5"="=a"@"J"J1a"P%)YY9*11	 &/ &" &55KY%W"%:;QS\%]"%;%C%CA%F%P%PQRTU%V"!II':&JPQRM$:J#GQOJ  +0L0X#23G#S #'??3G#H #7#?#?#B#L#LQPQ#R &*ii:+22	 '0 '# '66My&Y#&;<SU^&_#&=&E&Ea&H&R&RSTVW&X#!II';&KQRSM$;Z#HaPJ!.!4!4Q!7:Q!Q ,1K1K++qXK+/+B+B2TY ,C ,(G%q ''*11*>TVXYG **2w7aAr"
B(>C  6:5L5LX5m2m2%//G< ,T5O5O,22156yyQ+22:?OQST%**:q!R@GG
TUWgikl ,1K1K99gt_!4D!II'7&GQOM")11!Q1=M &002 5 @ @ B**,
  ""t'B'B $ A A!)!+! %  %!)!+! % mT;RSm4 &---/@BRTW^ace
 &--aAq!Q1E&&q!,44Q:BB1aH9'v66r1   )r  r`  r  r  r\  r_  r^  r]  r  r  ro  ))r=   r   r   re        r  i      i 6  re  TrY  ru   N),   *   r  g     @TTTFr   )	NNNNNNNTN)rv   rw   rx   ry   __doc__ _supports_gradient_checkpointing _skip_layerwise_casting_patterns_no_split_modules_keep_in_fp32_modules"_keys_to_ignore_on_load_unexpected_repeated_blocksr   r  r   _cp_planr   r   rz   r{   r(  rp   rb   r	   r   r|   
LongTensordictr   rs   r}   r~   )ru  ir   rh   s   000@r/   r[  r[    sT   "H (,$($ 23EF +9)9&01 2AQ]bc.!Z_`

 1AQ]bc
 1AQ]bc
 _ddf^g
h^gYZWQCv 5RS TT^g
h _ddf^g
h^gYZWQCv 5RS TT^g
h ]bbd\e
f\eWXWQCt3qPQRR\e
fH$  '0#%"% $5(,$0#$(&*,0#(",+P,#s(OP, !P,  	P,
 P, P, P, P, P, P, P, tP, P, :P, S/P,  !P," "#P,$  $%P,& &*'P,( !)P,*  +P,, 
-P, P,d () #&*$(%)" ! 26a7||a7 ""a7  %||	a7 a7 sCx.4/a7 
S%,,./	/a7 *a7s i
h
fs   F
8F
"F
r[  )9r  typingr   r   torch.nnr   torch.nn.functionalr    r   configuration_utilsr   r   loadersr   r   utilsr	   r
   utils.torch_utilsr   _modeling_parallelr   r   	attentionr   r   r   attention_dispatchr   cache_utilsr   
embeddingsr   r   r   modeling_outputsr   modeling_utilsr   normalizationr   
get_loggerrv   loggerr0   r5   r|   rI   rX   ModulerZ   r   rK   r   r   r:  r[  r   r1   r/   <module>r     s:         B ? . 5 L I I 6 $ P P 7 ' ) 
		H	%XN	&<<	&||	&0  fkfrfr (ryy 9 9x@
ehhoo'; @
F!:bii !:H,-299 ,-^ xRYY x xvB7-/EzSaB7r1   