
    
3j\                        S SK r S SKrS SKJr  S SKJs  Jr  SSKJrJ	r	  SSK
Jr  S\S\S\S\R                  4S	 jr " S
 S\R                   5      r " S S\R                   5      r " S S\R                   5      r " S S\R                   5      r " S S\R                   5      r " S S\\5      r " S S\R                   5      r " S S\R                   5      r " S S\\5      rg)    N   )ConfigMixinregister_to_config)
ModelMixincutoff
half_widthkernel_sizereturnc                    SU-  nUS-  nSUS-
  -  [         R                  -  U-  S-   nUS:  a	  SUS-
  -  nOUS	:  a  S
US-
  S-  -  SUS	-
  -  -   nOSn[        R                  " X&SS9nUS-  S:H  nU(       a  [        R                  " U* U5      S-   O[        R                  " U5      U-
  n	U S:X  a  [        R
                  " U	5      n
U
$ SU -  U	-  n	[        R                  " U	S:H  [        R                  " U	5      [        R                  " [         R                  U	-  5      [         R                  -  U	-  5      nSU -  U-  U-  n
XR                  5       -  n
U
$ )a  
Creates a Kaiser sinc kernel for low-pass filtering.

Args:
    cutoff (`float`):
        Normalized frequency cutoff (relative to the sampling rate). Must be between 0 and 0.5 (the Nyquist
        frequency).
    half_width (`float`):
        Used to determine the Kaiser window's beta parameter.
    kernel_size:
        Size of the Kaiser window (and ultimately the Kaiser sinc kernel).

Returns:
    `torch.Tensor` of shape `(kernel_size,)`:
        The Kaiser sinc kernel.
      gHzG@   g@g      I@gK46?gffffff!@g      5@gW2ı?   g?gUj@+0?g        F)betaperiodicr         ?)
mathpitorchkaiser_windowarange
zeros_likewhere	ones_likesinsum)r   r   r	   delta_f	half_size	amplituder   windoweventimefiltersincs               Z/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/pipelines/ltx2/vocoder.pykaiser_sinc_filter1dr&      s`   " *nGq IQ'$''1G;dBI4S)	d	RC//'Y=M2NN  %HF?aD8<5<<
I.4%,,{B[^gBgD}!!$' M 6zD {{AIOOD!IIdggn%/$6

 Vf$t+**,&M    c                      ^  \ rS rSrSr     SS\S\S-  S\S\S\4
U 4S	 jjjrS
\	R                  S\	R                  4S jrSrU =r$ )DownSample1d9   z.1D low-pass filter for antialias downsampling.Nratior	   use_paddingpadding_mode
persistentc                   > [         T	U ]  5         Xl        U=(       d    [        SU-  S-  5      S-  U l        U R                  S-  U R                  S-  -   S-
  U l        U R                  S-  U l        X0l        X@l        SU-  nSU-  n[        XgU R                  5      nU R                  SUR                  SSU R                  5      US9  g )N   r   r   r   333333?r#   r.   )super__init__r+   intr	   pad_left	pad_rightr,   r-   r&   register_bufferview)
selfr+   r	   r,   r-   r.   r   r   low_pass_filter	__class__s
            r%   r4   DownSample1d.__init__<   s     	
&A#a%i1n*=*A((A-1A1AA1EFJ))Q.&(u5[
.v4CSCSTX';';Aq$BRBR'S`jkr'   xr
   c                 &   UR                   S   nU R                  (       a5  [        R                  " XR                  U R
                  4U R                  S9n[        R                  " XR                  R                  USS5      U R                  US9nU$ )Nr   modestridegroups)shaper,   Fpadr6   r7   r-   conv1dr#   expandr+   )r:   r>   num_channels
x_filtereds       r%   forwardDownSample1d.forwardQ   sn    wwqza--8t?P?PQAXXa!3!3L"b!IRVR\R\eqr
r'   )r	   r6   r7   r-   r+   r,   )r   NT	replicateT)__name__
__module____qualname____firstlineno____doc__r5   boolstrr4   r   TensorrM   __static_attributes____classcell__r<   s   @r%   r)   r)   9   s|    8 "& 'll 4Zl 	l
 l l l* %,,  r'   r)   c                      ^  \ rS rSr     SS\S\S-  S\S\S\4
U 4S jjjrS	\R                  S
\R                  4S jr
SrU =r$ )
UpSample1dZ   Nr+   r	   window_typer-   r.   c                   > [         TU ]  5         Xl        X@l        US:X  a  SnSn[        R
                  " Xv-  5      nSU-  U-  S-   U l        Xl        SU-  U-  U l        U R                  U-
  U l	        [        R                  " U R                  5      U-  U-
  U-  n	U	R                  U* U5      n
[        R                  " U
[        R                  -  U-  S-  5      S-  n[        R                  " U	5      U-  U-  U-  R!                  SSS5      nOUc  [#        SU-  S-  5      S-  OUU l        U R                  U-  S-
  U l        U R                  U R                  -  U R                  U R                  -
  S-  -   U l        U R                  U R                  -  U R                  U R                  -
  S-   S-  -   U l	        [%        SU-  SU-  U R                  S	9nU R'                  S
UR!                  SSU R                  5      US9  g )NhanngGz?r0   r   r   rB   r   r1   )r   r   r	   r#   r2   )r3   r4   r+   r-   r   ceilr	   rH   r6   r7   r   r   clampcosr   r$   r9   r5   r&   r8   )r:   r+   r	   r^   r-   r.   rollofflowpass_filter_widthwidth	time_axistime_clampedr    sinc_filterr<   s                r%   r4   UpSample1d.__init__[   s    	
(& G#$ II2<=E 5y5014DHI-DM!--5DNd&6&67%?%G7RI$??,@+@BVWLYY|dgg58LLqPQUVVF ::i069GCeKQQRSUVXZ[K ;F:Ms1u9>2Q6S^D''5014DH HHtzz1T5E5E

5RWX4XXDM!XX

2d6F6F6SVW6W\]5]]DN.U{; ,,K 	X{'7'71d>N>N'O\fgr'   r>   r
   c                    UR                   S   n[        R                  " XR                  U R                  4U R                  S9nU R                  R                  UR                  UR                  S9R                  USS5      nU R                  [        R                  " XU R                  US9-  nUSU R                  U R                  * 24   $ )Nr   r@   )dtypedevicerB   rC   .)rF   rG   rH   r-   r#   torl   rm   rJ   r+   conv_transpose1dr6   r7   )r:   r>   rK   r;   s       r%   rM   UpSample1d.forward   s    wwqzEE!hh)0A0AB++..qwwqxx.HOOP\^`bdeJJ++AtzzZfggdmmt~~o5566r'   )r	   rH   r6   r7   r-   r+   )r   NkaiserrO   T)rP   rQ   rR   rS   r5   rV   rU   r4   r   rW   rM   rX   rY   rZ   s   @r%   r\   r\   Z   sz     "&#'&h&h 4Z&h 	&h
 &h &h &hP7 7%,, 7 7r'   r\   c                      ^  \ rS rSrSr  SS\\R                  -  S\S\4U 4S jjjr	S\
R                  S\
R                  4S	 jrS
rU =r$ )AntiAliasAct1d   z
Antialiasing activation for a 1D signal: upsamples, applies an activation (usually snakebeta), and then downsamples
to avoid aliasing.
act_fnr+   r	   c                   > [         TU ]  5         [        X#S9U l        [	        U[
        5      (       a:  US:X  a  [        S0 UD6nO(US:X  a  [        S0 UD6nO[        R                  " S0 UD6nXl	        [        X#S9U l        g )Nr+   r	   	snakebetasnake )r3   r4   r\   upsample
isinstancerV   	SnakeBetann	LeakyReLUactr)   
downsample)r:   ru   r+   r	   kwargsr<   s        r%   r4   AntiAliasAct1d.__init__   su     	"Hfc""$",V,7"",V,//&ULr'   r>   r
   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ N)r{   r   r   )r:   r>   s     r%   rM   AntiAliasAct1d.forward   s0    MM!HHQKOOAr'   )r   r   r{   )r      )rP   rQ   rR   rS   rT   rV   r~   Moduler5   r4   r   rW   rM   rX   rY   rZ   s   @r%   rs   rs      sc     	MbiiM M 	M M& %,,  r'   rs   c                      ^  \ rS rSrSr     SS\S\S\S\S\S\4U 4S	 jjjrSS
\	R                  S\S\	R                  4S jjrSrU =r$ )r}      z]
Implements the Snake and SnakeBeta activations, which help with learning periodic patterns.
channelsalphaepstrainable_paramslogscaleuse_betac                   > [         TU ]  5         X0l        XPl        X`l        [
        R                  " U R                  (       a  [        R                  " U5      O[        R                  " U5      U-  5      U l
        X@R                  l        U(       aj  [
        R                  " U R                  (       a  [        R                  " U5      O[        R                  " U5      U-  5      U l        X@R                  l        g g r   )r3   r4   r   r   r   r~   	Parameterr   zerosonesr   requires_gradr   )r:   r   r   r   r   r   r   r<   s          r%   r4   SnakeBeta.__init__   s     	  \\4==%++h"7ejjYaNbejNjk
#3

 dmmU[[%:QVQ[Q[\dQehmQmnDI&6II# r'   hidden_stateschannel_dimr
   c                    S/UR                   -  nSX2'   U R                  R                  U5      nU R                  (       a  U R                  R                  U5      nU R
                  (       a=  [        R                  " U5      nU R                  (       a  [        R                  " W5      nU R                  (       a  WOUnUSX`R                  -   -  [        R                  " X-  5      R                  S5      -  -   nU$ )Nr   rB         ?r   )ndimr   r9   r   r   r   r   expr   r   pow)r:   r   r   broadcast_shaper   r   r   s          r%   rM   SnakeBeta.forward   s    # 2 22')$

0==99>>/2D==IIe$E}}yy MMDu	%	HH0D)ES`ShIiImImnoIp(ppr'   )r   r   r   r   r   )r   g&.>TTT)r   )rP   rQ   rR   rS   rT   r5   floatrU   r4   r   rW   rM   rX   rY   rZ   s   @r%   r}   r}      s     !%77 7 	7
 7 7 7 7(U\\  ELL  r'   r}   c                      ^  \ rS rSr         SS\S\S\S\\S4   S\S\S	\S
\S\S\4U 4S jjjr	S\
R                  S\
R                  4S jrSrU =r$ )ResBlock   r   r	   rD   	dilations.ru   leaky_relu_negative_slope	antialiasantialias_ratioantialias_kernel_sizer-   c                   > [         TU ]  5         X@l        [        R                  " U Vs/ s H  n[        R
                  " XX#XS9PM     sn5      U l        [        R                  " 5       U l        [        [        U R                  5      5       He  nUS:X  a  [        USS9nO%US:X  a  [        USS9nO[        R                  " US9nU(       a
  [        XU	S9nU R                  R                  U5        Mg     [        R                  " [        [        U5      5       Vs/ s H  n[        R
                  " XX#S	U
S9PM     sn5      U l        [        R                  " 5       U l        [        [        U R                  5      5       He  nUS:X  a  [        USS9nO%US:X  a  [        USS9nO[        R                  " US9nU(       a
  [        WXS9nU R                  R                  W5        Mg     g s  snf s  snf )
N)rD   dilationpaddingrx   T)r   ry   Fnegative_sloperw   r   )r3   r4   r   r~   
ModuleListConv1dconvs1acts1rangelenr}   r   rs   appendconvs2acts2)r:   r   r	   rD   r   ru   r   r   r   r   r-   r   _r   r<   s                 r%   r4   ResBlock.__init__   s    	"mm !* )H 		(kS[r )
 ]]_
s4;;'(A$487"59ll2KL$SMbcJJc" ) mm s9~..A 		(kST^jk.
 ]]_
s4;;'(A$487"595NO$ScJJc" )3&s    G,!G1r>   r
   c                     [        U R                  U R                  U R                  U R                  5       H*  u  p#pEU" U5      nU" U5      nU" U5      nU" U5      nX-   nM,     U$ r   )zipr   r   r   r   )r:   r>   act1conv1act2conv2xts          r%   rM   ResBlock.forward  sb    (+DJJTZZQUQ\Q\(]$DaBrBbBrBA )^ r'   )r   r   r   r   r   )	r   r   r   r      
leaky_relu皙?Fr   r   same)rP   rQ   rR   rS   r5   tuplerV   r   rU   r4   r   rW   rM   rX   rY   rZ   s   @r%   r   r      s     %."+. %'"4#4# 4# 	4#
 c?4# 4# $)4# 4# 4#  #4# 4# 4#l %,,  r'   r   c                      ^  \ rS rSrSr\SSS/ SQ/ SQ/ SQ/ S	Q/ S	Q/ S	Q/S
SSSSSSS4S\S\S\S\\   S\\   S\\   S\\\      S\S\	S\
S\S\S\S-  S\
S \4U 4S! jjj5       rS'S"\R                  S#\
S$\R                  4S% jjrS&rU =r$ )(LTX2Vocoderi  zT
LTX 2.0 vocoder for converting generated mel spectrograms back to audio waveforms.
   i   r   )         r   r   r0   r   r   r   r   r         r   r   r   Fr   tanhTi]  in_channelshidden_channelsout_channelsupsample_kernel_sizesupsample_factorsresnet_kernel_sizesresnet_dilationsru   r   r   r   r   final_act_fnN
final_biasoutput_sampling_ratec                   > [         TU ]  5         [        U5      U l        [        U5      U l        X0l        [        R                  " U5      U l        Xl	        Xl
        Xl        U R                  [        U5      :w  a%  [        SU R                   S[        U5       S35      eU R                  [        U5      :w  a.  [        S[        U R                  5       S[        U5       S35      e/ SQnU R                  U;  a  [        SU R                   SU S35      e[        R                  " XS	S
SS9U l        [        R                   " 5       U l        [        R                   " 5       U l        Un['        [)        XT5      5       H  u  nu  nnUS-  nU R"                  R+                  [        R,                  " UUUUUU-
  S-  S95        [)        Xg5       H/  u  nnU R$                  R+                  [/        UUUUU	U
UUS95        M1     UnM     US:X  d  US:X  a  [1        WSS9n[3        UXS9U l        O US:X  a  [        R6                  " 5       U l        [        R                  " WUS	S
SUS9U l        g )Nza`upsample_kernel_sizes` and `upsample_factors` should be lists of the same length but are length z and z, respectively.z_`resnet_kernel_sizes` and `resnet_dilations` should be lists of the same length but are length )rx   ry   r   z!Unsupported activation function: z-. Currently supported values of `act_fn` are .r   r   r   )r	   rD   r   r   rD   r   )r   r	   r   ru   r   r   r   r   rx   ry   T)r   r   rw   r   )rD   r   bias)r3   r4   r   num_upsample_layersresnets_per_upsampler   r   prodtotal_upsample_factorru   r   r   
ValueErrorr~   r   conv_inr   
upsamplersresnets	enumerater   r   ConvTranspose1dr   r}   rs   act_outr   conv_out)r:   r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r   supported_act_fnsinput_channelsirD   r	   output_channelsr   r   r<   s                           r%   r4   LTX2Vocoder.__init__  sq   & 	#&'<#= $'(;$<!(%)YY/?%@"7(##s+;'<<,,-U37G3H2IZ 
 $$,<(==11235=M9N8O` 
 A;;//3DKK=@m$%Q( 
 yy1UV`ab--/}}((1#6F2^(_$A$,1OOO"""""#!(61a7 +..A*T&Y##!0$/"+%2K"+(7.C	 +U -N3 )`6 [ Fg$54HG)'lDL|#<<>DL		/<1VW^hir'   r   	time_lastr
   c           	         U(       d  UR                  SS5      nUR                  SS5      nU R                  U5      n[        U R                  5       H  nU R
                  S:X  a  [        R                  " XR                  S9nU R                  U   " U5      nX0R                  -  nUS-   U R                  -  n[        R                  " [        XE5       Vs/ s H  o`R                  U   " U5      PM     snSS9n[        R                  " USS9nM     U R                  U5      nU R!                  U5      nU R"                  S:X  a  [        R$                  " U5      nU$ U R"                  S	:X  a  [        R&                  " US
S5      nU$ s  snf )aB  
Forward pass of the vocoder.

Args:
    hidden_states (`torch.Tensor`):
        Input Mel spectrogram tensor of shape `(batch_size, num_channels, time, num_mel_bins)` if `time_last`
        is `False` (the default) or shape `(batch_size, num_channels, num_mel_bins, time)` if `time_last` is
        `True`.
    time_last (`bool`, *optional*, defaults to `False`):
        Whether the last dimension of the input is the time/frame dimension or the Mel bins dimension.

Returns:
    `torch.Tensor`:
        Audio waveform tensor of shape (batch_size, out_channels, audio_length)
r   r   r   r   r   r   dimr   rb   rB   )	transposeflattenr   r   r   ru   rG   r   r   r   r   r   stackr   meanr   r   r   r   rb   )r:   r   r   r   startendjresnet_outputss           r%   rM   LTX2Vocoder.forwardu  sY   $ )33Aq9M%--a3]3t//0A{{l* !]K^K^ _ OOA.}=M 111Eq5D555C"[[RWX]Rc)dRcQ,,q/-*HRc)djklN!JJ~1=M 1 ]3m4&!JJ}5M  ')!KKr1=M *es   F
)ru   r   r   r   r   r   r   r   r   r   r   r   )FrP   rQ   rR   rS   rT   r   r5   listrV   r   rU   r4   r   rW   rM   rX   rY   rZ   s   @r%   r   r     sH     #+<&5)3-6	9,M"+. %'#)$)!VjVj Vj 	Vj
  $CyVj s)Vj "#YVj tCy/Vj Vj $)Vj Vj Vj  #Vj DjVj Vj  "!Vj Vjp-U\\ -d -u|| - -r'   r   c                      ^  \ rS rSrSrSS\S\S\4U 4S jjjrS\R                  S\	\R                  \R                  4   4S	 jr
S
rU =r$ )
CausalSTFTi  a  
Performs a causal short-time Fourier transform (STFT) using causal Hann windows on a waveform. The DFT bases
multiplied by the Hann windows are pre-calculated and stored as buffers. For exact parity with training, the exact
buffers should be loaded from the checkpoint in bfloat16.
filter_length
hop_lengthwindow_lengthc                    > [         TU ]  5         X l        X0l        US-  S-   nU R	                  S[
        R                  " US-  SU5      SS9  U R	                  S[
        R                  " US-  SU5      SS9  g )Nr   r   forward_basisTr2   inverse_basis)r3   r4   r  r  r8   r   r   )r:   r  r  r  n_freqsr<   s        r%   r4   CausalSTFT.__init__  sv    $*1$q(_ekk'A+q-.Xeij_ekk'A+q-.Xeijr'   waveformr
   c                 4   UR                   S:X  a  UR                  S5      n[        SU R                  U R                  -
  5      n[
        R                  " XS45      n[
        R                  " XR                  U R                  SS9nUR                  S   S-  nUS S 2S U24   US S 2US 24   pe[        R                  " US-  US-  -   5      n[        R                  " UR                  5       UR                  5       5      R                  UR                  S9nXx4$ )Nr   r   r   r   )rl   )r   	unsqueezemaxr  r  rG   rH   rI   r  rF   r   sqrtatan2r   rn   rl   )	r:   r
  left_padspecr  realimag	magnitudephases	            r%   rM   CausalSTFT.forward  s    ==A))!,Hq$,,t>?55a=1xx"4"4T__VWX**Q-1$!XgX+&Q[(9dJJtQwq01	DJJL$**,7:::Lr'   )r  r  )   P   r  rP   rQ   rR   rS   rT   r5   r4   r   rW   r   rM   rX   rY   rZ   s   @r%   r  r    s[    kc kS kVY k k   u||U\\7Q1R    r'   r  c            	          ^  \ rS rSrSr    SS\S\S\S\4U 4S jjjrS\R                  S	\	\R                  \R                  \R                  \R                  4   4S
 jr
SrU =r$ )MelSTFTi  z
Calculates a causal log-mel spectrogram from a waveform. Uses a pre-calculated mel filterbank, which should be
loaded from the checkpoint in bfloat16.
r  r  r  num_mel_channelsc                    > [         TU ]  5         [        XU5      U l        US-  S-   nU R	                  S[
        R                  " XE5      SS9  g )Nr   r   	mel_basisTr2   )r3   r4   r  stft_fnr8   r   r   )r:   r  r  r  r  	num_freqsr<   s         r%   r4   MelSTFT.__init__  sM     	!-]K!Q&*	[%++6F*R_cdr'   r
  r
   c                     U R                  U5      u  p#[        R                  " USS9n[        R                  " U R                  R                  UR                  5      U5      n[        R                  " [        R                  " USS95      nXbX44$ )Nr   r   gh㈵>)min)	r  r   normmatmulr  rn   rl   logrb   )r:   r
  r  r  energymellog_mels          r%   rM   MelSTFT.forward  sj    <<1	I1-ll4>>,,Y__=yI))EKK67500r'   )r  )r  r  r  @   r  rZ   s   @r%   r  r    s     !  "ee e 	e
 e e1 1u||U\\SXS_S_afamam7m1n 1 1r'   r  c            F         ^  \ rS rSrSr\SSS/ SQ/ SQ/ SQ/ S	Q/ S	Q/ S	Q/S
SSSSSSSSS/ SQ/ SQ/ SQ/ S	Q/ S	Q/ S	Q/S
SSSSSSSSSSSS4"S\S\S\S\\   S\\   S\\   S\\\      S\S\	S \
S!\S"\S#\S-  S$\
S%\S&\S'\S(\\   S)\\   S*\\   S+\\\      S,\S-\	S.\
S/\S0\S1\S-  S2\
S3\S4\S5\S6\S7\S8\4DU 4S9 jjj5       rS:\R                  S;\R                  4S< jrS=rU =r$ )>LTX2VocoderWithBWEi  a  
LTX-2.X vocoder with bandwidth extension (BWE) upsampling. The vocoder and the BWE module run in sequence, with the
BWE module upsampling the vocoder output waveform to a higher sampling rate. The BWE module itself has the same
architecture as the original vocoder.
r   i   r   )r   r   r   r   r   r   )r   r   r   r   r   r   r   r   rx   r   Tr   NFr  )r   r   r   r   r   r   r  r+  i>  i  r   r   r   r   r   r   r   ru   r   r   r   r   r   r   bwe_in_channelsbwe_hidden_channelsbwe_out_channelsbwe_upsample_kernel_sizesbwe_upsample_factorsbwe_resnet_kernel_sizesbwe_resnet_dilations
bwe_act_fnbwe_leaky_relu_negative_slopebwe_antialiasbwe_antialias_ratiobwe_antialias_kernel_sizebwe_final_act_fnbwe_final_biasr  r  r  r  input_sampling_rater   c#                    > [         T#U ]  5         [        UUUUUUUUU	U
UUUUU!S9U l        [        UUUUUUUUUUUUUUU"S9U l        [        UUUU S9U l        [        U"U!-  SSS9U l        g )N)r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r   )r  r  r  r  r`   F)r+   r^   r.   )	r3   r4   r   vocoderbwe_generatorr  mel_stftr\   	resampler)$r:   r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r.  r/  r0  r1  r2  r3  r4  r5  r6  r7  r8  r9  r:  r;  r  r  r  r  r<  r   r<   s$                                      r%   r4   LTX2VocoderWithBWE.__init__  s    L 	"#+%"7- 3-&?+"7%!!4
" )'/)";1 71&C#/";)%!5
$  '!'-	
 $&*==
r'   mel_specr
   c                 N   U R                  U5      nUR                  u  p4nXPR                  R                  -  nUS:w  a&  [        R
                  " USU R                  U-
  45      nU R                  UR                  SS5      5      u  n    nUR                  SSU45      nUR                  SS5      n	U R                  U	5      n
U R                  U5      n[        R                  " X-   SS5      nXPR                  R                  -  U R                  R                  -  nUSS U24   nU$ )Nr   r   rB   r   r   .)r>  rF   configr  rG   rH   r@  r   	unflattenr   r?  rA  r   rb   r   r<  )r:   rC  r>   
batch_sizerK   num_samples	remainderr(  r   mel_for_bweresidualskipr
  output_sampless                 r%   rM   LTX2VocoderWithBWE.forward>  s   LL"01-
+  ++"8"88	>a!T__y89:A }}QYYq!_5Q1mmAL12 mmAq)%%k2 ~~a ;;xA6${{'G'GG4;;KjKjjC.01r'   )r?  r@  rA  r>  r   rZ   s   @r%   r-  r-    sb     #+>&8)3-6	9,M!+. %'#' "#& !/@*9-71:Iy0Q%/2"#$)+'+$   "#($)GU
U
 U
 	U

  $CyU
 s)U
 "#YU
 tCy/U
 U
 $)U
 U
 U
  #U
 DjU
 U
  !U
" !#U
$ %U
& $(9'U
( #3i)U
* "&c+U
, #49o-U
. /U
0 (-1U
2 3U
4 !5U
6 $'7U
8 *9U
: ;U
< =U
> ?U
@ AU
B CU
D !EU
F "GU
 U
n   r'   r-  )r   r   torch.nnr~   torch.nn.functional
functionalrG   configuration_utilsr   r   models.modeling_utilsr   r   r5   rW   r&   r   r)   r\   rs   r}   r   r   r  r  r-  rz   r'   r%   <module>rT     s         B /+ +E + +PUP\P\ +\299 B/7 /7dRYY @'		 'T>ryy >BK*k K\   >1bii 16v[ vr'   