
    
3j_                        S SK r S SKrS SKJr  S SKJs  Jr  SSKJrJ	r	  SSK
Jr  SSKJrJr   " S S\R                  5      r " S	 S
\R"                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r\	" SS 5      (       a  \R"                  rO " S! S"\R                  5      r " S# S$\R                  5      r " S% S&\R                  5      r " S' S(\R                  5      r " S) S*\R                  5      r      S2S+\!S,\"S-  S-\#S.\$S/\$S0\R                  4S1 jjr%g)3    N   )is_torch_npu_availableis_torch_version   )get_activation)CombinedTimestepLabelEmbeddings)PixArtAlphaCombinedTimestepSizeEmbeddingsc                      ^  \ rS rSrSr     SS\S\S-  S\S-  S\S\S	\4U 4S
 jjjr SS\	R                  S\	R                  S-  S\	R                  S-  S\	R                  4S jjrSrU =r$ )AdaLayerNorm   a  
Norm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`, *optional*): The size of the embeddings dictionary.
    output_dim (`int`, *optional*):
    norm_elementwise_affine (`bool`, defaults to `False):
    norm_eps (`bool`, defaults to `False`):
    chunk_dim (`int`, defaults to `0`):
Nembedding_dimnum_embeddings
output_dimnorm_elementwise_affinenorm_eps	chunk_dimc                 >  > [         TU ]  5         X`l        U=(       d    US-  nUb  [        R                  " X!5      U l        OS U l        [        R                  " 5       U l        [        R                  " X5      U l	        [        R                  " US-  XT5      U l        g Nr   )super__init__r   nn	EmbeddingembSiLUsiluLinearlinear	LayerNormnorm)selfr   r   r   r   r   r   	__class__s          X/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/normalization.pyr   AdaLayerNorm.__init__(   st     	"4=1#4
%||NBDHDHGGI	ii:LLq(T	    xtimesteptembreturnc                 P   U R                   b  U R                  U5      nU R                  U R                  U5      5      nU R                  S:X  a+  UR	                  SSS9u  pEUS S 2S S S 24   nUS S 2S S S 24   nOUR	                  SSS9u  pTU R                  U5      SU-   -  U-   nU$ )Nr   r   dimr   )r   r   r   r   chunkr   )r    r%   r&   r'   shiftscales         r"   forwardAdaLayerNorm.forward?   s     8888H%D{{499T?+>>Q  ::aQ:/LE!T1*%E!T1*%E::aQ:/LEIIaLAI&.r$   )r   r   r   r   r   )NNFh㈵>r   )NN)__name__
__module____qualname____firstlineno____doc__intboolfloatr   torchTensorr/   __static_attributes____classcell__r!   s   @r"   r   r      s    
 &*!%(-UU d
U $J	U
 "&U U U U0 bf).)<KP<<Z^K^	 r$   r   c                   N    \ rS rSrS\R
                  S\R
                  4S jrSrg)FP32LayerNormT   inputsr(   c                 L   UR                   n[        R                  " UR                  5       U R                  U R
                  b  U R
                  R                  5       OS U R                  b  U R                  R                  5       OS U R                  5      R                  U5      $ N)	dtypeF
layer_normr9   normalized_shapeweightbiasepsto)r    rB   origin_dtypes      r"   r/   FP32LayerNorm.forwardU   su    ||||LLN!!#';;#:DKK!%!6DIIOODHH
 "\
	r$    N)r2   r3   r4   r5   r:   r;   r/   r<   rO   r$   r"   r@   r@   T   s    ell u|| r$   r@   c            	          ^  \ rS rSrSrSS\S\S\SS4U 4S jjjr SS	\	R                  S
\	R                  S-  S\\	R                  S4   4S jjrSrU =r$ )SD35AdaLayerNormZeroX`   z
Norm layer adaptive layer norm zero (AdaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
r   	norm_typerJ   r(   Nc                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	N	   rJ   rG   Fư>elementwise_affinerK   Unsupported `norm_type` (z-) provided. Supported ones are: 'layer_norm'.
r   r   r   r   r   r   r   r   r   
ValueErrorr    r   rS   rJ   r!   s       r"   r   SD35AdaLayerNormZeroX.__init__i   sg    GGI	iiq=/@tL$]uRVWDI8Cpqrrr$   hidden_statesr   .c           	          U R                  U R                  U5      5      nUR                  SSS9u	  p4pVpxpnU R                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSU
S S 2S 4   -   -  U	S S 2S 4   -   nXXgXU4$ )NrU   r   r*   r   r   r,   r   )r    r_   r   	shift_msa	scale_msagate_msa	shift_mlp	scale_mlpgate_mlp
shift_msa2
scale_msa2	gate_msa2norm_hidden_statesnorm_hidden_states2s                 r"   r/   SD35AdaLayerNormZeroX.forwards   s    
 kk$))C.)lolulu1 mv m
i	h9
`i "YY}5*a)AtG2D.DE	RSUYRYHZZ0A
1d78K4KLzZ[]aZaObb	h]fffr$   r   r   r   rG   TrD   )r2   r3   r4   r5   r6   r7   strr8   r   r:   r;   tupler/   r<   r=   r>   s   @r"   rQ   rQ   `   s    sc sc sPT s`d s s $(g||g \\D g 
u||S 	!	g gr$   rQ   c                   `  ^  \ rS rSrSrSS\S\S-  4U 4S jjjr    SS\R                  S\R                  S-  S	\R                  S-  S
\R                  S-  S\R                  S-  S\\R                  \R                  \R                  \R                  \R                  4   4S jjrSrU =r$ )AdaLayerNormZero   
Norm layer adaptive layer norm zero (adaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
Nr   r   c                 V  > [         TU ]  5         Ub  [        X!5      U l        OS U l        [        R
                  " 5       U l        [        R                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l
        g US:X  a  [        USSS9U l
        g [        S	U S
35      e)N   rV   rG   FrW   rX   fp32_layer_norm)rY   rJ   rZ   @) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'.)r   r   r   r   r   r   r   r   r   r   r   r@   r\   )r    r   r   rS   rJ   r!   s        r"   r   AdaLayerNormZero.__init__   s    %6~UDHDHGGI	iiq=/@tL$]uRVWDI++%mTYZDI+I;6vw r$   r%   r&   class_labelshidden_dtyper   r(   c                     U R                   b  U R                  X#US9nU R                  U R                  U5      5      nUR                  SSS9u  pgppU R	                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nXXU4$ )N)r|   rw   r   r*   )r   r   r   r,   r   )r    r%   r&   r{   r|   r   rb   rc   rd   re   rf   rg   s               r"   r/   AdaLayerNormZero.forward   s     88((8(MCkk$))C.)ILSTZ[I\F	h9IIaLA	!T' 223i46HHI(::r$   )r   r   r   r   )NrG   T)NNNN)r2   r3   r4   r5   r6   r7   r   r:   r;   
LongTensorrE   rq   r/   r<   r=   r>   s   @r"   rs   rs      s    c 3:  * )-04+/#';<<; ,,%; &&-	;
 kkD(; \\D ; 
u||U\\5<<u||S	T; ;r$   rs   c                      ^  \ rS rSrSrSS\4U 4S jjjr SS\R                  S\R                  S-  S\	\R                  \R                  \R                  \R                  \R                  4   4S	 jjr
S
rU =r$ )AdaLayerNormZeroSingle   ru   r   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	N   rV   rG   FrW   rX   rZ   ry   r[   r]   s       r"   r   AdaLayerNormZeroSingle.__init__   sk    GGI	iiq=/@tL$]uRVWDI+I;6vw r$   Nr%   r   r(   c                     U R                  U R                  U5      5      nUR                  SSS9u  p4nU R                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nX4$ )Nr   r   r*   ra   )r    r%   r   rb   rc   rd   s         r"   r/   AdaLayerNormZeroSingle.forward   sg    
 kk$))C.)),1!)<&	hIIaLA	!T' 223i46HH{r$   rn   ro   rD   r2   r3   r4   r5   r6   r7   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   r   r      su    
c 
 
 $(<< \\D  
u||U\\5<<u||S	T	 r$   r   c                      ^  \ rS rSrSrS\S\S\4U 4S jjr SS\	R                  S	\	R                  S-  S
\\	R                  \	R                  \	R                  \	R                  4   4S jjrSrU =r$ )LuminaRMSNormZero   zx
Norm layer adaptive RMS normalization zero.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
r   r   r   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " [        US5      SU-  SS9U l        [        XS9U l	        g )Ni      TrV   rK   )
r   r   r   r   r   r   minr   RMSNormr   )r    r   r   r   r!   s       r"   r   LuminaRMSNormZero.__init__   sN    GGI	iit$

 M8	r$   Nr%   r   r(   c                     U R                  U R                  U5      5      nUR                  SSS9u  p4pVU R                  U5      SUS S 2S 4   -   -  nXXV4$ )Nr   r   r*   ra   )r    r%   r   rc   rd   rf   rg   s          r"   r/   LuminaRMSNormZero.forward   s\    
 kk$))C.)3699QA93F0	YIIaLA	!T' 223I//r$   rn   rD   )r2   r3   r4   r5   r6   r7   r9   r8   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   r   r      sy    9c 9U 9UY 9 $(	0<<	0 \\D 	0 
u||U\\5<<E	F		0 	0r$   r   c                   0  ^  \ rS rSrSrSS\S\4U 4S jjjr   SS\R                  S\
\\R                  4   S-  S	\S-  S
\R                  S-  S\\R                  \R                  \R                  \R                  \R                  4   4
S jjrSrU =r$ )AdaLayerNormSingle   a<  
Norm layer adaptive layer norm single (adaLN-single).

As proposed in PixArt-Alpha (see: https://huggingface.co/papers/2310.00426; Section 2.3).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    use_additional_conditions (`bool`): To use additional conditions for normalization or not.
r   use_additional_conditionsc                    > [         TU ]  5         [        XS-  US9U l        [        R
                  " 5       U l        [        R                  " USU-  SS9U l        g )Nr   )size_emb_dimr   rw   TrV   )	r   r   r	   r   r   r   r   r   r   )r    r   r   r!   s      r"   r   AdaLayerNormSingle.__init__   sM    <(:Vo
 GGI	iiq=/@tLr$   Nr&   added_cond_kwargs
batch_sizer|   r(   c                     U=(       d    S S S.nU R                   " U40 UDX4S.D6nU R                  U R                  U5      5      U4$ )N)
resolutionaspect_ratio)r   r|   r   r   r   )r    r&   r   r   r|   embedded_timesteps         r"   r/   AdaLayerNormSingle.forward   sN     .[VZ1[ HHXu1Buzu{{499%678:KKKr$   r   )F)NNN)r2   r3   r4   r5   r6   r7   r8   r   r:   r;   dictrp   rE   rq   r/   r<   r=   r>   s   @r"   r   r      s    Mc Md M M =A!%+/
L,,
L  U\\ 12T9
L $J	
L
 kkD(
L 
u||U\\5<<u||S	T
L 
Lr$   r   c                      ^  \ rS rSrSr SS\S\S\S\S-  S\4
U 4S	 jjjrS
\	R                  S\	R                  S\	R                  4S jrSrU =r$ )AdaGroupNormi  a  
GroupNorm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
    num_groups (`int`): The number of groups to separate the channels into.
    act_fn (`str`, *optional*, defaults to `None`): The activation function to use.
    eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability.
Nr   out_dim
num_groupsact_fnrK   c                    > [         TU ]  5         X0l        XPl        Uc  S U l        O[        U5      U l        [        R                  " XS-  5      U l        g r   )	r   r   r   rK   actr   r   r   r   )r    r   r   r   r   rK   r!   s         r"   r   AdaGroupNorm.__init__  sF     	$>DH%f-DHii{;r$   r%   r   r(   c                    U R                   (       a  U R                  U5      nU R                  U5      nUS S 2S S 2S S 4   nUR                  SSS9u  p4[        R                  " XR
                  U R                  S9nUSU-   -  U-   nU$ )Nr   r   r*   r   )r   r   r,   rF   
group_normr   rK   )r    r%   r   r.   r-   s        r"   r/   AdaGroupNorm.forward'  sz    88((3-Ckk#!Qd"#yyy*LLOO:UOe#r$   )r   rK   r   r   )Nr1   )r2   r3   r4   r5   r6   r7   rp   r9   r   r:   r;   r/   r<   r=   r>   s   @r"   r   r     sp    	 jn< <+.<<?<ILt<af< <	 	ELL 	U\\ 	 	r$   r   c                      ^  \ rS rSrSr    SS\S\4U 4S jjjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )AdaLayerNormContinuousi3  aq  
Adaptive normalization layer with a norm layer (layer_norm or rms_norm).

Args:
    embedding_dim (`int`): Embedding dimension to use during projection.
    conditioning_embedding_dim (`int`): Dimension of the input condition.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
    norm_type (`str`, defaults to `"layer_norm"`):
        Normalization layer to use. Values supported: "layer_norm", "rms_norm".
r   conditioning_embedding_dimc                 
  > [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!S-  US9U l        US:X  a  [        XX55      U l        g US:X  a  [        XU5      U l        g [        SU 35      e)Nr   rV   rG   rms_normunknown norm_type )r   r   r   r   r   r   r   r   r   r   r\   )r    r   r   rY   rK   rJ   rS   r!   s          r"   r   AdaLayerNormContinuous.__init__B  sw     	GGI	ii :A<MTXY$!-6HODI*$4FGDI1)=>>r$   r%   conditioning_embeddingr(   c                    U R                  U R                  U5      R                  UR                  5      5      n[        R
                  " USSS9u  pEU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU$ )Nr   r   r*   )r   r   rL   rE   r:   r,   r   )r    r%   r   r   r.   r-   s         r"   r/   AdaLayerNormContinuous.forwardZ  ss    kk$))$:;>>qwwGH{{3q1IIaLAIq$z22U1dA:5FFr$   rn   )Tr1   TrG   )r2   r3   r4   r5   r6   r7   r   r:   r;   r/   r<   r=   r>   s   @r"   r   r   3  s`    .  ?? %(? ?0 u|| PUP\P\  r$   r   c                      ^  \ rS rSr     SS\S\S\S-  4U 4S jjjrS\R                  S\R                  S	\R                  4S
 jrSr	U =r
$ )LuminaLayerNormContinuousib  Nr   r   r   c                 L  > [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!US9U l        US:X  a  [        XX55      U l        O$US:X  a  [        XUS9U l        O[        SU 35      eS U l        Ub  [        R
                  " XUS9U l        g g )NrV   rG   r   rK   rY   r   )r   r   r   r   r   r   linear_1r   r   r   r\   linear_2)	r    r   r   rY   rK   rJ   rS   r   r!   s	           r"   r   "LuminaLayerNormContinuous.__init__c  s     	 GGI			"<RVW$!-6HODI*$K]^DI1)=>>IIm4HDM r$   r%   r   r(   c                     U R                  U R                  U5      R                  UR                  5      5      nUnU R	                  U5      SU-   S S 2S S S 24   -  nU R
                  b  U R                  U5      nU$ Nr   )r   r   rL   rE   r   r   )r    r%   r   r   r.   s        r"   r/   !LuminaLayerNormContinuous.forward  sn     mmDII&<=@@IJIIaLAIq$z22==$a Ar$   )r   r   r   r   )Tr1   TrG   N)r2   r3   r4   r5   r7   r   r:   r;   r/   r<   r=   r>   s   @r"   r   r   b  sv      "II %(I tI I@<< !& 
	 r$   r   c                     ^  \ rS rSrSrS\S\4U 4S jjr SS\R                  S\R                  S	\R                  S-  S
\	\R                  \R                  \R                  \R                  \R                  4   4S jjr
SrU =r$ )%CogView3PlusAdaLayerNormZeroTextImagei  ru   r   r+   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  SS9U l        [        R                  " USSS9U l        [        R                  " USSS9U l	        g )N   TrV   Fr1   rX   )
r   r   r   r   r   r   r   r   norm_xnorm_c)r    r   r+   r!   s      r"   r   .CogView3PlusAdaLayerNormZeroTextImage.__init__  s[    GGI	iirCxdCll35dKll35dKr$   Nr%   contextr   r(   c                 2   U R                  U R                  U5      5      nUR                  SSS9u  nnnnnn	n
nnnnnU R                  U5      nU R	                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSUS S 2S 4   -   -  U
S S 2S 4   -   nXXxXXX4
$ )Nr   r   r*   )r   r   r,   r   r   )r    r%   r   r   rb   rc   rd   re   rf   rg   c_shift_msac_scale_msa
c_gate_msac_shift_mlpc_scale_mlp
c_gate_mlpnormed_xnormed_contexts                     r"   r/   -CogView3PlusAdaLayerNormZeroTextImage.forward  s     kk$))C.) IIbaI 	
;;q>W-Iag../)AtG2DD AAtG(<$<=AtG@TTI(Z^iuur$   )r   r   r   r   rD   r   r>   s   @r"   r   r     s    Lc L L $(	v<<v v \\D 	v
 
u||U\\5<<u||S	Tv vr$   r   c                      ^  \ rS rSr   SS\S\S\S\S\SS4U 4S	 jjjrS
\R                  S\R                  S\R                  S\
\R                  \R                  4   4S jrSrU =r$ )CogVideoXLayerNormZeroi  conditioning_dimr   rY   rK   rJ   r(   Nc                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        [        R                  " X$US9U l        g )Nrw   rV   r   )	r   r   r   r   r   r   r   r   r   )r    r   r   rY   rK   rJ   r!   s         r"   r   CogVideoXLayerNormZero.__init__  sJ     	GGI	ii 0!m2C$OLLL^_	r$   r_   encoder_hidden_statesr'   c                 P   U R                  U R                  U5      5      R                  SSS9u  pEpgpU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nXUS S 2S S S 24   U	S S 2S S S 24   4$ )Nrw   r   r*   ra   )
r    r_   r   r'   r-   r.   gate	enc_shift	enc_scaleenc_gates
             r"   r/   CogVideoXLayerNormZero.forward  s     >B[[SW=Y=_=_`agh=_=i:dy		-0AIq$z3JJUSTVZ\]S]M^^ $		*? @A	MSTVZ\]S]C^ ^ajklnrtukuav vT!T1*5ExPQSWYZPZG[[[r$   rn   )Tr1   T)r2   r3   r4   r5   r7   r8   r9   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   r   r     s    
 $(`` ` !	`
 ` ` 
` `\"\\\BG,,\V[VbVb\	u||U\\)	*\ \r$   r   z>=z2.1.0c                   F   ^  \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r   i  ap  
LayerNorm with the bias parameter.

Args:
    dim (`int`): Dimensionality to use for the parameters.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
rK   rY   rJ   c                   > [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       ah  [        R                  " [        R                  " U5      5      U l        U(       a*  [        R                  " [        R                  " U5      5      OS U l        g S U l        S U l        g rD   )r   r   rK   
isinstancenumbersIntegralr:   Sizer+   r   	ParameteronesrI   zerosrJ   r    r+   rK   rY   rJ   r!   s        r"   r   LayerNorm.__init__  s    GH#w//00fzz#DH! ll5::c?;>BBLLS)9:	" 	r$   c                     [         R                  " XR                  U R                  U R                  U R
                  5      $ rD   )rF   rG   r+   rI   rJ   rK   )r    inputs     r"   r/   LayerNorm.forward  s'    <<xxdiiRRr$   )rJ   r+   rK   rI   )r1   TTr2   r3   r4   r5   r6   r9   r8   r   r/   r<   r=   r>   s   @r"   r   r     s4    			!U 	!t 	!Z^ 	! 	!"	S 	Sr$   r   c                   F   ^  \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r   i  a  
RMS Norm as introduced in https://huggingface.co/papers/1910.07467 by Zhang et al.

Args:
    dim (`int`): Number of dimensions to use for `weights`. Only effective when `elementwise_affine` is True.
    eps (`float`): Small value to use when calculating the reciprocal of the square-root.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bool`, defaults to False): If also training the `bias` param.
rK   rY   rJ   c                   > [         TU ]  5         X l        X0l        [	        U[
        R                  5      (       a  U4n[        R                  " U5      U l	        S U l
        S U l        U(       ag  [        R                  " [        R                  " U5      5      U l
        U(       a0  [        R                  " [        R                  " U5      5      U l        g g g rD   )r   r   rK   rY   r   r   r   r:   r   r+   rI   rJ   r   r   r   r   r   s        r"   r   RMSNorm.__init__
  s    "4c7++,,&C::c?	,,uzz#7DKLLS)9:	  r$   c                    [        5       (       a  SS KnU R                  b]  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nUR                  XR                  U R                  S9S   nU R                  b  XR                  -   nU$ UR                  nUR                  [        R                  5      R                  S5      R                  SSS9nU[        R                  " X@R                  -   5      -  nU R                  b  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nXR                  -  nU R                  b  XR                  -   nU$ UR                  U5      nU$ )Nr   )epsilonr   Tkeepdim)r   	torch_npurI   rE   r:   float16bfloat16rL   npu_rms_normrK   rJ   float32powmeanrsqrt)r    r_   r   input_dtypevariances        r"   r/   RMSNorm.forward  sp   !##{{&;;$$(GG$1$4$4T[[5F5F$GM%22=++W[W_W_2`abcMyy$ -		 9   (--K$''6::1=BB2tBTH)EKK888K,LLM{{&;;$$(GG$1$4$4T[[5F5F$GM - ;99($1II$=M  !. 0 0 =r$   )rJ   r+   rY   rK   rI   )TFr   r>   s   @r"   r   r     s1    	; ;D ;t ; ;& r$   r   c                   >   ^  \ rS rSrSS\S\4U 4S jjjrS rSrU =r	$ )MochiRMSNormi<  rK   rY   c                 $  > [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       a0  [        R                  " [        R                  " U5      5      U l        g S U l        g rD   )r   r   rK   r   r   r   r:   r   r+   r   r   r   rI   )r    r+   rK   rY   r!   s       r"   r   MochiRMSNorm.__init__=  s^    c7++,,&C::c?,,uzz#7DKDKr$   c                 8   UR                   nUR                  [        R                  5      R	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  b  XR                  -  nUR                  U5      nU$ )Nr   r   Tr   )	rE   rL   r:   r   r   r   r   rK   rI   )r    r_   r   r  s       r"   r/   MochiRMSNorm.forwardL  s    #)) ##EMM266q9>>r4>P%Hxx4G(HH;;")KK7M%((5r$   )r+   rK   rI   )T)
r2   r3   r4   r5   r9   r8   r   r/   r<   r=   r>   s   @r"   r  r  <  s%     D  	 	r$   r  c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )GlobalResponseNormiX  z
Global response normalization as introduced in ConvNeXt-v2 (https://huggingface.co/papers/2301.00808).

Args:
    dim (`int`): Number of dimensions to use for the `gamma` and `beta`.
c                    > [         TU ]  5         [        R                  " [        R
                  " SSSU5      5      U l        [        R                  " [        R
                  " SSSU5      5      U l        g r   )r   r   r   r   r:   r   gammabeta)r    r+   r!   s     r"   r   GlobalResponseNorm.__init__a  sL    \\%++aAs";<
LLQ1c!:;	r$   c                     [         R                  " USSSS9nX"R                  SSS9S-   -  nU R                  X-  -  U R                  -   U-   $ )Nr   )r   r   T)pr+   r   r   )r+   r   rW   )r:   r   r   r  r  )r    r%   gxnxs       r"   r/   GlobalResponseNorm.forwardf  sO    ZZQFD977r470478zzQV$tyy0144r$   )r  r  )	r2   r3   r4   r5   r6   r   r/   r<   r=   r>   s   @r"   r
  r
  X  s    <
5 5r$   r
  c                   v   ^  \ rS rSrS
S\S\S\4U 4S jjjrS\R                  S\R                  4S jr	S	r
U =r$ )LpNormil  r  r+   rK   c                 F   > [         TU ]  5         Xl        X l        X0l        g rD   )r   r   r  r+   rK   )r    r  r+   rK   r!   s       r"   r   LpNorm.__init__m  s    r$   r_   r(   c                 j    [         R                  " XR                  U R                  U R                  S9$ )N)r  r+   rK   )rF   	normalizer  r+   rK   )r    r_   s     r"   r/   LpNorm.forwardt  s!    {{=FFdhhOOr$   )r+   rK   r  )r   r   g-q=)r2   r3   r4   r5   r7   r9   r   r:   r;   r/   r<   r=   r>   s   @r"   r  r  l  sF    #  u  PU\\ Pell P Pr$   r  rS   num_featuresrK   rY   rJ   r(   c                     U S:X  a  [        XX4S9nU$ U S:X  a  [        R                  " XX4S9nU$ U S:X  a  [        R                  " XUS9nU$ [	        SU < S35      e)Nr   )rK   rY   rJ   rG   
batch_norm)rK   affinez
norm_type=z is not supported.)r   r   r   BatchNorm2dr\   )rS   r  rK   rY   rJ   r   s         r"   get_normalizationr   x  s{     J|AS_ K 
l	"||LFXd
 K	 
l	"~~l<NO K JI<'9:;;r$   )r  Nr1   TT)&r   r:   torch.nnr   torch.nn.functional
functionalrF   utilsr   r   activationsr   
embeddingsr   r	   Moduler   r   r@   rQ   rs   r   r   r   r   r   r   r   r   r   r  r
  r  rp   r7   r9   r8   r   rO   r$   r"   <module>r(     s         < ' b6299 6r	BLL 	gBII gD(;ryy (;VRYY @0		 0<L LD#299 #L,RYY ,^.		 .b*vBII *vZ\RYY \0 D'""ISBII SB9bii 9|299 85 5(	PRYY 	P "##* 
 	
  YYr$   