3jT SSKJr SSKrSSKJs Jr SSKJr SSKJrJ r SSK J r J r SSK JrJr SS KJr SS KJrJr SS KJrJrJrJr SS KJr SS KJr SSKJrJ r \RB"\"5r#"SS\RH5r%"SS\RH5r&"SS\RH5r'"SS5r("SS\RH5r)"SS\\\\ \ 5r*g))AnyN)nn) ConfigMixinregister_to_config)FromOriginalModelMixinPeftAdapterMixin)apply_lora_scalelogging)AttentionMixin) AttentionSanaLinearAttnProcessor2_0) PatchEmbedPixArtAlphaTextProjectionTimestepEmbedding Timesteps)Transformer2DModelOutput) ModelMixin)AdaLayerNormSingleRMSNormc ^\rSrSrS S\S\S\S\S-S\SS4 U4S jjjrS \ RS\ R4S jr S r U=r $) GLUMBConv&N in_channels out_channels expand_ratio norm_typeresidual_connectionreturnc >[TU]5 [X1-5nX@lXPl[ R "5Ul[ R"XS-SSS5Ul [ R"US-US-SSSUS-S9Ul [ R"XbSSSSS9Ul SUl US:Xa[US S S S 9Ul gg) Nr rr)groupsFbiasrms_normh㈵>T)epselementwise_affiner%)super__init__intrrrSiLU nonlinearityConv2d conv_inverted conv_depth conv_pointnormr)selfrrrrrhidden_channels __class__s h/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/transformers/sana_transformer.pyr+GLUMBConv.__init__'s l89"#6 GGIYY{a4GAqQ))Oa$719LaQRTU^mpq^qr))O1aQVW  " $4VZ[DI # hidden_statescUR(aUnURU5nURU5nURU5n[R "USSS9upXRU5-nUR U5nURS:Xa1URURSS55RSS5nUR(aUW-nU$)Nr r"dimr&) rr0r.r1torchchunkr2rr3movedim)r4r:residualgates r7forwardGLUMBConv.forward>s  # #$H**=9 ))-8  6 #kk-B %(9(9$(??  6 >>Z ' IIm&;&;Ar&BCKKBPQRM  # #)H4Mr9)r1r0r2r.r3rr)NT)__name__ __module__ __qualname____firstlineno__r,floatstrboolr+r?TensorrD__static_attributes__ __classcell__r6s@r7rr&s~  $$( \\\ \ : \ " \ \\.U\\ellr9rc^\rSrSrS S\S\S\4U4SjjjrS\RS\RS\RS \R4S jr S r U=r $) SanaModulatedNormUr=r)r(cV>[TU]5 [R"XUS9Ulg)Nr)r()r*r+r LayerNormr3)r4r=r)r(r6s r7r+SanaModulatedNorm.__init__Vs! LLQTU r9r:tembscale_shift_tabler cURU5nUSUSS2S4RUR5-RSSS9upEUSU--U-nU$)Nr r"r<)r3todevicer@)r4r:rYrZshiftscales r7rDSanaModulatedNorm.forwardZsh -0 )$/$q$w-2B2BCTC[C[2\\ccdeklcm %U3e; r9)r3)Fư>) rGrHrIrJr,rMrKr+r?rNrDrOrPrQs@r7rSrSUscVCVTVVV"\\16RWR^R^ r9rSc~^\rSrSrU4SjrSS\R S\R S\R4SjjrSr U=r $) &SanaCombinedTimestepGuidanceEmbeddingscc>[TU]5 [SSSS9Ul[ SUS9Ul[SSSS9Ul[ SUS9Ul[R"5Ul [R"USU-SS9Ul g)NTr) num_channelsflip_sin_to_cosdownscale_freq_shift)rtime_embed_dimr$) r*r+r time_projrtimestep_embedderguidance_condition_projguidance_embedderrr-siluLinearlinear)r4 embedding_dimr6s r7r+/SanaCombinedTimestepGuidanceEmbeddings.__init__dsx "T`ab!2sS`!a'0cSWno'p$!2sS`!aGGI ii q=/@tL r9timestepguidance hidden_dtypec URU5nURURUS95nURU5nUR URUS95nXW-nUR UR U55U4$)N)dtype)rlrmr\rnrorrrp) r4rurvrwtimesteps_proj timesteps_emb guidance_proj guidance_emb conditionings r7rD.SanaCombinedTimestepGuidanceEmbeddings.forwardos1..~/@/@|/@/TU 44X> --m.>.>\.>.RS $3 {{499\23\AAr9)rnrorrrprlrmNN) rGrHrIrJr+r?rNryrDrOrPrQs@r7rcrccs< MB B B[`[f[fBBr9rcc \rSrSrSrSrS S\S\RS\RS-S\RS-S \R4 S jjr S r g) SanaAttnProcessor2_0zzk Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). cD[[S5(d [S5eg)Nscaled_dot_product_attentionzTSanaAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)hasattrF ImportError)r4s r7r+SanaAttnProcessor2_0.__init__s!q899tu u:r9Nattnr:encoder_hidden_statesattention_maskr c *Uc URO URupVnUb<URXFU5nURXQRSURS5nUR U5nUcUnUR U5n UR U5n URbURU5nURbURU 5n U RSn XR-n URUSURU 5RSS5nU RUSURU 5RSS5n U RUSURU 5RSS5n [R"XXSSS9nURSS5RUSURU -5nURUR5nURS"U5nURS"U5nX!R - nU$)Nr>r"r F) attn_mask dropout_p is_causalr)shapeprepare_attention_maskviewheadsto_qto_kto_vnorm_qnorm_k transposerrreshaper\ryto_outrescale_output_factor) r4rr:rr batch_sizesequence_length_querykeyvalue inner_dimhead_dims r7__call__SanaAttnProcessor2_0.__call__s$9#@M  F[FaFa ' Q  %!88ZdeN,00ZZ^MaMabdMefN -( ($1 !ii-. /0 ;; "KK&E ;; "++c"CIIbM  * :r4::x@JJ1aPhhz2tzz8<FFq!L :r4::x@JJ1aP66 3RW &//15==j"djj[cNcd %((5  A}5  A}5 %(B(BB r9r) rGrHrIrJ__doc__r+rr?rNrrOrr9r7rrzsiv6:.2 66||6 %||d2 6  t+ 6  66r9rch^\rSrSrSrSS\S\S\S\S\S-S \S-S \S-S \S \S \S\S\S\S-SS4U4Sjjjr SS\ RS\ RS-S\ RS-S\ RS-S\ RS-S\S\S\ R4Sjjr SrU=r$)SanaTransformerBlockzS Transformer block introduced in [Sana](https://huggingface.co/papers/2410.10629). Nr=num_attention_headsattention_head_dimdropoutnum_cross_attention_headscross_attention_head_dimcross_attention_dimattention_biasnorm_elementwise_affinenorm_epsattention_out_bias mlp_ratioqk_normr c >[TU]5 [R"USU S9Ul[ UUUU bUOSU UUS[ 5S9 UlUb>[R"XU S9Ul[ UU U bUOSUUUUSU [5S9 Ul [XU SSS9Ul [R"[R"SU5US-- 5Ulg) NFrV) query_dimrdim_headkv_headsrrr%r processorT) rrrrrrrr%out_biasr)rrrk?)r*r+rrWnorm1rrattn1norm2rattn2rff Parameterr?randnrZ)r4r=rrrrrrrrrrrrr6s r7r+SanaTransformerBlock.__init__s \\#%XN %',3,?(T $02     *c[cdDJ"6=6I2t$7/1+.0 DJCi4UZ[!#ekk!S.ACH.L!Mr9r:rrencoder_attention_maskruheightwidthcjURSnURSURUSS5-RSSS9upppUR U5nUSU --U -nUR UR 5nURU5nXU--nURbURUUUS9nUU-nURU5nUSU --U -nURSXg45RSSSS5nURU5nURSS5RSSS5nXU--nU$) Nrrkr>r"r<)rrrr )rrZrr@rr\ryrrr unflattenpermuterflatten)r4r:rrrrurrr shift_msa scale_msagate_msa shift_mlp scale_mlpgate_mlpnorm_hidden_states attn_output ff_outputs r7rDSanaTransformerBlock.forwardso#((+   " "4 (8+;+;J2+N N %q%/ G h9 "ZZ 6/1y=AIM/22=3F3FGjj!34 %;(>>  :: !**&;5%K (-7M"ZZ 6/1y=AIM/99!f_MUUVWYZ\]_`aGG./ %%a+33Aq!< %9(<< r9)rrrrrrZ) F rprTFraT@N)NNNNNN)rGrHrIrJrr,rKrMrLr+r?rN LongTensorrDrOrPrQs@r7rrs #%"$02/2*.#(-#'"3N 3N!3N 3N  3N $': 3N#&*3N!4Z3N3N"&3N3N!3N3Nt3N 3N3Np/3596:,0+||+ t++ %||d2 + !& t 3 + ""T) +++ ++r9rc..^\rSrSrSrSr/SQrSS/r\S+S\ S \ S-S \ S \ S \ S \ S-S\ S-S\ S-S\ S\ S\ S\ S\ S\ S\ S\ S\ S-S\ S\ S\ S-S\ SS4,U4Sjjj5r \"S5S,S \R S!\R S"\R S#\R S-S$\R S-S%\R S-S\\ \4S-S&\\R S-S'\ S\\R S(4\-4S)jj5rS*rU=r$)-SanaTransformer2DModeli$a A 2D Transformer model introduced in [Sana](https://huggingface.co/papers/2410.10629) family of models. Args: in_channels (`int`, defaults to `32`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `32`): The number of channels in the output. num_attention_heads (`int`, defaults to `70`): The number of heads to use for multi-head attention. attention_head_dim (`int`, defaults to `32`): The number of channels in each head. num_layers (`int`, defaults to `20`): The number of layers of Transformer blocks to use. num_cross_attention_heads (`int`, *optional*, defaults to `20`): The number of heads to use for cross-attention. cross_attention_head_dim (`int`, *optional*, defaults to `112`): The number of channels in each head for cross-attention. cross_attention_dim (`int`, *optional*, defaults to `2240`): The number of channels in the cross-attention output. caption_channels (`int`, defaults to `2304`): The number of channels in the caption embeddings. mlp_ratio (`float`, defaults to `2.5`): The expansion ratio to use in the GLUMBConv layer. dropout (`float`, defaults to `0.0`): The dropout probability. attention_bias (`bool`, defaults to `False`): Whether to use bias in the attention layer. sample_size (`int`, defaults to `32`): The base size of the input latent. patch_size (`int`, defaults to `1`): The size of the patches to use in the patch embedding layer. norm_elementwise_affine (`bool`, defaults to `False`): Whether to use elementwise affinity in the normalization layer. norm_eps (`float`, defaults to `1e-6`): The epsilon value for the normalization layer. qk_norm (`str`, *optional*, defaults to `None`): The normalization to use for the query and key. timestep_scale (`float`, defaults to `1.0`): The scale to use for the timesteps. T)rrrS patch_embedr3Nrrrr num_layersrrrcaption_channelsrrr sample_size patch_sizerrinterpolation_scaleguidance_embedsguidance_embeds_scalertimestep_scaler ct>[TU]5 U=(d UnX4-n[U U UUUUUbSOSS9UlU(a[ U5UlO[ U5Ul[U US9Ul[USSS9Ul [R"[U5Vs/sHn[UUUU UUUU UUU US9 PM sn5Ul[R "["R$"SU5US -- 5Ul[)US S S 9Ul[R,"UX-U-5UlS Ulgs snf) Nsincos)rrrr embed_dimrpos_embed_type) in_features hidden_sizer'T)r(r)) rrrrrrrrrr rFrarV)r*r+rrrc time_embedrrcaption_projectionr caption_normr ModuleListrangertransformer_blocksrr?rrZrSnorm_outrqproj_outgradient_checkpointing)r4rrrrrrrrrrrrrrrrrrrrrrrr6s r7r+SanaTransformer2DModel.__init__SsN2 #2{ '< &!# 3':'F8D  DYODO0;DO";HXfo"p#I4DQ#%-- z* +A%'&#.G-E(;#1,C%'# + # *"$ekk!Y.G)UX..X!Y))SWX  )Z-D|-ST &+#1 sD5attention_kwargsr:rrurvrrcontrolnet_block_samples return_dict.c UbBURS:Xa2SURUR5- S-nURS5nUbBURS:Xa2SURUR5- S-nURS5nURuppUR R nX-X-nnURU5nUbURX4URS9unnOURX:URS9unnURU5nURU SURS5nURU5n[R"5(atUR(ac[UR 5HIunnUR#UUUUUUUU5nUcM#SUs=:a[%U5::dM;O M?XUS- -nMK OX[UR 5H?unnU"UUUUUUU5nUcMSUs=:a[%U5::dM1O M5XUS- -nMA UR'UUUR(5nUR+U5nUR-XUUR R UR R S5nUR/SSSS SS 5nUR-U SX-UU-5nU (dU4$[1US 9$) Nr r"g)rvrw)rrwr>rrrF)sample)ndimr\ry unsqueezerconfigrrrrrrr?is_grad_enabledr enumerater_gradient_checkpointing_funclenrrZrrrr)r4r:rrurvrrrrrrrgrrppost_patch_heightpost_patch_widthembedded_timestep index_blockblockoutputs r7rDSanaTransformer2DModel.forwards.  %.*=*=*B  ."3"3M4G4G"HHHTN+55a8N " -2H2M2MQR2R&'*@*C*CMDWDW*X&X\d%d "%;%E%Ea%H "3@2E2E/ & KK " ".4k5:+((7  *.//-:M:M+:+ 'H'+///m>Q>Q+:+ 'H'!% 7 78M N 5 : ::r=K^K^_aKb c $ 1 12G H  " "t'B'B&/0G0G&H" U $ A A!")*%$ ! ,7A $#%#%"$02/2*. $$(-*. %'*" #-K,K,DjK,! K,  K,  K,$':K,#&*K,!4ZK,K,K,K,K,K,K, "&!K,"#K,$!4Z%K,&'K,( %)K,*t+K,,-K,. /K,K,Z() )-6:.226?C e7||e7 %||e7,, e7 ,,% e7 !& t 3 e7 t+e7sCx.4/e7#( "5"<e7e7 u||S !$< <e7*e7r9r)+typingrr?torch.nn.functionalr functionalrconfiguration_utilsrrloadersrr utilsr r attentionr attention_processorrr embeddingsrrrrmodeling_outputsrmodeling_utilsr normalizationrr get_loggerrGloggerModulerrSrcrrrrr9r7r%s B?.&]\7'7   H %, ,^  BRYYB.??De299ePc7ZFVXnc7r9