3jd` <SSKrSSKrSSKJr SSKrSSKJr SSKJs Jr SSKJ r SSK J r J r SSKJrJr SSKJr SS KJrJr SS KJrJr SS KJr SS KJr SS KJr \R>"\ 5r!S2Sjr"S3Sjr#S3Sjr$S4Sjr%S4Sjr&S5S\ S\ S\ S\'4Sjjr("SS\RR5r*"SS\RR5r+"SS\RR5r,"SS\RR5r-"S S!\RR5r."S"S#\RR5r/"S$S%5r0"S&S'\RR\5r1"S(S)\RR5r2"S*S+\RR5r3"S,S-\RR5r4"S.S/\RR5r5"S0S1\\ \\\\5r6g)6N)Any)Tensor) ConfigMixinregister_to_config)FromOriginalModelMixinPeftAdapterMixin)logging)AttentionMixinAttentionModuleMixin)_CAN_USE_FLEX_ATTNdispatch_attention_fn) CacheMixin)Transformer2DModelOutput) ModelMixinc [R"[R"U5*[R"SU[R S9-U- 5nU$)Nr)startenddtype)torchexpmathlogarangefloat32)dim max_periodfreqss m/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/transformers/transformer_kandinsky.py get_freqsr!'s@ IItxx ++ellSXS`S`.aadgg hE LcU(aCSn[XSXD4SS9n[XSXD4SS9nURSS5nURSS5nX4$URSS5nURSS5nX4$)Nrr r)local_patchingflatten)xropeshape block_mask pixel_sizes r fractal_flattenr.,s 1a%@a HdAz+FAN IIaO||Aq! 7N IIaO||Aq! 7Nr"cU(aCSnUR"URSSUS-/URSSQ76n[XSX34SS9nU$UR"/UQURSSQ76nU$)Nr$rr r%r&)reshaper+ local_merge)r)r+r,r-s r fractal_unflattenr39sx IIaggaj"j!m Baggabk B 1j"=1 E H II +u +qwwqr{ + Hr"c UupEpgUupn UR"/URSUQXX-PUPXi-PU PXz-PU PURUS-SQ76nUR"/[[ URSU55QUPUS-PUS-PUS-PUS-PUS-P[US-[ UR55Q76nUR X3S-5R US-US-5nU$)Nrr r%r1r+permuterangelenr( r)r+ group_sizer batch_sizedurationheightwidthg1g2g3s r r'r'CsS*/'J&JBB  #            q  A  s1774C=! "    a  a  a  a  a  sQwAGG % A #Qw''aq9A Hr"c UupEpgUupn UR"/URSUQXX-PXi-PXz-PUPU PU PURUS-SQ76nUR"/[[ URSU55QUPUS-PUS-PUS-PUS-PUS-P[US-[ UR55Q76nUR X3S-5R US-US-5R US-US-5nU$)Nr rr%r5r6r7r8r<s r r2r2^si*/'J&JBB  #            q  A  s1774C=! "    a  a  a  a  a  sQwAGG % A #Qw''aq9AA#'3QR7SA Hr"qkstathrc [(aSSKJn O [S5eUR SS5R 5nUR SS5R 5nUR upVpxUS-n URXVU SU5RS5n URXVU SU5RS5R SS5n X-n [R"U [R"U5- SS 9n U RS5upU RS5nUSU- :R5nUR!SUR#S55n[R$"UU5nUR'S5R)[R*5nUR#SS S 9R)[R*5nUR-[R."U5UUUSSS 9$) Nr) BlockMaskz=Nabla attention is not supported with this version of PyTorchr%r @r0r&T)r descending) BLOCK_SIZEmask_mod)r!torch.nn.attention.flex_attentionrK ValueError transpose contiguousr+r1meanrsoftmaxrsqrtsortcumsum_intgatherargsort logical_orsumtoint32from_kv_blocks zeros_like)rFrGrHrIrKBhSDs1qakamapvalsindscvalsmaskkv_nbkv_indss r nablaT_v2rqys ?XYY Aq$$&A Aq$$&AJA! bB 1R # ( ( ,B 1R # ( ( , 6 6r2 >B 'C --diil* 3C"JD LL E QW  ! ! #D ;;r4<<+ ,D   D# &D HHRLOOEKK (Ellrdl366u{{CG  # #E$4$4U$;WeWacnr # ssr"c2^\rSrSrSU4SjjrSrSrU=r$)Kandinsky5TimeEmbeddingsc<>[TU]5 US-S:XdeXlX0l[ URS-UR5Ul[ R"XSS9Ul[ R"5Ul [ R"X"SS9Ul g)Nr rTbias) super__init__ model_dimrr!rnnLinearin_layerSiLU activation out_layer)selfrztime_dimr __class__s r ry!Kandinsky5TimeEmbeddings.__init__sv 1}!!!"$t~~2DOOD  )DA '')8DAr"c[R"UR[R5URRUR S95n[R "[R"U5[R"U5/SS9nURURURU555nU$)N)devicer0r&) routerr_rrrcatcossinrrr})rtimeargs time_embeds r forward Kandinsky5TimeEmbeddings.forwards}{{4775==14::== =3TUYY $4ArJ ^^DOODMM*4M$NO r")rrr}rrzr@__name__ __module__ __qualname____firstlineno__ryr__static_attributes__ __classcell__rs@r rsrssBr"rsc.^\rSrSrU4SjrSrSrU=r$)Kandinsky5TextEmbeddingsc>[TU]5 [R"XSS9Ul[R "USS9Ulg)NTrvelementwise_affine)rxryr{r|r} LayerNormnorm)rtext_dimrzrs r ry!Kandinsky5TextEmbeddings.__init__s2  (DA LLtD r"cdURU5nURU5RU5$N)r}rtype_as)r text_embeds r r Kandinsky5TextEmbeddings.forwards*]]:. yy$,,Z88r")r}rrrs@r rrsE 99r"rc.^\rSrSrU4SjrSrSrU=r$)Kandinsky5VisualEmbeddingsc>[TU]5 X0l[R"[ R "U5U-U5Ulgr)rxry patch_sizer{r|rprodr})r visual_dimrzrrs r ry#Kandinsky5VisualEmbeddings.__init__s3 $ $))J"7*"DiP r"c hURup#pEnURUX0RS-URSX@RS-URSXPRS-URSU5RSSSSSSSS5R SS5nUR U5$) Nrr%r rr6r5r7)r+viewrr9r(r})rr)r>r?r@rArs r r"Kandinsky5VisualEmbeddings.forwards34770 fS FFOOA.."//!,,"++" WQ1aAq! , WQ] }}Qr")r}rrrs@r rrsQ   r"rc2^\rSrSrSU4SjjrSrSrU=r$)Kandinsky5RoPE1Dc>[TU]5 X0lXlX l[ US-U5n[ R"X$RS9nURS[ R"XT5SS9 g)Nr rrF persistent) rxryrrmax_posr!rrrregister_bufferr)rrrrfreqposrs r ryKandinsky5RoPE1D.__init__s] $ :.ll7**5 VU[[%;Nr"cURUn[R"U5n[R"U5n[R"X4*XC/SS9nUR "/UR SSQSPSP76nURS5$)Nr0r&r )rrrrstackrr+ unsqueeze)rrrcosinesiner*s r rKandinsky5RoPE1D.forwardstyy~4yy{{FE48bAyy0$**Sb/010a0~~b!!r")rrr)irrrs@r rrsO""r"rc6^\rSrSrSU4SjjrSSjrSrU=r$)Kandinsky5RoPE3Dc0>[T U]5 XlX lX0l[ [ X55H]unupV[US-U5n[R"XgRS9nURSU3[R"X5SS9 M_ g)Nr rargs_Fr) rxry axes_dimsrr enumeratezipr!rrrrr) rrrriaxes_dim ax_max_posrrrs r ryKandinsky5RoPE3D.__init__s " $)23y3J)K %A%X]J7D,,zr?r@rAargs_targs_hargs_wrrrr*s r rKandinsky5RoPE3D.forwardsP.3+ fSV$|A6SV$|A6SV$|A6yy AxAr299*aXYZ Aq&!R077 aQVXYZ Aq!UB/66zVUVXYZ    4yy{{FE48bAyy0$**Sb/010a0~~b!!r")rrr))rrr)?rrrrs@r rrs X""r"rc.^\rSrSrU4SjrSrSrU=r$)Kandinsky5ModulationicJ>[TU]5 [R"5Ul[R "XU-5UlUR RRR5 UR RRR5 gr) rxryr{r~rr|rweightdatazero_rw)rrrz num_paramsrs r ryKandinsky5Modulation.__init__sh '')8)-CD ""((*   &&(r"cBURURU55$r)rrrr)s r rKandinsky5Modulation.forwards~~dooa011r")rrrrs@r rrs)22r"rc,\rSrSrSrSrSrSSjrSrg)Kandinsky5AttnProcessoriNcr[[S5(d"[URRS35eg)Nscaled_dot_product_attentionz; requires PyTorch 2.0. Please upgrade your pytorch version.)hasattrF ImportErrorrr)rs r ry Kandinsky5AttnProcessor.__init__s3q899!8!8 99tuv v:r"c jURU5nUbURU5nURU5nURSSURSSpUR"/U QUR PSP76nUR"/U QUR PSP76nUR"/U QUR PSP76nOURU5nURU5nURSSn UR"/U QUR PSP76nUR"/U QUR PSP76nUR"/U QUR PSP76nUR UR55RU5nURUR55RU5nSn Ub.U "Xd5RU5nU "Xt5RU5nUb[UUUSUSS9n OSn [UUUU URURS9nURSS5nURU5n U $)Nr0cUR"/URSSQSPSPSP76R[R5nX-R SS9nUR"UR6R[R 5$)Nr0r%r r&)r1r+r_rrr^bfloat16)r)r*x_x_outs r apply_rotary6Kandinsky5AttnProcessor.__call__..apply_rotary5so3AGGCRL3"3a3366u}}EBYOOO+E==!''*--enn= =r"sta_maskP)rI) attn_maskbackendparallel_configrM)to_queryto_keyto_valuer+r1 num_heads query_normfloatrkey_normrqr_attention_backend_parallel_configr(r)rattn hidden_statesencoder_hidden_states rotary_emb sparse_paramsquerykeyvaluer+ cond_shaperrattn_outs r __call__ Kandinsky5AttnProcessor.__call__s+ m, ,++34CMM"78E % CR 0#))CR.:MM=5=$..="=E++>z>4>>>2>CMMB:Bt~~BrBE++m,CMM-0EKK$EMM=5=$..="=E++9u9dnn9b9CMM=5=$..="=E .66u=mmCIIK(005 >  ! 3;;EBEs/77>-0r"NNN) rrrrrrryrrrr"r rrsw:r"rc ^\rSrSr\r\/rS U4SjjrS S\RS\RS-S\RS-S\ \R\R4S-S\R4 S jjr S r U=r $) Kandinsky5AttentioniXNc>[TU]5 X-S:XdeX-Ul[R"XSS9Ul[R"XSS9Ul[R"XSS9Ul[R"U5Ul [R"U5Ul [R"XSS9Ul UcUR5nURU5 g)NrTrv)rxryrr{r|rrrRMSNormrrr_default_processor_cls set_processor)r num_channelshead_dim processorrs r ryKandinsky5Attention.__init__^s &!+++%1 ,4H ii F  ,4H **X. 8, J>41Q5I$!$>J~~   #8'!     v Ks! D 0D 7D  D+D)rrrrrrrrr)rrrrrr_available_processorsryrrtuplerrrrs@r rrXs4&&6:-1?C  ||  %||d2 ||d*  %,, 45<     r"rc.^\rSrSrU4SjrSrSrU=r$)Kandinsky5FeedForwardic>[TU]5 [R"XSS9Ul[R "5Ul[R"X!SS9Ulg)NFrv)rxryr{r|r}GELUrr)rrff_dimrs r ryKandinsky5FeedForward.__init__s?  #E: '')6U;r"c`URURURU555$r)rrr}rs r rKandinsky5FeedForward.forwards#~~doodmmA.>?@@r")rr}rrrs@r r5r5s< AAr"r5c.^\rSrSrU4SjrSrSrU=r$)Kandinsky5OutLayeric>[TU]5 X@l[X!S5Ul[ R "USS9Ul[ R"U[R"U5U-SS9Ul g)Nr FrTrv) rxryrr modulationr{rrr|rrr)rrzrrrrs r ryKandinsky5OutLayer.__init__sV $.xAFLLuE 9dii .Cj.PW[\r"c [R"URU5RSS9SSS9upEUR UR 55UR 5SS2SS4S--UR 5SS2SS4-R U5nURU5nURupxpn URUUU U SURSURSURS5RSSSSSS S S 5RSS5RSS 5RS S 5nU$) Nr%r&r r0rrr6r7rrr5) rchunkr?rrrrrr+rrr9r() r visual_embedrrshiftscaler)r>r?r@rAr.s r rKandinsky5OutLayer.forwards4{{4??:#>#H#HQ#H#OQRXZ[  IIl((* +u{{}Qd]/Kc/Q RUZU`U`UbcdfjlpcpUq q ',   NN< (12. fQ FF""" WQ1aAq! , WQ] WQ] WQ] r")r?rrrrrs@r r=r=s]r"r=c.^\rSrSrU4SjrSrSrU=r$)!Kandinsky5TransformerEncoderBlockic>[TU]5 [X!S5Ul[R "USS9Ul[X[5S9Ul [R "USS9Ul [X5Ul g)Nr7Frr) rxryrtext_modulationr{rself_attention_normrrself_attentionfeed_forward_normr5 feed_forwardrrzrr8rrs r ry*Kandinsky5TransformerEncoderBlock.__init__sc 3HK#%<< e#T 1)QhQjk!#iE!R1)Dr"cb[R"URU5RSS9SSS9upE[R"USSS9upgnUR UR 55UR 5S--UR 5-R U5n URXS9n UR 5UR 5U R 5--R U5n[R"USSS9upgnURUR 55UR 5S--UR 5-R U5n URU 5n UR 5UR 5U R 5--R U5nU$)Nr%r&r r0rr)r ) rrBrKrrLrrrMrNrO) rr)rr*self_attn_params ff_paramsrDrEgateouts r r)Kandinsky5TransformerEncoderBlock.forwards]&+kk$2F2Fz2R2\2\ab2\2cefln&o#"[[)91"Ed'' 2ekkmc6IJU[[]Zccdef!!#!7 WWY 3 3 < d%%aggi0EKKMC4GH5;;=Xaabcd$ WWY 3 3 < [TU]5 [X!S5Ul[R "USS9Ul[X[5S9Ul [R "USS9Ul [X[5S9Ul [R "USS9Ul [X5Ulg)N FrrJ)rxryrvisual_modulationr{rrLrrrMcross_attention_normcross_attentionrNr5rOrPs r ry*Kandinsky5TransformerDecoderBlock.__init__s !5h1!M#%<< e#T 1)QhQjk$&LLu$U!29RiRkl!#iE!R1)Dr"c[R"URU5RSS9SSS9upgn[R"USSS9upn UR UR 55U R 5S--U R 5-R U5n URXUS9n UR 5U R 5U R 5--R U5n[R"USSS9upn URUR 55U R 5S--U R 5-R U5n URXS9n UR 5U R 5U R 5--R U5n[R"USSS9upn URUR 55U R 5S--U R 5-R U5n URU 5n UR 5U R 5U R 5--R U5nU$)Nr%r&rr0r)r r )r ) rrBr\rrLrrrMr]r^rNrO) rrCrrr*r rScross_attn_paramsrTrDrErU visual_outs r r)Kandinsky5TransformerDecoderBlock.forwards29>  " ": . 8 8Q 8 ?: 6Y#[[)91"Ed..|/A/A/CD X[H[\_d_j_j_lluu  ((Ta(b $**,tzz|j>N>N>P/PPYYZfg "[[):A2Fd// 0B0B0DEY\I\]`e`k`k`mmvv  ))*)W $**,tzz|j>N>N>P/PPYYZfg "[[A2>d,,\-?-?-ABekkmVYFYZ]b]h]h]jjss  &&z2 $**,tzz|j>N>N>P/PPYYZfg r")r^r]rOrNrMrLr\rrs@r rYrYs Er"rYc^\rSrSrSrSS/r/SQrSr\S S\ S \ S \ S \ S \ S \ S\ S\ S\ S\ S\ 4U4Sjjj5r S!S\RS\RS\RS\RS\\ \ \ 4S\R"S\\ \ \ 4S\\ \4S-S\ S\\R*-4SjjrSrU=r$)"Kandinsky5Transformer3DModeliz7 A 3D Diffusion Transformer model for video-like data. rHrY)time_embeddingsr?r\rKTNattention_typeattention_causalattention_localattention_globattention_window attention_P attention_wT attention_wW attention_wHattention_add_staattention_methodc >[TU]5 [U 5nXlXplX`lXlXlU (aSU-S-OUn[Xt5Ul [X'5Ul [X45Ul [UXv5Ul[U5Ul[#U 5Ul[&R("[+U 5Vs/sHn[-XtUU5PM sn5Ul[&R("[+U 5Vs/sHn[1XtUU5PM sn5Ul[5XtXV5UlSUlgs snfs snf)Nr r%F)rxryr^ in_visual_dimrzr visual_condrgrsrfrtext_embeddingspooled_text_embeddingsrvisual_embeddingsrtext_rope_embeddingsrvisual_rope_embeddingsr{ ModuleListr:rHtext_transformer_blocksrYvisual_transformer_blocksr=rgradient_checkpointing)rrs in_text_dim in_text_dim2rout_visual_dimrrzr8num_text_blocksnum_visual_blocksrrtrgrhrirjrkrlrmrnrorprqrvisual_embed_dimr.rs r ry%Kandinsky5Transformer3DModel.__init__s86 y>*"$&,4?1},q0] 8 L7 O&>|&V#!;!&6y&A#(*}}_det_u v_uZ[ .yFH U_u v( $*,01 1A2)vxX1 * &,I\&+# w s EE r r timesteppooled_projectionsvisual_rope_pos text_rope_posrr  return_dictr!c *Un Un Un Un URU 5n URU 5nXRU 5-nURU 5nUR U5nUR SS9nUR HLn[R"5(a&UR(aURUXU5n MCU"XU5n MN URSSnURUXW5nUbUSOSn[UUUUS9unnURHQn[R"5(a)UR(aURUUU UUU5nMFU"XUUU5nMS [UUUS9nUR!XU5n U (dU $[#U S9$) a  Forward pass of the Kandinsky5 3D Transformer. Args: hidden_states (`torch.FloatTensor`): Input visual states encoder_hidden_states (`torch.FloatTensor`): Text embeddings timestep (`torch.Tensor` or `float` or `int`): Current timestep pooled_projections (`torch.FloatTensor`): Pooled text embeddings visual_rope_pos (`tuple[int, int, int]`): Position for visual RoPE text_rope_pos (`torch.LongTensor`): Position for text RoPE scale_factor (`tuple[float, float, float]`, optional): Scale factor for RoPE sparse_params (`dict[str, Any]`, optional): Parameters for sparse attention return_dict (`bool`, optional): Whether to return a dictionary Returns: [`~models.transformer_2d.Transformer2DModelOutput`] or `torch.FloatTensor`: The output of the transformer rr&Nr0 to_fractalF)r,)sample)rurfrvrwrxrr{ris_grad_enabledr}_gradient_checkpointing_funcr+ryr.r|r3rr)rr r rrrrrr rr)rrpooled_text_embedrrC text_ropetext_transformer_block visual_shape visual_ropervisual_transformer_blocks r r$Kandinsky5Transformer3DModel.forwardQs: * .))*5 ))$/ "="=>O"PP --a0 --m< ''A'. &*&B&B "$$&&4+F+F!>>*JI 4JIV 'C$))#2. 11,^ 4A4M]<0SX $3L+|hr$s! k(,(F(F $$$&&4+F+F#@@, !    8 j+}  )G)|PZ[ NNrs~  B?=J$7'   H %   6 > !t !t !t !t  !tHryy$9ryy9  2"ryy"&"ryy"B 2299 2BBJ. "))%9. bABIIA""J 4( (V^2 ^2r"