3j3E SSKJr SSKrSSKJr SSKJrJr SSKJ r SSK J r SSK J r Jr SSKJr S S KJr S S KJrJr S S KJr S S KJrJr S SKJr S SKJr S SKJ r J!r! \RD"\#5r$"SS\RJ5r&"SS\RJ5r'"SS\RJ5r(\"SS\RJ55r)"SS\RJ5r*\"SS\\\ \ \55r+g))AnyN) ConfigMixinregister_to_config)PeftAdapterMixin)FromOriginalModelMixin)apply_lora_scalelogging)maybe_allow_in_graph) FeedForward)MochiAttentionMochiAttnProcessor2_0) CacheMixin)%MochiCombinedTimestepCaptionEmbedding PatchEmbed)Transformer2DModelOutput) ModelMixin)AdaLayerNormContinuousRMSNormc:^\rSrSrS\4U4SjjrSSjrSrU=r$)MochiModulatedRMSNorm&epscR>[TU]5 Xl[SUS5UlgNrF)super__init__rrnorm)selfr __class__s i/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/transformers/transformer_mochi.pyrMochiModulatedRMSNorm.__init__'s# AsE* cURnUR[R5nUR U5nUbX-nURU5nU$N)dtypetotorchfloat32r)r hidden_statesscalehidden_states_dtypes r"forwardMochiModulatedRMSNorm.forward-sU+11%((7  -0  )1M%(()<= r$)rrr&) __name__ __module__ __qualname____firstlineno__floatrr.__static_attributes__ __classcell__r!s@r"rr&s+E+  r$rc^\rSrSrS S\S\4U4SjjjrS\RS\RS\R4SjrS r U=r $) MochiLayerNormContinuous; embedding_dimconditioning_embedding_dimc>[TU]5 [R"5Ul[R "X!US9Ul[US9Ulg)N)biasr) rrnnSiLUsiluLinearlinear_1rr)r r;r<rr>r!s r"r!MochiLayerNormContinuous.__init__<s? GGI  "?BB177KL IIa!eooa033EMMBB DttK  r$)rDrrB)h㈵>T) r0r1r2r3intrr)Tensorr.r5r6r7s@r"r9r9;sY   3 3%( 3 3 ! << !!&  !  ! !r$r9c ^\rSrSrSrSS\S\S\S\SS4 U4S jjjrS \ RS \ RS\ \ R\ R\ R\ R44S jr S r U=r$)MochiRMSNormZeroXzm Adaptive RMS Norm used in Mochi. Parameters: embedding_dim (`int`): The size of each embedding vector. r; hidden_dimrelementwise_affinerHNc>[TU]5 [R"5Ul[R "X5Ul[SUS5Ulgr) rrr@rArBrClinearrr)r r;rTrrUr!s r"rMochiRMSNormZero.__init__`s= GGI ii : AsE* r$r+embcZURnURURU55nURSSS9upEpgUR UR [ R55SUSS2S4R [ R5--nUR U5nXXg4$)NrJdim)r'rWrBchunkrr(r)r*)r r+rYr- scale_msagate_msa scale_mlpgate_mlps r"r.MochiRMSNormZero.forwardis,11kk$))C.)3699QA93F0 Y -"2"25=="ABa)TUW[T[J\J_J_`e`m`mJnFno %(()<=  ;;r$)rWrrB)rNF)r0r1r2r3__doc__rOr4boolrr)rPtupler.r5r6r7s@r"rRrRXsbg+ +.1+8=+Z^+ ++ <"\\ <05  < u||U\\5<<E F < [T U]5 XplSU-S-S-UlSU-S-S-Ul[ USU-USS9UlU(d[ USU-USS9UlO[UUUS9Ul[UUUSUSUUU[5SS9 Ul [US 9Ul UR(d [US 9OSUl[U5UlUR(d [US 9OSUl[#XRUSS 9UlSUlU(d[#UURUSS 9Ul[US 9Ul[US 9Ulg) Nr[r rF)rrU)r;r<rrN) query_dimheadsdim_headr>added_kv_proj_dimadded_proj_biasout_dimout_context_dimro processorrr?) inner_dimrnr>)rrro ff_inner_dimff_context_inner_dimrRnorm1 norm1_contextr9rrattn1rnorm2 norm2_contextnorm3 norm3_contextr ff ff_contextnorm4 norm4_context) r r]rjrkrlrmrnrorr!s r"rMochiTransformerBlock.__init__sf  0Wq[Q.%&)>%>%Bq$H!%c1s7PUV !1#q;P7PVYns!tD !93+."D  $%'3!1-+-   +s3 CGCXCX2s;^b*3/ CGCXCX2s;^bc->->mbgh)%33+ DO+s3 2s;r$r+encoder_hidden_statestembencoder_attention_maskimage_rotary_embcURX5upgpUR(dURX#5uppOURX#5n URUU UUS9upXR U[ R "U5RS55-nURUSURS5R[ R5-5nURU5nXRU[ R "U 5RS55-nUR(dX RU[ R "W 5RS55-nURUSW RS5R[ R5-5n URU 5nX R!U[ R "W 5RS55-nX4$)N)r+rrattention_maskrJ)r|ror}r~rr)tanhrKrr(r*rrrrrr)r r+rrrrnorm_hidden_statesr`rarbnorm_encoder_hidden_states enc_gate_msa enc_scale_mlp enc_gate_mlpattn_hidden_statescontext_attn_hidden_states ff_outputcontext_ff_outputs r"r.MochiTransformerBlock.forwards=AJJ}<[9i$$TXTfTf%U Q &m\*.););R>\>\]^>_(`` $$$9r&)r0r1r2r3rdrOstrrer4rr)rPrfr.r5r6r7s@r"rhrhvs4"%!&<< <<!<< << # <<  <<<<<<<< <<<[TU]5 X-Ulgr&)rr target_area)r rrr!s r"rMochiRoPE.__init__s &3r$cP[R"XUS-XES9nUSSUSS-S- $)NrJdevicer'r )r)linspace)r startstopnumrr'edgess r"_centersMochiRoPE._centerss4uC!GFPcr U12Y&!++r$ num_framesheightwidthrr'cXURX#-- S-n[R"XUS9nURU*U-S- X&-S- X$U5nURU*U-S- X6-S- X4U5n [R"XxU SS9upn [R "XU /SS9R SS5n U $) Ng?rr ij)indexingrr\r)rr)arangermeshgridstackview)r rrrrr'r,thwgrid_tgrid_hgrid_w positionss r"_get_positionsMochiRoPE._get_positions s!!V^4< LL% @ MM6'E/A-v~/A6SX Y MM5&5.1,ema.?PU V!&a$!GKK 8bAFFr1M r$freqsposc[R"URR[R5 [R "SUR [R5UR [R55nSSS5 [R"U5n[R"U5nX44$!,(df  N==f)Nz nd,dhf->nhf) r)autocastrtyper*einsumr(cossin)r rr freqs_cos freqs_sins r" _create_ropeMochiRoPE._create_ropes ^^ELL--u}} =LLu}}0EuxxPUP]P]G^_E>IIe$ IIe$ ## > =s AC Cpos_frequenciescTURX#XEU5nURX5upX4$r&)rr) r rrrrrr'rrope_cosrope_sins r"r.MochiRoPE.forward&s4!!*eUK!..D!!r$)r)r)NN)r0r1r2r3rdrOrr)rPrrr'rrrfr.r5r6r7s@r"rrs84C43444 ,5<<,'+$(    t#  {{T!  &$%,,$U\\$ell$'+$( " " " "  "  t# "{{T! " u||U\\) * " "r$rcZ^\rSrSrSrSrS/rSS/r\SS\ S \ S \ S \ S \ S \ S\ S-S\ S\ S\ S\ S\ SS4U4Sjjj5r \ "S5SS\ RS\ RS\ RS\ RS\\ \4S-S\S\ R4Sjj5rSrU=r$) MochiTransformer3DModeli4a A Transformer model for video-like data introduced in [Mochi](https://huggingface.co/genmo/mochi-1-preview). Args: patch_size (`int`, defaults to `2`): The size of the patches to use in the patch embedding layer. num_attention_heads (`int`, defaults to `24`): The number of heads to use for multi-head attention. attention_head_dim (`int`, defaults to `128`): The number of channels in each head. num_layers (`int`, defaults to `48`): The number of layers of Transformer blocks to use. in_channels (`int`, defaults to `12`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `None`): The number of channels in the output. qk_norm (`str`, defaults to `"rms_norm"`): The normalization layer to use. text_embed_dim (`int`, defaults to `4096`): Input dimension of text embeddings from the text encoder. time_embed_dim (`int`, defaults to `256`): Output dimension of timestep embeddings. activation_fn (`str`, defaults to `"swiglu"`): Activation function to use in feed-forward. max_sequence_length (`int`, defaults to `256`): The maximum sequence length of text embeddings supported. Trh patch_embedrN patch_sizerjrk num_layersrl in_channels out_channelsrmtext_embed_dimtime_embed_dimrnmax_sequence_lengthrHc >[TU]5 X#-n U=(d Un[UUU SS9Ul[ U UU U SS9Ul[ R"[R"SX#S-4S55Ul [5Ul [ R"[U5Vs/sHn[U UUUUU XS- :HS9PM sn5Ul[#U U S S S S 9Ul[ R&"XU-U-5UlS Ulgs snf) N)rr embed_dimpos_embed_type)r;rlrrrjrr grJ)r]rjrkrlrmrnroFr layer_norm)rUr norm_type)rrrrr time_embedr@ Parameterr)fullrrrope ModuleListrangerhtransformer_blocksrnorm_outrCproj_outgradient_checkpointing)r rrjrkrrlrrrmrrrnrryir!s r"r MochiTransformer3DModel.__init__Vs+ '< #2{ %!#  @#"7)) !   "||EJJ;NfgPg7hjm,noK "$--z* +A&!(;'9*?#"/%&q.%8+  # /  $"    )*-D|-ST &+#/ s#Dattention_kwargsr+rtimestepr return_dictc URupxpn URRn X-n X-nURUUUURS9upUR SSSSS5R SS5nURU5nURSUS45R SS5nURURU U UUR[RS9n[UR5HWunn[R "5(a+UR"(aUR%UUUUUU5upMKU"UUUUUS 9upMY UR'X5nUR)U5nUR+XyXXS5nUR SS SSSSS 5nUR+USXU 5nU(dU4$[-US 9$) N) hidden_dtyperr rJrr[rr)r+rrrr)sample)shapeconfigrrr'permuteflattenr unflattenrrrr)r* enumerateris_grad_enabledr_gradient_checkpointing_funcrrreshaper)r r+rrrrr batch_size num_channelsrrrppost_patch_heightpost_patch_widthrrrblockoutputs r"r.MochiTransformer3DModel.forwards?L>Q>Q; *e KK " ""K :&*oo  ! "&,, '6' #&--aAq!<DDQJ ((7 %//J3CDLLQPQR 99     ''-- % "$"9"9:HAu$$&&4+F+F7;7X7X!)*$ 84 48="/*?+A%5 84 4;$ m:  m4 %--jFWklqst %--aAq!QB &&z2z5Q9 'v66r$)rrrrrrrr) r 0i Nrirr)NT)r0r1r2r3rd _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsrrOrrr r)rP LongTensordictrrer.r5r6r7s@r"rr4sj8(,$01(5v'>$#%"%%)#'!"!%#&>,>,!>, >,  >, # >,>,Dj>,>,>,>,>,!>, >,>,@()37 >7||>7 %||>7"" >7 !& >7 sCx.4/ >7>7 >7*>7r$r),typingrr)torch.nnr@configuration_utilsrrloadersrloaders.single_file_modelrutilsr r utils.torch_utilsr attentionr attention_processorrr cache_utilsr embeddingsrrmodeling_outputsrmodeling_utilsr normalizationrr get_loggerr0loggerModulerr9rRrhrrr$r"r&s  B'?.5#G$J7';   H %BII*!ryy!: