3jMSSKrSSKrSSKJr SSKJs Jr SSKJrJ r SSK J r SSK J r SSKJrJrJr SSKJr SS KJr SS KJrJr \ R2"\5r"S S \R85r"S S\R85r"SS\R85r"SS5r "SS\R85r!"SS\\5r"g)N) ConfigMixinregister_to_config)logging) Attention)TimestepEmbedding Timestepsget_2d_sincos_pos_embed)Transformer2DModelOutput) ModelMixin) AdaLayerNormRMSNormcn^\rSrSrS\S\4U4SjjrS\RS\R4SjrSr U=r $) OmniGenFeedForward! hidden_sizeintermediate_sizec>[TU]5 [R"USU-SS9Ul[R"X!SS9Ul[R "5Ulg)NrFbias)super__init__nnLinear gate_up_proj down_projSiLU activation_fn)selfrr __class__s k/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/models/transformers/transformer_omnigen.pyrOmniGenFeedForward.__init__"sJ IIk17H3HuU#4NWWY hidden_statesreturncURU5nURSSS9up2X RU5-nURU5$)Nrdim)rchunkrr)r r% up_statesgates r"forwardOmniGenFeedForward.forward)sH%%m4 #//!/4 2 24 88 ~~i((r$)rrr) __name__ __module__ __qualname____firstlineno__intrtorchTensorr.__static_attributes__ __classcell__r!s@r"rr!s6'C'C')U\\)ell))r$rc^\rSrSrSS\S\S\S\S\S\S\4U4S jjjrS rS \ RS \S \ R4Sjr SS \ RS \S\ RS \ R4Sjjr Sr U=r$)OmniGenPatchEmbed0 patch_size in_channels embed_dimrinterpolation_scalepos_embed_max_size base_sizecT>[T U]5 [R"X#X4XS9Ul[R"X#X4XS9UlXlXPlX`l[UURUURSS9nURSUR5RS5SS9 g)N) kernel_sizestriderpt)rBr@ output_type pos_embedrT) persistent) rrrConv2doutput_image_projinput_image_projr=r@rAr register_bufferfloat unsqueeze) r r=r>r?rr@rArBrHr!s r"rOmniGenPatchEmbed.__init__1s !# 0HQ[" !# 0HQ[! %#6 "4+   # # $ 8 8   [)//*;*E*Ea*HUYZr$c0URc [S5eXR-nX R-nXR:a[SUSURS35eX R:a[SUSURS35eURU- S-nURU- S-nURR SURURS 5nUSS2X3U-2XDU-2SS24nUR SS UR S 5nU$) z2Crops positional embeddings for SD3 compatibility.Nz.`pos_embed_max_size` must be set for cropping.zHeight (z/) cannot be greater than `pos_embed_max_size`: .zWidth (rr()rA ValueErrorr=rHreshapeshape)r heightwidthtopleftspatial_pos_embeds r"_cropped_pos_embed$OmniGenPatchEmbed._cropped_pos_embedQs>  " " *MN N??*( ++ +6("QRVRiRiQjjkl  ** *% OPTPgPgOhhij &&/A5''%/A5 NN221d6M6MtOfOfhjk-aV|1CTSXLEXZ[.[\-55a=N=T=TUW=XY  r$r%is_input_imager&cU(aURU5nOURU5nURS5RSS5nU$)NrrS)rLrKflatten transpose)r r%r^s r"_patch_embeddings#OmniGenPatchEmbed._patch_embeddingshsH  11-@M 22=AM%--a0::1a@ r$padding_latentc[U[5(aUcS/[U5-n/n[X5HupVURSSupxUR XR5nUR Xx5n XY-nUb/[R"XVRUR5/SS9nURU5 M U$URSSupxUR Xx5n UR X5nX-nU$)Nr)) isinstancelistlenziprVrbr\r5cattodeviceappend) r r%r^rdpatched_latents sub_latentpaddingrWrXrHs r"r.OmniGenPatchEmbed.forwardps mT * *%"&#m*I 22=QM+7Or$)rLr@rKr=rA)riTrS@)N)r0r1r2r3r4boolrNrr\r5r6rbr.r7r8r9s@r"r;r;0s%&"%[[[ [  [ # [ [[[@!.u||TV[VbVbae"\\;?QVQ]Q] r$r;c4^\rSrSrSU4SjjrSrSrU=r$)OmniGenSuScaledRotaryEmbeddingc P>[TU]5 XlX lX@lSUR[ R "SURS[ RS9R5UR- -- nURSUSS9 USUl US Ul X0l g) N?rr)dtypeinv_freqF)tensorrI short_factor long_factor) rrr*max_position_embeddingsbaser5arangeint64rNrMrr original_max_position_embeddings)r r*rrr rope_scalingr}r!s r"r'OmniGenSuScaledRotaryEmbedding.__init__s '>$ $)) Q!5;;(W(](](_bfbjbj(jkl ZUK(8' 60P-r$cX[R"U5S-nX0R:a9[R"UR[R UR S9nO8[R"UR[R UR S9n[R"SURS[RUR S9R5UR- nSX@RU--- Ul URSSS2S4R5RURSSS5nUSS2SSS24R5nUR R n[#U[$5(aUS:waUOSn[R&"US S 9 UR5UR5-R)SS5n [R*"X4SS 9Sn UR,UR- n U S::aSn ON[.R0"S[.R2"U 5[.R2"UR5- -5n U R55U -n U R75U -nSSS5 X4$!,(df  W W4$=f) NrS)r|rmrrr{r(mpscpuF) device_typeenabledr))r5maxrr~rfloat32rmrrr*rrNrr}expandrVtypergstrautocastrarkrmathsqrtlogcossin)r r% position_idsseq_len ext_factorsinv_freq_shapeinv_freq_expandedposition_ids_expandedrfreqsembscalescaling_factorrrs r"r.&OmniGenSuScaledRotaryEmbedding.forwards)))L)A- :: :,,t'7'7u}}UbUiUijK,,t'8'8 VcVjVjkK LLDHHau{{=CWCW X ^ ^ `cgckck k {YY-FFG  MM$4-8>>@GG HZHZ[\H]_acde ,QaZ 8 > > @$**// %/ S%A%AkUZFZk`e ^^ U C&,,.1F1L1L1NNYYZ[]^_E))UN3A6C0043X3XXE|!$!%1txx$JoJoAp/p+p!q'')n,C'')n,CDxD CCxs ,C!J J))rr*r}rrrr)'N)r0r1r2r3rr.r7r8r9s@r"rxrxsswQ r$rxc \rSrSrSrSrS S\S\RS\RS\RS-S \RS-S \R4 S jjr S r g)OmniGenAttnProcessor2_0z Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is used in the OmniGen model. cD[[S5(d [S5eg)Nscaled_dot_product_attentionzPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.)hasattrF ImportError)r s r"r OmniGenAttnProcessor2_0.__init__s!q899pq q:r$Nattnr%encoder_hidden_statesattention_maskimage_rotary_embr&cURupgnURU5n URU5n URU5n U R 5upnU RSnXR -nUU-nU R USUR U5RSS5n U R USUU5RSS5n U R USUU5RSS5n UbSSKJ n U"XSS9n U"XSS9n [R"XXS9nURSS5RU 5nURXUR5nURS"U5nU$) Nr(rSr)apply_rotary_embrf)use_real_unbind_dim) attn_maskr)rVto_qto_kto_vsizeheadsviewra embeddingsrrrtype_asrUout_dimto_out)r rr%rrr batch_sizesequence_length_querykeyvaluebszq_len query_dim inner_dimhead_dimkv_headsrs r"__call__ OmniGenAttnProcessor2_0.__call__s_*7)<)<& Q -(ii-. /0 % IIIbM  *( :r4::x@JJ1aPhhz2x:DDQJ :r8X>HHAN  ' 5$URTUE"3bQC66u5c %//15==eD %--c$,,G  A}5 r$)NN) r0r1r2r3__doc__rrr5r6rr7rr$r"rrsw r/304 %%||% %|| %  t+ %  ,,- % %%r$rc ^\rSrSrS\S\S\S\S\SS4 U4S jjrS \RS \RS \RS\R4S jr Sr U=r $) OmniGenBlockrnum_attention_headsnum_key_value_headsr rms_norm_epsr&Nc >[TU]5 [XS9Ul[ UUX-UUSUS[ 5S9 Ul[XS9Ul[X5Ul g)NepsF) rcross_attention_dimdim_headrrrrout_bias processor) rrrinput_layernormrr self_attnpost_attention_layernormrmlp)r rrrrrr!s r"rOmniGenBlock.__init__se &{E"! + 7%(-/  )0 (N%%kEr$r%rrcURU5nURUUUUS9nX-nURU5nURU5nX-nU$)N)r%rrr)rrrr)r r%rrnorm_hidden_states attn_output ff_outputs r"r.OmniGenBlock.forwardsn"11-@nn,"4)- % &3 "::=IHH/0 %1 r$)rrrr) r0r1r2r3r4rNrr5r6r.r7r8r9s@r"rrsFF!F! F  F  F F2"\\;@<<[`[g[g r$rc(^\rSrSrSrSrS/r/SQr\S'S\ S\ S \ S \ S \ S \ S \ S\ S\ S\ S\ S\ S\ S\ S\ S\ S\ S\ S\ 4&U4Sjjj5rS\R S\\R S\ S\R S-4SjrS(S \R S!\ \ -\R&-S\R S\\R S\ \ \\ 4S"\R S#\R S$\ S\\\R -4S%jjrS&rU=r$))OmniGenTransformer2DModelia The Transformer model introduced in OmniGen (https://huggingface.co/papers/2409.11340). Parameters: in_channels (`int`, defaults to `4`): The number of channels in the input. patch_size (`int`, defaults to `2`): The size of the spatial patches to use in the patch embedding layer. hidden_size (`int`, defaults to `3072`): The dimensionality of the hidden layers in the model. rms_norm_eps (`float`, defaults to `1e-5`): Eps for RMSNorm layer. num_attention_heads (`int`, defaults to `32`): The number of heads to use for multi-head attention. num_key_value_heads (`int`, defaults to `32`): The number of heads to use for keys and values in multi-head attention. intermediate_size (`int`, defaults to `8192`): Dimension of the hidden layer in FeedForward layers. num_layers (`int`, default to `32`): The number of layers of transformer blocks to use. pad_token_id (`int`, default to `32000`): The id of the padding token. vocab_size (`int`, default to `32064`): The size of the vocabulary of the embedding vocabulary. rope_base (`int`, default to `10000`): The default theta value to use when creating RoPE. rope_scaling (`dict`, optional): The scaling factors for the RoPE. Must contain `short_factor` and `long_factor`. pos_embed_max_size (`int`, default to `192`): The maximum size of the positional embeddings. time_step_dim (`int`, default to `256`): Output dimension of timestep embeddings. flip_sin_to_cos (`bool`, default to `True`): Whether to flip the sin and cos in the positional embeddings when preparing timestep embeddings. downscale_freq_shift (`int`, default to `0`): The frequency shift to use when downscaling the timestep embeddings. timestep_activation_fn (`str`, default to `silu`): The activation function to use for the timestep embeddings. Tr)patch_embedding embed_tokensnormNr>r=rrrrr num_layers pad_token_id vocab_sizerr rope_baserrA time_step_dimflip_sin_to_cosdownscale_freq_shifttimestep_activation_fnc R>[TU]5 XlXl[ UUUUS9Ul[ UUU5Ul[UUU5Ul [UUU5Ul [R"XU 5Ul [X5-U U U US9Ul[R "[#U5Vs/sHn[%X5XgU5PM sn5Ul[)X4S9Ul[-USSSS9Ul[R0"X2U-UR-SS 9UlSUlgs snf) N)r=r>r?rA)rrrrrFgư>rS)norm_elementwise_affinenorm_eps chunk_dimTr)rrr> out_channelsr;rr time_projr time_token t_embedderr Embeddingrrxrope ModuleListrangerlayersrrrnorm_outrproj_outgradient_checkpointing)r r>r=rrrrrrrrrrrrrArrrrrr!s r"r"OmniGenTransformer2DModel.__init__Hs,. &'0!#!1  #=/CWX+M;H^_+M;H^_LL,O2  .$;-M%   mmz* *A[?Rgst*   K: $[%Z^jkl  +J/FIZIZ/Zaef &+# s0D$ input_idsinput_img_latentsinput_image_sizesr&cHUcgUVs/sHoDRUR5PM nnURU5nSnURUSS9nUR 5H7nX8H,upXvRUR5XXX24'US- nM. M9 U$s snf)NrTr^rS)rlr|rrkeys) r rrrxcondition_tokens input_img_inxinput_image_tokensb_inx start_inxend_inxs r"_get_multimodal_embeddings4OmniGenTransformer2DModel._get_multimodal_embeddingss  7HI7H!TT$**-7HI,,Y7 !112CTX1Y&++-E&7&>" =O=^=a=a$**> (9!9:" '?. Js$Br%timesteprr return_dictc  URuppURRn X-X-pURUSS9nUR S5nUR U5R U5nURU5RS5nURU5nURX4U5nUb[R"UUU/SS9nO[R"UU/SS9nUR S5nURSU5R5nUbhUR5S:XaTUR n[R""U5R$nSU- U-nURS5R U5nUR'X5nUR(HJn[R*"5(a&UR,(aUR/UXU5nMCU"XUS9nML UR1U5nUSS2U*S24nUR3UUS9nUR5U5nUR7XXU S5nUR9S S SSS S 5R;S S 5R;S S5nU(dU4$[=US 9$)NFr rSr)r(r)rr)tembrrrs)sample)rVconfigr=rrrrrrOrrr5rkrlongr*r|finfominrris_grad_enabledr_gradient_checkpointing_funcrrrrUpermuter`r )r r%rrrrrrrr num_channelsrWrXppost_patch_heightpost_patch_widthnum_tokens_for_output_image timestep_projrrr  seq_lengthr| min_dtyperblockoutputs r"r.!OmniGenTransformer2DModel.forwardsu3@2E2E/ & KK " ".4k5:+,,]5,Q &3&8&8&;#x088G __]3==a@ }-::9Yjk  '!II'7]&SYZ[M!IIz=&AqIM"''* #((Z8==?   %.*<*<*>!*C!''E E*..I.0I=N+55a8@@ON 99]A[[E$$&&4+F+F $ A A=:J! !&meu v ! -0 %a*E)E)F&FG  m$ ?  m4 %--jM]bcegh &&q!Q1a8@@AFNNqRST9 'v66r$) rrr>rrrrrrrrrr)rsri gh㈵> r-i r-i}i@}rrrNrtTrsilu)T)r0r1r2r3r _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsrr4rNdictrvrrr5r6rhr FloatTensorr tupler.r7r8r9s@r"rrs&P(,$'('R$"#%#%!%!'-04!"% $$%&,)9,9,9, 9,  9, ! 9,!9,9,9,9,9,"%9,+.9,9,9,  !9,"#9,$%9,&"'9,(!$)9,9,v  :>u||:L ae   8!=7||=7+ 1 11=7<< =7   - =7  T#Y/ =7 =7ll=7=7 "E%,,$7 7=7=7r$r)#rr5torch.nnrtorch.nn.functional functionalrconfiguration_utilsrrutilsrattention_processorrrr r r modeling_outputsr modeling_utilsr normalizationrr get_loggerr0loggerModulerr;rxrrrrr$r"rBs B+NN7'1   H % ) )U Up0RYY0f//d+299+\y7 Ky7r$