
    
3j'                        S SK r S SKrSSKJrJrJr  \R                  " \5      rS r	SS jr
SS jrS rS rS	 rS
 rS rS\S\\   S\4S jrS rS rS rS rS rS rSS jrSS jrSS jrS rS rS rS r g)     N   )is_peft_versionloggingstate_dict_all_zeroc                 V    U R                  SSS9u  p[        R                  " X!/SS9nU$ )Nr   r   dim)chunktorchcat)weightshiftscale
new_weights       a/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/loaders/lora_conversion_utils.pyswap_scale_shiftr      s.    <<q<)LEE>q1J    c                 	  ^ [        U R                  5       5      n/ SQn/ SQnSnSnU H?  m[        U4S jU 5       5      (       a  SnM!  [        U4S jU 5       5      (       d  M=  SnMA     U(       a?  U(       a8  U H0  m[        U4S jU 5       5      (       d  M  U R                  T5        M2     U $ Sn	U H!  m[        U4S jU 5       5      (       d  M  Sn	  O   U	(       d  U $ 0 n
/ S	Qn[	        5       [	        5       [	        5       pnU H  nS
U;   a  U R                  U5      X'   M  [        UR                  U5      S U S   5      nUS   U;   a  UR                  U5        MZ  US   U;   a  UR                  U5        Mv  US   U;   a  UR                  U5        M  [        SU S35      e   U VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU H  nUS-
  UR                  S-   -  nUS-
  UR                  S-   -  nUU    H  m[        TR                  U5      U   5      nST;  a  UU   OSnST;  a  [        U5      OSnUR                  TR                  U5      S US-
   [        U5      UU/-   TR                  U5      US-   S  -   5      nU R                  T5      U
U'   M     M     U H  nS nUS:X  a  US   S/nO+US:X  a  US   S/nOUS:X  a  US   S/nO[        SU S35      eUU    HU  mUR                  TR                  U5      S US-
   U-   TR                  U5      US  -   5      nU R                  T5      U
U'   MW     M     U H  nUUR                  S-   -  nUUR                  S-   -  nUU    H  m[        TR                  U5      U   5      nUU   nUS:  a  [        U5      OSnUR                  TR                  U5      S US-
   [        U5      UU/-   TR                  U5      US-   S  -   5      nU R                  T5      U
U'   M     M     U (       a  [        S5      eU
$ s  snf s  snnf s  snf s  snnf s  snf s  snnf )N)input_blocksmiddle_blockoutput_blocks)down_blocks	mid_block	up_blocksFc              3   ,   >#    U  H	  oT;   v   M     g 7fN .0pkeys     r   	<genexpr>5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>)        .ACx   Tc              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r"   r#   +   s     4#3ac#3r%   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r"   r#   2   s     2\8\r%   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r"   r#   9   r$   r%   )resnets
attentions
upsamplerstextr      r   z'Checkpoint not supported because layer z not supported.r   r   r   opdownsamplers01zInvalid middle block id .z:At this point all state dict entries have to be converted.)listkeysanypopsetintsplitadd
ValueErrorlayers_per_blockstrjoin)
state_dictunet_config	delimiterblock_slice_posall_keyssgm_patternsnot_sgm_patternscontains_sgm_patternscontains_not_sgm_patternsis_in_sgm_formatnew_state_dictinner_block_mapinput_block_idsmiddle_block_idsoutput_block_idslayerlayer_idr!   r   middle_blocksr   iblock_idlayer_in_block_idinner_block_idinner_block_keyinner_layers_in_blocknew_keykey_parts                    `           r   "_maybe_map_sgm_blocks_to_diffusersrZ      sg   JOO%&HDL@ " %....$(!4#3444(,%	  !:C2\222s#   ....# 
  N=O ;>%'7OU?$.NN5$9N!5;;y12B?CBGHHA%'##H-aE) $$X.aE) $$X. #J5'Q`!abb   ('H 	*\*3,yk(0TX[0[3*\\'   )(H 	*\*3,yk(0TX[0[3*\\(   )(H 	*]*3-	{8*0UY\0\3*]](   E{;;a?@U{'C'Ca'GH?C 9!5o!FGNAESon=VdO>B#oC(9$:SV!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7# #	  6'*C0H!V'*C0H!V'*C0H7s!<== #Cnn		)$%:':;hFS\I]^m^nIooG '1nnS&9N7#	 $ " 559:!=!=!AB #C 9!5o!FGN-n=O>Lq>PC(9$:VY!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7# $	  UVV ]
 ]
 ^sZ   1R,<R'R'R,$R7/R2R2R7S"R=5R=;S'R,2R7=Sc                    0 n0 n0 n0 n[        S U  5       5      n[        S U  5       5      n[        S U  5       5      n	U(       d  U(       d  U	(       a  [        SS5      (       a  [        S5      e[        U R	                  5       5      n
U
 GHj  nUR                  S5      (       d  M  UR                  S5      S	   nUS
-   nUS-   nUR                  S5      (       a  [        U5      nU R                  U5      X?'   U R                  U5      X?R                  SS5      '   U(       a=  SU;   a  SOSnU R                  UR                  SS5      5      X?R                  US5      '   GOPUR                  S5      (       Ga9  [        X5      nUR                  S5      (       a7  U R                  U5      XO'   U R                  U5      XOR                  SS5      '   O6U R                  U5      X_'   U R                  U5      X_R                  SS5      '   U(       d  U	(       a  SU;   a  SOSnUR                  S5      (       a4  U R                  UR                  SS5      5      XOR                  US5      '   OIUR                  S5      (       a3  U R                  UR                  SS5      5      X_R                  US5      '   X;   d  GM/  U R                  U5      R                  5       nUR                  [        UWU5      5        GMm     [        U 5      S	:  a+  [        SSR!                  U R	                  5       5       35      e["        R%                  S5        UR'                  5        VVs0 s H  u  nnU SU 3U_M     nnnUR'                  5        VVs0 s H  u  nnU SU 3U_M     nnn[        U5      S	:  a*  UR'                  5        VVs0 s H  u  nnSU 3U_M     snnOSnUb  UR                  U5        0 UEUEnUU4$ s  snnf s  snnf s  snnf )a  
Converts a non-Diffusers LoRA state dict to a Diffusers compatible state dict.

Args:
    state_dict (`dict`): The state dict to convert.
    unet_name (`str`, optional): The name of the U-Net module in the Diffusers model. Defaults to "unet".
    text_encoder_name (`str`, optional): The name of the text encoder module in the Diffusers model. Defaults to
        "text_encoder".

Returns:
    `tuple`: A tuple containing the converted state dict and a dictionary of alphas.
c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f)
dora_scale
lora_unet_Nr   r   ks     r   r"   ;_convert_non_diffusers_lora_to_diffusers.<locals>.<genexpr>   s$     [PZ1|q0F\Q5FFPZ    c              3   \   #    U  H"  nS U;   =(       a    SU;   =(       d    SU;   v   M$     g7f)r]   lora_te_	lora_te1_Nr   r_   s     r   r"   ra      s/     mbl]^\Q.XJ!O4W{VWGWXbl   *,c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f)r]   	lora_te2_Nr   r_   s     r   r"   ra      s"     Yjla/DK14DDjrb   <z0.9.0zeYou need `peft` 0.9.0 at least to use DoRA-enabled LoRAs. Please upgrade your installation of `peft`.lora_down.weightr3   r   .lora_up.weight.alphar^   .down..up.z_lora.down.z.lora.down.r]   z.lora_magnitude_vector.)rd   re   rh   rd   re   .lora_linear_layer.rh   z7The following keys have not been correctly renamed: 

 , z"Non-diffusers checkpoint detected.text_encoder_2.N)r6   r   r<   r4   r5   endswithr:   
startswith_convert_unet_lora_keyr7   replace_convert_text_encoder_lora_keyitemupdate_get_alpha_namelenr?   loggerinfoitems)r@   	unet_nametext_encoder_nameunet_state_dictte_state_dictte2_state_dictnetwork_alphasdora_present_in_unetdora_present_in_tedora_present_in_te2all_lora_keysr!   	lora_namelora_name_uplora_name_alphadiffusers_namedora_scale_key_to_replacedora_scale_key_to_replace_tealphamodule_nameparamsrJ   s                         r   (_convert_non_diffusers_lora_to_diffusersr      s    OMNN [PZ[[mblmmYjYY15H3((w 
 *+M||.// IIcN1%	 !#44#h. --3C8N /9nnS.AO+HRWcHdO228VDE $=Jn=\Mbo)NN3;;/A<#PQ   6 67PRk lm
 !!"HII;CKN ##$=>>0:s0C-JT..YeJf44XvFG1;1D.KU>>ZfKg55hGH "%8%2n%DMJ_ - ''(ABB"s{{3E|'TU ""8"89UWp"qr ))+66"s{{3E|'TU ##9#9:VXq#rs
 (NN?388:E!!//>SX"YZo t :TUYU^U^_i_n_n_pUqTrstt
KK45 XgWlWlWnoWn@SV)Ak]3V;WnOo]j]p]p]rs]rFYkSY)*!K=96A]rMs ~" M[L`L`LbcLb5H[&?;-	(&	0Lbc 
 !^,99=9N>)) pscs   O8O>Pc                 x  ^ U R                  SS5      R                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS	5      mTR                  S
S	5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mST;   a  ST;  a  S n[        R                  " USTS!S"9mS#T;   a  TR                  S$S%5      mS&T;   a  TR                  S'S(5      mS)T;   d  S*T;   a  TR                  S+S,5      mS-T;   a  TR                  S.S/5      mST;   a  TR                  SS5      mS0T;   a  TR                  S0S/5      mST;   a;  S1T;   d  S2T;   a&  TR                  S1S35      mTR                  S2S45      mT$ S5T;   a   T$ [        U4S6 jS7 5       5      (       a   T$  T$ )8z:
Converts a U-Net LoRA key to a Diffusers compatible key.
r^    _r3   zinput.blocksr   zdown.blockszmiddle.blockr   z	mid.blockzoutput.blocksr   z	up.blocksztransformer.blockstransformer_blocksz	to.q.lora	to_q_loraz	to.k.lora	to_k_loraz	to.v.lora	to_v_lorazto.out.0.lorato_out_lorazproj.inproj_inzproj.outproj_outz
emb.layerstime_emb_projembztime.emb.projz\.\d+(?=\D*$)r.   )countz.in.zin.layers.2conv1z.out.zout.layers.3conv2r0   r+   r/   convskipzskip.connectionconv_shortcutzconv.shortcutattn1attn2zattn1.processorzattn2.processorffc              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r   r!   r   s     r   r"   )_convert_unet_lora_key.<locals>.<genexpr>4  s     F.EsN".Er%   )r   r   )rv   resubr6   )r!   patternr   s     @r   ru   ru     sx    [[r2::3DN $++NMJN#++M=IN#++NKHN#++KEN#++O[IN#++KEN#++,@BVWN#++KEN#++KEN#++KEN#++O]KN#++IyAN#++J
CN#++L/JN ?.#H"^1E'//wG. '//H'<>+I'//f='//0A?S .('//Q.('//Q ~-n$>(A+33G=NON+33G=NON  ^#  
F.EF	F	F  	r   c                    UR                  S5      (       a  UR                  S5      (       a  SOSnOSnU R                  US5      R                  SS5      nUR                  SS	5      nUR                  S
S5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nSU;   d  SU;   a   U$ SU;   a  UR                  SS5      nU$ )zA
Converts a text encoder LoRA key to a Diffusers compatible key.
ro   rd   re   rh   r   r   r3   z
text.model
text_modelz	self.attn	self_attnzq.proj.lorar   zk.proj.lorar   zv.proj.lorar   zout.proj.lorar   ztext.projectiontext_projectionmlp.lora.rp   )rt   rv   )r!   r   key_to_replacer   s       r   rw   rw   <  s    566'0';';J'G'G[$[[4<<S#FN#++L,GN#++KEN#++M;GN#++M;GN#++M;GN#++O]KN#++,=?PQNn$(9^(K  
.	  (//:OPr   c                     U R                  S5      (       a  SnOU R                  S5      (       a  SnOSnX1R                  S5      S   -   S-   nXB0$ )	z6
Gets the correct alpha name for the Diffusers model.
r^   zunet.ro   text_encoder.rr   r   r   rl   )rt   r:   )r   r   r   prefixnew_names        r   rz   rz   X  s\     !!,//		#	#$=	>	> ",,X6q99HDHr   c                   ^^ S mS*S jmUU4S jnS n[        S U  5       5      nU(       a_  U R                  5        VVs0 s H@  u  pEUR                  S5      (       d  M  UR                  SS5      R                  S	S
5      U_MB     n nnU $ [        S U  5       5      nU(       GdX  U R                  5        VVs0 s H  u  pEUR                  SS5      U_M     n nnU R                  5        VVs0 s H  u  pEUR                  SS5      U_M     n nn[        S U  5       5      nU(       aj  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pESU;  d  M  XE_M     n nn[        S U  5       5      n	U	(       az  [	        U S5      n
U
(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pEUR                  S5      (       a  M  XE_M!     n nn[        S U  5       5      nU(       aj  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pESU;  d  M  XE_M     n nn[        S U  5       5      nU(       ar  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S 5        U R                  5        VVs0 s H  u  pES!U;  d  M  SU;  d  M  XE_M     n nnS"S#/n[        S$ U  5       5      (       a  UR                  S%5        U R                  5        VVs0 s H)  u  pEUR                  S&5      (       d  M  [        XO5      U_M+     n nn[        S' U  5       5      (       aA  [
        R                  S(5        U R                  5        VVs0 s H  u  pES)U;  d  M  XE_M     n nnU(       a  U" U 5      $ U" U 5      $ s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf )+Nc                    US-   U ;  a  g U R                  US-   5      nUR                  S   n[        R                  " XTR                  UR
                  SS9nU R                  US-   U5      R                  5       nXu-  nUn	Sn
U	S-  U
:  a  U	S-  n	U
S-  n
U	S-  U
:  a  M  XI-  XS-   '   U R                  US	-   5      U
-  XS
-   '   g N.lora_down.weightr   Fdtypedevicerequires_gradrl         ?r   .lora_A.weightrk   .lora_B.weightr7   shaper   tensorr   r   rx   sds_sdait_sdsds_keyait_keydown_weightrankdefault_alphar   r   
scale_downscale_ups              r   _convert_to_ai_toolkitE_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkiti  s    ((6jj+>!>?   #T1B1B;K]K]mrs

7X-}=BBD 
1nx'!OJMH 1nx' .9-E))*-3ZZBS8S-TW_-_))*r   c                    US-   U ;  a  g U R                  US-   5      nU R                  US-   5      nUR                  S   n[        R                  " XuR                  UR
                  SS9nU R                  US-   U5      n	X-  n
U
nSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  X[-  nXl-  n[        U5      nUc  UR                  S   U-  /U-  nO[        U5      UR                  S   :X  d   eSnX}-  S:X  a  X}-  nS	nSn[        [        U5      5       Hh  n[        [        U5      5       HE  nUU:X  a  M  U=(       a1    [        R                  " UUUUU   -   2UU-  US
-   U-  24   S:H  5      nMG     UUU   -  nMj     U(       a  [        R                  SU 35        U Vs/ s H  nUS-   PM
     nnU Vs/ s H  nUS-   PM
     nnU(       dk  UR                  [        R                  UU5      5        UR                  [        U[        R                   " XdSS95       VVs0 s H	  u  nnUU_M     snn5        g UR                  [        U[        R"                  " X]SS95       VVs0 s H	  u  nnUU_M     snn5        Sn[        [        U5      5       H9  nUUUUU   -   2UW-  US
-   U-  24   R%                  5       UUU   '   UUU   -  nM;     g s  snf s  snf s  snnf s  snnf Nr   rk   r   Fr   rl   r   r   Tr.   zweight is sparse: r   r   r   r7   r   r   r   r   r   r{   sumrangeallr|   r}   ry   dictfromkeyszipr:   r
   
contiguousr   r   r   ait_keysdimsr   	up_weightsd_lora_rankr   r   r   r   r   
num_splits	is_sparseait_rankrR   jr`   ait_down_keysait_up_keysvs                         r   _convert_to_ai_toolkit_catI_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkit_cat~  s$   ((6jj+>!>?JJw)::;	"((+  1 1+:L:L\a
 

7X-}=$ 
1nx'!OJMH 1nx' ".(	 ]
<OOA&*45
BDt9	 2222 	$)#1HIA3t9%s4y)AAv  ) !eii!!a$q'k/1x<1q5HBT3T"TUYZZ/I * T!W & 0	:; 8@@x!--x@5=>Xq++X>MM$--{CD MMCU[[^_=`,ab,aDAq1a4,abc MMCu{{;hi?j,kl,kDAq1a4,klm A3t9%)21q47{?ALTUXYTY]eSeDe3e)f)q)q)s{1~&T!W & A> c m   'K<K&K"
,K(
c                 R  >^ 0 m[        S5       H  nT" U TSU S3SU S35        T" U TSU S3SU S3SU S3SU S	3/5        T" U TSU S
3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S3SU S3SU S3/5        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        M     [        S5       HM  nT" U TSU S3SU S3SU S3SU S	3SU S 3// S!QS"9  T" U TSU S#3SU S$35        T" U TSU S%3SU S&35        MO     U4S' jn[        S( U  5       5      (       a  T" U TS)S*5        T" U TS+S,5        [        S- U  5       5      (       a  T" U TS.S/5        [        S0 U  5       5      (       a  T" U TS1S25        [        S3 U  5       5      (       a  T" U TS4S55        T" U TS6S75        [        S8 U  5       5      (       a  T" U TS9S:5        T" U TS;S<5        [        S= U  5       5      (       a  U" S>S?[        4S@/U 5        [        U R	                  5       5      n0 nU(       GaR  [        SA U 5       5      (       d  [        SBSCR                  U5       35      eU GH  nUR                  SD5      (       d  M  UR                  SE5      SF   nU SG3nU SH3n[        XV5      n	UR                  SI5      (       aG  U R                  U5      n
U
R                  SF   nXU	'   U R                  U5      XIR                  SJSK5      '   X;   d  M  U R                  U5      R                  5       nUW-  nUnSLnUSM-  U:  a  USM-  nUSM-  nUSM-  U:  a  M  XI==   U-  ss'   XIR                  SJSK5      ==   U-  ss'   GM     [!        U 5      SF:  a&  ["        R%                  SNU R	                  5        35        U(       a)  UR'                  5        VVs0 s H  u  nnSOU 3U_M     nnn0 TEUEnU$ s  snnf )PN   lora_unet_double_blocks__img_attn_projtransformer.transformer_blocks..attn.to_out.0_img_attn_qkv
.attn.to_q
.attn.to_k
.attn.to_v
_img_mlp_0z.ff.net.0.proj
_img_mlp_2z	.ff.net.2_img_mod_linz.norm1.linear_txt_attn_proj.attn.to_add_out_txt_attn_qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj
_txt_mlp_0z.ff_context.net.0.proj
_txt_mlp_2z.ff_context.net.2_txt_mod_linz.norm1_context.linear&   lora_unet_single_blocks__linear1&transformer.single_transformer_blocks.z	.proj_mlp)   r  r  i 0  )r   _linear2	.proj_out_modulation_linz.norm.linearc                    > S H\  nUS:X  a  SOSnU  HI  u  pEnUR                  US9nUR                  US9nUR                  U5      n	U(       a  U" U	5      n	U	T
U'   MK     M^     g )Nlora_Alora_Br  	lora_downlora_up)lora_key)orig_lora_key)formatr7   )assignmentssourcer  r  
target_fmt
source_fmt	transform
target_key
source_keyvaluer   s             r   assign_remaining_weightsr_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.assign_remaining_weights$  sv    0/78/C9D5JI!+!2!2H!2!EJ!+!2!2!2!OJ"JJz2E  )% 0).F:& :E 1r   c              3   ,   #    U  H
  nS U;   v   M     g7f)guidance_inNr   r_   s     r   r"   c_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>/       26a}!6   lora_unet_guidance_in_in_layer*time_text_embed.guidance_embedder.linear_1lora_unet_guidance_in_out_layer*time_text_embed.guidance_embedder.linear_2c              3   ,   #    U  H
  nS U;   v   M     g7f)img_inNr   r_   s     r   r"   r  >       -fx1}fr   lora_unet_img_in
x_embedderc              3   ,   #    U  H
  nS U;   v   M     g7f)txt_inNr   r_   s     r   r"   r  F  r'  r   lora_unet_txt_incontext_embedderc              3   ,   #    U  H
  nS U;   v   M     g7f)time_inNr   r_   s     r   r"   r  N  s     .v!yA~vr   lora_unet_time_in_in_layer*time_text_embed.timestep_embedder.linear_1lora_unet_time_in_out_layer*time_text_embed.timestep_embedder.linear_2c              3   ,   #    U  H
  nS U;   v   M     g7f)	vector_inNr   r_   s     r   r"   r  \  s     0A{ar   lora_unet_vector_in_in_layer&time_text_embed.text_embedder.linear_1lora_unet_vector_in_out_layer&time_text_embed.text_embedder.linear_2c              3   ,   #    U  H
  nS U;   v   M     g7f)final_layerNr   r_   s     r   r"   r  j  r  r   z!norm_out.linear.{lora_key}.weightz?lora_unet_final_layer_adaLN_modulation_1.{orig_lora_key}.weight)zproj_out.{lora_key}.weightz3lora_unet_final_layer_linear.{orig_lora_key}.weightNc              3   B   #    U  H  oR                  S 5      v   M     g7f))lora_telora_te1Nrt   r_   s     r   r"   r  {  s     Un||$;<<n   zIncompatible keys detected: 

 rq   rj   r3   r   rk   rl   ro   rm   rn   r   r   z!Unsupported keys for ai-toolkit: r   )r   r6   r   r4   r5   r   r<   r?   rs   r:   rw   rt   r7   r   rv   rx   r{   r|   warningr~   )r   rR   r  remaining_keysr   r!   r   r   r   r   r   r   r   r   r   r   r   r   rJ   r   r   r   s                      @r   !_convert_sd_scripts_to_ai_toolkitP_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit  s   rA"*1#^<1!NC	 '*1#];5aS
C5aS
C5aS
C		 #*1#Z81!NC	 #*1#Z81!I>	 #*1#\:1!MB	 #*1#^<1!4DE	 '*1#];5aS8HI5aS8HI5aS8HI		 #*1#Z81!4JK	 #*1#Z81!4EF	 #*1#\:1!4IJ	 L rA&*1#X6<QCzJ<QCzJ<QCzJ<QCyI	 / #*1#X689E	 #*1#_=8<H	' 6		/ 26222"0<	 #1<	 -f---""	 -f---"""	 .v...",<	 #-<	 0000".8	 #/8	 26222$ <Y(
 p 
 fkkm,UnUUU #DTYY~E^D_!`aa%||$677IIcN1-	"+O<%.Kv"6!?!O''(ABB"(**S/K#.#4#4Q#7L4?.1NTjjYeNfM"8"86"JK","JJ7<<>E!L0E!&J"H$q.83"a
 A %q.83 "1Z?1!"8"86"JKxWK5 &8 v;?NN>v{{}oNO^k^q^q^st^sGZ{TZ}[M:FB^sMt4F4m4 us   P#c                 	  ^ 0 nS nU  Vs1 s HM  nUR                  S5      (       a  M  UR                  SS5      R                  SS5      R                  SS5      iMO     nn[        S U 5       5      (       d   SU< 35       eS	nU GHj  mTR                  S
5      (       a6  [        TR	                  S
5      S   R	                  S5      S   5      nSU 3nGOTR                  S5      (       a6  [        TR	                  S5      S   R	                  S5      S   5      nSU 3nGO9TR                  S5      (       a  SnM  TR                  S5      (       a  SnGOTR                  S5      (       a  SnOTR                  S5      (       a  SnOTR                  S5      (       a  SnOTR                  S5      (       a#  [        TR	                  S5      S   5      nSU 3nOTR                  S5      (       a#  [        TR	                  S5      S   5      nSU 3nOHTR                  S5      (       a#  [        TR	                  S5      S   5      nS U 3nO[        S!T S"35      eS#T;   a  S$T;   a  US%-  nOyS&T;   a  US'-  nOm[        U4S( jS) 5       5      (       a  TR	                  S#5      S   nUS*U 3-  nO6[        U4S+ jS, 5       5      (       a  TR	                  S#5      S   nUS*U 3-  nU" TXpU5        GMm     U(       a  [        R                  " S-5      n	S.S/S0S1S2.n
S3S4S5.nU H  mTR                  S5      (       d  M  U	R                  T5      nU(       d  M5  [        UR                  S65      5      nS7U 3nS8T;   a&  U
R                  5        H  u  pUT;   d  M  X~-  n  O.   O+S9T;   a%  UR                  5        H  u  pUT;   d  M  X~-  n  O   U" TXpU5        M     S	nU (       a  [        S: U  5       5      nU(       a3  [        U R                  5       5      nU H  mU R                  T5        M     [        U 5      S:  a&  [!        S;[        U R                  5       5       S<35      eUR                  5        VVs0 s H$  u  nnUR                  S=5      (       a  M  S>U 3U_M&     nnnUR                  5        VVs0 s H$  u  nnUR                  S=5      (       d  M  S?U 3U_M&     nnn0 UEUE$ s  snf s  snnf s  snnf )@Nc                 <   U  S3nUR                  U5      nUR                  S   nU  S3nUR                  U5      nU  S3n	UR                  U	5      n
X-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  X\-  nX-  nU S3nXSU'   XUR                  SS	5      '   g )
Nr   r   rk   rl   r   r   r   z.lora_A.z.lora_B.)r7   r   rv   )original_keydiffusers_keyr@   rJ   down_keyr   	lora_rankup_weight_keyr   	alpha_keyr   r   r   r   diffusers_down_keys                  r   _converti_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>._convert  s    &'89H$..2K#))!,I+nO<M"}5I'./INN9-E %EJHq.8+a
A q.8+ &2K!,I$1?.!A1<-.QZ-55j*MNr   r^   r   r   rk   rl   c              3   B   #    U  H  oR                  S 5      v   M     g7f))lora_transformer_re   Nr?  r_   s     r   r"   j_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     ]_<< BCC_r@  zall_unique_keys=F+lora_transformer_single_transformer_blocks_r-   r   r   single_transformer_blocks.$lora_transformer_transformer_blocks_transformer_blocks.re   T!lora_transformer_context_embedderr-   lora_transformer_norm_out_linearnorm_out.linearlora_transformer_proj_outr   lora_transformer_x_embedderr)  :lora_transformer_time_text_embed_guidance_embedder_linear_z)time_text_embed.guidance_embedder.linear_6lora_transformer_time_text_embed_text_embedder_linear_z%time_text_embed.text_embedder.linear_:lora_transformer_time_text_embed_timestep_embedder_linear_z)time_text_embed.timestep_embedder.linear_zHandling for key (z) is not implemented.attn_	_to_out_0r   _to_add_outr   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r   qkvr`   s     r   r"   rR    s     F-Ec-Er%   to_qto_kto_v.attn.c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r   add_qkvr`   s     r   r"   rR    s     `5_'A5_r%   
add_q_proj
add_k_proj
add_v_projz(lora_te1_text_model_encoder_layers_(\d+)z.self_attn.q_projz.self_attn.k_projz.self_attn.v_projz.self_attn.out_proj)q_projk_projv_projout_projz.mlp.fc1z.mlp.fc2)fc1fc2r.   ztext_model.encoder.layers.attnr   c              3   B   #    U  H  oR                  S 5      v   M     g7fr^   Nr?  r_   s     r   r"   rR    s     $TA\\,%?%?r@  z\Expected an empty state dict at this point but its has these keys which couldn't be parsed: r3   ztext_model.transformer.r   )rt   rv   r   r9   r:   NotImplementedErrorr6   r   compilesearchgroupr~   r4   r5   r7   r{   r<   )r@   rJ   rN  r`   all_unique_keyshas_te_keysrR   rH  	remaininglayer_patternattn_mappingmlp_mappingmatchkey_fragmentsuffixremaining_all_unetr5   r   transformer_state_dictr   s      `                r   (_convert_mixture_state_dict_to_diffusersW_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers  s4   	[8  
<</ dAII)2.667H"MUUV^`bc 	 

 ]_]]]tbrbqas_tt] A||IJJ MNrRXXY\]^_`a"<QC @DEE FGKQQRUVWXYZ"5aS 9k**"ABB 2@AA 19:: *;<< ,Z[[ \]^`ab"KA3 OVWW XYZ\]^"Gs KZ[[ \]^`ab"KA3 O),>qcAV*WXX!|!#!%55M"a'!%77MF-EFFF ! 0 4I!vi[%99M`5_``` ! 0 4I!vi[%99MQ>BU !X JJ'RSM---1	L #-Z@K$||K00%,,Q/A'"<QC @Q;0<0B0B0D,'1,)3M! 1E aZ0;0A0A0C,'1,)3M! 1D
 M~F+ %. #!$$T$T!T
)*Dq!  z?Qnost~  uD  uD  uF  pG  oH  HI  J 
 /=.B.B.D"
.DdaALLYfLg!l1#!.D 	 "
 =K<P<P<Rr<RDAqVWVbVbcpVq/=,a/<Rr:(:M::K
B"
 ss"   S,6S,7S1
S18S7
S7c              3   B   #    U  H  oR                  S 5      v   M     g7f)rx  Nr?  r_   s     r   r"   8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>)  s     OJqll>::Jr@  rx  rj   zlora_A.weightzlora_up.weightzlora_B.weightc              3      #    U  H9  oR                  S 5      =(       a    SU;   =(       d    SU;   =(       d    SU;   v   M;     g7f)rQ  r  r  r   Nr?  r_   s     r   r"   r  3  s@      lvgh()b{a/?/a9PQ>/aU\`aUablvs   AAdiffusion_model.r^   z!text_encoders.clip_l.transformer.rd   c              3   ,   #    U  H
  nS U;   v   M     g7f)position_embeddingNr   r_   s     r   r"   r  <  s     $S
1%9Q%>
r   r  zThe `position_embedding` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zThe state_dict has position_embedding LoRA params and we currently do not support them. Open an issue if you need this supported - https://github.com/huggingface/diffusers/issues/new.c              3   B   #    U  H  oR                  S 5      v   M     g7f) text_encoders.t5xxl.transformer.Nr?  r_   s     r   r"   r  L  s     ]R\Q%GHHR\r@  ztext_encoders.t5xxlzThe `t5xxl` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zT5-xxl keys found in the state dict, which are currently unsupported. We will filter them out.Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r  c              3   \   #    U  H"  nS U;   =(       a    UR                  S5      v   M$     g7f)diff_br^   rd   re   Nr?  r_   s     r   r"   r  [  s)     tisdeA_!,,7^*__isrf   .diff_bzThe `diff_b` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.z`diff_b` keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f).norm.diffNr   r_   s     r   r"   r  k  s      N:aGqL9W\9:rb   r  zThe `diff` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zNormalization diff keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r  r  r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r   Nr   r_   s     r   r"   r  |  s     0Zw!|Zr   r   r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r   Nr   r_   s     r   r"   r    s     :z! A%zr   z`text_projection` keys found in the `state_dict` which are unexpected. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r   r   )	r6   r~   rt   rv   r   r|   r}   append_custom_replace)r@   rC  r  has_peft_state_dictr`   r   has_mixturehas_position_embeddingzero_status_pe	has_t5xxlzero_status_t5	has_diffbzero_status_diff_bhas_norm_diffzero_status_difflimit_substringsr   r   s                   @@r   %_convert_kohya_flux_lora_to_diffusersr  h  s!   `*AF_BB;N OJOO #((*
*||N+ iAII(/:BBCSUdeghh* 	 

   lv K
 Q[QaQaQcdQcaii 2LA1DQc
d`j`p`p`rs`rX\XYaii CZPRSS`r
s!$$S
$S!S!0=QRNe v ,6+;+;+=_+=41AU]^A^$!$+=J_]R\]]	0=RSNe
 p ,6+;+;+=v+=41Q\\RtEu$!$+=Jvtistt	!4Z!K!e
 K
 ,6+;+;+=T+=41RSAS$!$+=JTN:NN2:wGe
 K
 ,6+;+;+=g+=41PQAQ$V]efVf$!$+=Jg'30Z000##G, #((*
*||CD 4OA0!3* 	 
 :z:::KKG
 ,6+;+;+=\+=41ARZ[A[$!$+=J\7
CC,Z88K
 es  ` w  U  h
 ]sf   P!'P:P/P";P(P(>P.P.P4!P4P:$P:,P:4Q Q &Q6Qc                    0 n[        U R                  5       5      nS S jnU GHk  nUR                  S5      (       a  [        R                  " SU5      R                  S5      nSU 3nSU;   a  US-  nOYSU;   a  US	-  nOMS
U;   a!  SU;  a  U" U UUSU S3SU S3SU S3/5        O&SU;   a   SU;  a  U" U UUSU S3SU S3SU S3/5        SU;   a  US-  nOSU;   a  US-  nOUR                  S5      (       au  [        R                  " SU5      R                  S5      nSU 3nSU;   a  US-  nO&SU;   a   SU;  a  U" U UUSU S3SU S3SU S3/5        SU;   a  US-  nOSU;   a  US-  nOUnSU;  d  GMX  U R                  U5      X'   GMn     [        U 5      S:  a&  [        S[        U R                  5       5       S35      eU$ )!Nc                 ,   U R                  U5      nU R                  UR                  SS5      5      n[        U5      nUc  UR                  S   U-  /U-  nO[	        U5      UR                  S   :X  d   eU Vs/ s H  oS-   PM	     n	nU Vs/ s H  oS-   PM	     n
nUR                  [        R                  X5      5        UR                  [        U
[        R                  " XdSS95       VVs0 s H  u  pX_M	     snn5        g s  snf s  snf s  snnf )Nz.down.weightz
.up.weightr   r   r   r   )r7   rv   r{   r   r   ry   r   r   r   r   r:   )r   r   r   r   r   r   r   r   r`   r   r   r   s               r   
handle_qkv9_convert_xlabs_flux_lora_to_diffusers.<locals>.handle_qkv  s    jj)JJw~|LM	 ]
<OOA&*45
BDt9	 2222 8@@x!--x@5=>X++X> 	dmmM?@ 	KYZ[9\(]^(]qt(]^_ A> _s   :DD/D
)zdiffusion_model.double_blocksdouble_blockszdouble_blocks\.(\d+)r.   r   zprocessor.proj_lora1r   zprocessor.proj_lora2r   zprocessor.qkv_lora2upr   r   r   zprocessor.qkv_lora1r   r   r   downr   r   )zdiffusion_model.single_blockssingle_blockszsingle_blocks\.(\d+)r  	proj_lorar  qkv_lorarc  r   z2`old_state_dict` should be at this point but has: r3   r   )	r4   r5   rt   r   r{  r|  r7   r{   r<   )old_state_dictrJ   	orig_keysr  old_key	block_numrX   s          r   %_convert_xlabs_flux_lora_to_diffusersr    sk   N^((*+I`* PQQ		"97CII!LI7	{CG%0++'72--&'1d'6I""9)DTU9)DTU9)DTU		 ''1d'6I""9)JO9)JO9)JO		  ++++  RSS		"97CII!LI>ykJGg%;&w&4w+>""@:V@:V@:V		  ++++ G &4&8&8&AN#Q T >QMdSaSfSfShNiMjjklmmr   r!   
substringsreturnc                 (   SSR                  S U 5       5      -   S-   n[        R                  " X 5      nU(       aF  UR                  5       nUS:  a  XS-
     S:X  a  US-
  nOUnU S U R	                  SS5      nXS  nXg-   $ U R	                  SS5      $ )	N(|c              3   N   #    U  H  n[         R                  " U5      v   M     g 7fr   )r   escape)r   r   s     r   r"   "_custom_replace.<locals>.<genexpr>  s     BzRYYs^^zs   #%)r   r.   r3   r   )r?   r   r{  startrv   )r!   r  r   r  	start_subboundaryleftrights           r   r  r    s     CHHBzBBBSHGIIg#EKKM	q=SQ/36 1}H H9H~%%c3/I|{{3$$r   c                    0 n[        U R                  5       5      nSnSnSnSnS GH0  nU R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  S
U S35      USU S3'   S
U S	3U;   a  U R                  S
U S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   [        S U  5       5      nU(       a  U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   d  GM  U R                  SU S	35      USU S	3'   GM3     [	        U5       GH'  n	SU	 S3n
S GH  nU R                  SU	 SU S35      X SU S3'   SU	 SU S	3U;   a   U R                  SU	 SU S	35      X SU S	3'   U R                  SU	 SU S35      X SU S3'   SU	 SU S	3U;   a   U R                  SU	 SU S	35      X SU S	3'   US :X  a  U R                  SU	 S!U S35      n[
        R                  " U/5      X S"U S3'   [
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   U R                  SU	 S%U S35      n[
        R                  " U/5      X S&U S3'   [
        R                  " U/5      X S'U S3'   [
        R                  " U/5      X S(U S3'   GO[
        R                  " U R                  SU	 S!U S35      S)S*S+9u  pn[
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U R                  SU	 S%U S35      S)S*S+9u  nnn[
        R                  " U/5      X S&U S3'   [
        R                  " U/5      X S'U S3'   [
        R                  " U/5      X S(U S3'   SU	 S!U S	3U;   a  [
        R                  " U R                  SU	 S!U S	35      S)S*S+9u  nnn[
        R                  " U/5      X S#U S	3'   [
        R                  " U/5      X S$U S	3'   [
        R                  " U/5      X S"U S	3'   SU	 S%U S	3U;   a  [
        R                  " U R                  SU	 S%U S	35      S)S*S+9u  nnn[
        R                  " U/5      X S&U S	3'   [
        R                  " U/5      X S'U S	3'   [
        R                  " U/5      X S(U S	3'   U R                  SU	 S,U S35      X S-U S3'   SU	 S,U S	3U;   a   U R                  SU	 S,U S	35      X S-U S	3'   U R                  SU	 S.U S35      X S/U S3'   SU	 S.U S	3U;   a   U R                  SU	 S.U S	35      X S/U S	3'   U R                  SU	 S0U S35      X S1U S3'   SU	 S0U S	3U;   a   U R                  SU	 S0U S	35      X S1U S	3'   U R                  SU	 S2U S35      X S3U S3'   SU	 S2U S	3U;   a   U R                  SU	 S2U S	35      X S3U S	3'   U R                  SU	 S4U S35      X S5U S3'   SU	 S4U S	3U;   a   U R                  SU	 S4U S	35      X S5U S	3'   U R                  SU	 S6U S35      X S7U S3'   SU	 S6U S	3U;   d  GM  U R                  SU	 S6U S	35      X S7U S	3'   GM     U R                  SU	 S835      X S93'   U R                  SU	 S:35      X S;3'   U R                  SU	 S<35      X S=3'   U R                  SU	 S>35      X S?3'   GM*     [	        U5       GH  n	S@U	 S3n
S GHX  nU R                  SAU	 SBU S35      X SCU S3'   SAU	 SBU S	3U;   a   U R                  SAU	 SBU S	35      X SCU S	3'   [        XV-  5      nXUUU4nUS :X  Ga7  U R                  SAU	 SDU S35      n[
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U/5      X SEU S3'   SAU	 SDU S	3U;   a  U R                  SAU	 SDU S	35      n[
        R                  " U/5      X S#U S	3'   [
        R                  " U/5      X S$U S	3'   [
        R                  " U/5      X S"U S	3'   [
        R                  " U/5      X SEU S	3'   GOg[
        R                  " U R                  SAU	 SDU S35      US*S+9u  nnnn [
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U /5      X SEU S3'   SAU	 SDU S	3U;   a  [
        R                  " U R                  SAU	 SDU S	35      US*S+9u  n!n"n#n$[
        R                  " U!/5      X S#U S	3'   [
        R                  " U"/5      X S$U S	3'   [
        R                  " U#/5      X S"U S	3'   [
        R                  " U$/5      X SEU S	3'   U R                  SAU	 SFU S35      X SGU S3'   SAU	 SFU S	3U;   d  GM8  U R                  SAU	 SFU S	35      X SGU S	3'   GM[     U R                  SAU	 SH35      X S93'   U R                  SAU	 SI35      X S;3'   GM     S H  nU R                  SJU S35      USGU S3'   SJU S	3U;   a  U R                  SJU S	35      USGU S	3'   [        U R                  SKU S35      5      USLU S3'   SKU S	3U;   d  Mv  [        U R                  SKU S	35      5      USLU S	3'   M     [        U 5      S*:  a  [        SMU R                  5       < S35      e[        UR                  5       5       H  n%UR                  U%5      USNU% 3'   M     U$ )ONr   r  r        @r
  ztime_in.in_layer..weightz+time_text_embed.timestep_embedder.linear_1..biasztime_in.out_layer.z+time_text_embed.timestep_embedder.linear_2.zvector_in.in_layer.z'time_text_embed.text_embedder.linear_1.zvector_in.out_layer.z'time_text_embed.text_embedder.linear_2.c              3   ,   #    U  H
  nS U;   v   M     g7f)guidanceNr   r_   s     r   r"   >_convert_bfl_flux_control_lora_to_diffusers.<locals>.<genexpr>?  s     H4Gq:?4Gr   zguidance_in.in_layer.z+time_text_embed.guidance_embedder.linear_1.zguidance_in.out_layer.z+time_text_embed.guidance_embedder.linear_2.ztxt_in.zcontext_embedder.zimg_in.zx_embedder.rV  r3   double_blocks..img_mod.lin.norm1.linear..txt_mod.lin.norm1_context.linear.r  .img_attn.qkv.
attn.to_v.
attn.to_q.
attn.to_k..txt_attn.qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj.   r   r   .img_mlp.0.ff.net.0.proj..img_mlp.2.	ff.net.2..txt_mlp.0.ff_context.net.0.proj..txt_mlp.2.ff_context.net.2..img_attn.proj.attn.to_out.0..txt_attn.proj.attn.to_add_out.z.img_attn.norm.query_norm.scalezattn.norm_q.weightz.img_attn.norm.key_norm.scalezattn.norm_k.weightz.txt_attn.norm.query_norm.scalezattn.norm_added_q.weightz.txt_attn.norm.key_norm.scalezattn.norm_added_k.weightrT  single_blocks..modulation.lin.norm.linear.	.linear1.	proj_mlp.	.linear2.	proj_out.z.norm.query_norm.scalez.norm.key_norm.scalefinal_layer.linear.zfinal_layer.adaLN_modulation.1.znorm_out.linear.W`original_state_dict` should be empty at this point but has original_state_dict.keys()=rx  )r4   r5   r7   r6   r   r   r   r
   r9   r:   r   r{   r<   )&original_state_dictconverted_state_dictoriginal_state_dict_keys
num_layersnum_single_layers	inner_dim	mlp_ratior  has_guidancerR   block_prefixsample_lora_weightcontext_lora_weightsample_qsample_ksample_v	context_q	context_k	context_vsample_q_biassample_k_biassample_v_biascontext_q_biascontext_k_biascontext_v_biasmlp_hidden_dim
split_sizelora_weight	lora_biasqr`   r   r   q_biask_biasv_biasmlp_biasr!   s&                                         r   +_convert_bfl_flux_control_lora_to_diffusersr    s   #$7$<$<$>?JII(  ##&7z$IJ 	J8*T[\] xj.2JJ#''*;H:U(KL !#NxjX]!^_
  ##&8
'$JK 	J8*T[\]  z/3KK#''*<XJe(LM !#NxjX]!^_
 ]p\s\s!(73]
FxjPWXY !
%04LL^q^u^u%hZu5_ #J8*TY!Z[ ]p\s\s"8*G4]
FxjPWXY "(515MM^q^u^u&xj6_ #J8*TY!Z[
 H4GHH#''*?z(QR !#NxjX_!`a 'xj6:RR'++.CH:U,ST %'RS[R\\a%bc
 $''*@
'(RS !#NxjX_!`a (z7;SS'++.DXJe,TU %'RS[R\\a%bc
 GZF]F]hZw'G
0
'BC XJe$(@@H[H_H_(5)I #4XJe!DE
 AT@W@WZabjakkrXs@t{8*G<=XJe$(@@BUBYBY\cdlcmmrZsBt ;xj!>?E )J :,QCq1,HTgTkTk =
'BU >xj!PQ  s-z?C[[ViVmVm$QC}XJeDW$~]8*E%RS ]p\s\s =
'B] >1FxjPW!XY  s-z?C[[^q^u^u$QC}XJeD_$~5J8*TY%Z[
 8#%8%<%<~aSP^_g^hho=p%q"UZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=qcQ_`h_iip>q&r#[`[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++nQC~hZW^,_`bcij0,H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++nQC~hZW^,_`bcij3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++nQC~hZW\,]^`agh?;}m TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++nQC~hZW\,]^`aghB> Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl ;xj@V >z!QR  s+hZu=AYYWjWnWn$QC{8*EBX$~^H:U%ST QdPgPg ;xj@Q >8*G!LM  s+hZu=AYYReRiRi$QC{8*EBS$~Yxj%NO ^q]t]t ;xj@^ >1GzQX!YZ  s+hZu=AYY_r_v_v$QC{8*EB`$~5KH:UZ%[\ YlXoXo ;xj@Y >1B8*G!TU  s+hZu=AYYZmZqZq$QC{8*EB[$~5FxjPU%VW
 ViUlUl ?8*GDV >z!QR  s/(5AE]]WjWnWn$QCxjFX$~^H:U%ST XkWnWn ?8*GDX >1A(7!ST  s/(5AE]]YlYpYp$QCxjFZ$~5EhZu%UV_ -h EXD[D[QC>?E
~-?@A EXD[D[QC<=E
~-?@A K^JaJaQC>?K
~-EFG K^JaJaQC<=K
~-EFGA J $%3A3a8,HSfSjSj #3H:WET >hZw!OP  s"28*EBF^^UhUlUl$QC'7zGV$~\(5%QR
 !!67N#	>JJ8#155qcS[R\\c6deUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ#A3iz?C[[ 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STV[V_V_aj`kVl(>8*E)RS${{'++nQCy
RY,Z[]gmn 1a V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ#A3iz?C[[7<{{+//.9XJV[0\]_iop84FFFH X]W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STV[V_V_ai`jVk(>8*E)RS QdPgPg 9XJg>Q >8*G!LM  s)H:U;?WWReRiRi$QCy
%@S$~Yxj%NOe -n EXD[D[QC56E
~-?@A EXD[D[QC34E
~-?@A{ &B )>Q>U>U!(73?
y
':; !
%04LL@S@W@W%hZu5A 9XJe!<= FV##&EhZw$WXF
/zAB -XJe<@XXGW#''*I(SX(YZH #3H:U!CD )" !#sXkXpXpXrWttuvww(--/05I5M5Mc5R|C512 1  r   c                    0 n[        U R                  5       5      nSnSnSnSn[        U5       GH  nSU S3nSn	S GH  n
U R                  U	 S	U S
U
 S35      X SU
 S3'   S	U S
U
 S3U;   a"  U R                  U	 S	U S
U
 S35      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U
S:X  a  U R                  U	 S	U SU
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   U R                  U	 S	U SU
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   GO[        R                  " U R                  U	 S	U SU
 S35      SSS9u  pn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   S	U SU
 S3U;   a  [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   S	U SU
 S3U;   a  [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U	 S	U SU
 S3U;   a"  U R                  U	 S	U SU
 S35      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U	 S	U SU
 S3U;   a"  U R                  U	 S	U SU
 S35      X SU
 S3'   U R                  U	 S	U S U
 S35      X S!U
 S3'   U	 S	U S U
 S3U;   a"  U R                  U	 S	U S U
 S35      X S!U
 S3'   U R                  U	 S	U S"U
 S35      X S#U
 S3'   U	 S	U S"U
 S3U;   a"  U R                  U	 S	U S"U
 S35      X S#U
 S3'   U R                  U	 S	U S$U
 S35      X S%U
 S3'   U	 S	U S$U
 S3U;   a"  U R                  U	 S	U S$U
 S35      X S%U
 S3'   U R                  U	 S	U S&U
 S35      X S'U
 S3'   U	 S	U S&U
 S3U;   d  GM  U R                  U	 S	U S&U
 S35      X S'U
 S3'   GM     GM     [        U5       GH  nS(U S3nS GHn  n
U R                  W	 S)U S*U
 S35      X S+U
 S3'   U	 S)U S*U
 S3U;   a"  U R                  U	 S)U S*U
 S35      X S+U
 S3'   [        XV-  5      nXUUU4nU
S:X  Ga;  U R                  U	 S)U S,U
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X S-U
 S3'   U	 S)U S,U
 S3U;   a  U R                  S)U S,U
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X S-U
 S3'   GOm[        R                  " U R                  U	 S)U S,U
 S35      USS9u  nnnn [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U /5      X S-U
 S3'   U	 S)U S,U
 S3U;   a  [        R                  " U R                  U	 S)U S,U
 S35      USS9u  n!n"n#n$[        R
                  " U!/5      X SU
 S3'   [        R
                  " U"/5      X SU
 S3'   [        R
                  " U#/5      X SU
 S3'   [        R
                  " U$/5      X S-U
 S3'   U R                  U	 S)U S.U
 S35      X S/U
 S3'   U	 S)U S.U
 S3U;   d  GML  U R                  U	 S)U S.U
 S35      X S/U
 S3'   GMq     GM     S HM  n
U R                  W	 S0U
 S35      US/U
 S3'   U	 S0U
 S3U;   d  M/  U R                  U	 S0U
 S35      US/U
 S3'   MO     [        U 5      S:  a  [        S1U R                  5       < S35      e[        UR                  5       5       H  n%UR                  U%5      US2U% 3'   M     U$ )3Nr   r  r  r  rV  r3   zbase_model.model.r
  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r  r  r  r  r  r  r  r  r  r  r  r  rT  r  r  r  r  r  r  r  r  r  rx  )r4   r5   r   r7   r   r   r
   r9   r:   r{   r<   )&r  r  r  r  r  r  r  rR   r  original_block_prefixr  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r	  r
  r  r`   r   r   r  r  r  r  r!   s&                                         r   &_convert_fal_kontext_lora_to_diffusersr  @  s   #$7$<$<$>?JII :,QCq1 3,HTgTkTk()s-zQXYU >xj!PQ  s-z?C[[ViVmVm,-^A3mH:UZ[W$~]8*E%RS ]p\s\s()s-zQXY] >1FxjPW!XY
 8#%8%<%<,-^A3nXJV]^&" V[U^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=,-^A3nXJV]^'# \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++01s.QYPZZab 0,H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++01s.QYPZZab 3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++/D.E^TUSVVdemdnns,tu?;}m
 TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++/D.E^TUSVVdemdnns,tuB>
 Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl()s+hZwWV >z!QR ((qcXJeTXppWjWnWn,-^A3k(SXYX$~^H:U%ST QdPgPg()s+hZwWQ >8*G!LM ((qcXJeTXppReRiRi,-^A3k(SXYS$~Yxj%NO ^q]t]t()s+hZwW^ >1GzQX!YZ ((qcXJeTXpp_r_v_v,-^A3k(SXY`$~5KH:UZ%[\ YlXoXo()s+hZwWY >1B8*G!TU ((qcXJeTXppZmZqZq,-^A3k(SXY[$~5FxjPU%VW
 ViUlUl()s/(SZ[V >z!QR ((qc
RWX\ttWjWnWn,-^A3ohZW\]X$~^H:U%ST XkWnWn()s/(SZ[X >1A(7!ST ((qc
RWX\ttYlYpYp,-^A3ohZW\]Z$~5EhZu%UVw -	 H $%3A3a8,HSfSjSj()s:J8*T[\T >hZw!OP ((qc9I(SXY]uuUhUlUl,-^A3>NxjX]^V$~\(5%QR
 !!67N#	>JJ8#155,-^A3izQXY V[U^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ+,N1#YxjPUVZrr 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STV[V_V_aj`kVl(>8*E)RS${{'++/D.E^TUSVV_`h_iip,qr 1a
 V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ+,N1#YxjPUVZrr7<{{+//3H2IXYWZZcdlcmmr0st"84FFFH
 X]W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STV[V_V_ai`jVk(>8*E)RS QdPgPg()s)H:WUQ >8*G!LM ((qc8*ERVnnReRiRi,-^A3izQVWS$~Yxj%NOq - &~ )>Q>U>U$%%8
'J?
y
':; $$$7zGKcc@S@W@W())<XJeLA 9XJe!<= ) !#sXkXpXpXrWttuvww(--/05I5M5Mc5R|C512 1  r   c                    [        U R                  5       5       Vs0 s H  oU R                  U5      _M     nnS nS nS nS nS n0 SS_SS	_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_S S!_S"S#_S$S%_S&S'_S(S)S%S*S+SS,S-S.S/S0S1S2S3.EnUUUUUS4.n	[        UR                  5       5       Ho  n
U
R                  S55      (       a  UR                  U
5      X*[	        S55      S  '   U
R                  S65      (       d  MP  UR                  U
5      X*[	        S65      S  '   Mq     [        UR                  5       5       HE  n
U
S S  nUR                  5        H  u  pUR                  X5      nM     UR                  U
5      X+'   MG     [        UR                  5       5       H+  n
U	R                  5        H  u  pX;  a  M  U" X5        M     M-     [        UR                  5       5       H  n
UR                  U
5      US5U
 3'   M     U$ s  snf )7Nc                     UR                  U 5      nUR                  SSS9u  p4[        R                  " XC/SS9nXQU R	                  SS5      '   g )Nr   r   r   final_layer.adaLN_modulation.1rY  )r7   r
   r   r   rv   )r!   r@   r   r   r   r   s         r   remap_norm_scale_shift_I_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_norm_scale_shift_!  sK    $||A1|-YY~15
Wa3;;?ARSTr   c                 .   S nSU ;   as  UR                  U 5      nUR                  SSS9u  pEnXAU" U R                  SS5      5      '   XQU" U R                  SS5      5      '   XaU" U R                  SS5      5      '   g UR                  U 5      X" U 5      '   g )	Nc                    U R                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      nUR                  SS5      nU$ )Nzindividual_token_refiner.blocksztoken_refiner.refiner_blockszadaLN_modulation.1rY  r+  r-  zt_embedder.mlp.0r1  zt_embedder.mlp.2r3  
c_embedderztime_text_embed.text_embedderr   r   )rv   )r!   rX   s     r   
rename_keyS_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_.<locals>.rename_key(  s    kk"CEcdGoo&:<MNGooh0BCGoo&8:fgGoo&8:fgGool4STGooeT2GNr   self_attn_qkvr  r   r   	attn.to_q	attn.to_k	attn.to_v)r7   r
   rv   )r!   r@   r  r   re  rf  rg  s          r   remap_txt_in_?_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_'  s    	 c!^^C(F%||A1|5DPTz#++o{"KLMPTz#++o{"KLMPTz#++o{"KLM*4..*=Jz#'r   c                 J   UR                  U 5      nSU ;   a=  X!U R                  SS5      '   X!U R                  SS5      '   X!U R                  SS5      '   g UR                  SSS9u  p4nX1U R                  SS5      '   XAU R                  SS5      '   XQU R                  SS5      '   g )	Nr  img_attn_qkvr  r   r!  r  r   r   r7   rv   r
   r!   r@   r   re  rf  rg  s         r   remap_img_attn_qkv_E_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_img_attn_qkv_;  s    $s?CIs{{>;?@CIs{{>;?@CIs{{>;?@%||A1|5DCGs{{>;?@CGs{{>;?@CGs{{>;?@r   c                 J   UR                  U 5      nSU ;   a=  X!U R                  SS5      '   X!U R                  SS5      '   X!U R                  SS5      '   g UR                  SSS9u  p4nX1U R                  SS5      '   XAU R                  SS5      '   XQU R                  SS5      '   g )	Nr  txt_attn_qkvzattn.add_q_projzattn.add_k_projzattn.add_v_projr  r   r   r&  r'  s         r   remap_txt_attn_qkv_E_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_attn_qkv_G  s    $s?IOs{{>3DEFIOs{{>3DEFIOs{{>3DEF%||A1|5DIMs{{>3DEFIMs{{>3DEFIMs{{>3DEFr   c                    SnSU ;   d  SU ;   a  UR                  U 5      nSU ;   a>  U R                  SS5      R                  S5      nX1U S3'   X1U S	3'   X1U S
3'   X1U S3'   g X"X#R                  S5      SU-  -
  4n[        R
                  " X5SS9u  pgpU R                  SS5      R                  S5      nXaU S3'   XqU S3'   XU S3'   XU S3'   g SU ;   d  SU ;   a  UR                  U 5      n
SU ;   a>  U R                  SS5      R                  S5      nXU S3'   XU S3'   XU S3'   XU S3'   g X"X*R                  S5      SU-  -
  4n[        R
                  " XSS9u  ppU R                  SS5      R                  S5      nXU S3'   XU S3'   XU S3'   XU S3'   g U R                  SS5      nUR                  S S!5      nUR                  S"S#5      nUR                  S$S%5      nUR                  U 5      X'   g )&Nr  zlinear1.lora_A.weightzlinear1.lora_B.weightr  r  single_transformer_blocksz.linear1.lora_A.weightz.attn.to_q.lora_A.weightz.attn.to_k.lora_A.weightz.attn.to_v.lora_A.weightz.proj_mlp.lora_A.weightr   r  r   z.linear1.lora_B.weightz.attn.to_q.lora_B.weightz.attn.to_k.lora_B.weightz.attn.to_v.lora_B.weightz.proj_mlp.lora_B.weightzlinear1.lora_A.biaszlinear1.lora_B.biasz.linear1.lora_A.biasz.attn.to_q.lora_A.biasz.attn.to_k.lora_A.biasz.attn.to_v.lora_A.biasz.proj_mlp.lora_A.biasz.linear1.lora_B.biasz.attn.to_q.lora_B.biasz.attn.to_k.lora_B.biasz.attn.to_v.lora_B.biasz.proj_mlp.lora_B.biaslinear2r   q_normattn.norm_qk_normattn.norm_k)r7   rv   removesuffixsizer   r:   )r!   r@   hidden_sizelinear1_weightrX   r  r  r`   r   r   linear1_biasr  r  r  r  s                  r    remap_single_transformer_blocks_R_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_single_transformer_blocks_S  s   "c)-D-K'^^C0N3++o7RS``, DRgY&>?@CQgY&>?@CQgY&>?@BPgY&=>?)EXEXYZE[^_bm^mEmn
${{>1Ma++o7RS``, DEgY&>?@CDgY&>?@CDgY&>?@BEgY&=>?"c)-Bc-I%>>#.L3++o7RS``* BNgY&<=>AMgY&<=>AMgY&<=>@LgY&;<=)EVEVWXEY\]`k\kEkl
38;;|]^3_0++o7RS``* BHgY&<=>AGgY&<=>AGgY&<=>@HgY&;<= kk/3NOGooi<Gooh>Gooh>G",.."5Jr   r&  r)  ztime_in.mlp.0r1  ztime_in.mlp.2r3  zguidance_in.mlp.0r"  zguidance_in.mlp.2r$  zvector_in.in_layerr7  zvector_in.out_layerr9  r  r   img_attn_q_normr2  img_attn_k_normr4  img_attn_projattn.to_out.0txt_attn_q_normzattn.norm_added_qtxt_attn_k_normzattn.norm_added_ktxt_attn_projattn.to_add_outzimg_mod.linearznorm1.linear	img_norm1z
norm1.norm	img_norm2norm2r   znorm1_context.linearnorm2_context
ff_contextznorm.linearz	norm.normznorm_out.normr   
net.0.projnet.2r   )img_mlpztxt_mod.linear	txt_norm1	txt_norm2txt_mlpself_attn_projzmodulation.linearpre_normzfinal_layer.norm_finalfinal_layer.linearrs  rt  input_embedder)r+  r%  r+  r  r  rx  r  )r4   r5   r7   rt   r{   r~   rv   )r  r`   r  r  r"  r(  r,  r:  TRANSFORMER_KEYS_RENAME_DICTTRANSFORMER_SPECIAL_KEYS_REMAPr!   rX   replace_keyr  special_keyhandler_fn_inplaces                   r   (_convert_hunyuan_video_lora_to_diffusersrX    s   CGH[H`H`HbCcdCca266q99Ccdb>(
H
N26h$,$E$ 	E$ 	I	$
 	I$ 	F$ 	G$ 	-$ 	=$ 	=$ 	$ 	.$ 	.$ 	*$ 	.$  	\!$" 	W#$$ 0!$)*"1(#=$ D  ++9*A&" (--/0>>.))?S?W?WX[?\ S%8%:!;<>>,--CWC[C[\_C` S);%<%>!?@	 1 (--/0a&'C'I'I'K#Kook>G (L(<(@(@(E%	 1 (--/0/M/S/S/U+K%s9 0V 1 (--/05I5M5Mc5R|C512 1  W es   Hc                   ^ ^ T R                  5        VVs0 s H  u  pU[        S5      S  U_M     snnm 0 mS nUU 4S jnSnU" T R                  5       U5      n[        U5       H  nU" SUSS9  M     SnU" T R                  5       U5      n	[        U	5       H  nU" S	US
S9  M     Sn
U" T R                  5       U
5      n[        U5       H  nU" SUSS9  M     [        T 5      S:  a  [	        ST R                  5       < 35      e[        TR                  5       5       H  nTR                  U5      TSU 3'   M     T$ s  snnf )Nr  c                     [        5       nU  HK  n[        R                  " X5      nU(       d  M"  UR                  [	        UR                  S5      5      5        MM     [        U5      $ )Nr.   )r8   r   r{  r;   r9   r|  r{   )r5   r   layersr!   r  s        r   get_num_layersH_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.get_num_layers  sK    CIIg+Eu

3u{{1~./  6{r   c           	        > T	R                  U  SU S35      nT	R                  U  SU S35      nS H  nUTU  SU SU S3'   M     [        / SQ[        R                  " U/ SQSS	95       H  u  pVUTU  SU SU S
3'   M     T	R                  U  SU S35      TU  SU S3'   T	R                  U  SU S35      TU  SU S3'   [	        SS5       HO  nT	R                  U  SU SU S35      TU  SU SU S3'   T	R                  U  SU SU S
35      TU  SU SU S
3'   MQ     U(       aA  T	R                  U  SU S35      TU  SU S3'   T	R                  U  SU S35      TU  SU S3'   g g )Nr3   z.attention.qkv.lora_A.weightz.attention.qkv.lora_B.weightrd  rh  r   )i 	     r_  r   r   r   z.attention.out.lora_A.weightz.attn.to_out.0.lora_A.weightz.attention.out.lora_B.weightz.attn.to_out.0.lora_B.weightr.      z.feed_forward.wz.feed_forward.linear_z!.adaLN_modulation.1.lora_A.weightz.norm1.linear.lora_A.weightz!.adaLN_modulation.1.lora_B.weightz.norm1.linear.lora_B.weight)r7   r   r   r:   r   )
r   indexconvert_normr  r  attn_keyr   rO   r  r@   s
           r   process_blockG_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.process_block  s,   NNfXQug5Q#RS	..F81UG3O!PQ0HV_ F81UG6(>!RS 1 #$<ekk'Scij>k lHV\ F81UG6(>!RS !m Q[P^P^haw:;Q
xq/KLM Q[P^P^haw:;Q
xq/KLM
 1a[Eblbpbp(!E7/%Gc F81UG3H~!^_ cmbpbp(!E7/%Gc F81UG3H~!^_	 ! S]SaSa(!E7"CDT F81UG3N!OP T^SaSa(!E7"CDT F81UG3N!OP	 r   znoise_refiner\.(\d+)\.noise_refinerT)rb  zcontext_refiner\.(\d+)\.context_refinerFzlayers\.(\d+)\.r[  r   E`state_dict` should be empty at this point but has state_dict.keys()=rx  )r~   r{   r5   r   r<   r4   r7   )r@   r`   r   r\  rd  noise_refiner_patternnum_noise_refiner_layersrR   context_refiner_patternnum_context_refiner_layerscore_transformer_patternnum_core_transformer_layersr!   r  s   `            @r   0_convert_non_diffusers_lumina2_lora_to_diffusersro    sb   >H>N>N>PQ>Pda!C*+-.1>PQJ D 6-joo.?AVW+,oqt< - :!/
0ACZ![-.'? /  2"01BD\"]./h5 0 :azO`Nbcdd(--/05I5M5Mc5R|C512 1  E Rs   D?c           
        ^$^%^&^'^( 0 nU R                  5        VVs0 s H  u  p#U[        S5      S  U_M     snnm'T' Vs1 s H7  o"R                  S5      (       d  M  [        UR	                  S5      S   5      iM9     nn[        U5      n[        U5      n[        S T' 5       5      =(       a    [        S T' 5       5      n[        S T' 5       5      (       a  SOS	m%[        S
 T' 5       5      (       a  SOSm&[        S T' 5       5      nU'4S jn	[        T'R                  5       5       Hk  n
U
R                  S5      (       a0  SU
;   a*  T'R                  U
5        [        R                  SU
 S35        SU
;   d  MQ  U(       a  MZ  T'R                  U
5        Mm     [        XVS-   5       GH  n[        / SQ/ SQ5       H  u  pSU SU S3nUT';   nSU SU ST% S3nSU SU S3nSU SU ST& S3nSU SU S3nU(       a?  T'R                  U5      nT'R                  U5      nU	" UU5      u  nnUU-  UU'   UU-  UU'   O4UT';   a  T'R                  U5      UU'   UT';   a  T'R                  U5      UU'   SU SU S3nSU SU S3nUT';   d  M  T'R                  U5      UU'   M     [        / SQ/ SQ5       H  u  pSU SU S3nUT';   nSU SU ST% S3nSU SU S3nSU SU ST& S3nSU SU S3nUT';   a  T'R                  U5      nUUU'   UT';   a  T'R                  U5      nUUU'   U(       a&  U	" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU SU S3nUT';   d  M  T'R                  U5      UU'   M     U(       a  [        S S!/S"S#/5       H  u  pSU SU S3nUT';   nSU SU ST% S3nSU SU S3nSU SU ST& S3nSU SU S3nUT';   a  T'R                  U5      nUUU'   UT';   a  T'R                  U5      nUUU'   U(       a&  U	" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU SU S3nUT';   d  M  T'R                  U5      UU'   M     [        S$S%/S&S'/5       H  u  pSU SU S3nUT';   nSU SU ST% S3nSU S(U S3nSU SU ST& S3nSU S(U S3nUT';   a  T'R                  U5      nUUU'   UT';   a  T'R                  U5      nUUU'   U(       a&  U	" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU S(U S3nUT';   d  M  T'R                  U5      UU'   M     GM     T'(       Ga.  [        S) T' 5       5      (       a^  S*T% S3nS+nUT';   a  T'R                  U5      UU'   S*T& S3nS,nUT';   a  T'R                  U5      UU'   S-T';   a  T'R                  S-5      US.'   [        S/ T' 5       5      (       Ga*  [        U%4S0 jU  5       5      (       a  T'R                  S1T% S35      US2'   [        U&4S3 jU  5       5      (       a  T'R                  S1T& S35      US4'   S5T';   a  T'R                  S55      US6'   [        S7 U  5       5      (       a  S1T% S3U ;   a  [        R!                  S8T% S935        T'R                  S:5      US2'   US2   nUR"                  S;   US6   R"                  S;   4n[$        R&                  " UUR(                  UR*                  S<.6R,                  US4'   S= H  m([        U(4S> jT' 5       5      (       d  M  S? H  m$T$S;:X  a  SOS@nT(SA:X  a  SBOSCn[        U$U(4SD jT' 5       5      (       aF  T'R                  T( ST$ ST% S35      UU SEU S3'   T'R                  T( ST$ ST& S35      UU SEU S3'   T( ST$ S3T';   d  M  T'R                  T( ST$ S35      UU SEU S3'   M     M     SF H  u  nnU ST% S3nSGU S3nUT';   a  T'R                  U5      UU'   U ST& S3nSGU S3nUT';   d  MD  T'R                  U5      UU'   UR/                  ST& S35      S-   n U T';   d  Mx  UR/                  S5      SH-   n!T'R                  U 5      UU!'   M     [        T'5      S;:  a  [1        SI T' 5       5      n"U"(       a\  T' Vs1 s H  o"R                  SJ5      (       d  M  UiM     n#n[1        SK U# 5       5      (       d  [2        e[        R!                  SL5        O[3        SMT'R                  5       < 35      e[        UR                  5       5       H  n
UR                  U
5      USNU
 3'   M     U$ s  snnf s  snf s  snf )ONr  blocks.r3   r.   c              3   ,   #    U  H
  nS U;   v   M     g7fk_imgNr   r_   s     r   r"   ?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>       @,?qgl,?r   c              3   ,   #    U  H
  nS U;   v   M     g7fv_imgNr   r_   s     r   r"   ru         Hs_rZ[TU_rr   c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r_   s     r   r"   ru    s     #O;NaHM;Nr   r  r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r_   s     r   r"   ru    s     !M9LA(a-9Lr   r  r  c              3   r   #    U  H-  oR                  S 5      =(       a    UR                  S5      v   M/     g7f)time_projectionr  N)rt   rs   r_   s     r   r"   ru    s-      %K^a&'AAJJy,AAK^s   57c                    > U R                   S   nTR                  U5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ Nr   r   r   r   r7   rx   )r   rL  r   r   r   r   r   r  s          r   get_alpha_scalesF_convert_non_diffusers_wan_lora_to_diffusers.<locals>.get_alpha_scales#  st      ##''	2779
1nx'!OJMH 1nx' ##r   )r  r  normz	Removing zG key from the state dict as it is a norm diff key. This is unsupported.r~  r  r`   r   ore  rf  rg  zto_out.0z.self_attn.rl   r  .attn1.r   r   r  z.lora_B.biasz.cross_attn..attn2.rt  ry  rm  rn  zffn.0zffn.2rI  rJ  .ffn.c              3   ,   #    U  H
  nS U;   v   M     g7f)r~  Nr   r_   s     r   r"   ru    s     C/B! A%/Br   ztime_projection.1.z*condition_embedder.time_proj.lora_A.weightz*condition_embedder.time_proj.lora_B.weightztime_projection.1.diff_bz(condition_embedder.time_proj.lora_B.biasc              3   ,   #    U  H
  nS U;   v   M     g7f)	head.headNr   r_   s     r   r"   ru    s     =)<A{a)<r   c              3   6   >#    U  H  nS T S3U;   v   M     g7f
head.head.r  Nr   )r   r`   lora_down_keys     r   r"   ru    s     PZZg6!;Z   r  zproj_out.lora_A.weightc              3   6   >#    U  H  nS T S3U;   v   M     g7fr  r   )r   r`   lora_up_keys     r   r"   ru    s     N:aZ}G49:r  zproj_out.lora_B.weightzhead.head.diff_bzproj_out.lora_B.biasc              3   \   #    U  H"  nS U;   =(       a    UR                  S5      v   M$     g7f)r  r  N)rs   r_   s     r   r"   ru    s&     PZ;!#;

7(;;Zrf   zEThe state dict seems to be have both `head.head.diff` and `head.head.z#.weight` keys, which is unexpected.zhead.head.diffr   )r   r   )text_embeddingtime_embeddingc              3   .   >#    U  H
  nTU;   v   M     g 7fr   r   )r   r`   	text_times     r   r"   ru    s     ?+>a9>+>   )r   r   r   r  z condition_embedder.text_embedderz condition_embedder.time_embedderc              3   8   >#    U  H  nT S T 3U;   v   M     g7fr3   Nr   )r   r`   b_nr  s     r   r"   ru    s#     R>Qi[#/14>Qs   z.linear_))zff.net.0.projzimg_emb.proj.1)zff.net.2zimg_emb.proj.3z"condition_embedder.image_embedder.r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r_   s     r   r"   ru    s     =)<A7a<)<r   r  c              3   ,   #    U  H
  nS U;  v   M     g7f)loraNr   r_   s     r   r"   ru    s     :	1vQ	r   zThe remaining `state_dict` contains `diff` keys which we do not handle yet. If you see performance issues, please file an issue: https://github.com/huggingface/diffusers//issues/newN`state_dict` should be empty at this point but has original_state_dict.keys()=rx  )r~   r{   rt   r9   r:   minmaxr6   r4   r5   rs   r7   r|   debugr   r   r}   r   r   eyer   r   Tr5  r   r<   ))r@   r  r`   r   block_numbers	min_block	max_blockis_i2v_lorahas_time_projection_weightr  r!   rR   r  crL  	has_alphaoriginal_key_Aconverted_key_Aoriginal_key_Bconverted_key_Br   r   r   r   rG  converted_keydown_matrix_headup_matrix_shapediffusers_b_nr   img_ours
img_theirsbias_key_theirsbias_keydiff	diff_keysr  r  r  r  r  s)                                       @@@@@r   ,_convert_non_diffusers_wan_lora_to_diffusersr    s   GQGWGWGYZGYtq1S!3467:GYZ3Fb3Fa,,W`Ja)Sa)3FMbM"IM"I@,?@@sSHs_rHsEsK ##O;N#O O OHU`M!!M9L!MMM(S\K!$ %K^% "	$ ',,./<<,--&C-  ##C(LL9SE)pqr#,F,F  ##C( 0  9!m,,.RSDA!!Ks&9I!%88I&qcQCqwON 's'!NCO&qcQCqWMN 's'!NCO155nE/33NC	'7Y'O$
H8Cj8P$_58AH8L$_5 "%88<O<S<STb<c(9!%88<O<S<STb<c(9$QC{1#W=L%aSs,?M226I6M6Ml6[$]33 T8 ,.RSDA!!L6:I!%88I&qcaS-PN 's'!NCO&qcaS+gNN 's'!NCO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QC|A3g>L%aSs,?M226I6M6Ml6[$]3/ T2 Wg.|0LM%aSQCv>	%)<<	#*1#\!Am_G!T$+A3gaS"G#*1#\!Ak]'!R$+A3gaS"G!%88"5"9"9."IK<G(9!%88 3 7 7 GI<E(9+;K+S(J(9ZG9(9XE9!(<s'B")!GA3l C#66:M:Q:QR^:_(7/ N4 '*\7,CDDA!!AaS/I!%88I&qc1#Q}oWEN 's%s.AO&qc1#Q{m7CN 's%s.AO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QCq73L%aSaS=M226I6M6Ml6[$]3/ Ee -X C/BCCC/gFLHM226I6M6Ml6[$]3/}GDLHM226I6M6Ml6[$]3)-@@SfSjSj.T$%OP =)<===PZPPPATAXAX w7B$%=> N:NNNATAXAX W5B$%=> "%88?R?V?VWi?j$%;< PZPPPg6*DKK_`m_n  oR  S BUAXAXYiAj$%=>#78P#Q #3#9#9!#<>RSi>j>p>pqr>s"tAF$,<,B,BK[KbKbB! %%=> >I?+>???!C),AqM %(88 ;? #
 R>QRRR/33yk3%qW^4_` -/?xVd-ef 033yk3%qU\4]^ -/?xVd-ef $AcU'26II/33yk3%w4OP -/?xVb-cd " >*%
 Hj )\=/AL@
.YM226I6M6Ml6[$]3(\;-w?L@
.YM226I6M6Ml6[$]3".";";a}G<T"UXa"a"&99,99)DwNH5H5L5L_5](2!%
$ !#=)<==$7O$7q::g;N$7IO:	:::  KKG
 nSfSkSkSmRopqq(--/05I5M5Mc5R|C512 1  q [bR Ps   e(e.!e.:e3e3c           	        ^ 0 nU R                  5        VVs0 s H  u  p#U[        S5      S  U_M     snnm[        T Vs1 s H(  o"R                  S5      S   R                  S5      S   iM*     sn5      n[        S T 5       5      =(       a    [        S T 5       5      nU4S jn[	        U5       GH  n[        / S	Q/ S
Q5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     [        / S	Q/ S
Q5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     U(       av  [        SS/SS/5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     [        SS/SS/5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     GM     [        T5      S:  a  [        STR                  5       < 35      e[        UR                  5       5       H  nUR                  U5      USU 3'   M     U$ s  snnf s  snf )Nr^   blocks_r.   r   r   c              3   ,   #    U  H
  nS U;   v   M     g7frs  r   r_   s     r   r"   8_convert_musubi_wan_lora_to_diffusers.<locals>.<genexpr>  rv  r   c              3   ,   #    U  H
  nS U;   v   M     g7frx  r   r_   s     r   r"   r    rz  r   c                    > U R                   S   nTR                  US-   5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ )Nr   rl   r   r   r  )r   r!   r   r   r   r   r   r  s          r   r  ?_convert_musubi_wan_lora_to_diffusers.<locals>.get_alpha_scales  sx      ##''h7<<>
1nx'!OJMH 1nx' ##r   r  r  _self_attn_r   rk   rq  r  r   r   _cross_attn_r  rt  ry  rm  rn  ffn_0ffn_2rI  rJ  r  r  rx  )
r~   r{   r:   r6   r   r   r7   r<   r5   r4   )r@   r  r`   r   
num_blocksr  r  rR   r  r  r   r   r   r   r!   r  s                  @r   %_convert_musubi_wan_lora_to_diffusersr    s   AKAQAQASTAS1S.0114ASTCVWCVaggi(+11#6q9CVWXJ@,?@@sSHs_rHsEsK	$ :,.RSDA-11GA3k!L]2^_K+//'!Ks/0Z[I#3K71#[YZX[A\#] JJUJb 71#WQC~!FGJSJ^ 71#WQC~!FG T ,.RSDA-11GA3l1#M^2_`K+//'!L?0[\I#3K71#\Z[Y\A]#^ JJUJb 71#WQC~!FGJSJ^ 71#WQC~!FG T Wg.|0LM155s,qcQb6cd/33gaSQC4_`	'7wqcQ]^_]`Ea'b$
NYNf$wqc>%JKNWNb$wqc>%JK N '*\7,CDDA-11GA3asBS2TUK+//'!AaS0PQI#3K71#QqcAR#S JHSH` 71#U1#^!DEHQH\ 71#U1#^!DE E5 B !#jObOgOgOiNklmm(--/05I5M5Mc5R|C512 1  o UWs
   K>/Lc                 (  ^ [        U4S jU  5       5      (       d  [        S5      eU R                  5        VVs0 s H  u  p#UR                  T S35      U_M     nnnUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf )Nc              3   D   >#    U  H  oR                  T5      v   M     g 7fr   r?  r   r`   non_diffusers_prefixs     r   r"   C_convert_non_diffusers_hidream_lora_to_diffusers.<locals>.<genexpr>O  s     F:a||011:s    z$Invalid LoRA state dict for HiDream.r3   rx  r   r<   r~   removeprefixr@   r  r`   r   r  s    `   r   0_convert_non_diffusers_hidream_lora_to_diffusersr  N  s    F:FFF?@@V`VfVfVhiVhdaANN.B-C1+EFIVhi>R>X>X>Z[>Zdal1#.1>Z[ j[   !B1Bc                 (  ^ [        U4S jU  5       5      (       d  [        S5      eU R                  5        VVs0 s H  u  p#UR                  T S35      U_M     nnnUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf )Nc              3   J   >#    U  H  oR                  T S 35      v   M     g7fr  r?  r  s     r   r"   @_convert_non_diffusers_ltxv_lora_to_diffusers.<locals>.<genexpr>W  s$     LA||34A677s    #z&Invalid LoRA state dict for LTX-Video.r3   rx  r  r  s    `   r   -_convert_non_diffusers_ltxv_lora_to_diffusersr  V  s    LLLLABBV`VfVfVhiVhdaANN.B-C1+EFIVhi>R>X>X>Z[>Zdal1#.1>Z[ j[r  c                 D   U R                  5        VVs0 s H"  u  p#UR                  U S35      (       d  M   X#_M$     n nnU R                  5        VVs0 s H  u  p#UR                  U S35      U_M     nnnUS:X  a  SSSSSSS	S
SSSSS.nOSS0n0 nUR                  5        H8  u  pxUS S  n	UR                  5        H  u  pU	R                  X5      n	M     XU	'   M:     0 nUR                  5        He  u  pxUR                  S5      (       a  UR                  SS5      n	XU	'   M3  UR                  S5      (       a  UR                  SS5      n	XU	'   Ma  XU'   Mg     US:X  a  SOSnUR                  5        VVs0 s H  u  p#U SU 3U_M     nnnU$ s  snnf s  snnf s  snnf )Nr3   diffusion_modelr   audio_proj_inav_cross_attn_video_scale_shiftav_cross_attn_video_a2v_gateav_cross_attn_audio_scale_shiftav_cross_attn_audio_v2a_gate&video_a2v_cross_attn_scale_shift_table&audio_a2v_cross_attn_scale_shift_tablenorm_qnorm_kaudio_prompt_adalnprompt_adaln)patchify_projaudio_patchify_proj$av_ca_video_scale_shift_adaln_singleav_ca_a2v_gate_adaln_single$av_ca_audio_scale_shift_adaln_singleav_ca_v2a_gate_adaln_singlescale_shift_table_a2v_ca_videoscale_shift_table_a2v_ca_audior1  r3  audio_prompt_adaln_singleprompt_adaln_singleaggregate_embedtext_proj_inzadaln_single.ztime_embed.zaudio_adaln_single.zaudio_time_embed.transformer
connectors)r~   rt   r  rv   )r@   r  r`   r   r  rename_dictrenamed_state_dictr!   r  rX   old_patternnew_patternfinal_state_dictr   s                 r   -_convert_non_diffusers_ltx2_lora_to_diffusersr  ^  s   #-#3#3#5b#541I]H^^_F`9a$!$#5JbV`VfVfVhiVhdaANN.B-C1+EFIVhi00&#24U+I4U+I.V.V)=#1
  ).9 *002
a&(3(9(9(;$Kook?G )<&+7#	 3 (..0
>>/**kk/=AG(-W%^^122kk"79LMG(-W%$)S! 1 36GG]\F7G7M7M7OP7Otq6(!A3*7OP] ciV Qs   FF!F7Fc                   ^ ^^^^ [        S T  5       5      nU(       a5  T R                  5        VVs0 s H  u  p#UR                  S5      U_M     snnm [        S T  5       5      nU(       ar  T R                  5        VVs0 s H  u  p#UR                  S5      U_M     snnm S[        S[        4S jnT R                  5        VVs0 s H  u  p#U" U5      U_M     snnm [        S T  5       5      nU(       a6  T R                  5        VVs0 s H  u  p#UR	                  S	S
5      U_M     snnm 0 n[        T R                  5       5      nSmSmSmSm[        UU4S jU 5       5      n	[        UU4S jU 5       5      n
U	(       a  U 4S jnU H  nUR                  T5      (       d  M  UR	                  TS5      nUR	                  TT5      R	                  TS5      nUR	                  TS5      nT R                  U5      nT R                  UR	                  TT5      5      nU" X5      u  nnUU-  X|'   UU-  X}'   M     OJU
(       aC  U H=  nTU;   d  TU;   a  T R                  U5      Xr'   M$  SU;   d  M,  T R                  U5        M?     [        T 5      S:  a  [        ST R                  5       < 35      eUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf s  snnf s  snnf s  snnf )Nc              3   B   #    U  H  oR                  S 5      v   M     g7fr  Nr?  r_   s     r   r"   @_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.<genexpr>       S
1ll+=>>
r@  r  c              3   B   #    U  H  oR                  S 5      v   M     g7frw  r?  r_   s     r   r"   r         GJq\22Jr@  r^   r!   r  c           	      *   SnSU ;   a  U R                  SS5      u  p#OU Sp2U S3nU[        U5      S  nSU;   a  UR                  SS5      u  pgSU-   nOUSpv1 Skn0 n	U H5  n
U	R                  [        U
5      [	        5       5      R                  U
5        M7     UR                  S5      n/ nSn[        U	R                  5       SS	9nU[        U5      :  a  S
nU H\  nUU-   [        U5      ::  d  M  [        XUU-    5      U	U   ;   d  M0  UR                  SR                  XUU-    5      5        UU-  nSn  O   U(       d  UR                  X   5        US-  nU[        U5      :  a  M  SR                  U5      nU SU U 3nUU(       a  SU-   -   $ S-   $ )Nr   r3   r.   r   r   >   tor`   r  r  r  r   r  outr;   r`   r;   r  r;   r   imgr   r  modtxtr   r  r  r  r;   r  r;   r`   projr;   r  r  r;   r   r  r   TreverseF)rsplitr{   r:   
setdefaultr8   r;   sortedr5   tupler  r?   )r!   r   baser  r  restheadtail	protectedprot_by_lenngpartsmergedrR   lengths_descmatchedLhead_convertedconverted_bases                      r   convert_keyB_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.convert_key  s   )Fcz"zz#q1f"BfhaLEE
%Dd{!ZZQ/
Tz!2dI( K&&s2w6::2>   JJsOEFA!+"2"2"4dCLc%j.%A1uE
*uUq1u5E/F+VW./XchhuQ/?&@AQ"& & MM%(+FA c%j. !XXf-N &xq(8?N!vcFlFF2FFr   c              3   ,   #    U  H
  nS U;   v   M     g7fdefault.Nr   r_   s     r   r"   r         :z!jAozr   r.  r   r   rk   r   r   c              3   F   >#    U  H  nTU;   =(       d    TU;   v   M     g 7fr   r   r   r`   rI  up_keys     r   r"   r    !     #S(QHM$@Vq[$@(   !c              3   F   >#    U  H  nTU;   =(       d    TU;   v   M     g 7fr   r   r   r`   a_keyb_keys     r   r"   r    !     K(Q
 8eqj 8(r4  c                    > U R                   S   nTR                  U5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ r  r  )r   rL  r   r   r   r   r   r@   s          r   r  G_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.get_alpha_scales  sr    $$Q'DNN9-224ELEJHq.8+a
A q.8+ ''r   rl   r   rh  rx  )r6   r~   r  r>   rv   r4   r5   rs   r7   r{   r<   )r@   has_diffusion_modelr`   r   has_lora_unetr*  has_defaultr  rD   has_non_diffusers_lora_idhas_diffusers_lora_idr  rM  diffusers_up_keyrL  r   r   r   r   r7  r8  rI  r2  s   `                  @@@@r   -_convert_non_diffusers_qwen_lora_to_diffusersrB    s   S
SSHRHXHXHZ[HZann%78!;HZ[
GJGGMBLBRBRBTUBT$!ann\2A5BTU
<	GS <	GS <	G| 5?4D4D4FG4FDAk!na'4FG
:z::K?I?O?O?QR?Qtqaii
B/2?QR
JOO%&H"HFEE ##S(#S SK(KK 		( Azz(##%&YYx9I%J"#$99Xv#>#F#FvO_#` IIh9	(nnQ/&NN199Xv+FG	'7'O$
H;F;S$89BX9M$6  
AzUaZ*4..*;$'Qq!	  :azO`Nbcdd>R>X>X>Z[>Zdal1#.1>Z[s \ V@ H S` \s   K K#K)K/K5c                 "
   0 nSnU R                  5        VVs0 s H  u  p4U[        U5      S  U_M     nnn[        S UR                  5        5       5      nU(       aC  0 nUR                  5        H+  u  p4UR	                  SS5      R	                  SS5      nXGU'   M-     UnSSSS	.n	[        UR                  5       5       HW  n
U	R                  5        H@  u  pU
R                  U5      (       d  M  UR                  U
5      XU
[        U5      S  -   '     MU     MY     S
nS
nUR                  5        H  n
U
R                  S5      (       a,  [        U[        U
R                  S5      S   5      S-   5      nME  U
R                  S5      (       d  M]  [        U[        U
R                  S5      S   5      S-   5      nM     SnSn[        U5       Hi  nSU 3nSU S3nU HU  nU SU S3nUU;   a  UR                  U5      UU SU S3'   U SU S3nUU;   d  M;  UR                  U5      UU SU S3'   MW     Mk     [        U5       GH  nSU 3nU GH]  nU GHR  nU S3nSU SU SU S3nUU;  a  M  UR                  U5      nUS:X  a<  US:X  a  / SQO/ SQnU H&  n[        R                  " U/5      UU SU SU S3'   M(     Mp  [        R                  " USS
S9u  nnn US:X  ab  [        R                  " U/5      UU SU S3'   [        R                  " U/5      UU SU S3'   [        R                  " U /5      UU SU S3'   M  [        R                  " U/5      UU S U S3'   [        R                  " U/5      UU S!U S3'   [        R                  " U /5      UU S"U S3'   GMU     GM`     S#S$/n!U! HB  u  n"n#U H6  nSU SU" SU S3n$U$U;   d  M  U SU# SU S3n%UR                  U$5      UU%'   M8     MD     / S%Qn&U& HB  u  n'n(U H6  nSU SU' SU S3n$U$U;   d  M  U SU( SU S3n%UR                  U$5      UU%'   M8     MD     GM     S&S'S(S)S*S+S,S-S.S/S0S1.n)U)R                  5        H9  u  n*n+U H-  nU* SU S3n$U$U;   d  M  UR                  U$5      UU+ SU S3'   M/     M;     [        U5      S
:  a  [        S2UR                  5       < S35      e[        UR                  5       5       H  n
UR                  U
5      US3U
 3'   M     U$ s  snnf )4Nr  c              3   D   #    U  H  nS U;   =(       d    SU;   v   M     g7f)r  r  Nr   r_   s     r   r"   A_convert_non_diffusers_flux2_lora_to_diffusers.<locals>.<genexpr>	  s#     bGa!;!+=yA~=Garb   r  r  r  r  rT  rV  )rT  rV  zsformer_blocks.r   r  r3   r.   r  r
  )img_attntxt_attnz.attnr  r  z.to_qkv_mlp_proj.r  z.to_out.z.qkv.rF  rd  rk  r  r   z.to_q.z.to_k.z.to_v.z.add_q_proj.z.add_k_proj.z.add_v_proj.)zimg_attn.projr?  )ztxt_attn.projrC  ))z	img_mlp.0zff.linear_in)z	img_mlp.2zff.linear_out)z	txt_mlp.0zff_context.linear_in)z	txt_mlp.2zff_context.linear_outr)  r-  z.time_guidance_embed.timestep_embedder.linear_1z.time_guidance_embed.timestep_embedder.linear_2z.time_guidance_embed.guidance_embedder.linear_1z.time_guidance_embed.guidance_embedder.linear_2r   rY  zsingle_stream_modulation.linearz#double_stream_modulation_img.linearz#double_stream_modulation_txt.linear)r&  r+  ztime_in.in_layerztime_in.out_layerzguidance_in.in_layerzguidance_in.out_layerrQ  r  zsingle_stream_modulation.linz double_stream_modulation_img.linz double_stream_modulation_txt.linr  rx  )r~   r{   r6   r5   rv   r4   rt   r7   r  r9   r:   r   r   r   r
   r<   ),r@   r  r   r`   r   r  has_lora_down_uptemp_state_dictrX    possible_expanded_block_prefixesr!   source_prefixtarget_prefixnum_double_layersr  	lora_keys
attn_typesslsingle_block_prefixattn_prefixr  linear1_keylinear2_keydltransformer_block_prefix	attn_typeqkv_keyfused_qkv_weightdiff_attn_proj_keysproj_keyr  r  r  proj_mappingsorg_proj	diff_projrG  rH  mlp_mappingsorg_mlpdiff_mlpextra_mappingsorg_keydiff_keys,                                               r   ._convert_non_diffusers_flux2_lora_to_diffusersre  	  sW   F;E;K;K;MN;M411S[]+Q.;MNbGZG_G_Gabb'--/DAiiX6>>y(SG'(G$ 0 . 'C40($
 ',,./,L,R,R,T(M~~m,,ReRiRijmRn$S]9K9M5N%NO -U 0 "'')>>*++ #$5s399S>!;L7MPQ7Q R^^,-- #$5s399S>!;L7MPQ7Q R	 * %I)J%& .rd322$e<!H018*GLK11[n[r[r\$}4EhZw%WX 118*GLK11ReRiRijuRv$}HXJg%NO "	 ' %&%8#= !H'	!9 :%@*2$a	{%zQ"55#6#:#:7#C x' %
2 1G (
 %8^c^g^g-._,}AhZq
RY-Z[ %8
 49;;?OQRXY3Z0Hh J.X]XaXackblXm,}F8*G-TUX]XaXackblXm,}F8*G-TUX]XaXackblXm,}F8*G-TU^c^g^giqhr^s,}L
RY-Z[^c^g^giqhr^s,}L
RY-Z[^c^g^giqhr^s,}L
RY-Z[; ( "B /0
 $1Hi%!/t1XJazQ#66'?&@)AhZW^$_M:M:Q:QR^:_(7	 & $1
 ".GX%!/t1WIQxjP#66'?&@(1XJV]$^M:M:Q:QR^:_(7	 & ".i 'x $LM P!Q(*;(I,Q,QN ,113!H%Yaz9L22H[H_H_`lHm$z8*G%DE " 4 !#sXkXpXpXrWttuvww(--/05I5M5Mc5R|C512 1  O Os   Tc                    S nS*S jnSnSnU R                  5        H  nUR                  S5      (       a-  [        UR                  S5      S   5      n[	        X6S-   5      nMF  UR                  S5      (       d  M^  [        UR                  S5      S   5      n[	        XFS-   5      nM     0 n[        U5       H  nU" U USU S	3S
U S35        U" U USU S3S
U S3S
U S3S
U S3/5        U" U USU S3S
U S35        U" U USU S3S
U S3S
U S3S
U S3/5        U" U USU S3S
U S35        U" U USU S3S
U S35        U" U USU S3S
U S35        U" U USU S3S
U S35        M     [        U5       H)  nU" U USU S3SU S 35        U" U USU S!3SU S"35        M+     S#S$S%S&S'S(.n	U	R                  5        H  u  pU" XX5        M     [        U R                  5       5      nU(       a  [        R                  S)U 35        U$ )+Nc                    US-   U ;  a  g U R                  US-   5      nUR                  S   n[        R                  " XTR                  UR
                  SS9nU R                  US-   U5      R                  5       nXu-  nUn	Sn
U	S-  U
:  a  U	S-  n	U
S-  n
U	S-  U
:  a  M  XI-  XS-   '   U R                  US	-   5      U
-  XS
-   '   g r   r   r   s              r   r   F_convert_kohya_flux2_lora_to_diffusers.<locals>._convert_to_ai_toolkit	  s    ((6jj+>!>?   #T1B1B;K]K]mrs

7X-}=BBD
1nx'!OJMH 1nx' .9-E))*-3ZZBS8S-TW_-_))*r   c                    US-   U ;  a  g U R                  US-   5      nU R                  US-   5      nUR                  S   n[        R                  " XuR                  UR
                  SS9nU R                  US-   U5      n	X-  n
U
nSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  X[-  nXl-  n[        U5      nUc  UR                  S   U-  /U-  nO[        U5      UR                  S   :X  d   eSnX}-  S:X  a  X}-  nS	nSn[        [        U5      5       Hh  n[        [        U5      5       HE  nUU:X  a  M  U=(       a1    [        R                  " UUUUU   -   2UU-  US
-   U-  24   S:H  5      nMG     UUU   -  nMj     U(       a  [        R                  SU 35        U Vs/ s H  nUS-   PM
     nnU Vs/ s H  nUS-   PM
     nnU(       dk  UR                  [        R                  UU5      5        UR                  [        U[        R                   " XdSS95       VVs0 s H	  u  nnUU_M     snn5        g UR                  [        U[        R"                  " X]SS95       VVs0 s H	  u  nnUU_M     snn5        Sn[        [        U5      5       H9  nUUUUU   -   2UW-  US
-   U-  24   R%                  5       UUU   '   UUU   -  nM;     g s  snf s  snf s  snnf s  snnf r   r   r   s                         r   r   J_convert_kohya_flux2_lora_to_diffusers.<locals>._convert_to_ai_toolkit_cat	  s   ((6jj+>!>?JJw)::;	"((+ 1 1+:L:L\a
 

7X-}=$
1nx'!OJMH 1nx' ".(	]
<OOA&*45
BDt9	 2222 	$)#1HIA3t9%s4y)AAv  ) !eii!!a$q'k/1x<1q5HBT3T"TUYZZ/I * T!W & 0	:;7?@x!--x@5=>Xq++X>MM$--{CDMMCU[[^_=`,ab,aDAq1a4,abcMMCu{{;hi?j,kl,kDAq1a4,klmA3t9%)21q47{?ALTUXYTY]eSeDe3e)f)q)q)s{1~&T!W & A> clr   r   r   r   r`  r.   r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   z.ff.linear_inr   z.ff.linear_outr   z.ff_context.linear_inr   z.ff_context.linear_outr  r  z.attn.to_qkv_mlp_projr  z.attn.to_outztransformer.x_embedderztransformer.context_embedderz:transformer.time_guidance_embed.timestep_embedder.linear_1z:transformer.time_guidance_embed.timestep_embedder.linear_2ztransformer.proj_out)r(  r,  r0  r2  lora_unet_final_layer_linearz2Unsupported keys for Kohya Flux2 LoRA conversion: r   )
r5   rt   r9   r:   r  r   r~   r4   r|   rA  )r@   r   r   rM  r  r!   	block_idxr   rR   rb  r   r   rB  s                r   &_convert_kohya_flux2_lora_to_diffusersrm  	  s   `(7t  >>455CIIcN1-.I #$51} E^^677CIIcN1-.I #$51} E ! F$%&qc8-aS?		
 	#&qc71!J?1!J?1!J?			
 	&qc8-aS0@A		
 	#&qc71!4DE1!4DE1!4DE			
 	&qc4-aS>		
 	&qc4-aS?		
 	&qc4-aS0EF		
 	&qc4-aS0FG		
k &x $%&qc24QC7LM		
 	&qc24QC|D		
 &$ 5:&b'c(>N +002z7D 3 *//+,NKNK[\]Mr   c                 @  ^ ^^^^^  [        S T  5       5      nU(       a5  T R                  5        VVs0 s H  u  p#UR                  S5      U_M     snnm [        S T  5       5      nU(       ar  T R                  5        VVs0 s H  u  p#UR                  S5      U_M     snnm S[        S[        4S jnT R                  5        VVs0 s H  u  p#U" U5      U_M     snnm S[        S[        4S	 jnT R                  5        VVs0 s H  u  p#U" U5      U_M     snnm [        S
 T  5       5      nU(       a6  T R                  5        VVs0 s H  u  p#UR	                  SS5      U_M     snnm T R                  5        VVs0 s H(  u  p#UR	                  SS5      R	                  SS5      U_M*     snnm 0 n[        T R                  5       5      n	SmSm SmSm[        UU 4S jU	 5       5      n
[        UU4S jU	 5       5      nU 4S jnU
(       a  U	 H  nUR                  T5      (       d  M  UR	                  TS5      nUR	                  TT 5      R	                  T S5      nUR	                  TS5      nT R                  U5      nT R                  UR	                  TT 5      5      nU" UU5      u  nnUU-  X'   UU-  X'   M     OU(       a  U	 H{  nUR                  T5      (       d  M  UR	                  TT5      nUR	                  TS5      nT R                  U5      nT R                  U5      nU" UU5      u  nnUU-  X'   UU-  X'   M}     SmSn[        U4S jT  5       5      nU(       Ga  [        T R                  5       5      nU GH  nTU;  a  M  UT ;  a  M  US[        T5      *  nUR                  S5      (       aJ  T R                  U5        T R                  UR	                  TU5      S5        T R                  US-   S5        M  [        R                  " SU5      (       a;  SU;  a5  T R                  U5        T R                  UR	                  TU5      S5        M  [        R                  " S [        R                  " T5      -   S!-   S"T-   U5      nUS[        T5      *  nUS-   nUR	                  TS5      nUR	                  TS5      nT R                  U5      nT R                  UR	                  TU5      5      nU" UU5      u  nnUU-  UU'   UU-  UU'   GM     [        T 5      S#:  a  [        S$T R                  5       < 35      eUR                  5        VVs0 s H  u  p#S%U 3U_M     nnnU$ s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf )&u+  
Convert non-diffusers ZImage LoRA state dict to diffusers format.

Handles:
- `diffusion_model.` prefix removal
- `lora_unet_` prefix conversion with key mapping
- `default.` prefix removal
- `.lora_down.weight`/`.lora_up.weight` → `.lora_A.weight`/`.lora_B.weight` conversion with alpha scaling
c              3   B   #    U  H  oR                  S 5      v   M     g7fr  r?  r_   s     r   r"   C_convert_non_diffusers_z_image_lora_to_diffusers.<locals>.<genexpr>a
  r  r@  r  c              3   B   #    U  H  oR                  S 5      v   M     g7frw  r?  r_   s     r   r"   rp  e
  r  r@  r^   r!   r  c           	         SU ;   a  U R                  SS5      u  pOU Sp!1 Skn0 nU H5  nUR                  [        U5      [        5       5      R	                  U5        M7     UR                  S5      n/ nSn[        UR                  5       SS9n	U[        U5      :  a  S	n
U	 HW  nX-   [        U5      ::  d  M  [        XhX-    5      XK   ;   d  M-  UR                  SR                  XhX-    5      5        X-  nSn
  O   U
(       d  UR                  Xh   5        US-  nU[        U5      :  a  M  SR                  U5      nX(       a  SU-   -   $ S-   $ )
Nr3   r.   r   >   r  r  r  r  feedforwardr   r   Tr  F)r  r  r{   r8   r;   r:   r  r5   r  r  r?   )r!   r  r  r   r!  r"  r#  r$  rR   r%  r&  r'  r)  s                r   r*  E_convert_non_diffusers_z_image_lora_to_diffusers.<locals>.convert_keyi
  sG    cz"zz#q1f"BfI K&&s2w6::2>   JJsOEFA!+"2"2"4dCLc%j.%AuE
*uUqu5E/F+./Xchhu/?&@A"& & MM%(+FA c%j. !XXf-N!vcFlFF2FFr   r`   c                 B    SU ;   a  U $ [         R                  " SSU 5      $ )N.to_outz2\.out(?=\.(?:lora_down|lora_up)\.weight$|\.alpha$)z	.to_out.0)r   r   )r`   s    r   normalize_out_keyK_convert_non_diffusers_z_image_lora_to_diffusers.<locals>.normalize_out_key
  s(    >HvvA
 	
r   c              3   ,   #    U  H
  nS U;   v   M     g7fr-  r   r_   s     r   r"   rp  
  r/  r   r.  r   zcontext.refiner.zcontext_refiner.znoise.refiner.znoise_refiner.r   rk   r   r   c              3   F   >#    U  H  nTU;   =(       d    TU;   v   M     g 7fr   r   r1  s     r   r"   rp  
  r3  r4  c              3   F   >#    U  H  nTU;   =(       d    TU;   v   M     g 7fr   r   r6  s     r   r"   rp  
  r9  r4  c                    > U R                   S   nTR                  US 5      nUc  gUR                  5       U-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ )Nr   )r   r   r   r   r  )r   rL  r   alpha_tensorr   r   r   r@   s          r   r  J_convert_non_diffusers_z_image_lora_to_diffusers.<locals>.get_alpha_scales
  s      #!~~i6$& 	 
1nx'!OJMH 1nx' ##r   rl   z.lora.down.weightz.lora.up.weightc              3   .   >#    U  H
  nTU;   v   M     g 7fr   r   )r   r`   lora_dot_down_keys     r   r"   rp  
  s     Ij/14jr  Nz.qkvz\.out$rx  z\.to\.([qkv])$z.to_\1r   rh  rx  )r6   r~   r  r>   rv   r4   r5   rs   r7   r{   r   r{  r   r  r<   )!r@   r<  r`   r   r=  r*  ry  r>  r  rD   r?  r@  r  rM  rA  rL  r   r   r   r   lora_dot_up_keyhas_lora_dot_formatdot_keysr  r  	norm_basediffusers_downdiffusers_upr7  r8  rI  r  r2  s!   `                           @@@@@r   0_convert_non_diffusers_z_image_lora_to_diffusersr  W
  s    S
SSHRHXHXHZ[HZann%78!;HZ[
GJGGMBLBRBRBTUBT$!ann\2A5BTU
*	GS *	GS *	GX 5?4D4D4FG4FDAk!na'4FG

S 
S 
 7A6F6F6HI6Hda#A&)6HIJ:z::K?I?O?O?QR?Qtqaii
B/2?QR
 $$&&DA 	
		$&89AABRTdeghh&J
 JOO%&H"HFEE ##S(#S SK(KK$ !Azz(##%&YYx9I%J"#$99Xv#>#F#FvO_#` IIh9	(nnQ/&NN199Xv+FG	'7Y'O$
H;F;S$89BX9M$6  
Azz%  #$99UE#: IIeX6	(nnQ/&NN+;<	'7Y'O$
H*5
*B$'9BX9M$6 " ,'OIjII
)*A )
".-../D }}V$$q!qyy):OLdSth5 yyD))it.Cq!qyy):OLdS VV 299->#??$F--F
 8#&7"8!89I!H,I#^^,=?OPN!>>*;=MNL$..+K"qyy1BO'TUI#3K#K J3>3K 01:X1E .K N :azO`Nbcdd>R>X>X>Z[>Zdal1#.1>Z[i \ V\ H J S\ \s)   U6U<VV	V?/VV)r      )unettext_encoder)r  )!r   r   utilsr   r   r   
get_logger__name__r|   r   rZ   r   ru   rw   rz   r  r  r>   r4   r  r  r  rX  ro  r  r  r  r  r  rB  re  rm  r  r   r   r   <module>r     s    
  A A 
		H	%vri*X5p8 h9\fR% %$s) % %(j Z	[ |l ^D Nz z: z  0f| ~K \vr@ r   