
    
3j.              	          S SK r S SKJrJr  S SKJr  S SKrSSKJr  SSK	J
r
  SSKJrJrJrJr  SS	KJrJr  SS
KJrJr  \" \5      rSr\ " S S5      5       r " S S\R4                  R6                  5      r " S S\5      r " S S\5      r " S S\5      rS\R@                  RB                  S\SS4S jr"SS\R@                  RB                  S\S\#S-  SS4S jjr$g)    N)asdict	dataclass)Callable   )
get_logger)unwrap_module   )"_ALL_TRANSFORMER_BLOCK_IDENTIFIERS_ATTENTION_CLASSES_FEEDFORWARD_CLASSES_get_submodule_from_fqn)AttentionProcessorRegistryTransformerBlockRegistry)HookRegistry	ModelHooklayer_skip_hookc                       \ rS rSr% Sr\\   \S'   Sr\	\S'   Sr
\\S'   Sr\\S	'   Sr\\S
'   Sr\\S'   S rS r\S\SS 4S j5       rSrg)LayerSkipConfig(   ay  
Configuration for skipping internal transformer blocks when executing a transformer model.

Args:
    indices (`list[int]`):
        The indices of the layer to skip. This is typically the first layer in the transformer block.
    fqn (`str`, defaults to `"auto"`):
        The fully qualified name identifying the stack of transformer blocks. Typically, this is
        `transformer_blocks`, `single_transformer_blocks`, `blocks`, `layers`, or `temporal_transformer_blocks`.
        For automatic detection, set this to `"auto"`. "auto" only works on DiT models. For UNet models, you must
        provide the correct fqn.
    skip_attention (`bool`, defaults to `True`):
        Whether to skip attention blocks.
    skip_ff (`bool`, defaults to `True`):
        Whether to skip feed-forward blocks.
    skip_attention_scores (`bool`, defaults to `False`):
        Whether to skip attention score computation in the attention blocks. This is equivalent to using `value`
        projections as the output of scaled dot product attention.
    dropout (`float`, defaults to `1.0`):
        The dropout probability for dropping the outputs of the skipped layers. By default, this is set to `1.0`,
        meaning that the outputs of the skipped layers are completely ignored. If set to `0.0`, the outputs of the
        skipped layers are fully retained, which is equivalent to not skipping any layers.
indicesautofqnTskip_attentionFskip_attention_scoresskip_ff      ?dropoutc                     SU R                   s=::  a  S::  d  O  [        SU R                    S35      e[        R                  " U R                   S5      (       d  U R                  (       a  [        S5      eg g )Nr   r	   z6Expected `dropout` to be between 0.0 and 1.0, but got .r   bCannot set `skip_attention_scores` to True when `dropout` is not 1.0. Please set `dropout` to 1.0.)r   
ValueErrormathiscloser   selfs    T/home/wildlama/miniconda3/lib/python3.13/site-packages/diffusers/hooks/layer_skip.py__post_init__LayerSkipConfig.__post_init__I   sg    T\\&Q&UVZVbVbUccdeff||DLL#..43M3Mt  4N.    c                     [        U 5      $ N)r   r$   s    r&   to_dictLayerSkipConfig.to_dictQ   s    d|r)   datareturnc                     [        S0 U D6$ )N )r   )r.   s    r&   	from_dictLayerSkipConfig.from_dictT   s    &&&r)   r1   N)__name__
__module____qualname____firstlineno____doc__listint__annotations__r   strr   boolr   r   r   floatr'   r,   staticmethoddictr2   __static_attributes__r1   r)   r&   r   r   (   ss    0 #YCND"'4'GTGU ' '!2 ' 'r)   r   c                       \ rS rSrSS jrSrg)AttentionScoreSkipFunctionModeY   r1   Nc                 V   Uc  0 nU[         R                  R                  R                  L av  UR	                  SS 5      nUR	                  SS 5      nUR	                  SS 5      nUb  UOUS   nUb  UOUS   nUb  UOUS   nUR
                  S   UR
                  S   :X  a  U$ U" U0 UD6$ )Nquerykeyvaluer   r	   r   )torchnn
functionalscaled_dot_product_attentiongetshape)r%   functypesargskwargsrF   rG   rH   s           r&   __torch_function__1AttentionScoreSkipFunctionMode.__torch_function__Z   s    >F588&&CCCJJw-E**UD)CJJw-E".EDGE#d1gC".EDGE {{1~Q/T$V$$r)   )r1   N)r4   r5   r6   r7   rS   rA   r1   r)   r&   rC   rC   Y   s    %r)   rC   c                   d    \ rS rSrS
S\S\S\4S jjrS\R                  R                  4S jrSrg	)AttentionProcessorSkipHookn   skip_processor_output_fnr   r   c                 (    Xl         X l        X0l        g r+   )rX   r   r   )r%   rX   r   r   s       r&   __init__#AttentionProcessorSkipHook.__init__o   s    (@%%:"r)   modulec                 &   U R                   (       ab  [        R                  " U R                  S5      (       d  [	        S5      e[        5          U R                  R                  " U0 UD6nS S S 5        U$ [        R                  " U R                  S5      (       a  U R                  " U/UQ70 UD6nU$ U R                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ ! , (       d  f       W$ = f)Nr   r    p)r   r"   r#   r   r!   rC   fn_reforiginal_forwardrX   rI   rJ   rK   r%   r\   rQ   rR   outputs        r&   new_forward&AttentionProcessorSkipHook.new_forwardt   s    %%<<c22 x  0155tFvF 2  ||DLL#..66vOOO  55tFvF,,44V||4L 21 s   D
D)r   r   rX   N)Fr   )r4   r5   r6   r7   r   r=   r>   rZ   rI   rJ   Modulerd   rA   r1   r)   r&   rV   rV   n   s2     RV in 
%((// r)   rV   c                   f   ^  \ rS rSrS\4U 4S jjrS\R                  R                  4S jr	Sr
U =r$ )FeedForwardSkipHook   r   c                 .   > [         TU ]  5         Xl        g r+   superrZ   r   r%   r   	__class__s     r&   rZ   FeedForwardSkipHook.__init__       r)   r\   c                 l   [         R                  " U R                  S5      (       a@  UR                  SS 5      nUc  UR                  SS 5      nUc  [	        U5      S:  a  US   nU$ U R
                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ )Nr   hidden_statesxr   r^   )
r"   r#   r   rM   lenr`   ra   rI   rJ   rK   rb   s        r&   rd   FeedForwardSkipHook.new_forward   s    <<c**ZZ6F~C.~#d)a-a  [[114B6BFXX((00<<0HFr)   )r   )r4   r5   r6   r7   r>   rZ   rI   rJ   rf   rd   rA   __classcell__rn   s   @r&   rh   rh      s(     
%((// 
 
r)   rh   c                   l   ^  \ rS rSrS\4U 4S jjrS rS\R                  R                  4S jr
SrU =r$ )TransformerBlockSkipHook   r   c                 .   > [         TU ]  5         Xl        g r+   rk   rm   s     r&   rZ   !TransformerBlockSkipHook.__init__   rp   r)   c                 b    [         R                  " [        U5      R                  5      U l        U$ r+   )r   rM   r   rn   	_metadata)r%   r\   s     r&   initialize_hook(TransformerBlockSkipHook.initialize_hook   s$    155mF6K6U6UVr)   r\   c                    [         R                  " U R                  S5      (       aY  U R                  R	                  SX#5      nU R                  R
                  c  UnU$ U R                  R	                  SX#5      nXF4n U$ U R                  R                  " U0 UD6n[        R                  R                  R                  XPR                  S9nU$ )Nr   rr   encoder_hidden_statesr^   )r"   r#   r   r~   _get_parameter_from_args_kwargs"return_encoder_hidden_states_indexr`   ra   rI   rJ   rK   )r%   r\   rQ   rR   original_hidden_statesrc   original_encoder_hidden_statess          r&   rd   $TransformerBlockSkipHook.new_forward   s    <<c**%)^^%S%STcei%r"~~@@H/  261_1_+T2. 1Q  [[114B6BFXX((00<<0HFr)   )r~   r   )r4   r5   r6   r7   r>   rZ   r   rI   rJ   rf   rd   rA   rv   rw   s   @r&   ry   ry      s-     %((//  r)   ry   r\   configr/   c                     [        X5        g)ag  
Apply layer skipping to internal layers of a transformer.

Args:
    module (`torch.nn.Module`):
        The transformer model to which the layer skip hook should be applied.
    config (`LayerSkipConfig`):
        The configuration for the layer skip hook.

Example:

```python
>>> from diffusers import apply_layer_skip_hook, CogVideoXTransformer3DModel, LayerSkipConfig

>>> transformer = CogVideoXTransformer3DModel.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16)
>>> config = LayerSkipConfig(layer_index=[10, 20], fqn="transformer_blocks")
>>> apply_layer_skip_hook(transformer, config)
```
N)_apply_layer_skip_hook)r\   r   s     r&   apply_layer_skipr      s    ( 6*r)   namec                    U=(       d    [         nUR                  (       a  UR                  (       a  [        S5      e[        R
                  " UR                  S5      (       d  UR                  (       a  [        S5      eUR                  S:X  a0  [         H  n[        X5      (       d  M  X1l          O   [        S5      e[        XR                  5      nUb)  [        U[        R                  R                  5      (       d  [        SUR                   S35      e[        UR                   5      S:X  a  [        S	5      eS
n[#        U5       GHQ  u  pgXaR                   ;  a  M  SnUR                  (       au  UR$                  (       ad  [&        R)                  SUR                   SU S35        [*        R,                  " U5      n[/        UR                  5      n	UR1                  X5        GO	UR                  (       d  UR                  (       a  UR3                  5        H  u  p[        U[4        5      (       d  M  UR6                  (       a  M/  [&        R)                  SUR                   SU SU
 S35        [8        R:                  " UR<                  R>                  5      R@                  n[*        R,                  " U5      n[C        XR                  UR                  5      n	UR1                  X5        M     UR$                  (       d  GM  UR3                  5        H  u  p[        U[D        5      (       d  M  [&        R)                  SUR                   SU SU
 S35        [*        R,                  " U5      n[G        UR                  5      n	UR1                  X5        M     GMT     U(       d&  [        SUR                    SUR                   S35      eg )NzXCannot set both `skip_attention` and `skip_attention_scores` to True. Please choose one.r   r    r   zCould not find a suitable identifier for the transformer blocks automatically. Please provide a valid `fqn` (fully qualified name) that identifies a stack of transformer blocks.zCould not find z in the provided module, or configured `fqn` (fully qualified name) does not identify a `torch.nn.ModuleList`. Please provide a valid `fqn` that identifies a stack of transformer blocks.r   zTLayer index list is empty. Please provide a non-empty list of layer indices to skip.FTz&Applying TransformerBlockSkipHook to 'r   'z(Applying AttentionProcessorSkipHook to 'z!Applying FeedForwardSkipHook to 'zDCould not find any transformer blocks matching the provided indices z and fully qualified name 'z4'. Please check the indices and fqn for correctness.)$_LAYER_SKIP_HOOKr   r   r!   r"   r#   r   r   r
   hasattrr   
isinstancerI   rJ   
ModuleListrt   r   	enumerater   loggerdebugr   check_if_exists_or_initializery   register_hooknamed_modulesr   is_cross_attentionr   rM   	processorrn   rX   rV   r   rh   )r\   r   r   
identifiertransformer_blocksblocks_foundiblockregistryhooksubmodule_name	submodule	output_fns                r&   r   r      s   ##D!=!=stt<<,,1M1Mp
 	
 zzV<Jv**'
 =
 ^ 
 1D!4FH[H[)\)\fjj\ *s t
 	
 6>>aoppL01NN"  V^^LLA&**QqcQRST#AA%HH+FNN;D""4.""f&B&B-2-@-@-B)i);<<YEaEaEaLL#KFJJ<WXYZX[[\]k\llm!no : > >y?R?R?\?\ ] v vI+II)TH5iA]A]_e_m_mnD**46 .C >>>-2-@-@-B)i)=>>LL#DVZZLPQRSQTTUVdUeef!gh+II)TH.v~~>D**46 .C- 2: RSYSaSaRb c%%+ZZL0df
 	
 r)   r+   )%r"   dataclassesr   r   typingr   rI   utilsr   utils.torch_utilsr   _commonr
   r   r   r   _helpersr   r   hooksr   r   r4   r   r   r   	overridesTorchFunctionModerC   rV   rh   ry   rJ   rf   r   r<   r   r1   r)   r&   <module>r      s     )    -  K * 
H	$ 
 -' -' -'`%U__%F%F %* .) $y 2+UXX__ +o +$ +.@
588?? @
O @
SVY]S] @
im @
r)   