
    3j_]                   
   % S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKJrJr  SSKJr  SSKJrJrJrJrJrJr  SSKJr  SSKJrJrJrJ r   SS	K!J"r"J#r#  SSK$r$SSK%r$SSK&r$SSK'J(s  J)r*  SS
K+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6J7r7  SSK8J9r9  SSK:J;r;J<r<  SSK=J>r>  SSK?J@r@  SSKAJBrB  SSKCJDrE  SSKFJGrGJHrH  SSKIJJrJ  SSKKJLrLJMrM  SSKNJOrO  SSKPJDrD  SSKQJRrR  SSKSJTrT  \	R                  " \V5      rW\$R                  R                  rY\$R                  R                  rZ\r[\[\$R                  R                  -  r]\R                  R                  SS5      r`SqaS \bS!'   SqcS"\bS#'   SS$ jrd " S% S&\5      re " S' S(\5      rf " S) S*\5      rg\ " S+5      rh\$R                  R                  R                  \k-  rl " S, S-5      rm\m" 5       rn S       SS. jjro " S/ S05      rp " S1 S2\q5      rr\p\r-  rsSS3 jrt " S4 S55      ru " S6 S7\5      rv " S8 S9\v5      rw " S: S;\v5      rx " S< S=\v5      ry " S> S?\v5      rz " S@ SA\v5      r{\|\SB4   r} " SC SD\{5      r~ " SE SF\~5      r " SG SH\~5      r " SI SJ\~5      r " SK SL\{5      r " SM SN\5      r " SO SP\5      r " SQ SR\5      r " SS ST\v5      r " SU SV\v5      r " SW SX\v5      r " SY SZ5      r " S[ S\\5      r\GR                   " S] S^5      5       r\GR                   " S_ S`\5      5       r\GR                   " Sa Sb\5      5       r\GR                   " Sc Sd\5      5       rSSe jrSSf jr S         SSh jjr\SSiSSgS\R4                         SSj jjr\B" 5       rSk\bSl'               SSm jr\" \5      GR.                  Sn-  So-  r/ rSp\bSq'   \SSiSg\R4                       SSr jjr\EGR6                  " SgSs9SSi\R4             SSt jj5       r  S           SSu jjr\4SgSv.         SSw jjjr\4SgSv.         SSx jjjrSSy jr\GRB                  " Sz5      rSS{ jrSS| jrSS} jrSS~ jrSS jrSS jrSS jr " S S5      rSS jr    S           SS jjr\$GRZ                  " 5       S\RS.         SS jj5       r\$GR^                  " 5       \RS.       SS jj5       rSS jrSS jrSS jrSS jrSS jr\B" 5       rSk\bS'    S       SS jjrSS jrSS jrg)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)Callable
Collection	GeneratorIterableMappingSequence)Path)AnyNoReturnProtocolTypeVar)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fx)guard_or_falsestatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)preserve_node_meta)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_type#TORCHINDUCTOR_PATTERN_MATCH_BACKENDinductorzbool | OrderedSet[str] | None_debug_nodes_cache
str | None_debug_nodes_env_value_cachec                P    SS jnU" 5       n[        U[        5      (       a  U$ X;   $ )Nc                     SS jn [         R                  R                  S5      nU[        :w  d  [        c
  U " U5      qUq[        $ )Nc                T    U (       d  gU S:X  a  g[        U R                  S5      5      $ )NFallT,)r    split)	env_values    Y/home/wildlama/miniconda3/lib/python3.13/site-packages/torch/_inductor/pattern_matcher.pyparse_debug_envE_should_debug_node.<locals>._get_debug_nodes.<locals>.parse_debug_enve   s'    E!iooc233    !TORCHINDUCTOR_PATTERN_MATCH_DEBUG)r8   r0   returnbool | OrderedSet[str])osenvirongetr1   r/   )r:   current_envs     r9   _get_debug_nodes,_should_debug_node.<locals>._get_debug_nodesb   sD    	4 jjnn%HI 66:L:T!0!=+6(!!r<   )r>   r?   )
isinstancebool)	node_namerD   debug_nodess      r9   _should_debug_noderJ   a   s-    "& #$K+t$$##r<   c                  *    \ rS rSr% S\S'   SS jrSrg)SearchFn{   str__name__c                    g N selfargskwargss      r9   __call__SearchFn.__call__~       #r<   rR   NrU   r   rV   r   r>   r   )rO   
__module____qualname____firstlineno____annotations__rW   __static_attributes__rR   r<   r9   rL   rL   {   s
    M=r<   rL   c                      \ rS rSrSS jrSrg)	ReplaceFn   c                    g rQ   rR   rS   s      r9   rW   ReplaceFn.__call__   rY   r<   rR   NrZ   rO   r[   r\   r]   rW   r_   rR   r<   r9   ra   ra      s    =r<   ra   c                  .    \ rS rSr        SS jrSrg)TraceFn   c                    g rQ   rR   )rT   fnrU   rV   s       r9   rW   TraceFn.__call__   s    "r<   rR   N)rj   zSearchFn | ReplaceFnrU   r   rV   r   r>   torch.fx.GraphModulere   rR   r<   r9   rg   rg      s'    #&#/2#>A#	#r<   rg   Tc                      \ rS rSrSS jrSrg)Multiple   c                8    S[        5       ;  d  U [        L d   eg g )NMULTIPLE)globalsrr   rT   s    r9   __init__Multiple.__init__   s    *dh.>>>.>*r<   rR   Nr>   None)rO   r[   r\   r]   ru   r_   rR   r<   r9   ro   ro      s    ?r<   ro   c                   SSK JnJn  [        R                  R
                  S:X  aw  U R                  S/ 5      R                  5       nUR                  U" XUR                  5      5        U R                  S UR                  R                  5        5       5        XPS'   O0U R                  S UR                  R                  5        5       5        SUR                  ;   a  UR                  S   U S'   SU ;  a"  SUR                  ;   a  UR                  S   U S'   S	U ;  a$  S	UR                  ;   a  UR                  S	   U S	'   g g g )
Nr   )
NodeSourceNodeSourceActionr*   	from_nodec              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7frQ   torchfxproxy_COPY_META_FIELDS.0kvs      r9   	<genexpr>!_transfer_meta.<locals>.<genexpr>   3      
-EHHNN444 QF-   .>
>c              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7frQ   r~   r   s      r9   r   r      r   r   stack_tracevaltensor_meta)torch.fx.tracebackrz   r{   r#   traceprovenance_tracking_levelrB   copyappendREPLACEupdatemetaitems)new_metaold_node	pass_namerz   r{   new_from_nodes         r9   _transfer_metar      s    @ ||--2 ["5::<Z=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	

 %"*--"> H(--!7"--.H$(--)G"*--"> *H$r<   c                    ^  \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'   S\S'   S\S'   S\S'     S         SU 4S jjjr\SS j5       rSS jr	S S jr
S!S jrS"S jrS#S jrS$S jr      S%S jr  S&         S'S jjrSrU =r$ )(Match   ao  
Represents a successfully matched pattern.

The `Match` object is returned to represent a successfully matched
pattern. Included in the Match are the pattern that was matched, the graph
nodes matched, and any args that were used during the matching.

The args and kwargs are specific to the type of pattern that was matched and
provide hints about what was matched.
PatternExprpattern	list[Any]rU   dict[str, Any]rV   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxztorch.fx.GraphModule | Nonereplacement_graphc                   > [         TU ]  5         X l        [        U=(       d    / 5      U l        U=(       d    0 U l        / U l        0 U l        Xl        S U l	        g rQ   )
superru   r   listrU   rV   r   r   r   r   )rT   r   r   rU   rV   	__class__s        r9   ru   Match.__init__   sK     	$	l
!%r<   c                .    U R                   R                  $ rQ   )r   graphrt   s    r9   r   Match.graph   s    xx~~r<   c                B   U R                   (       az  [        U R                   R                  5       5      [        UR                   R                  5       5      -   H/  nU R                   U   UR                   U   :w  d  M%  [        SU5      e   U R                  R                  UR                  5        U R                  R                  UR                  5        U R                   R                  UR                   5        U R                  R                  UR                  5        g )Nzkwarg mismatch: {})	rV   r    keysFailedMatchrU   extendr   r   r   )rT   otherkeys      r9   r   Match.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW;;s#u||C'88%&:C@@ X 			$

%++&5<<(EMM*r<   c                p    U R                   (       a  [        U R                   5      /U l         U $ / U l         U $ rQ   )rU   tuplert   s    r9   bundleMatch.bundle   s.    *.))U499%&	 :<	r<   c                <    SU R                    SU R                   S3$ )NzMatch(..., , )rU   rV   rt   s    r9   __repr__Match.__repr__   s    TYYKr$++a88r<   c                    U R                   n[        U R                  5       H:  nUR                  (       a  M  UR                  (       a  M)  UR                  U5        M<     g rQ   )r   reversedr   _erasedusers
erase_node)rT   r   ns      r9   erase_nodesMatch.erase_nodes   s=    

$**%A999QWWW  # &r<   c                    U R                   R                   Vs/ s H!  nUb  U R                   R                  U   OS PM#     sn$ s  snf rQ   )r   outputspattern_to_node)rT   ps     r9   output_nodesMatch.output_nodes   sK     XX%%
% -.MTXX%%a(tC%
 	
 
s   (Ac                B    [        S U R                  5        5       5      $ )Nc              3  6   #    U  H  o(       d  M  Uv   M     g 7frQ   rR   )r   r   s     r9   r   $Match.output_node.<locals>.<genexpr>  s     82!aAA2s   
	)nextr   rt   s    r9   output_nodeMatch.output_node  s    8t002888r<   c                X    [         R                  X R                  R                  X5        g rQ   )ReplacementPatternEntryreplace_with_graphr   r   )rT   r   rU   s      r9   r   Match.replace_with_graph  s      	 22((.."3	
r<   c                `  ^^ SSK JnJn  [        UR                  U5      (       a  UR                  c  UR                  O[
        R                  " 5       nSS jnU   Uc  [        R                  " [        US9nU" U R                  5      (       Ga}  0 mU R                  S   R                  S   u  p0 U
En
[        U R                  5      U R                  pSU4S jjn[        R                   R"                  R%                  XU4X45        [        R&                  R)                  UU4S j5      nU" X5      n[        R&                  R)                  US	 5      nU" X5      n[+        UR,                  R                  5      [+        UR,                  R                  5      :X  d   e[/        UR,                  R                  UR,                  R                  5       H4  u  nnSUR                  ;   d  M  UR                  S   UR                  S'   M6     O[        R&                  R)                  US
 5      n[        R0                  R                   R3                  U5      mTb0  U4S jn[        R&                  R4                  R7                  UU5      nU" X5      n[+        U R                  5      S:X  a?  UR,                  R                   H%  n[9        UR                  U R                  S   SS9  M'     [:        R=                  U U R>                  R,                  UU5        SSS5        g! , (       d  f       g= f)zReplace with a graph generated by tracing the replacement_fn.

Args:
    run_functional_passes (bool). If we should run passes that
        assume functional IR (like DCE, remove_noop_ops), on the
        replacement graph.

r   )NullHandlerVNc                P   [        U 5      S:w  a  gU S   nSUR                  ;  a  gUR                  [        [        R
                  R                  R                  [        R
                  R                  R                  [        R
                  R                  R                  /5      ;   $ )Nr*   Fr   eager_input_vals)
lenr   targetr    r   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r   nodes     r9   !should_propagate_eager_input_valsCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals&  s~    5zQ8D!2;;*II**KKII**>>II**AA#  r<   )run_functional_passesr   c                d   > [        U [        R                  R                  5      (       a  UTU '   g g rQ   )rF   r   r   Node)r   r   node_to_vals     r9   record(Match.replace_by_example.<locals>.recordI  s'    !$66,/D) 7r<   c                   > TU    $ rQ   rR   )argr   s    r9   <lambda>*Match.replace_by_example.<locals>.<lambda>Q  s
    +cBRr<   c                     U R                   S   $ Nr   r   r   s    r9   r   r   W  s    #((5/r<   c                ^    SU R                   ;   a  U R                   S   $ U R                   S   $ )Nr   example_valuer   r   s    r9   r   r   m  s1    ( !$ !3/2!3r<   c                Z   > [        U [        5      (       a  Tb  TR                  U 5      $ U $ rQ   )rF   r'   from_tensor)it	fake_modes    r9   _convert_to_fake_mode7Match.replace_by_example.<locals>._convert_to_fake_modet  s+    %b*55):O#,#8#8#<<!	r<   r*   replace_by_exampler   r   r   )r   r   r>   rG   )r   torch.fx.Noder   r   r>   rx   ) torch._inductor.virtualizedr   r   rF   r   
contextlibnullcontext	functoolspartialfwd_onlyr   r   r   rU   rV   r   utils_pytreetree_mapr   map_argr   r   zip_dynamodetect_fake_moder   map_aggregater   r   r   r   )rT   replacement_fnrU   trace_fnr   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementr   new_noder   r   r   r   s                        @@r9   r   Match.replace_by_example  s    	? q{{K88Q[[=P KK'') 		 $,,4I 1<< !)-A););<N)O&	-o+0+;T[[L0 ##,,68P  %xx//6RS )1(N%  %xx//6QR&'<K 066<<=%%++B    +.)//55{7H7H7N7N+&Hh *X]]:<DMM.=&89	+  %xx//3  "MM//@@N	("
 $)88==#>#>$&;$L '~D4::!#$**00A"!"!%A"6 1 $66	a WWs   FL,D*L
L-)rU   r   rV   r   r   r   r   NN)
r   r   r   r   rU   zSequence[Any] | NonerV   zdict[str, Any] | Noner>   rx   )r>   torch.fx.Graph)r   r   r>   rx   )r>   r   r>   rN   rw   )r>   zlist[torch.fx.Node | None])r>   r  )r   r  rU   Sequence[Any]r>   rx   NT)
r  ra   rU   r   r  zTraceFn | Noner   rG   r>   rx   )rO   r[   r\   r]   __doc__r^   ru   propertyr   r   r   r   r   r   r   r   r   r_   __classcell__r   s   @r9   r   r      s   	 
O44	22 &*(,&& & #	&
 && 
& &&  +
9$
9
!/
7D
	
 $(&*{!{ { !	{
  ${ 
{ {r<   r   c                  B    \ rS rSr% SrS\S'   S
S jrSS jrSS jrSr	g	)r   i  zs
Represents a unsuccessful match.

The `FailedMatch` object is returned to represent a failure to match a
pattern.
rN   format_stringc                b    Xl         [        U5      S:  a  [        SU 35      eX l        X0l        g )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r'  r   RuntimeErrorrU   rV   )rT   r'  rU   rV   s       r9   ru   FailedMatch.__init__  s<    * }#hivhwx  	r<   c                b    U R                   R                  " U R                  0 U R                  D6$ rQ   )r'  formatrU   rV   rt   s    r9   __str__FailedMatch.__str__  s&    !!(($))Ct{{CCr<   c                    gNFrR   rt   s    r9   __bool__FailedMatch.__bool__      r<   )rU   r'  rV   N)r'  rN   rU   r   rV   r   r>   rx   r  r>   rG   )
rO   r[   r\   r]   r"  r^   ru   r.  r2  r_   rR   r<   r9   r   r     s     	Dr<   r   c                    [        U 5      $ )zp
TypeIs cannot act on `self`. Thus this function exists to let mypy
recognize FailedMatch.__bool__ as a TypeIs.
)rG   )ms    r9   is_matchr8    s    
 7Nr<   c                  t    \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'    S       SS jjrSS jrSS jrSr	g)r   i  z;
Internal state needed while running PatternExpr._match().
list[PatternExpr | None]r   z'dict[PatternExpr, torch.fx.Node | None]r   r  r   zlist[NodeOrConstant]exclusive_node_setNc               T    Xl         Uc  0 O
[        U5      U l        X0l        / U l        g rQ   )r   dictr   r   r;  )rT   r   r   r   s       r9   ru   MatchContext.__init__  s*     %4%<r$BW
"$r<   c                    XR                   ;   a)  U R                   U   U:X  a  [        X5      $ [        S5      $ UR                  X 5      nXR                   ;  d   eU(       a  UOSU R                   U'   U$ )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r   r   _match)rT   r   r   r7  s       r9   matchMatchContext.match  sq    ***##G,4T++"#=>>NN4&2222201tW%r<   c                    U R                   R                  5        VVs0 s H#  u  pUR                  5       (       d  M  Uc  M!  X_M%     snn$ s  snnf rQ   )r   r   has_multiple_users)rT   r   r   s      r9   filter_multi_user_patterns'MatchContext.filter_multi_user_patterns  sR     "&!5!5!;!;!=
!=))+ 04 GM!=
 	
 
s   AAA)r;  r   r   r   rQ   )r   r:  r   z'dict[PatternExpr, torch.fx.Node] | Noner   r  r>   rx   )r   r   r   NodeOrConstantr>   MatchResult)r>   z dict[PatternExpr, torch.fx.Node])
rO   r[   r\   r]   r"  r^   ru   rA  rE  r_   rR   r<   r9   r   r     s^     &%<<,,
 DH
%)
% A
%
 
% 

%

r<   r   c                  j    \ rS rSrSr\SS j5       rSS jrSS jrSS jr	      SS jr
SS jrS	rg
)r   i  z#
Base class for types of patterns.
c                    g rQ   rR   rT   r   r   s      r9   r@  PatternExpr._match  s    MPr<   c                ~     [        U /UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = fNr   )r   r   rA  r   rT   r   es      r9   rA  PatternExpr.match  s:    	djj9??KK 	H	s   #& 
<7<<c                    gr1  rR   rt   s    r9   rD  PatternExpr.has_multiple_users  r4  r<   c                4    U R                   R                  S-   $ )Nz())r   rO   rt   s    r9   r   PatternExpr.__repr__  s    ~~&&--r<   c              #  N   #    XR                   ;   a  UR                   U    v   g g 7frQ   )r   rT   r   searcheds      r9   find_anchor_nodesPatternExpr.find_anchor_nodes  s)      &&&%%d++ 's   #%c                ,    [        XR                  5      $ )z
Compare two `PatternExpr`s and return true if they are the
same. Note this is NOT matching a pattern - it is comparing the pattern
structures (for debugging).
)rF   r   )rT   r   s     r9   
pattern_eqPatternExpr.pattern_eq  s     %00r<   rR   Nr   r  r   r   r>   rH  r   r  r>   rH  r5  r  r   r   rY  OrderedSet[torch.fx.Node]r>   z+Generator[torch.fx.Node | None, None, None]r   r   r>   rG   )rO   r[   r\   r]   r"  r   r@  rA  rD  r   rZ  r]  r_   rR   r<   r9   r   r     sH     P P.,,+D,	4,1r<   r   c                  "    \ rS rSrSrSS jrSrg)Argi  zb
Capture an arg which will become an input to the handler.  Args are
passed in depth first order.
c                    [        X U/S9$ )N)rU   r   rK  s      r9   r@  
Arg._match  s    SdV,,r<   rR   Nr   rG  r   r   r>   rH  rO   r[   r\   r]   r"  r@  r_   rR   r<   r9   re  re    s    
-r<   re  c                  6    \ rS rSrSrSS jrS	S jrS
S jrSrg)Ignoredi  z,
Match an arg, but don't pass it to handler
c                    [        X 5      $ rQ   rg  rK  s      r9   r@  Ignored._match  s    Sr<   c                    g)N*rR   rt   s    r9   r   Ignored.__repr__  s    r<   c                    g)Nz	Ignored()rR   )rT   pps     r9   pretty_printIgnored.pretty_print  s    r<   rR   Nri  r  rs  PatternPrettyPrinterr>   rN   )	rO   r[   r\   r]   r"  r@  r   rt  r_   rR   r<   r9   rl  rl    s     r<   rl  c                  T   ^  \ rS rSrSrSU 4S jjrS	S jrS
S jrSU 4S jjrSr	U =r
$ )
KeywordArgi  <
Capture a kwarg which will become an input to the handler.
c                .   > [         TU ]  5         Xl        g rQ   r   ru   namerT   r}  r   s     r9   ru   KeywordArg.__init__      	r<   c                $    SU R                   < S3$ )NzKeywordArg(r   r}  rt   s    r9   r   KeywordArg.__repr__#  s    TYYM++r<   c                .    [        X U R                  U0S9$ )NrV   )r   r}  rK  s      r9   r@  KeywordArg._match&  s    S		4'899r<   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ rQ   typingcastr   r   r]  r}  rT   r   r   s     r9   r]  KeywordArg.pattern_eq)  5    D%(w!%(DTYY%**-DDr<   r  r}  rN   r>   rx   r  ri  rc  rO   r[   r\   r]   r"  ru   r   r@  r]  r_   r$  r%  s   @r9   ry  ry    s#    ,:E Er<   ry  c                  `   ^  \ rS rSr% SrS\S'   S
U 4S jjrSS jrSS jrSU 4S jjr	S	r
U =r$ )ExclusiveKeywordArgi.  rz  rN   r}  c                .   > [         TU ]  5         Xl        g rQ   r|  r~  s     r9   ru   ExclusiveKeywordArg.__init__5  r  r<   c                $    SU R                   < S3$ )NzExclusiveKeywordArg(r   r  rt   s    r9   r   ExclusiveKeywordArg.__repr__9  s    %dii]!44r<   c                    XR                   ;   a  [        S5      $ UR                   R                  U5        [        X U R                  U0S9$ )Nzexclusive arg appears twicer  )r;  r   r   r   r}  rK  s      r9   r@  ExclusiveKeywordArg._match<  sD    )))<==%%d+S		4'899r<   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ rQ   r  r  s     r9   r]  ExclusiveKeywordArg.pattern_eqC  r  r<   r  r  r  ri  rc  )rO   r[   r\   r]   r"  r^   ru   r   r@  r]  r_   r$  r%  s   @r9   r  r  .  s*     I5:E Er<   r  c                     ^  \ rS rSr% SrS\S'   S\S'    S     SU 4S jjjr\\SS j5       5       r	SS	 jr
SS
 jrSS jr      SS jrSS jrSS jrSU 4S jjrSrU =r$ )_TargetExpriH  z/
Base class for filtering match by node.target
zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                |  >^ [         TU ]  5         [        U5      (       d  [        U[        5      (       a  U/O
[        U5      nU HW  m[        T[        R                  R                  5      (       d  M.  UR                  U4S jTR                  5        5       5        MY     Xl        [        U5      U l        X l        g )Nc              3  <   >#    U  H  n[        TU5      v   M     g 7frQ   )getattr)r   overloadrj   s     r9   r   '_TargetExpr.__init__.<locals>.<genexpr>W  s     PX72x00s   )r   ru   callablerF   rN   r   r   _opsOpOverloadPacketr   	overloadsr  r    r  r   )rT   r  r   rj   r   s      @r9   ru   _TargetExpr.__init__P  s     	}}
3(<(<se$s)B"ejj99::

PPP  !#
r<   c                    g rQ   rR   rt   s    r9   op_TargetExpr.op]  s    r<   c                   U R                   S   n[        U[        5      (       d  UR                  n[	        U R                   5      S:  a  SU S3$ U R                   S   [        [        US 5      L a  SU 3$ U R                   S   [        [        US 5      L a  SU 3$ [        U R                   S   [        R                  R                  5      (       a  [        U R                   S   5      $ U$ )Nr   r*   [z, ...]torch.z	operator.)
r  rF   rN   rO   r   r  r   operatorr  
OpOverload)rT   
first_reprs     r9   fns_repr_TargetExpr.fns_repra  s    XXa[
*c**#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;;txx{##r<   c                    U R                   [        L a  SnO#U R                   S:w  a  SU R                    S3nOSnU R                  R                   SU R	                  5        U S3$ )Nz
, MULTIPLEr*   r   r    ()r   rr   r   rO   r  )rT   comma_userss     r9   r   _TargetExpr.__repr__q  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr<   c                b    [        U R                  [        5      =(       d    U R                  S:  $ )Nr*   )rF   r   ro   rt   s    r9   rD  _TargetExpr.has_multiple_usersz  s     $**h/A4::>Ar<   c                    [         erQ   NotImplementedErrorrX  s      r9   rZ  _TargetExpr.find_anchor_nodes}  s
     "!r<   c                    [        U[        R                  R                  5      =(       a8    UR                  U R                  :H  =(       a    [        U5      U R                  ;   $ rQ   )rF   r   r   r   r  extract_targetr  )rT   r   s     r9   
_match_fns_TargetExpr._match_fns  sD    tUXX]]+ 5477"5t$4	
r<   c                    XR                   ;   =(       d;    U R                  [        L =(       d"    [        UR                  5      U R                  :H  $ rQ   )r   r   rr   r   rK  s      r9   _match_users_TargetExpr._match_users  s;    KK -zzX%-4::$**,	
r<   c                  > [         R                  " [        U5      n[        TU ]  U5      =(       aY    U R
                  UR
                  :H  =(       a9    U R                  UR                  :H  =(       a    U R                  UR                  :H  $ rQ   )r  r  r   r   r]  r  r  r   r  s     r9   r]  _TargetExpr.pattern_eq  sf    D%(Gu% *588#*EII%* 

ekk)		
r<   )r  r  r   )r*   )r  zFnsType | Sequence[FnsType]r   zMultiple | intr>   rx   r  r5  ra  r   r  r>   rG   )r   r  r   r   r>   rG   rc  )rO   r[   r\   r]   r"  r^   ru   r#  r   r  r  r   rD  rZ  r  r  r]  r_   r$  r%  s   @r9   r  r  H  s     
   IJ.7E	     LB""+D"	4"



 
r<   r  .c                     ^  \ rS rSrSrSS.         SU 4S jjjr\      SS j5       r\      SS j5       rSS jr	SS	 jr
SS
 jr      SS jrSU 4S jjrSrU =r$ )_TargetArgsExpri  z=
Base class for filtering match by node.{target,args,kwargs}
r*   )_usersc               r  > [         TU ]  X5        [        U5      U l        [	        U5      U l        [        S [        R                  " X4R                  5       5       5       5      (       a  U R                  U l        OU R                  U l        U R                  U R                  U R
                  5      U l        g )Nc              3  X   #    U  H   n[        U[        [        [        45      v   M"     g 7frQ   )rF   r=  r   r   r   xs     r9   r   +_TargetArgsExpr.__init__.<locals>.<genexpr>  s(      
; q4u-..;s   (*)r   ru   r   rU   r=  rV   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)rT   r  r  rU   rV   r   s        r9   ru   _TargetArgsExpr.__init__  s     	%$K	6l 
__T==?;
 
 
  ..DL..DL $TYY Dr<   c                l    / U QUR                  5       Q7n[        U 5      /UR                  5       Q7nX#4$ rQ   )r  r   r   )rU   rV   r  specs       r9   r  _TargetArgsExpr.simple_flatten  s8     +4*&--/*D	*FKKM*|r<   c                   ^^ [         [        [        [        [        [        0mSUU4S jjm[
        R                  " TX4U4S jS9n[
        R                  " U5      u  p4X44$ )Nc                   > [        U 5      nTR                  U5      nUb   [        R                  " TU" U 5      U4S jS9$ U $ )Nc                    > [        U 5      T;   $ rQ   typer  type_mappings    r9   r   F_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1g&=r<   is_leaf)r  rB   pytreer  )r  cls
convert_fnconvert_typer  s      r9   r  4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sG    q'C%))#.J% qM= 
 Hr<   c                    > [        U 5      T;   $ rQ   r  r  s    r9   r   0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1g5r<   r  )r  r   r>   r   )r   r   r   r   r=  r  r  tree_flatten)rU   rV   normalized_args_treeflatr  r  r  s        @@r9   r  _TargetArgsExpr.pytree_flatten  s^    
 E%D*
		 		  &N5 

 (()=>
zr<   c                   U R                  5       /[        [        U R                  5      QU R                  R                  5        VVs/ s H  u  pU SU 3PM     snnQnU R                  [        L a  UR                  S5        O.U R                  S:w  a  UR                  SU R                   35        U R                  R                   SSR                  U5       S3$ s  snnf )N=_users=MULTIPLEr*   _users=r  r   r   )r  mapreprrU   rV   r   r   rr   r   r   rO   join)rT   r   r   rU   s       r9   r   _TargetArgsExpr.__repr__  s    MMO
tyy!
 &*[[%6%6%89%8TQ1QCj%89

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
       ^ U R                  5       /U4S jU R                   5       QU R                  R                  5        VVs/ s H  u  p#U STR	                  U5       3PM     snnQnU R
                  [        L a  UR                  S5        O.U R
                  S:w  a  UR                  SU R
                   35        SnU R                  R                   SUR                  U5       S3$ s  snnf )	Nc              3  F   >#    U  H  nTR                  U5      v   M     g 7frQ   )rt  )r   r  rs  s     r9   r   /_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s     4)Qbooa  )s   !r  r  r*   r  r   r  r   )r  rU   rV   r   rt  r   rr   r   r   rO   r  )rT   rs  r   r   rU   
joiner_strs    `    r9   rt  _TargetArgsExpr.pretty_print  s    MMO
4$))4
 7;kk6G6G6IJ6Ida1R__Q'()6IJ

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks   "C,c                   U R                  U5      (       a,  [        UR                  5      [        U R                  5      :w  a  [        SX5      $ U R	                  X5      (       d  [        SU 5      $ UR                  nUR
                  n[        U5      [        U R
                  5      :  a  SSKJn  [        UR                  5      (       d   eU" UR                  UR                  UR
                  5      nUc  [        SX5      $ Uu  p4[        U5      [        U R                  5      :X  aI  [        U5      [        U R
                  5      :  a'  U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnO2[        SX5      $ U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnU R                  X45      u  pU R                  u  pX:w  a  [        SX5      $ [        U5      [        U
5      :X  d   e[        X 5      n[        X5       H  u  p[        U[        5      (       a8  UR!                  X5      n[#        U5      (       d  Us  $ UR%                  U5        MR  [        U[&        R(                  R*                  5      (       d  X:w  d  M  [        SUUUS9s  $    UR,                  R/                  U5        UR                  UR0                  U '   U$ s  snf s  snf )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r   )r  r   rU   r   r  rV   torch.fx.operator_schemasr  r  r   r  r  r   r  rF   r   rA  r8  r   r   r   r   r   r   r   )rT   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr7  r   
child_nodechild_matchs                   r9   r@  _TargetArgsExpr._match  s[   t$$DII#dii.(HGTT  ++2D99		++w<#dkk**DDKK(((();TYY*& *1"#KTXX!;u:TYY/CLCDT4T6=RgdkkAQ}q!*}gGRG&@$  /6Jgdkk9I}q!*}gGJ $U <
 $ 5 5
!5yLL:#j/111##&z#>G';//!ii<,,&&%J66*:O"9#	  $? 	
t++		$= S Ks   K'	KK	Kc              #    #    XR                   ;   a  UR                   U    v   gU R                  S    H  n[        U[        5      (       d  M  UR	                  X5       Hu  n[        U[
        R                  R                  5      (       d  M.  UR                   H7  nXR;  d  M
  U R                  U5      (       d  M"  Uv   UR                  U5        M9     Mw     M     g7f)z
This is used when we are matching a pattern with multiple outputs.
There is a partial match (stored in ctx) and we want to walk
this pattern to find a connection to an already-matched node.

Yields candidate nodes that `self._match` might like.
Nr   )r   r  rF   r   rZ  r   r   r   r   r  add)rT   r   rY  r   
other_noder   s         r9   rZ  !_TargetArgsExpr.find_anchor_nodes*  s      &&&%%d++,,Q/G';//")";";C"JJ%j%((--@@  * 0 0/#t44&*
 (T 2	 !1 #K 0s   ACAC"C:"Cc                  > [         R                  " [        U5      n[        TU ]  U5      =(       a\    U R
                  S   UR
                  S   :H  =(       a6    [        S [        U R
                  S   UR
                  S   5       5       5      $ )Nr*   c              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7frQ   rF   r   r]  r   abs      r9   r   -_TargetArgsExpr.pattern_eq.<locals>.<genexpr>H  s4      TDA $.a#=#=Q16IT   9;r   )r  r  r   r   r]  r  r5   r  r  s     r9   r]  _TargetArgsExpr.pattern_eqC  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r<   )rU   r  r  rV   )
r  z*torch.fx.node.Target | str | Sequence[Any]rU   r   r  zint | MultiplerV   r   r>   rx   )rU   r   rV   zMapping[Any, Any]r>   z3tuple[Sequence[Any], _SimpleSpec | pytree.TreeSpec]r  rv  r_  ra  rc  )rO   r[   r\   r]   r"  ru   staticmethodr  r  r   rt  r@  rZ  r]  r_   r$  r%  s   @r9   r  r    s     "#	E7E E 	E
 E 
E E& %6	<  %6	< 8
?E4l33+D3	432	
 	
r<   r  c                      \ rS rSrSrSrSrg)CallFunctioniO  zJ
Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
call_functionrR   NrO   r[   r\   r]   r"  r  r_   rR   r<   r9   r  r  O  s     
Br<   r  c                      \ rS rSrSrSrSrg)
CallMethodiW  zO
Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
call_methodrR   Nr  rR   r<   r9   r  r  W       
Br<   r  c                      \ rS rSrSrSrSrg)
CallModulei_  zH
Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
call_modulerR   Nr  rR   r<   r9   r#  r#  _  r!  r<   r#  c                  "    \ rS rSrSrSS jrSrg)_TargetExprVarArgsig  zS
Matches a call_function node with any arguments which are passed into the pattern
c                   U R                  U5      (       d  [        S5      $ U R                  X5      (       d  [        S5      $ [        X 5      nUR                  R                  U5        UR                  UR                  U '   UR                  R                  UR                  5        UR                  R                  UR                  5        U$ )Nfunction_mismatchmultiple_users)r  r   r  r   r   r   r   r   rU   r   rV   r   )rT   r   r   r7  s       r9   r@  _TargetExprVarArgs._matchl  s    t$$233  ++/00#	t++		$	dii 	$r<   rR   Nr_  rj  rR   r<   r9   r&  r&  g  s    r<   r&  c                      \ rS rSrSrSrg)CallFunctionVarArgsi{  r  rR   NrO   r[   r\   r]   r  r_   rR   r<   r9   r,  r,  {  s    	Br<   r,  c                      \ rS rSrSrSrg)CallMethodVarArgsi  r   rR   Nr-  rR   r<   r9   r/  r/        	Br<   r/  c                      \ rS rSrSrSrg)CallModuleVarArgsi  r$  rR   Nr-  rR   r<   r9   r2  r2    r0  r<   r2  c                  X   ^  \ rS rSrSrSS	U 4S jjjrS
S jrSS jrSU 4S jjrSr	U =r
$ )ListOfi  z
Matches a repeated pattern
c                h   > [         TU ]  5         [        U[        5      (       d   eXl        X l        g rQ   )r   ru   rF   r   r   r  )rT   r   r  r   s      r9   ru   ListOf.__init__  s+    ';////r<   c                N    U R                   R                   SU R                   S3$ Nr  r   )r   rO   r   rt   s    r9   r   ListOf.__repr__  $    ..))*!DLL>;;r<   c                J   [        U[        [        45      (       a  [        U5      S:X  a  [	        S5      $ [        X 5      nUR                  5       nSn[        U5       H  u  pg[        UR                  XGR                  S9nUR                  U R                  U5      n	UR                  5       n[        U	5      (       d!  U R                  (       d  [	        SXi5      s  $ M  SnUR                  U	R!                  5       5        M     U(       d  [	        S5      $ UR!                  5       $ )Nr   non_listFrO  zlist[{}]: {}Tzlist: no_match)rF   r   r   r   r   r   rE  	enumerater   r   r   rA  r   r8  r  r   r   )
rT   r   r   r7  r   matchedr  r	  	child_ctxr
  s
             r9   r@  ListOf._match  s    $u..#d)q.z**# 88:&t_MA$_4D4DI $//$,,
CK'BBDOK((||&~qFFGHH['')* - /00xxzr<   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       aE    U R
                  R	                  UR
                  5      =(       a    U R                  UR                  :H  $ rQ   )r  r  r   r   r]  r   r  r  s     r9   r]  ListOf.pattern_eq  sU    D%(Gu% .''6.-	
r<   )r  r   F)r   r   r  rG   r>   rx   r  )r   r   r   r   r>   rH  rc  r  r%  s   @r9   r4  r4    s&     <0
 
r<   r4  c                     ^  \ rS rSr% S\S'   SU 4S jjr\SS j5       rSS jrSS jr	SS jr
      SS	 jrSS
 jrSU 4S jjrSrU =r$ )MultiOutputPatterni  r:  r   c                   > [         TU ]  5         [        US   [        5      (       d   e[	        S U 5       5      (       d   U5       e[        U5      U l        US   R                  U l        g )Nr   c              3  V   #    U  H  oS L =(       d    [        U[        5      v   M!     g 7frQ   )rF   r   r  s     r9   r   .MultiOutputPattern.__init__.<locals>.<genexpr>  s!     LGq9:
1k ::Gs   '))r   ru   rF   r  r5   r   r   r  )rT   r   r   s     r9   ru   MultiOutputPattern.__init__  s[    '!*k2222LGLLLUgULG}!*--r<   c                j    [         R                  " [        U R                  S   5      nUR                  $ Nr   )r  r  r  r   r  )rT   outputs     r9   r  MultiOutputPattern.fns  s&     [$,,q/:zzr<   c                N    U R                   R                   SU R                   S3$ r8  )r   rO   r   rt   s    r9   r   MultiOutputPattern.__repr__  r:  r<   c                    U R                    Vs/ s H  o!R                  U5      PM     nnSS 3nU R                  R                   SUR	                  U5       3nU S3nU$ s  snf )Nz,
z  z([z
]))r   rt  r   rO   r  )rT   rs  r  rU   r  str_outs         r9   rt  MultiOutputPattern.pretty_print  sg    ,0LL9Lq"L94&\
^^,,-R
0E/FGIT"	 :s   A!c                >   [         R                  " [        U R                  S   5      nUR	                  X15      n[        U5      (       d  U$ U R                  SS   H>  nUc  M  U R                  XR5      n[        U5      (       d  Us  $ UR                  U5        M@     U$ )Nr   r*   )r  r  r  r   rA  r8  _match_from_anchorsr   )rT   r   r   rL  r7  r   r
  s          r9   r@  MultiOutputPattern._match  s    [$,,q/:IIf#{{H||AB'G227@KK((""HH[! ( r<   c                    [        UR                  5      n[        S5      nUR                  U[	        5       5       H8  nUR                  X5      n[        U5      (       a  Us  $ [        U5      Ul        M:     U$ )Nzno anchor found)r=  r   r   rZ  r    rA  r8  )rT   r   r   priorr7  r   s         r9   rT  &MultiOutputPattern._match_from_anchors  sh     S(()$%67--c:<@D		'(A{{"&u+C A r<   c                     [        U R                  UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = frN  )r   r   r   rA  r   rP  s      r9   rA  MultiOutputPattern.match  s<    	DJJ?EEdQQ 	H	s   ,/ 
AA A Ac                *  > [         R                  " [        U5      n[        TU ]  U5      =(       ab    [        U R                  5      [        UR                  5      :H  =(       a0    [        S [        U R                  UR                  5       5       5      $ )Nc              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7frQ   r  r  s      r9   r   0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  s4      <DA $.a#=#=Q16I<r  )	r  r  r   r   r]  r   r   r5   r  r  s     r9   r]  MultiOutputPattern.pattern_eq  sm    D%(Gu% DLL!S%77 emm< 	
r<   )r  r   )r   zSequence[PatternExpr | None]r>   rx   )r>   z(Callable[..., Any] | str | Sequence[Any]r  rv  r_  )r   r   r   r   r>   rH  r`  rc  )rO   r[   r\   r]   r^   ru   r#  r  r   rt  r@  rT  rA  r]  r_   r$  r%  s   @r9   rE  rE    s]    %%   
< ")5		
 	
r<   rE  c                  ^   ^  \ rS rSrSrSU 4S jjr\S	S j5       rS
S jrSU 4S jjr	Sr
U =r$ )RepeatedExpri  zh
Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
c                P   > [         TU ]  5         Xl        UR                  U l        g rQ   )r   ru   inner_patternr  )rT   rb  r   s     r9   ru   RepeatedExpr.__init__  s!    *""r<   c                .    U R                   R                  $ rQ   )rb  r  rt   s    r9   r  RepeatedExpr.fns	  s    !!%%%r<   c                   UR                  U R                  U5      n[        U5      (       d  U$ UR                  R	                  U R                  5        U R                  R                  U[        U/5      5       HW  n[        U /UR                  S9R                  U R                  U5      n[        U5      (       d  Us  $ UR                  U5        MY     U$ rN  )
rA  rb  r8  r   poprZ  r    r   r   r   )rT   r   r   r7  anchor_nodeanchor_ms         r9   r@  RepeatedExpr._match  s    IId(($/{{H	
  --??TF#
K $TF$**=CC""KH H%%HHX
 r<   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a%    U R
                  R	                  UR
                  5      $ rQ   )r  r  r   r   r]  rb  r  s     r9   r]  RepeatedExpr.pattern_eq   sF    D%(w!%( 
T-?-?-J-J.
 	
r<   )rb  r  )rb  r  r>   rx   )r>   zSequence[FnsType]r_  rc  )rO   r[   r\   r]   r"  ru   r#  r  r@  r]  r_   r$  r%  s   @r9   r`  r`    s0    #
 & &&
 
r<   r`  c                  l    \ rS rSrSrS	S jr\\R                  S
SS jj5       5       r	SS jr
SS jrSrg)rw  i'  z
Serializes Patterns to executable python.
XXX: currently only used and tested for fuse attention patterns. May not cover
all patterns.
c                z    [         R                  R                  R                  5       U l        0 U l        0 U l        g rQ   )r   r   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprt   s    r9   ru   PatternPrettyPrinter.__init__.  s*    224;= 8:r<   c                (   [        5       n[        U S5      (       d   eU R                  US9nUR                   Vs/ s H$  nUR                  U    SUR                  U    3PM&     nnUR                  U SU 35        SR                  U5      $ s  snf )zE
Serializes obj to python code with obj written out to `output_name`
rt  )rs  z = 
)rw  hasattrrt  rq  rr  r   r  )objoutput_namers  out_strr   rL  s         r9   runPatternPrettyPrinter.run3  s     "#sN++++""b") --
- %%c*+3r/B/B3/G.HI- 	 

 	S	23yy  
s   +Bc                    [        U[        5      (       a5  U R                  R                  U5      =n(       a  U$ U R	                  U5      $ [        US5      (       a  UR                  U 5      $ [        U5      $ )Nrt  )rF   r  rq  rB   memoizerv  rt  r  )rT   rw  memoized_names      r9   rt  !PatternPrettyPrinter.pretty_printG  sh    c?++ $ 8 8 < <S AA}A$$||C((3''##D))Cyr<   c                    UR                  U 5      nUR                  5       nS H  nUR                  US5      nM     U R                  R	                  US 5      nXPR
                  U'   X R                  U'   U$ )N)zaten.r  zprims.r  )rt  r  replacerp  create_namerq  rr  )rT   rw  obj_strobj_nameprefixtmp_names         r9   r}  PatternPrettyPrinter.memoizeR  sr    ""4(<<>3F''3H 4 >>--h=(0  %%,c"r<   )rq  rr  rp  Nrw   )rL  )rw  r   rx  rN   r>   rN   )rw  r   r>   rN   )rw  r  r>   rN   )rO   r[   r\   r]   r"  ru   r  r  cacherz  rt  r}  r_   rR   r<   r9   rw  rw  '  s6    ;
 __!  !$		r<   rw  c                  &    \ rS rSr    SS jrSrg)_PassDictsTypei^  c                    g rQ   rR   )rT   r   s     r9   __getitem___PassDictsType.__getitem___  s     r<   rR   N)r    tuple[str, torch.fx.node.Target]r>   list[PatternEntry])rO   r[   r\   r]   r  r_   rR   r<   r9   r  r  ^  s    !1!	!r<   r  c                  T    \ rS rSr% S\S'   S\S'   S
S jr  S       SS jjrS	rg)PatternEntryid  r   r   Callable[[Match], bool]extra_checkc                    [         erQ   r  rT   rA  r   r   s       r9   applyPatternEntry.applyi  s    !!r<   Nc                >   UcK  [        U R                  S5      (       d   eU R                  R                   H  nU R                  XUS9  M     g [	        U[
        [        45      (       ay  [        U R                  S5      (       d   eU(       a+  XR                  R                  U4   R                  SU 5        g XR                  R                  U4   R                  U 5        g [        R                  " [        [           U5      nU H  nU R                  XRUS9  M     g )Nr  prependr  r   )rv  r   r  registerrF   r=  PatternMatcherPassr  insertr   r  r  r   r  )rT   
pass_dictsr   r  rj   r  s         r9   r  PatternEntry.registerl  s     >4<<////ll&&jg> '
T+=$>??4<<....LLOOV45<<QELLOOV45<<TBXn%=zJJa9  r<   rR   rA  r   r   r  r   r  r>   rx   r1  )r  )_PassDictsType | Sequence[_PassDictsType]r   ztorch.fx.node.Target | Noner  rG   r>   rx   )rO   r[   r\   r]   r^   r  r  r_   rR   r<   r9   r  r  d  sK    ((" /3	:=: ,: 	:
 
: :r<   r  c                  *    \ rS rSr% S\S'   SS jrSrg)LoweringPatternEntryi  Callable[..., Any]handlerc                   [         R                  " U R                  5      " [         R                  " U R                  U5      5      nUR	                  U5         UR                  U[        UR                  5      UR                  5      n[        UR                  U5        UR                  U5        S S S 5        UR                  S   UL d   eUR                  5         g ! , (       d  f       N3= f)N)r  wrapsr  r  inserting_beforer  r   rU   rV   r   r   replace_all_uses_withr   r   )rT   rA  r   r   r  r  s         r9   r  LoweringPatternEntry.apply  s    //$,,/	0A0A$,,PU0VW##D)--guUZZ7H%,,WK;++T2&&{3 * {{2$&&& *)s   AC
C*rR   Nr  )rO   r[   r\   r]   r^   r  r_   rR   r<   r9   r  r    s    r<   r  c                  .    \ rS rSr% SrS\S'   SS jrSrg)	GraphPatternEntryi  z0
A pattern that runs a function on the FX graph
r  r  c                    UR                  U5         U R                  " U/UR                  Q70 UR                  D6  S S S 5        g ! , (       d  f       g = frQ   )r  r  rU   rV   r  s       r9   r  GraphPatternEntry.apply  s:    ##D)LL<<u||< *))s   +A
ArR   Nr  )rO   r[   r\   r]   r"  r^   r  r_   rR   r<   r9   r  r    s      =r<   r  c                  l    \ rS rSr% SrS\S'   SrS\S'   \ S           SS jj5       rSS	 jr	S
r
g)r   i  z'
The replacement pattern for the graph
zCallable[..., list[Any]]normalize_argsNr0   pattern_namec           	     :  ^^^^^^^ / m " UUU4S jS[         R                  R                  5      nU R                  5       n[	        U5      S:X  a  US   nOUS   (       d   e[        US   R                  R                  5      nU V	s/ s HA  n	[        U	[         R                  R                  5      (       d  M.  UR                  U	5      U	4PMC     n
n	[        U
[        R                  " S5      S9S   n          S
S jmTR                  U5         [        U[         R                  R                  5      (       d   eU" U5      R                   " T6 n[        U[         R                  R                  5      (       a  U/nSS jm      SUUUUUUU4S jjm[	        U5      [	        U5      :X  a  [#        Xk5       H  u  pT" X5        M     O[	        U5      S:X  d   eT" US   U5        S	S	S	5        U R%                  5         ['        T5       Hs  nUR(                  (       a  M  UR+                  5       (       a  M-  [        UR,                  [         R.                  R0                  5      (       a  Mb  TR3                  U5        Mu     g	s  sn	f ! , (       d  f       N= f)zD
Inserts the replacement graph into the toplevel graph at the match
c                  @   >^  \ rS rSrSrSrSrSU UUU4S jjrSrU =r	$ )<ReplacementPatternEntry.replace_with_graph.<locals>.Replaceri  Nc                  > UR                   S;   a  [        TU ]	  U5      $ UR                  nU R	                  U5      u  p4UR                   S:X  a  [        U5      (       d   eTR                  X#U5      nTR                  U5        [        UR                  UT=(       d    SS9  SUR                  ;   a  UR                  S   UR                  S'   U$ UR                   S:X  a  SSK
Jn  [        TU ]1  X#U5      n[        U[        R                  R                   5      (       d  [#        S	U S
U S35      eTR$                  c   eS nTR$                  R'                  5        H  u  pXzL d  M  U	n  O   UcG  [        U[(        5      (       d   eU" TR$                  U5      u  pTR$                  R+                  X5        TR                  U5      nTR                  U5        U$ [#        SU 35      e)N)placeholderrL  r  r  r  r   get_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )r  r   run_noder   fetch_args_kwargs_from_envr  r  r   r   r   torch._higher_order_ops.utilsr  r  rF   r   r   GraphModuler  owning_modulenamed_modulesrN   register_module)rT   r   r   rU   rV   resultr  sub_gm
graph_namer   mod_getattr_noder   added_replacement_nodesr   r   s                r9   r  EReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node  s   7777 7+D11#>>tD77o-#F++++"00vFF+226:"!'!%"+/r *TYY6:>))DV:W$67!M77j( #W-fFCF%fehh.B.BCC15fX=YZ`Yaabc  !..:::!%J"'"5"5"C"C"E!=)*J! #F "))&#6666(C!//") ++;;JO#(>>*#=L+22<@'')Jtf*=>>r<   rR   r   r  r>   r   )
rO   r[   r\   r]   r   r$  r  r  r_   r$  )r   r  r   r   s   @r9   Replacerr    s!    KKH5? 5? 5?r<   r  r*   r   )r   c                B   U /n[         [        R                  R                     " 5       nU(       ao  UR	                  5       nXe;  aP  Xc;  aK  [        US5      (       a:  UR                  U5        X&R                  U'   UR                  UR                  5        U(       a  Mn  g g )Nr   )
r    r   r   r   rg  rv  r  r   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r9   percolate_tagsBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  sx     FE /1Giik&.V,,KK$)2HHX&LL!4!45 %r<   c                    U R                   S:w  a  g U R                  [        R                  :w  a  g [	        U R
                  5      S:X  d   eU R
                  S   $ )Nr  r!   r*   )r  r   r  getitemr   rU   r   s    r9   maybe_getitemAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sJ    77o-;;("2"22499~***yy|#r<   c           	     <  > SU4S jjnU c  Ub   eg [        U [        R                  R                  5      (       d   eUc;  U R	                  S US9  [        U R                  5      S:X  a  T	R                  U 5        g [        U[        R                  R                  5      (       a  [        UR                  U T=(       d    SS9  S H4  nX0R                  ;   d  M  T" XU R                  U   [        T5      5        M6     U R	                  XS9  [        U R                  5      S:X  a  T	R                  U 5        g [        U R                  R                  5       5      nU H,  nT
" U5      nUc  U R	                  U5        M!  T" XQU   5        M.     T	R                  U 5        g )Nc                   > U T;  $ rQ   rR   )r   r  s    r9   !filter_nodes_in_newly_added_nodesfReplacementPatternEntry.replace_with_graph.<locals>.replace.<locals>.filter_nodes_in_newly_added_nodes$  s      '>>>r<   )delete_user_cbr   r  r   )	recomputeac_graph_idr  )rF   r   r   r   r  r   r   r   r   r   r    r   r   )oldnewr  r  old_usesuseridxr  rU   r   r  r   r  r  s          r9   r  ;ReplacementPatternEntry.replace_with_graph.<locals>.replace   sm   ?
 ;;&;!#uxx}}5555;--'H .  399~*((-c588==11"388SIOL %A#xx/* #sxx/A:dCS %A -- .  399~*((-:  		 01$D'-C{ 11#6#h/ %   %r<   N)
r   r  r  rN   r  rN   r  rb  r>   rx   r  )r  ztorch.fx.Node | Noner  z.torch.fx.Node | Sequence[torch.fx.Node] | Noner>   rx   )r   r   Interpreterr   r   r   r   r   rF   r   indexminr  
itemgetterr  r  rz  r  r   r   r   	is_impurer   r  HigherOrderOperatorr   )rA  r   r   rU   r   r  r   	last_noder   r   indicesr  r  r  r   r  r  r  r  s    ` ``          @@@@r9   r   *ReplacementPatternEntry.replace_with_graph  s[    8::	? :	?uxx++ :	?x ))+|!$QI?"?a..445E &%Aa/ $Q#%  
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I./1E1EFFFF"#45994@K+uxx}}55*m$Q&)Q&CQ& Q& Q&f < C$44 #L >HCC% !? <(A---Q5M /P 	 45DJJJ(("4;;

0N0NOO  & 6O6 /.s   	-J:JCJ
Jc           
         UR                   c   eU R                  UUUR                   U R                  " UR                  0 UR                  D6U R
                  =(       d    SS9  g )Nr   r  )r   r   r  rU   rV   r  r  s       r9   r  ReplacementPatternEntry.apply  s`    &&222##<u||<''?+? 	  	
r<   rR   rQ   )rA  r   r   r  r   z%torch.fx.Graph | torch.fx.GraphModulerU   zSequence[torch.fx.Node]r   r0   r>   rx   r  )rO   r[   r\   r]   r"  r^   r  r  r   r  r_   rR   r<   r9   r   r     sw     -,#L*# !%]']']' A]' &	]'
 ]' 
]' ]'~
r<   r   c                    gr!  rR   )rA  s    r9   _return_truer    s    r<   c                F    [         R                  SU R                  U5        g )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginforO   )	search_fnrQ  s     r9   log_trace_failurer    s    HHJ	r<   Fc                  ^^ [         R                  U 5      mUR                  T5      nU(       d'  UT   R                  U(       a  [	        U5      OS5        gUc#  U(       a  g[
        R                  " SU4S j5        [	        U5      nU H0  mUT:w  a  M  U(       a    g[
        R                  " SUU4S j5        M2     UR                  U5        g)a.  
Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
in matching, use the graph to distinguish equivalent search patterns.

Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
`skip_duplicates` is False and a duplicate is found.
NFTc                    > ST  S3$ )NDuplicate pattern: z with no graphrR   )pattern_reprs   r9   r   1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),~Fr<   c                    > ST ST  S3$ )Nr  z with duplicated match graph  rR   )	graph_strr  s   r9   r   r    s    ),7TU^T__`ar<   )rw  rz  rB   r   rN   r   _check)r   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strr  r  s         @@r9   check_and_add_duplicate_patternr    s     (++G4L'++L9l#**3u:DI}F	

 JM(	I%a	
 ) }-r<   rR   c                  ^ ^^^^^^^^^ / [         R                  " T 5      R                  R                  5       Qm[         R                  " T 5      (       a  [        T T5      m [         R                  " T5      (       a<  / [         R                  " T5      R                  R                  5       Qn[        TU5      m[        U[        [        45      (       d  [        S[        U5       35      eS
UUUUUUUU UU4
S jjnSU4S jjnT[        L a  [        R                  " 5       (       a  g[        R                  " SS9   U Vs/ s H0  n[        U[        R                   5      =(       a    UR"                  PM2     snmTc  [%        T UTTTTS9u  nnOTnSn[        U[&        5      (       a  UOU/ HR  n[        U[(        5      (       d  M  [+        UU(       a  UR,                  OSUR.                  U	S9(       d  MJ    SSS5        g   [1        UUUU
S	9nUR3                  U5        UR4                  sSSS5        $ s  snf ! , (       d  f       g= f)a  
Create a replacement rule based on example functions that get traced
to create patterns.  This supports both training and inference when
run on a joint forward+backward graph.

Args:
    search_fn: traced to give original pattern
    replace_fn: traced to give replacement graph
    example_inputs: example inputs for initial trace
    trace_fn: fwd_only or joint_fwd_bwd
    pass_dict: dict of passes to register to
    extra_check: additional check to run on match(using real shapes)
z,example_inputs must be a list or tuple, got c           
     	  >
^^ [        T5      nU H-  nX R                  ;  d  M  [        SU SU R                   35      e   [        [        R                  R                  U Vs/ s H  o R                  U   PM     snS 5      5      m/ n[        R                  R                  R                  T5      nUc   eU   [        T5       GH9  u  pV[        TU   [        R                  5      (       d  M*  U(       a'  [        TU   R                  5      (       a
    SSS5        g[        R                  " TU   R                  5       TU   R!                  5       TU   R                  TU   R"                  US9TU'   [$        R&                  " TU   R(                  TU   R!                  5       5       HQ  m[        T[        R*                  5      (       d  M$  [-        U4S jU 5       5      (       d  M@  UR/                  T5        MS     GM<     TnU(       Gd6  U(       Ga  SUU4S jjn T" UUT-   TS	9n	/ n[3        [5        [7        U5      [7        T5      -   5      U	R8                  R:                  5       H  u  p\U[7        U5      :  a  UR/                  UR<                  5        M1  U	R8                  R?                  U5         U	R8                  RA                  X[7        U5      -
     5      nURB                  Ul        URE                  U5        U	R8                  RG                  U5        SSS5        M     X-   nO	 T" TTTS	9n	[I        U	UTTS
9nU RK                  5       S   nUc   eURM                  U5      n[O        URB                  5      (       a#  [P        RS                  SUURT                  UU5        [W        U5      (       a  T" U5      (       ay  T" TTTS	9U l,        [7        U R:                  5      S:X  aI  U RX                  R8                  R:                   H%  n[[        UR\                  U R:                  S   SS9  M'      SSS5        g SSS5        gs  snf ! [         a  n
[1        TU
5         Sn
A
SSS5        gSn
A
ff = f! , (       d  f       GM  = f! [         a  n
[1        TU
5         Sn
A
SSS5        gSn
A
ff = f! , (       d  f       g= f)z
Often shapes get burned into the pattern, so our initial match ran with
`ignore_types=(int, ...)`.

Recheck the match with the correct shapes.
z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     U R                   S   $ r   r   )r   s    r9   r   8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE]r<   NF)dtypedevicerequires_gradc              3  @   >#    U  H  n[        TU:g  5      v   M     g 7frQ   )r   )r   r  r   s     r9   r   9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s!      ?CKa1!q&998s   c                 >   > T" U [        U 5      [        T5      -
  S  6 $ rQ   )r   )args_newr  rU   r  s     r9   search_fn_new=register_replacement.<locals>.check_fn.<locals>.search_fn_new/  s#    ((3x=3t93L3N*OPPr<   get_decomp_fn)argnamesexclusive_arg_namesscalar_workaroundr   z"Specific pattern match: %s%s %s %sr*   r  r  T)r  r   r  r   r>   r   )/r   rV   r*  r   r   r  r  r	  r  r=  rF   Tensorr   r  empty_stridedsizestrider  r  r  shapeSymIntr5   r   r  r  ranger   r   r   r   inserting_afterr  r}  r  r   fx_to_patternr   rA  rJ   r  warningrU   r8  r   r   r   )rA  r  r}  sym_argsr   r  gradspecific_patternr  specific_graphrQ  sym_arg_namesr  r  r   specific_pattern_matchr   rU   r   argnames_staticr  r  r  
replace_fnr  r  r  search_fn_patternr  s                    @@r9   check_fn&register_replacement.<locals>.check_fn  sI    (D<<'"99A
/RWR^R^Q_a   HH089d#9;R
 (*MM''88>	$$$$]3d1gu||44 0a ? ?$ Y $11QQ("1gmm#Aw~~&*
DG '__T!W]]DGNN<LM%a663 ?CK? < < %OOA.	 N% 46  1#Q Q%)1)$tO*7* %'M*-c(mc$i78&,,22+ s8},)001C1CD$+11AA+N'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KH ON+   -7H%)1%t=* $1"%(;&7	$  %%'*D###%5%;%;D%A"!$)),,8II*$ .//K@V4W4W*2M+' u{{#q("44::@@&%&VV%*[[^&3 A a Yb c Y :n ( %))Q7$e Y`% ON ( %))Q7$] YX%Y Ys   $Q;2R91%R9B,R9R9+5R9!Q,BR98A'Q;R91R9C?R9R9
Q8Q3&R93Q88R9;
R	R9
R6R1$R91R66R99
Sc                   > T Vs/ s H  oR                  U5      PM     nn[        S[        U 5      S-   5       H1  nSU 3U ;  a    O'UR                  U R                  SU 35      5        M3     U (       a   SU < 35       eU$ s  snf )Nr*   	tangents_zleftover kwargs: )rg  r   r   r   )rV   r}  rU   r  r*  s       r9   r  ,register_replacement.<locals>.normalize_args{  s    -<=_T

4 _=q#f+/*A1#f,KK

Yqc?34 + 9.vj99z >s   BFfunctionalize_rng_opsNr  )r  )r   r  r  r  rA  r   r>   rG   )rV   r   r>   r   )inspect	signature
parametersr   ismethod_wrap_bound_methodrF   r   r   	TypeErrorr  joint_fwd_bwdr   is_inference_mode_enabledfunctorch_configpatchr  r  gen_pattern_and_search_gmr   r  r  r   r  r   r  r   )r  r+  example_inputsr  r  r  r  r  r,  r  r  r  replace_argnamesr-  r  r  r   gmpattern_matcher_passr*  r  s   `` ` ````  `       @@r9   register_replacementrD    s   6 H)))4??DDFGO	""&y/B	
##MW..z:EEJJLM'
4DE
ntUm44:4;O:PQ
 	
I I IV =  **,, 
		e	<ES%
ESJq%,,';AOO;^%
 $3!#+KGR (GB %Z::JL ! .0BCC2 "BHH(66$3	  !7 
=	<$ M * )%	
 	$I 
=	<%
 
=	<s1   <H;7H68AH;,H;8H;*H;6H;;
I	zOrderedSet[str]_serialized_patternsc                l   SS jn[         R                  5       (       d  [        S[          35      eUR                  nSSKJn  UR                  " SS9   [        XX45      nS S S 5        [        R                  WU S9n	U[        ;  a  Sn
[        R                  U5        OS	n
U" 5       n[        [         U S
3-  U
5       nU
S:X  a  UR                  U5        OUR                  S5        UR                  U	5        UR                  S5        S S S 5        U$ ! , (       d  f       N= f! , (       d  f       U$ = f)Nc                    [         R                  " S5      n [         R                  " S5      R                  U S9n/ n[        [        R
                  R                  5       Hm  n[        [        R
                  R                  U5      n [        U[        5      (       a0  [        U[        [        45      (       a  UR                  U5        Mk  Mm  Mo     SR                  U5      nSU S3nU U 3$ ! [         a     M  f = f)Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr-  dirr   	_inductorpattern_matcherr  rF   r  
issubclassr   r  r   r:  r  )auto_generated_msgfile_templatepattern_matcher_importsr}  attrformatted_importss         r9   get_file_template-_serialize_pattern.<locals>.get_file_template  s    %__
 !
 &'&
( 	  #%778D5??::DADdD))j;4/ / ,2248	/) 9 %MM*ABPQbPccij!2 344  s   AC--
C;:C;z0Could not find serialized patterns directory at r   r"   Fr2  )rx  wr  z.pyz

ru  r  )SERIALIZED_PATTERN_PATHis_dirr*  rO   torch._functorchr#   r>  gen_patternrw  rz  rE  r  openwrite)unique_namer  r@  r  r  rT  r  r=  r   serialized_pattern
write_moderP  fs                r9   _serialize_patternra    s   '5R #))++>?V>WX
 	
 %%L;			e	<iU 
= .11'{1S//
  .
%'M	%<.(<<j	IQGGM"GGFO	"#	 
J N) 
=	< 
J	I Ns   D=AD$
D!$
D3	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc                   [        U5      nS[        R                  ;   a  [        XX4U5      nO]UR                  n[
        R                  " SU 35      nU(       a  [        X5      (       d  [        R                  SU 5        [        X5      n[        R                  " U5       H0  n[        U[        5      (       d  M  UR                  c  M)  S Ul        M2     [         R#                  XXGU45        [%        UUUUUUUUUU	U U
S9  g )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r,  r  r  r  )r   r@   rA   ra  rO   	importlibimport_modulerv  r  r#  r  r  	tree_iterrF   r'   constantrd  r   rD  )r]  r  r+  r@  r  r  r  r  r  r  r  patr  r7  r   s                  r9   gen_register_replacementrl    s     >*N+ N>O
 !))##<\NK
 //KKV a%/c:&&3<<+C
  CL 0  &&	HE ' #r<   r2  c                p   / [         R                  " U 5      R                  R                  5       QnUc  0 n/ nSnU H5  n	X;   a  UR	                  X9   5        M  UR	                  X   5        US-  nM7     U" XUS9n
[        U
[        [        [        [        R                  [        R                  4UUUS9U
4$ )Nr   r*   r  )ignore_typesr  r  r  )r5  r6  r7  r   r   r"  intfloatr   r   r  r  )r  r@  r  r  r  r  r  flat_inputs	input_idxargname	search_gms              r9   r?  r?  P  s     A""9-88==?@H KI'09:~89NI  }MIudELL%++F/ 3	
 		 	r<   c                "    [        XX#U5      S   $ rK  )r?  )r  r@  r  r  r  s        r9   rZ  rZ  t  s!     %8@S	 	r<   r  c               $   ^ ^^^ SUUU U4S jjnU$ )z
Register an aten to inductor IR replacement pattern.  The decorated
function is saved and then called a lowering time allowing direct
pattern to inductor IR conversion.
c                l   > [        U 5      (       d   e[        TTU S9R                  TTS9  SU l        U $ )Nr   r  r  r  T)r  r  r  _inductor_lowering_functionr  r  	pass_dictr   r  s    r9   	decorator,register_lowering_pattern.<locals>.decorator  s>        g	

(9g(
..2+r<   r  r  r>   r  rR   r   r  r{  r  r|  s   ```` r9   register_lowering_patternr    s      r<   c               $   ^ ^^^ SUUU U4S jjnU$ )z_
Register a pattern that runs a function on the FX graph, allowing
custom transformation code.
c                ^   > [        U 5      (       d   e[        TTU S9R                  TTS9  U $ )Nrx  r  )r  r  r  rz  s    r9   r|  )register_graph_pattern.<locals>.decorator  s6        g	

(9g(
.r<   r~  rR   r  s   ```` r9   register_graph_patternr    s      r<   c                B    U[        [        U R                  5      5      L $ rQ   )r   iterr   )r   r   s     r9   is_start_of_fx_graphr    s    4U[[)***r<   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    U R                   S:w  a  gU [        R                  R                  R                  R
                  [        R                  R                  R                  R
                  4;   $ )Nr.   F)rp  r   r   r.   accumulate_grad_defaultresize_storage_bytes_)r  s    r9   "fixme_incorrect_inductor_schema_opr    sW    	||z! 		++33		0088  r<   c                   [        U R                  [        R                  R                  5      (       a:  [        U R                  5      (       d   U R                  R                  R                  $ [        U R                  [        R                  R                  R                  5      (       a  gU R                  S:X  aL  [        U R                  5      (       d   e[        R                  U R                  R                  5      (       a  gOVU R                  S:X  aF  [        U R                  [         5      (       d   e[        R                  U R                  5      (       a  gU R"                  R%                  S5      S L$ )NFr  Tr   out)rF   r   r   r  r  r  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedr  r  _mutation_op_researchrO   rN   rV   rB   r  s    r9   is_mutation_opr    s   UZZ** 0=={{""---	U,,??RR
 
 ww/!$$$$!!$++"6"677 8	M	!$++s++++!!$++..;;??5!--r<   c                    SU R                   ;   d   eSUR                   ;   d   eU R                   S   UR                   S   :H  $ Nmutation_region_idr   )r  r  s     r9   same_mutation_regionsr    sE    166)))166)))66&'1662F+GGGr<   c                P   UnSUR                   ;  a>  [        X5      (       d.  UR                  nSUR                   ;  a  [        X5      (       d  M.  UR                   R                  SS5      nX!La5  UR                  n[        U5      (       a  US-  nX2R                   S'   X!La  M5  U$ )Nr  r   r*   )r   r  prevrB   r   r  )r   r   r   r  s       r9   get_mutation_region_idr    s    A
aff
,5I%5S5SFF aff
,5I%5S5S$8!<
-FF!!#'9#$	 -
 r<   c                X    S[        [        U R                  5      5      R                  ;  $ r  )r   r  r   r   rO  s    r9   "should_compute_mutation_region_idsr    s!    tD,='>'C'CCCr<   c                t    SnU R                    H&  n[        U5      (       a  US-  nXR                  S'   M(     g )Nr   r*   r  )r   r  r   )r   r  nds      r9   compute_mutation_region_idsr    s9    kk"!#(:$% r<   c                   ^  SU 4S jjnU Vs/ s H2  n[         R                  " U[         R                  R                  5      PM4     nn[         R                  " U5      Ul        U$ s  snf )zI
Wrap a bound method to remove 'self' from its signature for FX tracing.
c                    > T" U 0 UD6$ rQ   rR   )rU   rV   rj   s     r9   wrapper#_wrap_bound_method.<locals>.wrapper  s    4"6""r<   rZ   )r5  	ParameterPOSITIONAL_OR_KEYWORD	Signature__signature__)rj   r  r  r}  paramss   `    r9   r9  r9    sb    
#
 D 	$ 1 1 G GH   $--f5GNs   9A&c                  `   ^  \ rS rSrSr  S     S	U 4S jjjrS
S jrSS jrSS jrSr	U =r
$ )r  i  z9
Registry of patterns to match and replace in FX graphs.
c                   > [         TU ]  5         [        [        5      U l        Xl        X l        [        [        5      U l        g rQ   )r   ru   r   r   patternsr   	subsystemr  )rT   r   r  r   s      r9   ru   PatternMatcherPass.__init__  s=    
 	  	 #" ;Fd:Kr<   c                     U R                   U   $ rQ   )r  )rT   items     r9   r  PatternMatcherPass.__getitem__  s    }}T""r<   c                   U R                   (       d  g[        U[        R                  R                  5      (       a  UR
                  nOO[        U[        R                  R                  5      (       a  UnUR                  nO[        S[        U5       35      e[        U5      (       a  [        U5        [        R                  " [        U5      nSn/ nSnU R                    H.  u  pxUS:X  a  SnM  UR                  UR!                  XxSS95        M0     U(       a  UR                  UR!                  SSS95        U R"                  b  U R"                  OS	n	[        U[        R                  R                  5      (       d   e[%        XU R&                  5         [)        [*        R,                  R/                  U5      SS
9 GH  n
[1        U
5      nU
R2                  S:X  a  U
R2                  U4U R                   ;  a  M=  [5        U
SS9(       a  MN  U R                   U
R2                  U4    GH  nU
R6                  (       a    M  UR8                  R;                  U
5      n[=        U5      (       a-  [?        [A        [C        X<RD                  5      5      5      S:w  a  Mp  [=        U5      (       a+  [?        [A        S URD                   5       5      5      S:w  a  M  [G        U
RH                  5      (       a+  [J        RM                  SXRN                  XR8                  5        [=        U5      (       d  GM  [Q        URS                  U5      5      (       d  GM%  US-  nURU                  XU
5        [V        [X           S==   S-  ss'   [V        [X           S==   [?        URD                  5      -  ss'   [Z        R\                  R_                  S5      (       d  GM  [a        USS5      (       a  URb                  nO<UR8                  Rd                  Rf                  n[i        U
Rj                  5      nU SU 3n[X         S3n[V        U   U==   S-  ss'   GM!     GM     SSS5        U$ ! , (       d  f       U$ = f)zLApply all registered patterns to the graph, returning the number of matches.r   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr$  T)r  r   sort)r  r  NrM  )reverse)allow_cpu_inputsr*   c              3  z   #    U  H1  nUR                   R                  S 0 5      R                  SS5      v   M3     g7f)customstreamr   N)r   rB   )r   r   s     r9   r   +PatternMatcherPass.apply.<locals>.<genexpr>]  s6      ')0A !"

8R 8 < <Xq I I)0r  z
%s%s %s %spattern_matcher_countpattern_matcher_nodesr=   r  r  _pattern_matcher_per_pattern)6r  rF   r   r   r  r   Graphr  r*  r  r  r  r  r  r  r   
find_nodesr   r   r  sortedr  r  from_iterabler  r  r,   r   r   rA  r8  r   r    r  r   rJ   r}  r  r#  rU   r   r  r  r   backendr@   rA   rB   r  r  r   rO   rN   r   )rT   rB  r   get_mutation_region_id_partialcountr   has_call_moduler  r   r   r   entryr7  r  pattern_classpattern_keys                   r9   r  PatternMatcherPass.apply!  s   }}b%((..//HHEEHHNN++E$$B\]abd]e\fg  .e44'.)2):):"E*
& --JB]""&U---OP	 (
 LL))])GH&*nn&@DNNFW	"ehh223333#B4>>By<<UCTR'-77m+(= 
 9PUV!]]DGGV+<=E||++D1A !&s+I77'ST 
 ! !& ')*'   !)$))44L$		1mmT{{~e6G6G6J'K'K
Ad3 )*ABaGB )*ABc!''lRB ::>>*MNN&undCC/4/A/A 160G0G0P0P),T[[)92?&/J-4I5Q*RK$[1,?1D?[ > S Cv w CBv s!   FQ(Q
A<QBQ
Q&c                8    U R                   R                  5         g rQ   )r  clearrt   s    r9   r  PatternMatcherPass.clear|  s    r<   )r   r  r  r  r  )r   r0   r  r0   r>   rx   )r  r  r>   r  )rB  %torch.fx.GraphModule | torch.fx.Graphr>   ro  rw   )rO   r[   r\   r]   r"  ru   r  r  r  r_   r$  r%  s   @r9   r  r    sQ     !% $LL L 
	L L$#Yv r<   r  c                     [         erQ   r  r   s     r9   _not_implementedr    s    
r<   c                <  ^^^^	^
^ U=(       d    0 nUR                  5        VVs0 s H  u  pVXe_M	     snnm
[        T
5      [        U5      :X  d   e S     SUU
4S jjjm[        R                  " 5       m	 " UU	UUU4S jS[        R
                  R                  5      n[        U [        R
                  R                  5      (       d   eU" U 5      R                  5       n[        U[        5      (       d  [        [        R                  " U5      5      $ U$ s  snnf )z
Convert an FX graph into a PatternExpr.  This is useful for simple
patterns that can only match single functions and fixed-length lists.
c                   > Ub  UOTn[        U [        [        45      (       a  U T;   a  [        TU    5      $ [	        U 5      U;   a
  [        5       $ [        U [        5      (       a(  [        S U  5       5      (       a  U (       a
  [        5       $ U $ )Nc              3  B   #    U  H  n[        U[        5      v   M     g 7frQ   )rF   rl  )r   ys     r9   r   5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s     &Iq!z!W'='=qs   )rF   rp  ro  ry  r  rl  r   r5   )r  ignore_types_overridecurrent_ignore_typesrn  inv_scalar_workarounds      r9   process_arg"fx_to_pattern.<locals>.process_arg  s     &;%F!L 	 a%&&10E+E3A6777**9a3&Iq&I#I#Ia9r<   c                     >^  \ rS rSr\r\r\r        SUUU4S jjr        SUU4S jjr	SU 4S jjr
SrU =r$ )	 fx_to_pattern.<locals>.Converteri  c                   > [        T5      nU[        T5      :  a  TU   nO<T(       a  UR                  S5      (       d   eUnO[        R                  " SSU5      nUnUT;   a  [        U5      $ [        U5      $ )Ntangentz_\d+$r  )r   r   
startswithresubr  ry  )	rT   r   rU   rV   r   r}  r  argnumr  s	         r9   r  ,fx_to_pattern.<locals>.Converter.placeholder  sx     VA3x= {((3333"f5***400!$''r<   c                p  > T	nU[         R                  L a"  [        S T 5       5      4     SU	4S jjjnUn[        R                  " XBU45      u  p#[
        T;   aA  U Vs/ s H
  od" U5      PM     nnUR                  5        VVs0 s H  u  pvXt" U5      _M     nnn[        U/UQ70 UD6$ s  snf s  snnf )Nc              3  :   #    U  H  o[         Ld  M  Uv   M     g 7frQ   )ro  )r   ts     r9   r   Afx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      N#/aC<<s   	c                   > T" X5      $ rQ   rR   )r  r  r  s     r9   process_arg_fn_implKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q@@r<   r  rm   r  zSequence[type[Any]] | Noner>   zT | KeywordArg | Ignored)r  r  r   r  r  r   r   r  )
rT   r   rU   rV   process_arg_fnr  r  r   rn  r  s
           r9   r  .fx_to_pattern.<locals>.Converter.call_function  s     )N))) IN N#/N IAA+EA
 .A A "5!??>&>JLD|#3784aq)48;A<<>J>41!^A..>J8888 9Js   !B-B2c                  > [         TU ]  U5      nUR                  S:X  a  [        U[        5      (       ap  UR
                  S   n[        U[        5      (       d   e[        U5      [        U5      :X  d   e[        X#5       H  u  pE[        UR                  5      Ul	        M!     U$ [        UR                  5      Ul	        U$ )NrL  r   )
r   r  r  rF   r   rU   r   r   r  r   )rT   r   rvrU   rr   r   s         r9   r  )fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$9$9vvay!$
33332w#d)+++!"mFA!#))nAG ,
 I qww<Ir<   rR   )r   rN   rU   r   rV   Mapping[str, Any]r>   z ExclusiveKeywordArg | KeywordArg)r   rN   rU   r   rV   r  r>   r   )r   r  r>   r   )rO   r[   r\   r]   r  r   r$  r  r  r  r  r_   r$  )r   r  r  r  rn  r  s   @r9   	Converterr    s|    &&#	(	(  	( &		(
 .	( 	(*	9	9  	9 &		9
 	9 	96	 	r<   r  rQ   r  )r   r   r  r  r   r   r  rF   r  rz  r   rE  r  tree_leaves)rB  rn  r  r  r  r   r   r  r   r  r  r  s    `` `    @@@r9   r"  r"    s    */R.?.E.E.GH.GdaQT.GH$%->)???? CG%?	!  __FD DEHH(( DL b%((..////m!Gg{++!&"4"4W"=>>N{ Is   DT)r   r  c               L   [        5          [        5          [        X" 5       SS9" U6 nSSS5        SSS5        SSKJn  U(       a,  U" WR
                  5        UR
                  R                  5         WR                  5         U$ ! , (       d  f       Na= f! , (       d  f       Nj= f)z>Build a normalized inference graph, for use with fx_to_patternreal)tracing_modeNr*   remove_noop_ops)r   r   r   fx_passes.post_gradr  r   eliminate_dead_code	recompile)rj   rU   r   r  rB  r  s         r9   r  r    sq     
"	#%7%9Rv>E &:	# 5! 	$$&LLNI% &:%9	#	#s!   BBB
B	B
B#r  c          
       ^ Sm        S
U4S jjn[         R                  R                  S5         [        U S UU" 5       SSS9" U6   SSS5        T(       d   eSSKJn  U" TR                  5        SS	KJn  UR                  TR                  5        [         R                  R                  R                  5       TR                  l        TR                  R                  5         TR                  5         T$ ! , (       d  f       N= f)z=Build a normalized training graph, for use with fx_to_patternNc                D   > T(       a   e[        U 5      m[        X40 UD6$ rQ   )clone_graphr&   )joint_graphinputsrV   rB  s      r9   record_joint_graph)joint_fwd_bwd.<locals>.record_joint_graph	  s&     v% ???r<   c                    [        U 5      $ rQ   )r%   )rB  r@  s     r9   r   joint_fwd_bwd.<locals>.<lambda>'	  s	    r':r<   TF)partition_fndecompositionskeep_inference_input_mutations
enable_logr*   r  )early_patterns)r   rl   r  r   rV   r   r>   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])r   _guardstracingr$   r  r  r   fx_passes.joint_graphr
  r  r   CodeGen_codegenr  r  )rj   rU   r  r  r  r
  rB  s         @r9   r;  r;  	  s     '+B@)@3@@LO@	:@ 
		t	$:+(?+/	
 	 
% I24BHH5" ..0BHHHH  "LLNI1 
%	$s   C88
Dc                    / n[         R                  R                  U R                  U R                  4UR
                  5        U$ rQ   )r   r   r  rU   rV   r   )r   rU   s     r9   r  r  >	  s1    )+D	HHaffahh'5Kr<   c                   [        [        U R                  5      5      n[        [        R
                  R                     " 5       n[        [         5      nS nU(       a  UR                  5       n[        U5       Vs/ s H  ofU;  d  M
  UPM     nnU(       a  X7S      R                  U5        OdUR                  U5        U(       a   UR                  ULa  UR                  U5        UnUR                  [        UR                  US5      5      5        U(       a  M  U(       d"  [        U5      [        U R                  5      :X  d   eg s  snf )Nr  rR   )r   r   r   r    r   r   r   r   rg  r  r   r  r   r   r   )r   pendingreadywaitingcursorr   r  waiting_fors           r9   stable_topological_sortr  D	  s     8EKK()G uxx}}%'E $G F
{{}"'+@+Q%q+@ O$++D1IIdO&++T1d#F NN8GKKb$9:; '  3u:U[[)99999 As   :	EEc                   ^ ^ [         R                  " T 5      R                  m[        R                  [        R
                  " T 5      S[        4     SUU 4S jjj5       5       nU$ )z0Wrapper around lazy init functions in fx_passes/Nc                  > [         [           R                  5       n0 nST;   a  XS'   ST;   a  XS'   [        R                  R                  S 5         [        5          [        5          T" S0 UD6nS S S 5        S S S 5        S S S 5        U[         [        '   W$ ! , (       d  f       N-= f! , (       d  f       N6= f! , (       d  f       N?= f)Ninput_devicer  rR   )r   r  r   r   r  r  r   r(   )r  r  counters_refrV   r  
_fn_paramsrj   s        r9   	lazy_init%init_once_fakemode.<locals>.lazy_initm	  s      (--/!#Z'%1>"j(&3?#]]""4(*@*BNDT\&\F EU*B( ) EUDT*B*B((s<   B:B)(	B1B)9B:
B&"B))
B7	3B::
C)r  z
Any | Noner  ,Callable[..., dict[Any, Callable[..., Any]]]r>   r   )r5  r6  r7  r  r  r  r+   )rj   r  r  s   ` @r9   init_once_fakemoder   h	  ss     ""2&11J____R#'  	 

 
   , r<   c                   ^  SU 4S jjnU$ )z2Function for extra_check to put pass behind a flagc                $   > [        [        T5      $ rQ   )r  r#   )rA  r}  s    r9   
flag_checkconfig_flag.<locals>.flag_check	  s    vt$$r<   )rA  r   r>   r   rR   )r}  r#  s   ` r9   config_flagr%  	  s    % r<   c                L     " S S[         5      nU" U 5      R                  5       $ )Nc                  ,   ^  \ rS rSrSU 4S jjrSrU =r$ )clone_graph.<locals>.CopyGraphi	  c                T  > [         TU ]  U5      n[        U[        R                  R
                  5      (       an  UR                  R                  R                  UR                  5        U R                  R                  R                  UR                  S 5      UR                  l        U$ rQ   )r   r  rF   r   r   Proxyr   r   r   	new_graph_graph_namespacer  r}  )rT   r   r  r   s      r9   r  'clone_graph.<locals>.CopyGraph.run_node	  st    w'1H(EHHNN33""))(--8%)^^%D%D%P%PMM4&" Or<   rR   )r   r  r>   r  )rO   r[   r\   r]   r  r_   r$  r%  s   @r9   	CopyGraphr(  	  s    	 	r<   r.  )r)   	transform)input_graphr.  s     r9   r  r  	  s$    	K 	 [!++--r<   _seen_patternsc                    [        U R                  5      U:  a  U R                  U   $ Uc  g U R                  R                  U5      $ rQ   )r   rU   rV   rB   )r   
arg_number
kwarg_names      r9   get_arg_valuer5  	  s?     499~
"yy$$		{{z**r<   c           	     (   U/n[        U[        R                  R                  5      (       a9  UR	                  UR                  5        Vs/ s H  n[        X5      PM     sn5        U  Vs/ s H  oDR                  U;   d  M  UPM     sn$ s  snf s  snf rQ   )rF   r   r  r  r   r  r  r   )r   rj   r  r  r   s        r9   filter_nodesr7  	  sm    $C"ejj1122

",,.I.hGB).IJ"9UTkkS&8DU99 J9s   
B
*BBc                    U R                   S:X  aU  [        U R                  [        5      (       d   e[	        U R
                  R                  U R                  5      R                  $ U R                  $ )zFor call_function and call_method, we directly use the target function;
For call_module, the target is string, and we treat the module class
 as a function.
r$  )r  rF   r   rN   r   r   r  r   r  s    r9   r  r  	  sR    
 ww-$++s++++114;;?III;;r<   )rH   rN   r>   rG   )r  )r   r   r   r  r   rN   r>   rx   )r7  rH  r>   zTypeIs[Match]r4  )r  r  rQ  r*  r>   rx   rC  )
r   r   r   ztorch.fx.Graph | Noner  zdict[str, list[str | None]]r  rG   r>   rG   )r  rL   r+  ra   r@  zlist[Any] | tuple[Any, ...]r  rg   r  r  r  r  r  dict[str, float | int] | Noner  Sequence[str]r,  zPatternExpr | Noner  rG   r  r0   r  r  r>   rG   )r]  rN   r  rL   r@  r   r  rg   r  r9  r>   r   )r]  rN   r  rL   r+  ra   r@  zIterable[Any]r  rg   r  r  r  r  r  r9  r  r:  r  rG   r  r  r>   rx   )r  rL   r@  r   r  rg   r  r9  r  r:  r  r  r>   z(tuple[PatternExpr, torch.fx.GraphModule])NrR   )r  rL   r@  r   r  rg   r  r9  r  r:  r>   r   )
r   r   r  r  r{  r  r  rG   r>   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r  r   r  r>   rG   )r  ztorch._ops.OpOverloadr>   rG   r  )r  r  r  r  r>   rG   )r   r  r   r  r>   ro  )r   r  r>   rG   )r   r  r>   rx   )rj   r   r  z	list[str]r>   r   )rU   r   rV   r   r>   r   )rR   rR   NrR   )rB  r  rn  zSequence[type[Any]]r  r:  r  r9  r  r:  r>   r   )
rj   r  rU   r   r   rG   r  r  r>   rl   )rj   r  rU   r   r  r  r>   rl   )r   r  r>   zlist[torch.fx.node.Argument])rj   r  r>   r  )r}  rN   r>   zCallable[[Match], Any])r0  rl   r>   rl   rQ   )r   r  r3  ro  r4  r0   r>   r   )r   zIterable[torch.fx.Node]rj   r   r>   r   )r   r  r>   ztorch.fx.node.Target)r"  
__future__r   r  dataclassesr  rg  r5  r  loggingr  r@   r  rI  r  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   r   pathlibr   r   r   r   r   typing_extensionsr   r   r   torch._guardstorch.fxtorch.utils._pytreer	  r
  r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   r   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr   r   r   torch.utils._ordered_setr    
_functorchr#   r=  _functorch.aot_autogradr$   r%   _functorch.partitionersr&   _subclassesr'   r(   r   r)   r  decompositionr+   loweringr,   	getLoggerrO   r  r   atenprimsConstantr   rG  rA   rB   r  r/   r^   r1   rJ   rL   ra   rg   rm   r   TargetrN   FnsTypero   rr   r   r   r*  r   rH  r8  r   r   re  rl  ry  r  r  r   _SimpleSpecr  r  r  r#  r&  r,  r/  r2  r4  rE  r`  rw  r  	dataclassr  r  r  r   r  r  r  rD  rE  ra  __file__parentrW  rd  rl  r>  r?  rZ  r  r  r  compiler  r  r  r  r  r  r  r9  r  r  r"  no_gradr  enable_gradr;  r  r  r   r%  r  r1  r5  r7  r  rR   r<   r9   <module>rc     s  !F #         	 	   # # X X  3 3 *    $ $ ; ( 0 @ 6 W + I K 1 / 3 C 7 4   . ; !yy~~		EHHMM)
**..>

K 59 1 8+/ j /$4>x >> >#h # CL ((--


$? ? : IK??(5?BE?	?BN Nb, 8 k!'
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHoo
k o
d?    (, * * ,
[ ,
^F
 F
R%
; %
P4 4n!X ! : : :: 
< 
 
 	= 	= 	= p
l p
 p
f "	'' ' /' 	'
 
'` ,87;)+,0!#BUhhh 0h 	h
 :h )h 5h 'h *h h h @h 
hV )3 o 4MMM "M 	M
 5M M` x.//+=@UU     $ ,87;)+!BU888 8 "	8
 8 :8 )8 58 '8 8 @8 
8v e4
 8<)+BU  !    5	 
 '  @  .  5 N 8<)+				!		 		 5			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ **VW	.(H
D; v vr )+ 7;)+k-k%k k 5	k
 'k k\ 
 #'(;
  	
 &  < 
 )<	))
) &	)
 ) )X!:H@.  #-, . DH+
+%(+6@++:r<   