
    3j                     F4   % S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJrJ	r	J
r
  S SKJr  S SKrS SKJr  S SKJr  S SKrS SKJrJrJr  S SKJrJrJrJrJr  S SK J!r!  S SK"J#r#  S S	K$J%r%  S S
K&J'r'  S SK(J)r)  S SKJ*r*J+r+J,r,J-r-  S SK.J/r0  S SK1J2r2J3r3  SSK4J5r5J6r6  SSKJ7r7J8r8J9r9  \
" S5      r:\" S5      r;\Rx                  Rz                  \Rx                  R|                  -  r?\	\@S'   \R                  " \B5      rC\R                  R                  rE\R                  R                  rF\R                  R                  rG\R                  R                  rH\R                  R                  rI\" / \ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER^                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\ER                  P\'P\ER                  P\ER                  P\ER                  P\ER                  R                  P\ER                  R                  P\ER                  P\GR                  P\HR                  P5      ru0 \" 5       E\uErv\ER                  \ER                  \ER                  \ER                  R                  \ER                  \ER                  \ER                  \ER8                  \ER^                  \ER                  \EGR                   \EGR                  \EGR                  \EGR                  \EGR                  GR
                  \EGR                  \EGR                  \EGR                  \EGR                  \EGR                  \EGR                  \EGR                  GR                  \EGR                  \EGR                  \EGR                   \EGR"                  GR                  \EGR$                  \EGR&                  \EGR(                  /r\\Rx                  GR.                  \Rx                  R|                  -     \@S'   \" \v\5        S\?\\?   -  S\\\;\:4   /\\;\:4   4   4S jr\" \EGR&                  GR                  /5      S\GR
                  S\GR
                  S\S\GR
                  4S j5       r\" \EGR&                  GR
                  /5      S\GR
                  S\GR
                  S\GR
                  S\GR
                  4S j5       r\" \ER8                  /5      S\GR
                  S\GR
                  S \oS!\oS"\S\GR
                  4S# j5       r\" \ER^                  5      S$\GR
                  S%\GR<                  S\GR
                  S-  S&\GR
                  S-  S'\S\\GR
                  \GR
                  \GR
                  4   4S( j5       r\" \EGRB                  R                  /5      SSS).S*\GRD                  S+\GRF                  GRH                  S-  S,\GRF                  GRH                  S-  SS4S- jj5       r\" \EGRJ                  /5      \  SS.\GR
                  S+\GRF                  GRH                  S-  S,\GRF                  GRH                  S-  S\GR
                  4S/ jj5       5       r\" \EGR                  /5      \S.\GR
                  S\GR
                  4S0 j5       5       r\" \EGRL                  /5      S1\\o\GRD                  -     S2\GRF                  GRH                  S3\S\GR
                  4S4 j5       r\" \ER                  /5      SS5.S.\GR
                  S6\oS7\GR
                  S8\GR
                  S9\GRF                  GRH                  S\GR
                  4S: jj5       r\" \EGRN                  R                  /5      S1\\o\GRD                  -     S;\\o   S3\S\GR
                  4S< j5       r\" \EGRP                  /5      S\GR
                  S$\GR
                  S\GR
                  S=\\o   S>\o\\o   -  S?\o\\o   -  S@\o\\o   -  SA\SB\\o   SC\oSD\\   S\\GR
                  \GR
                  \GR
                  4   4SE j5       r\" \EGRR                  GRT                  /5      SS.\GR
                  SF\oS\GR
                  4SG jj5       r\" \EGRX                  /5      \ SSH\GR
                  SI\GR
                  SJ\GRZ                  S-  S\GR
                  4SK jj5       5       r\" \EGR\                  /5      \   SSH\GR
                  SL\GR
                  SM\GR
                  SJ\GRZ                  S-  SN\GRF                  GRH                  S9\GRF                  GRH                  S\GR
                  4SO jj5       5       r\" \EGR^                  /5      \ SSH\GR
                  SP\GR
                  SJ\GRZ                  S-  S\GR
                  4SQ jj5       5       r\" \EGR`                  R                  /5       SSR\\GR
                     S6\oS\GR
                  4SS jj5       r\" \EGRb                  /5      S.\GR
                  S\GR
                  4ST j5       r\" \EGRd                  /5      SS5.S.\GR
                  SU\GR
                  S9\GRF                  GRH                  S-  S\GR
                  4SV jj5       r\" \EGRf                  /5      SH\GR
                  S\GR
                  4SW j5       r\" \EGRh                  \EGRj                  /5      SH\GR
                  S\GR
                  4SX j5       r\" \EGRl                  \FGRl                  /5      SH\GR
                  SY\GR
                  S\GR
                  4SZ j5       r\" \EGRn                  \FGRn                  /5      SH\GR
                  SY\GR
                  S\GR
                  4S[ j5       r\" \EGRp                  5        SSH\GR
                  S6\oS-  S]\S\GR
                  4S^ jj5       r\" \EGRr                  5        SSH\GR
                  S6\oS-  S]\S\GR
                  4S_ jj5       r\" \EGRt                  /5      SH\GR
                  S6\oS\oS`\oS\GR
                  4
Sa j5       r\" \EGRv                  R                  /5      SH\GR
                  S1\\o\GRD                  -     S\GR
                  4Sb j5       r\" \EGRv                  GRZ                  /5      SH\GR
                  Sc\GRZ                  S\GR
                  4Sd j5       rSH\GR
                  S\\GR<                  \GR|                  4   4Se jr\" \EGR                  5      SSSS\S\\GR                  Sf.SH\GR
                  S2\o\-  Sc\GRZ                  S-  Sg\GR                  S-  Sh\GR                  S-  Si\Sj\Sk\GR                  S\GR
                  4Sl jj5       rSS\GR                  Sm.Sn\So\GR
                  4   SH\GR
                  Sc\GRZ                  S-  Sh\GR                  S-  Sk\GR                  S3\S\GR
                  4Sp jjr\GR0                  " \EGR                  /\!5      SH\GR
                  S3\S\GR
                  4Sq j5       r\GR0                  " \EGR                  /\!5      SH\GR
                  S3\S\GR
                  4Sr j5       r\GR0                  " \EGR                  R                  /\!5      SH\GR
                  Ss\oS3\S\GR
                  4St j5       r\GR0                  " \EGR                  GR                  /\!5      SH\GR
                  Su\oSs\oS3\S\GR
                  4
Sv j5       r\" \EGR                  R                  5      Ss\oS1\\o\GRD                  -     S3\S\GR
                  4Sw j5       r\" \GR                  R                  5       SS$\GR
                  S\GR
                  S&\GR
                  S-  S\GR
                  4Sx jj5       rs\" \HR                  R                  5      S$\GR
                  Sy\GR
                  Sz\GR
                  S\GR
                  S{\GR
                  S|\GR
                  S&\GR
                  S}\GR
                  S~\GR
                  S\oS\GR
                  4S j5       rt\" \R                  R                  GR                  5      S\GR
                  S\GR
                  4S j5       r\" \EGR                  /5      \   SS\GR
                  S\GR
                  S\oS\oS\S\GR
                  4S jj5       5       r\" \EGR                  GR                  5      S\\GR
                     S\\GR
                     S\GRF                  GRH                  S\\GR
                     4S j5       r\" \EGR                  GR                  5      S\\GR
                     S\\GR
                     S\\GRF                  GRH                     S\\GR
                     4S j5       r\EGR                  R                  GR                  \GR                  GR                  GR                  5      \" \EGR                  5      S$\GR
                  S\GR
                  S&\GR
                  S-  S\GR
                  S-  S\GR
                  S-  S\S\S\S\\GR
                  \GR
                  \GR
                  4   4S j5       5       r\ GR                  S\\\So\4   4   4S j5       rS\\\So\4   4   4S jr\" \EGR                  5      SH\GR
                  S\GR
                  S\GR
                  S\GR
                  4S j5       r\" \IGR                  GR                  5      S$\GR
                  S\oS\oS'\Sc\GRZ                  S\\GR
                  \GR
                  4   4S j5       r\" \EGR                  5       SSH\GR
                  S7\GR
                  S\GR
                  S\S\GR
                  4
S jj5       r\" \EGR                  5       SSH\GR
                  S7\GR
                  S\GR
                  S\S\GR
                  4
S jj5       r\" \ER                  R                  5      \S\GR
                  S\GR
                  S6\oS\GRZ                  S\GR
                  4
S j5       5       r|\" \EGR                  5      SS.SH\GR
                  S6\oS7\GR
                  S\GR
                  S\S\S\GR
                  4S jj5       rS.\GR
                  S\\o   S>\o\\o   -  S-  S?\o\\o   -  S@\o\\o   -  S\S6\oS\\GR
                  \GR
                  4   4S jr\" \EGR                  5          SS.\GR
                  S\\o   S>\o\\o   -  S-  S?\o\\o   -  S@\o\\o   -  S\S\\GR
                  \GR
                  4   4S jj5       r\" \EGR                  5          SS.\GR
                  S\\o   S>\o\\o   -  S-  S?\o\\o   -  S@\o\\o   -  S\S\\GR
                  \GR
                  4   4S jj5       r\" \EGR                  5      S.\GR
                  S\\o   S\\GR
                  \GR
                  4   4S j5       r\" \EGR                  GR                  5      S\S\SSS.S\GR
                  SH\GRF                  GRH                  S\S\S\S-  S\GR
                  S-  S\GR
                  4S jj5       r\" \EGR                  GR                  5      S\S\S.SH\GRF                  GRH                  S\GR
                  S\S\S\GR
                  4
S jj5       r\" \EGR                  5          SSH\GR
                  S\GR
                  S\S\S\S\GR                  S-  S\\GR
                  \GR
                  4   4S jj5       r\" \EGR                  GR
                  5       SS\GR
                  S\oS-  S\GR
                  4S jj5       r     SS$\GR
                  S\GR
                  S&\GR
                  S-  S>\\o   S?\\o   S@\\o   SC\oS\GR
                  4S jjrg)    N)Callable)Any	TypeAliasTypeVar)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDsuggest_memory_formattype_to_dtype)native_layer_norm)guard_or_falsestatically_known_true   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     [        U [        5      (       a  U OU / H%  nU[        ;   d  M  [        R	                  SU 5        M'     [
        R                  " U [        5      $ )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r&   ops     W/home/wildlama/miniconda3/lib/python3.13/site-packages/torch/_inductor/decomposition.pyr/   r/      sM      T**c5KK.4 6 ((n==    startendweightc                     [        U 5      nU[        R                  :w  a  UR                  US9nX-
  nUS:  d  US::  a  [        R                  " XSU-
  * S9$ [        R                  " XUS9$ )Nmemory_format      ?g            ?alpha)r   torchcontiguous_format
contiguousadd)r3   r4   r5   fmtdiffs        r1   _lerp_scalarrC      sh      
&C
e%%%nn3n/;D}$yyC&L/::99U//r2   c                     [        U 5      nU[        R                  :w  a  UR                  US9nX-
  nUR	                  5       S:  nSU-
  * n[        R
                  " XVU5      n[        R
                  " XQU 5      n[        R                  " XUSS9$ )Nr7   r9   r:   r   )value)r   r=   r>   r?   abswhereaddcmul)	r3   r4   r5   rA   rB   maskneg_omwwbases	            r1   _lerp_tensorrM      s      
&C
e%%%nn3n/;D::<3DfoGD6*A;;t%(D==$a00r2   grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 J    U R                   (       a  [        $ [        XX#U5      $ N)is_xpuNotImplementeddecomp_embedding_dense_backward)rN   rO   rP   rQ   rR   s        r1   _embedding_dense_backwardrX      s(     *k8J r2   inputnormalized_shapebiasepsc                 J    U R                   (       a  [        $ [        XX#U5      $ rT   )is_mtiarV   decomp_native_layer_norm)rY   rZ   r5   r[   r\   s        r1   _native_layer_normr`      s      }}#EV3OOr2   )minmaxsymbolra   rb   c                    g rT    )rc   ra   rb   s      r1   sym_constrain_range_for_sizerf      s     r2   xc                 V    Ub  U R                  U5      n Ub  U R                  U5      n U $ rT   )	clamp_min	clamp_max)rg   ra   rb   s      r1   clamprk      s/     KK
KKHr2   c                 J    U SU R                  5       R                  5       -   -  $ )Nr   )negexp)rg   s    r1   siluro      s      AEEGKKM!""r2   size
fill_valuekwargsc                     UR                  S5      nUc.  [        [        U5      5      US'   [        R                  " X40 UD6$ [
        $ )Ndtype)getr   typer=   fullrV   )rp   rq   rr   rt   s       r1   rw   rw      sD     JJwE}'Z(89wzz$5f55r2   r;   dimindextensorr<   c          	          [        5       (       d$  U R                  [        R                  :X  a  [        $ [        XX#SUS9$ )NF)inplacer<   )r   rt   r=   bfloat16rV   r   )rg   rx   ry   rz   r<   s        r1   	index_addr~     s1     ;;177enn4!%eLLr2   physical_layoutc                 Z   [        U5      [        [        [        U5      5      5      :H  nU(       a  [        R                  " U 40 UD6$ S/[        U 5      -  n[        U5       H	  u  pVXTU'   M     [        R                  " U Vs/ s H  o`U   PM	     sn40 UD6R                  U5      $ s  snf Nr   )r*   rangelenr=   empty	enumeratepermute)rp   r   rr   is_identitypermpls          r1   empty_permutedr   !  s     '4c/6J0K+LLK{{4*6**sSYo.DAG /{{_=_G_=HHPPQUVV=s   B(
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 >   U
S   (       a$  [        U R                  R                  5      (       d  [        $ [        R                  U S/[        [        SU R                  5       5      5      -   5      n[        R                  U UUUUUUUUU	U
S   U
S   S/5      u  pnXU4$ )N   r   r   F)
r   devicerv   rV   atensumr*   r   rx   convolution_backward)rN   rY   r5   r   r   r   r   r   r   r   r   	grad_biasgrad_inpgrad_weight_s                  r1   r   r   2  s     q>(:(:(?(?!@!@qcDq+//:K1L,M&MNI#88	QQ/ H1 9--r2   decimalsc                 F    SU-  n[         R                  X-  5      SU-  -  $ )Ng      $@r:   )r   round)rg   r   ten_pow_decimalss      r1   	round_decr   S  s)    X~::a*+s5E/EFFr2   selfbatch2r   c                 d   [        U R                  S   S:H  5      (       d   [        UR                  S   S:H  5      (       a  X-  R                  5       $ [        R                  (       a  U R
                  R                  S;  ar  [        U R                  S   S:H  5      (       d   [        UR                  S   S:H  5      (       a2  U R                  S5      UR                  S5      -  R                  SS9nU$ U R
                  R                  S:X  a  [        U R                  S5      S:H  5      (       a|  [        UR                  S5      S:H  5      (       aZ  [        S   S==   S-  ss'   [        R                  " U R                  S5      UR                  S5      -  SS	S
9R                  S5      $ [        $ )Nr   r   cpumpsrx   r   inductordecompose_bmmTrx   keepdim)r   shaper?   r   coordinate_descent_tuningr   rv   	unsqueezer   rp   r   r=   squeezerV   )r   r   r   outs       r1   bmmr   Y  sm    TZZ]a/004IQ15 5 ))++ ''DKK,<,<N,R A!!3448MLLOq 9
 9
 >>"%(8(8(;;@@Q@GCJ{{5  1!2338MKKOq 9
 9
 Z 1Q6199Q&.."44!Til r2   mat1mat2betac                    UR                   R                  S;  a  [        UR                  S5      S:H  5      (       ae  [        UR                  S5      S:g  5      (       aC  [        UR                  S5      S:g  5      (       a!  [        S   S==   S-  ss'   X-  nXV-  X@-  -   $ U R                   R                  S:X  GaG  [        UR                  S5      S:H  5      (       a  [        UR                  S5      S:H  5      (       ac  [        S   S==   S-  ss'   [
        R                  " UR                  S5      UR                  S5      -  SSS	9R                  S5      nXV-  X@-  -   $ [        UR                  S5      S:H  5      (       a~  [        UR                  S5      S
:*  5      (       a\  [        UR                  S5      S
:*  5      (       a:  [        S   S==   S-  ss'   UR                  U-  R                  SSS	9nXV-  X@-  -   $ [        $ )Nr   r   r   r   r   decompose_addmmr   Tr      )r   rv   r   rp   r   r=   r   r   r   r   TrV   )r   r   r   r   r   r<   r   s          r1   addmmr   z  s    {{~-!$))B-1"455%diila&788%diila&788Z !23q83+C;,,{{5  1!2338MIIbMQ9
 9
 Z !23q83))Q$,,r"224il  ;,,!$))A,!"344tyy|r122tyy|r122Z !23q8366D=%%!T%:C;,,r2   input2c                 \   [         R                  (       a  U R                  R                  S;  ap  [	        U R
                  S   S:H  5      (       d   [	        UR
                  S   S:H  5      (       a0  U R                  S5      UR                  S5      -  R                  SS9$ [	        U R                  S5      S:H  5      (       Ga-  [	        U R                  S5      S:g  5      (       Ga
  [	        UR                  S5      S:g  5      (       a  U R                  R                  S;  d  U R                  R                  S:X  a  [	        U R                  S5      S:  5      (       a  [	        UR                  S5      S:H  5      (       ap  U R                  UR                  :X  aV  [        [        R                  " U 5      [        R                  " U5      -   S:*  5      (       a  [        S	   S
==   S-  ss'   X-  $ U R                  R                  S:X  a  [	        U R                  S5      S:H  5      (       a|  [	        UR                  S5      S:H  5      (       aZ  [        S	   S
==   S-  ss'   [        R                  " U R                  S5      UR                  S5      -  SSS9R                  S5      $ [        $ )Nr   r   r   r   r   r   r       r   decompose_mmTr   )r   r   r   rv   r   r   r   r   rp   rt   r   r=   numelr   r   rV   )r   r   r   s      r1   mmr     s    ''DKK,<,<N,R A!!3448MLLOq 9
 9
 NN1%(8(8(;;@@Q@GG 	diimq011!$))A,!"344!&++a.A"566;;>1KK%%diilQ&677%fkk!n&9::v||+D 1EKK4G GBNOOZ 0A50= {{5  1!2338MKKOq 9
 9
 Z 0A5099Q&.."44!Til r2   tensorsc                 0  ^^ S[         R                  S[        4U4S jjn[        [	        X 5      5      m[        T5      S:X  aO  [        U S[        R                  06S   nTS   nX4R                  :X  a  UR                  5       $ UR                  US9$ S[        T5      s=:  a  [        U 5      :  a#  O  O [        R                  R                  TT5      $ [        T5      S:  a  [        U4S jT 5       5      (       a  TS   n[        UR                   5      nTS:  a  T[        UR                   5      -   OTmUR#                  T[        T5      5        UR%                  T5      R&                  " U6 R)                  TTS-   5      R                  5       $ [*        $ )	Nrg   r'   c                    > [        U R                  5      S:X  a!  [        U R                  S   S:H  5      (       a  gT[        U R                  5      :  a!  [        U R                  T   S:H  5      (       a  gg)Nr   r   FT)r   r   r   )rg   rx   s    r1   non_empty_tensorcat.<locals>.non_empty_tensor  s[    " qww<1
a!@!@QWW.1B"C"Cr2   r   type_promotion_kindr   rt   c              3   0   >#    U  H  oTS    L v   M     g7f)r   Nre   ).0tfiltered_tensorss     r1   	<genexpr>cat.<locals>.<genexpr>  s      )*:Qa  *:s   )r=   Tensorboolr*   filterr   r   r   DEFAULTrt   clonetor   catdefaultallr   insertr   expandflattenrV   )r   rx   r   promoted_dtype
filtered_tinpr   r   s    `     @r1   r   r     s   
ELL T 2 F#3=>
!+
 ? G G
  &a(
 !1!11 	
 ^4	

 
S!"	1S\	1xx 0#66 q S )*:) & & q!SYY&)AgcC		N"3S#./0}}S!((%088cAgFLLNN r2   c                 *   U R                  5       (       ah  [        R                  " [        R                  " U R                  5      [        S5      [        R                  " U R                  U R                  5      5      $ [        U [        R                  S9u  p[        R                  " [        R                  X R                  S9n[        R                  " U S:  US5      n[        R                  " [        R                  " U 5      [        S5      U5      $ )Nnan)r   rt   r   r   g        )
is_complexr=   rG   isnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpir   )rg   r   rt   r   rets        r1   angler     s    ||~~{{KKuu{{166166/J
 	
 "	;HHHA 
		TWWE((	CB
++a!eR
%C;;u{{1~uU|S99r2   yc                   [         R                  " U 5      =(       a    U R                  5       n[         R                  " U5      =(       a    UR                  5       nU(       a  U(       d  [        $ S[         R                  S[
        4S jnSnU R                  S:X  a  UR                  S:X  a  SnU R                  S:X  a  U R                  S5      n UR                  S:X  a  UR                  S5      nUnUb  X!-  n[         R                  " U R                  UR                  5      nU" U 5      (       d  U" U5      (       a  [        $ S[         R                  S[         R                  4S jn	U S-   n US-   nU	" U R                  U R                  R                  5      5      n
U	" UR                  UR                  R                  5      5      n[         R                  " X-   S	S
9R                  U5      nU(       a  US   $ U$ )Nrz   r'   c                 P    U R                   S:X  a  gU R                  5       S   S:g  $ )Nr   Fr   r   )ndimr   )rz   s    r1   _requires_fallbackadd.<locals>._requires_fallback0  s'    ;;!}}r"a''r2   Fr   Tr   c                     U R                   Gt pUS-  S:w  a  [        S5      e/ UQUS-  PSP7nU R                  U5      nU$ )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]r   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)rz   initial_dimslast_dim	new_shapereshaped_tensors        r1   reshape_tensor_complex#add.<locals>.reshape_tensor_complexI  s\     #),, a<1 c 
 6l5HM515	 ++i0r2   )	start_dim)r=   	is_tensorr   rV   r   r   r   reshapepromote_typesrt   r   r   r   )rg   r   r<   x_is_complex_tensory_is_complex_tensorr   output_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults                r1   r@   r@   #  s     //!,?//!,?&9(5<< (D ( vv{qvv{vv{IIaLvv{IIaL	AI&&qww8L! 21 5 5u||  $ 	
AA	AA'qvv||(<=J'qvv||(<=J]]:2bAFF|TFayMr2   c                 <    U R                  5       (       a  [        $ U $ rT   )r   rV   r   s    r1   conj_physicalr  g  s    Kr2   c                     U $ rT   re   r  s    r1   liftr  n  s    Kr2   otherc                 b    [         R                  " [         R                  " U5      X:  -  X5      $ rT   r=   rG   r   r   r  s     r1   fminr  s  #    ;;u{{5)U\:DHHr2   c                 b    [         R                  " [         R                  " U5      X:  -  X5      $ rT   r  r  s     r1   fmaxr  x  r  r2   Fr   c                 t    U R                   [        R                  :X  a  [        R                  " XUS9$ [        $ Nr   )rt   r=   r   anyrV   r   rx   r   s      r1   amaxr  }  +     zzUZZyy88r2   c                 t    U R                   [        R                  :X  a  [        R                  " XUS9$ [        $ r  )rt   r=   r   r   rV   r  s      r1   aminr    r  r2   lengthc                 f    [         R                  " XX#5      R                  [         R                  S9$ )Nr7   )r=   narrowr   r>   )r   rx   r3   r  s       r1   narrow_copyr"    s0     <<5177-- 8  r2   c                 H    [         R                  X5      R                  5       $ rT   )r   r   r   )r   rp   s     r1   view_copy_defaultr$    s    
 99T &&((r2   rt   c                 @    U R                  5       R                  U5      $ rT   )r   r   )r   rt   s     r1   view_copy_dtyper&    s    
 ::<U##r2   c                     [         R                  " U 5      u  pU Vs/ s H  o0R                  U   PM     nnS/[        U5      -  n[	        U5       H	  u  pcXeU'   M     XE4$ s  snf r   )utils3compute_elementwise_output_logical_to_physical_permr   r   r   )r   r   r   r   r   permutationr   s          r1   _get_shape_permutation_liker+    sj     RRSWXO$34OqZZ]OE4#E
"K/*A +  5s   A!)rt   layoutr   
pin_memoryrequires_gradr8   r,  r   r-  r.  r8   c          
         Uc  U R                   OUnUc  U R                  OUnUc  U R                  OUnU[        R                  :w  a3  [        R
                  " U R                  UUUUUUS9nUR                  US9$ U[        R                  :X  d   e[        U 5      u  p[        R
                  " U	UUUUUUS9nU
[        [        [        U
5      5      5      :X  a  U$ UR                  U
5      R                  5       $ )N)rt   r,  r   r-  r.  r7   )rt   r,  r   r=   preserve_formatrw   r   r   stridedr+  r*   r   r   r   r   )r   rq   rt   r,  r   r-  r.  r8   r	  r   r*  s              r1   	full_liker2    s      -DJJUE"NT[[F"NT[[F---JJ!'
 yy}y55 &&&8>!'
 $uS%5677M~~k*0022r2   )rt   r   r8   rand_fn.c                j   Uc  UR                   OUnUc  UR                  OUnU[        R                  :w  a#  U " UR                  4UUS.UD6R                  US9$ [        U5      u  pgU " U4UUS.UD6nU[        [        [        U5      5      5      :X  a  U$ UR                  U5      R                  5       $ )Nr   r7   )rt   r   r=   r0  r   r   r+  r*   r   r   r   r   )	r3  r   rt   r   r8   rr   r   r*  r	  s	            r1   
_rand_liker5    s      -DJJUE"NT[[F---JJ

 	

 "="
)	* 5T:E 	F d5[!1233>>+&,,..r2   c                 8    [        [        R                  U 40 UD6$ rT   )r5  r=   randr   rr   s     r1   	rand_liker9    s    ejj$1&11r2   c                 8    [        [        R                  U 40 UD6$ rT   )r5  r=   randnr8  s     r1   
randn_liker<    s    ekk42622r2   highc                 x    [        [        R                  " [        R                  R
                  SU5      U 40 UD6$ r   r5  	functoolspartialr   randintlow)r   r=  rr   s      r1   randint_likerD    s-    i''(8(8!TBDSFSSr2   rC  c                 v    [        [        R                  " [        R                  R
                  X5      U 40 UD6$ rT   r?  )r   rC  r=  rr   s       r1   randint_like_lowrF    s-     i''(8(8#DdUfUUr2   c                 F    [         R                  R                  " SX40 UD6$ r   )r   rB  rC  )r=  rp   rr   s      r1   rB  rB  !  s      <<At4V44r2   c                     [         R                  R                  R                  U5      n[         R                  R                  R	                  XX!R                  5       S   5      $ r   )r=   r&   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrp   )rY   r5   r[   packed_weights       r1   #linear_dynamic_fp16_unpacked_weightrM  *  sK     II((MMfUM99AAdKKM!$4 r2   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          [         R                  R                  R                  X4XV5      n
[         R                  R                  R	                  U UUU
UUU	5      $ rT   )r=   r&   rI  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)rY   rN  rO  r5   rP  rQ  r[   rR  rS  rT  rL  s              r1   wrapped_quantized_linearrX  6  sX     II((@@/M 99CC r2   packedc                     S[         R                  S[         R                  4S jnU" U SSS24   5      nU" U SSS 24   5      nU SS S24   R                  [         R                  5      U-  U-   $ )Nu8r'   c                   ^  U 4S jS 5       u  pp4[         R                  S:X  a3  XS-  -   US-  -   US-  -   R                  [        R                  5      S   $ US-  US-  -   US-  -   U-   R                  [        R                  5      S   $ )Nc              3   l   >#    U  H)  nTS U4   R                  [        R                  5      v   M+     g7f.N)r   r=   int32)r   nr[  s     r1   r   Pq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>T  s)     G,QbajmmEKK00,s   14)r   r   r      little   r      r^  )sys	byteorderr   r=   float32)r[  rg   r   r  rK   s   `    r1   bitcast_u8_to_f32=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32S  s    G,G
a==H$aLAG,R8>>u}}MiXX"Wb)Q!V4q8>>u}}MiXXr2   .i)r=   r   r   rh  )rY  ri  scalesoffsetss       r1   "q_embedding_bag_byte_unpack_decomprn  Q  sz    Yell Yu|| Y vc2b5j12FsBCx 01G#ss(u}}-6@@r2   agridinterpolation_modepadding_modealign_cornersc           	          U R                   [        R                   " S5      :H  =(       a)    US:H  =(       a    U R                  [        R                  S9(       + n[	        U UUUUUS9nU$ )Nr   r   r7   )rp  rq  rr  rs  _expand_grid)r   r=   is_contiguousr>   decomp_grid_sampler_2d)ro  rp  rq  rr  rs  ru  outputs          r1   grid_sampler_2dry  _  sm     	
ELL'' 	C!#	COO%*A*AOBL $	-!#!F Mr2   start_tensorsend_tensorsc           	          [         R                  R                  U [         R                  R	                  [         R
                  R                  X5      U5      5      $ rT   )r   _foreach_addList_foreach_mulScalar_foreach_sub)rz  r{  r5   s      r1   _foreach_lerp_scalarr    sJ     !!  "";>	
 r2   scalarsc           	          [         R                  R                  U [         R                  R	                  [         R
                  R                  X5      U5      5      $ rT   )r   r}  r~  r  
ScalarListr  )rz  r{  r  s      r1   _foreach_lerp_scalarlistr    sJ     !!$$"";>	
 r2   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          [         R                  U UUUUUUU5      u  pn
U(       a  XU
4$ UUR                  S5      UR                  S5      4$ )Nr   )r   native_batch_norm	new_zeros)rY   r5   r[   r  r  r  r  r  ro  bcs              r1   miopen_batch_normr    sh     $$"	GA! ay	 r2   c                      0 [         E[        E$ rT   )r+   r   re   r2   r1   fast_random_decompsr    s    5n5 455r2   c                  
   [         R                  (       a  [        $ [         R                  (       aH  [        R	                  [
        R                  R                  R                  R                  S5        [        $ [        5       n U $ )z"decomps can change based on configN)r   fallback_randomr+   "fallback_embedding_bag_byte_unpackpopr=   r&   	quantizedembedding_bag_byte_unpackr   r  )r	  s    r1   select_decomp_tabler    sT    00599..HHPPRVW "FMr2   rI   sourcec                    SSK JnJn  U" U R                  UR                  5      (       a  [
        R                  X/5      u  pUR                  S5      R                  S5      S-
  nS XU4 5       u  pgn[
        R                  XU/S5      n	[        R                  " XyU5      R                  U R                  5      $ [        $ )Nr   )BackendFeaturehas_backend_featurer   r   c              3   @   #    U  H  oR                  5       v   M     g 7frT   )r   )r   rg   s     r1   r   !masked_scatter.<locals>.<genexpr>  s     ,WBVQYY[[BVs   )codegen.commonr  r  r   MASKED_SCATTER_WITH_INDEXr   broadcast_tensorsr   cumsum_unsafe_masked_indexr=   rG   r   r   rV   )
r   rI   r  r  r  
source_idx	self_flat	mask_flatsource_flatr	  s
             r1   masked_scatterr    s     D4;;(P(PQQ ++TL9
\\"%,,Q/!3
,W4vBV,W)	k**;J<QRS{{9i8==djjIIr2   	quant_min	quant_maxc                    [         R                  " U 5      u  pVXe-
  [        X!-
  5      -  n[         R                  " U[         R                  " U/5      5      nU[         R
                  " XW-  5      R                  [         R                  5      -
  n[         R                  " XU5      nUR                  [         R                  5      UR                  [         R                  5      4$ rT   )r=   aminmaxr   rb   r   r   r   intrk   float64int64)	rY   r  r  r\   rt   min_valmax_valscale
zero_points	            r1   choose_qparams_tensorr    s     }}U+G%	(=">>EIIeU\\3%01EU[[9<<UYYGGJZI>J88EMM"JMM%++$>>>r2   
accumulatec                     U R                  5       n[        R                  " XA/UR                  UR                  5      U5      nUR                  U R                  5      $ rT   )r   r=   	index_putr   r   )r   ry   r  r  	flatteneds        r1   putr    sJ     I7FNN5;;7I TZZ((r2   c                 L    [         R                  XX#S9nU R                  U5      $ )N)r  )r   r  copy_)r   ry   r  r  r   s        r1   put_r    s$     ((4(
>C::c?r2   rx  input_dtypec                     X-  n[         R                  " XBSS9n[        R                  " U* XT5      nU R                  U:w  a  UR                  U5      nUR                  5       $ )NTr   )r=   r   r   fmart   r   r?   )rN   rx  rx   r  new_grad_outputsum_new_grad
grad_inputs          r1   _softmax_backward_datar    s]     "*O99_tDL##VG\KJ K']];/
  ""r2   Tinclude_selfsrcreduction_typer  c                   US:X  Ga  [        U R                  5      (       d  U R                  R                  =(       d    U R                  R                  n[        R
                  " U5      nU(       a)  U n[        R
                  " U 5      R                  XU5      n	OMU R                  XS5      n[        R                  " U 5      R                  XU5      n	U	R                  U	S:  S5      n	UR                  XU5      nU(       a  X-  $ X-  $ [        [        R                  R                  UU R                  UR                  UR                  R                  S5      (       a  [         $ U R"                  US-   S  R%                  5       U R"                  S U R%                  5       -  n
UR%                  5       /U R"                  US-   S  QU R"                  S U Q7n/ ['        U R(                  U-
  U R(                  5      QSP['        SU R(                  U-
  5      Q7nUR+                  [        R,                  5      R/                  U
5      R1                  U5      R3                  U5      nU R5                  UUUUUS9$ )Nmeanr   r   Tr  )r    rt   is_floating_pointr   r=   	ones_liker~   
index_fill
zeros_likemasked_fillr!   r   scatter_reduce_twor   rv   rV   r   r   r   r   r   r  repeat_interleaver   r   scatter_reduce)r   rx   ry   r  r  r  true_divisiononesr   countsrepeatsindex_shaper   scatter_indexs                 r1   index_reducer  "  s    (T

) ) 

44M

8M8Ms#C__T*44SFF//#a0C%%d+55c$GF''
A6FmmC,,s|?#-?  

		

   jjq#))+djj#.>.D.D.FFG;;=L4::cAgi#8L4::ds;KLKNU499s?DII.NNE!TYY_4MND		7	#				  !   r2   kernel_size	ceil_modec                 <   US:X  a  S/U-  nUS:X  a  S/U-  nU(       d  Un[        X5      n[        XF5      n[        X65      n[        X&5      n[        R                  " [        R                  U5      n[
        R                  R                  R                  XS9(       d2  U[
        R                  " [
        R                  5      R                  :  a  [        $ [        R                  U UUUUU5      u  p[        R                  U	UU R                   U* S  UUU5      n
X4$ )Nr   r   )n_dim)r   r@  reduceoperatormulr=   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8rb   rV   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rg   r  r   r   r   r  rx   window_sizevalsrm  rO   s              r1   _max_pool_with_indicesr  X  s    1}39!|#) {0KH*H7(G&&F""8<<=K 	  FF 	G 	
 UZZ0444;;	MD ;;	G =r2   c           
          [        XX#XESS9$ )Nr   r   r  rg   r  r   r   r   r  s         r1   max_pool2d_with_indicesr         "	! r2   c           
          [        XX#XESS9$ )Nrb  r   r  r  s         r1   max_pool3d_with_indicesr    r  r2   output_sizec                    U R                   Gt p#nUu  pVUS:X  d  US:X  a7  / UQUPUPnU R                  U5      U R                  U[        R                  S94$ X5-  S:X  a%  XF-  S:X  a  X5-  XF-  /n[        R                  X5      $ [        $ )Nr   r   )r   	new_emptyr=   r  r   r  rV   )	rg   r  batchh_inw_inh_outw_outo_sizer  s	            r1   adaptive_max_pool2dr    s     U$LEzUaZ'5'%''{{6"AKKekkK$JJJ|qT\Q.}dm4++A;;r2   	out_int32rightsidesortersorted_sequencer  r  r  r  c          	      t    [         R                  U [        R                  " U/U R                  S9UUUUS9S   $ )Nr   r   r   )r   searchsortedr=   rz   r   )r  r   r  r  r  r  s         r1   searchsorted_scalarr	    sK     dVO$:$:;   	 	r2   r  r  
boundariesc                    [         R                  [        R                  " U /UR                  S9UUUS9R                  S5      $ )Nr  r
  r   )r   	bucketizer=   rz   r   r   )r   r  r  r  s       r1   bucketize_scalarr    sD     >>dVJ$5$56	  
 gajr2   noiselowerupper	generatorc                    U(       aM  U S:*  n[         R                  XX5S9n[        R                  " X`U-  U 5      n[        R                  " XgS5      n	X4$ X#-   S-  n
[         R	                  X
5      [        R
                  " 5       4$ )Nr   )r  r   r   )r   uniformr=   rG   
leaky_relur   )r   r  r  r  r  r  not_positiverrx  	noise_outnegative_slopes              r1   rrelu_with_noise_functionalr    su     qyLLeLA\!8T:KK3	  -1,t4ellnDDr2   repeatc                 R   [         R                  R                  (       a  [        $ Ub  [	        U5      [
        La  [        $ U R                  R                  S:X  a  [        $ U R                  [        R                  [        R                  4;   d   eU R                  S:X  d   eU R                  S5      n[        R                  " XR                  S9n[        R                  " X#U R                  [        R                  :H  SS9n[        R                  " X@R!                  S5      S-
  S9$ )Nr   r   r   r  Tr
  )rb   )r   tritonautotune_at_compile_timerV   rv   r  r   rt   r=   r_  r  r   r  aranger  rk   rp   )r  r  r  posrO   s        r1   repeat_interleave_Tensorr!    s    
 }}-- d;/s:}}U"<<EKK5555;;!]]1F
,,{==
9C   ;DG ;;wKKNQ$677r2   r  c           
      6   U R                  5       S:X  a  UR                  5       S:X  d   S5       eUS   nUS   nUS   nU R                  S5      nUR                  S5      n[        R                  R	                  UUUUS4US4US4US9n	U	R                  S5      $ )Nrb  z,Expect (N,C_in,L) and (C_out,C_in//groups,K)r   r   r   )r   r   r   r   )rx   r   r   conv2dr   r   )
rY   r5   r[   r   r   r   r   input_2d	weight_2dout_2ds
             r1   conv1d_to_conv2dr'    s     99;!

 1 61
 AYFajG{H r"H  $I [[  {!A ! F >>"r2   )NNrT   )Nr   r   )NF)r   r   F)F)Nr   r   F)g      ?gUUUUUU?FN)Nr   r  r(  r   )r@  loggingr   r  rf  collections.abcr   typingr   r   r   typing_extensionsr   r=   torch._decomp_decompr.   torch._prims_common_prims_commonr(  $torch.ao.quantization.fx._decomposedr   r	   r
   torch._decomp.decompositionsr   rw  r   r   rW   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   r   torch._refsr   r_   %torch.fx.experimental.symbolic_shapesr   r    r   r   r   r    r!   r"   r#   _opsOperatorBaseOpOverloadPacketr$   __annotations__	getLogger__name__r,   r&   r   r  r  rI  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvr  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardr  native_group_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvechann_windowrM  rX  inductor_decompositionsr+   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rj   ri   r~   gluselect_scatterslice_scatterro   splitr   r   r   unbindbaddbmmrH   addcmul__foreach_addcmulr  _foreach_addcmul_addcdivaddcdiv__foreach_addcdiv_foreach_addcdiv_lerplerp_r%   r*   
OpOverloadr/   r   rC   rM   r   rX   	ShapeTypetupler`   rf   SymInttypesNumberrk   rw   r   r   r   r   r   r   rt   r   r   r   r   r@   r  r  detach_r  r  r  r  r"  	view_copyr$  r&  
StrideTyper+  r2  r0  r,  r   r8   r5  r9  r<  rD  	low_dtyperF  rB  r  rn  ry  _foreach_lerpr  r  r  r  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsrz   r  r  r  r  strr  r  r  r  r  r	  r  r  r  	Generatorr  r!  r'  re   r2   r1   <module>r     s       
 $ * * '   # + 
  F ( ( 7 .  F W $  T]t_#jj55

8S8SS ) S!yy~~		II	YY!!
yy55 ,,**,, 	

, 		,
 	, 	, 	, 			, 	, 	, 			, 			, 	, 	, 	,  	!," 	#,$ 	--%,& 	%%',( 	00),* 	11+,, 	$$-,. 	///,0 	""1,2 	  3,4 	5,6 	7,8 	9,: 	  ;,< 	=,> 	&&?,@ 	A,B 			C,D 	

E,F 	G,H 	I,J 	K,L 	M,N 	O,P 	  $$Q,R 	S,T 	55U,V 	++W,. ^ K,.J2IJ
 	,,44<<NNNN!!NNHHIIJJLLHHKKLLLLMM  LLMM  IIJJ=Q D..1L1LLM B n&8 9>	D!12	2>xB (2r6"223> ))*+0 05<< 0 05<< 0 ,0 ))*+1<<1#ll149LL1
\\1 ,1  6678\\  	
  \\ 9" ../
P<<
Poo
P LL4
P ,,
	
P
 

P 5<<u||34
P 0
P ::BBCD &*%)	LL 
		d	" 
		d	"	
 
 E %# &*%)	||				d	"	 
		d	"	 \\		 $ &	 $#ELL #U\\ #  %# $	
sU\\!
"	""	 	 \\		 %	 () !"M||M	M <<M LL	M ;;M \\M *M* ,,4456W
sU\\!
"W#YW W \\	W 7W  2234..<<. LL. S		.
 $s)O. 49_. DIo. . I. . d. 5<<u||34. 5.@ ,,-.G G GU\\ G /G
 
# %)
,,LL {{T! \\	  $> %
 %)  !#
,,#
,,# ,,# {{T!	#
 ++

# ;;# \\#  &#L 	" %)&
,,&LL& {{T!& \\	&  #&Z ))*+ :%,,:	: \\: ,:z %:U\\ :ell : &:& 
#
 (,	@||@||@ ;;$	@
 \\@ $@F ++,-   . DLL12u||   3 EJJ/0Iu|| IELL IU\\ I 1I EJJ/0Iu|| IELL IU\\ I 1I 		" 
,,	t  \\	 # 		" 
,,	t  \\	 # ))*+

,,
	
 
 	

 \\
 ,
 //01)
,,)
sU\\!
") \\) 2) --./$
,,$;;$ \\$ 0$
 
,,
 
5??E,,,-
  '
 !%"&"&).)>)>)3
,,)3e)3 ;;	)3
 LL4)3 LL4)3 )3 )3 &&)3 \\)3 ()3` !%"&).)>)>/c5<<'(/
,,/ ;;	/
 LL4/ &&/ / \\/@ /1EF2ELL 2C 2ELL 2 G2 02FG3U\\ 3S 3U\\ 3 H3  1 1 9 9:<PQTu|| T3 T# T%,, T RT  1 1 ; ;<>RSV
,,V V(+V7:V
\\V TV ,,-5
5
sU\\!
"5 5 \\	5 .5 	EEMMN !%<<LL ,,
 \\	 O 
;;CCD<< ll LL	
 ,, || ,, || LL  \\ E4 		++EEF
Au|| 
A 
A G
A --./  ||
,,  	
  \\  0@ **112
%
ell#
 KK
 
%,,	
 3
 **556
%
ell#
 %++$$%
 
%,,	
 7
 ''(<(<(E(EF../<<LL ,,
 ,,%	
 $  !&  5<<u||34 0 G< 6T#xS'9"9: 6 6	T#xS'9"9: 	 ++,
,,
,, LL \\	 -$ ,;;BBC?<<?? ? 
	?
 ;;? 5<<%&? D? !
 	
)
,,
)<<
) LL
) 	
)
 \\
) "
) 		"
 	
,,<< LL 	
 \\ # 33;;<##LL# 
# 	#
 \\#  =#( ))* 2
,,2	2 <<2 
	2
 2 2 \\2 +2j5||5c5 $s)Od"5 49_	5
 DIo5 5 
5 5<<%&5p 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 001||"&s)
5<<%& 2" ))001
 "&	\\	
++

	 		
 	 *	 LL4	 \\	 2	& --.
 
++

 	
  \\ / 889 %(,E
,,E<<E E 	E
 E %E 5<<%&E :E& ..556 #8LL8t8 \\8 784 !%)<<)LL) ,,
) #J	)
 3Z) Cj) ) \\)r2   