
    3ja                   ~   % S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKJr  S SKJrJrJrJr  S SKJrJrJrJrJr  S SKJr  S SKJr  S SKrS SKrS SKrS SK rS SK!J"s  J#r$  S SK%J&r&  S S	K'J(r(  S S
K)J*r*  S SK+J,r,  S SK-J.r.  S SK/J0r0  S SK1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<  S SK=J>r>J?r?  S SK@JArAJBrBJCrC  S SKDJErE  S SKFJGrGJHrHJIrIJJrJJKrK  SSKLJMrM  SSKNJOrOJPrPJQrQJRrR  SSKSJTrTJUrU  SSKQJVrVJWrWJXrXJYrYJZrZJ[r[J\r\J]r]J^r^J_r_J`r`JaraJbrbJcrcJdrdJere  SSK"JfrfJgrgJhrhJiriJjrjJkrkJlrlJmrmJnrnJoroJprpJqrq  SSKrJsrsJtrt  \(       a  SSKuJvrv  \" S5      rw\" S5      rx\E" SS/5      ry\R                  " \{5      r|0 r}S \~S!'   0 rS"\~S#'   0 rS$\~S%'   \E\GR                  GR                     " 5       r\R                  GR                  r\R                  GR
                  r\R                  GR                  r\E\GR                  GR                     " 5       r\E\GR                  GR                     " \GR                  GR                  /5      r\E\GR                  GR                     " 5       r0 rS&\~S''   \R                  GR                  rGSS( jr    GSS) jrGSS* jr    GSS+ jrGSS, jr    GSS- jr      GSS. jr\" \GR*                  \GR,                  \GR.                  \GR0                  \GR2                  \GR4                  \GR6                  \GR8                  \GR:                  \GR<                  \GR>                  \GR@                  \GRB                  \GRD                  /5        \GRF                  \GRH                  \GRJ                  \GRL                  \GRN                  \GRP                  \GRR                  \GRT                  \GRV                  \GRX                  \GRV                  \GRZ                  \GR\                  S/.rGSS0 jrGSS1 jrGSS2 jrS3S4.       GSS5 jjrS6 r      GSS7 jrGSS8 jr            GSS9 jr      GSS: jr          GSS; jrS3\7GRt                  S3\}4   GSS< jjrS= rGSS> jrS? r      GSS@ jrGSSA jrSB r GS       GSSD jjr\" \GR                  GR                  SSE9SF 5       r\" \GR                  SSE9GSSG j5       rS3SH.GSSI jjr\" \GR                  GR                  SSE9GSSJ j5       rS3S3SK.GSSL jjr\" \GR                  SSE9GSGSSM jj5       rSSC\7GRt                  S3SSS3S3S4	SN jr\n" SO\7GR                  SSP9  \" \GR                  SCSSQ9GSSR j5       rSS r\" 5           GSST jr\" \GR                  S3SSQ9SU 5       r\" \GR                  S3SSQ9SV 5       r\" \GR                  \GR                  \GR                  \GR                  \GR                  /5      SW 5       r\" \SX5      (       a  \" \GR                  5      " \5        \" \GR                  SSE9GSSY j5       r\" \GR                  SSE9GSSZ j5       r\" \GR                  /5      GSS[ j5       r\" \GR                  5      S\ 5       r\" \GR                  5      S] 5       r\" \GR                  5      S^ 5       r\" \GR                  5      S_ 5       r\" \GR                  GR                  5      S` 5       r\" \GR                  5      Sa 5       r\" \GR                  SSE9Sb 5       r\" \GR                  SSE9Sc 5       r\" \GR                  SSE9Sd 5       r\" \GR                  5      Se 5       r\" \GR                  SSE9\" \GR                  SSE9\" \GR                  SSE9GSSf j5       5       5       r\" \GR                  SSE9Sg 5       r\" \GR                  SSE9S S \
GR                  SSC4Sh j5       r\" \GR*                  SSE9GSSi j5       r\" \GR                  SSE9GSSj j5       r\" \GR,                  SSE9GSSk j5       rGSSl jr\" \GR                  SSE9                GSSm j5       rSn r\" \GR                  GR                  5      So 5       r\" \GR                  GR                  5      Sp 5       r\" \GR                  SSE9SSq.                 GSSr jj5       r\" \GR                  GR                  SSE9              GSSs j5       r\" \GR                  GR                  SSE9SSq.               GSSt jj5       r\" \GR                  GR                  SSE9              GSSu j5       Gr \" \GR                  GR                  SSE9SSq.               GSSv jj5       Gr\" \GR                  5      GSSw j5       Gr\" \GR                  SSE9GSGSSx jj5       Gr\" \GR                  SSE9GSGSSy jj5       Gr\" \GR
                  SSE9GSGSSz jj5       Gr\" \GR                  SSE9S{ 5       Gr\" \GR                  SSE9GSS| j5       Gr\" \GR                  SSE9GSS} j5       Gr\" \GR                  SSE9GSS~ j5       Gr	\" \GR                  SSE9S 5       Gr
\" \GR                  SSE9S 5       Gr\" \GR                  SSE9S 5       GrGSS jGr\" \GR                  5      GSS j5       GrGSS jGr\GR                   S 5       GrGSGSS jjGrGSGSS jjGrGSGSS jjGr    GSS jGrS Gr\" \R                  GR.                  GR0                  SSE9S 5       Gr\" \GR2                  SSE9S 5       Gr\" \GR4                  SSE9S 5       Gr\" \GR6                  GR8                  SSE9S 5       Gr\" \GR<                  5      S 5       Gr\GR>                  " S5      S 5       Gr S Gr!G\" \GRD                  GR                  5      Gr#G\" \GRD                  GRH                  5      Gr%G\" \GRL                  GR                  5      Gr'G\" \GRL                  GRH                  5      Gr(G\" \GRR                  5        G\" \GRT                  SCS9  G\" \GRV                  SCS9  G\" \GRX                  SCS9  G\" \R                  GRZ                  GR\                  GR                  5        G\" \R                  GRZ                  GR^                  GR                  5        G\" \R                  GRZ                  GR`                  GR                  5        G\" \R                  GRZ                  GRb                  GR                  5        \" \GRD                  5      S 5       Gr"\" \GRL                  5      S 5       Gr&\" \PGRd                  SSE9S 5       Gr3\" \PGRh                  SSE9GSS j5       Gr5\" \PGRl                  SSE9S 5       Gr7\" \PGRp                  SSE9S 5       Gr9GSS jGr:\" \PGRv                  SSE9S \GRR                  S.         GSS jj5       Gr<G\" \PGRz                  5        G\" \PGR|                  5        \" \PGRR                  SSE9S S.         GSS jj5       Gr?GSS jGr@GSS jGrA\" \GR                  GR                  SSE9S3S3SSS.             GSS jj5       GrB\" \GR                  GR                  \7GR                  SE9S3S3S.       GSS jj5       GrDS GrFS GrGS GrHS GrIS GrJS GrKS GrLS GrM\" \GR2                  GR                  G\M5        S GrNG\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  S3S9  G\" \GR                  GR                  S3S9  G\" \GR                  G\H5        G\" \GR                  S3S9  G\" \GR                  S3S9  \GR                  GR                  5       (       a  G\" \GR                  S3S9  \GR                  GR                  5       (       a  G\" \GR                  S3S9  G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  GR                  5        G\" \GR                  GR                  5        G\" \GR                  5        G\" \GR                  GR                  5        G\" \GR                  GR                  5        G\" \GR                  GR                  5        G\" \GR                  5        G\" \GR                  S3S9  G\" \GR                  G\G5        G\" \GR6                  G\M5        G\" \GR                  G\G5        G\" \GR                  G\H5        G\" \GR                  G\G5        G\" \GR                  G\H5        G\" \GR                  G\H5        G\" \GR                  G\H5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  G\G5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  G\H5        G\" \GR                  5        G\" \GR                  5        G\" \GR                   G\H5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR
                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                  5        G\" \GR                   5        G\" \GR"                  5        G\" \GR$                  5        G\" \GR&                  5        G\" \GR(                  5        G\" \GR*                  5        G\" \GR,                  GR.                  5        G\" \GR0                  5        G\" \GR2                  5        G\" \GR4                  5        G\" \GR6                  5        G\" \GR8                  5        G\" \GR:                  5        G\" \GR<                  5        G\" \GR>                  GR                  5        G\" \GR@                  GR                  S3S9  G\" \GRB                  G\G5        G\" \GRD                  GRF                  GRH                  5        G\" \GRD                  GRF                  GRJ                  5        G\" \GRD                  GRF                  GRL                  5        G\" \GRD                  GRF                  GRN                  5        G\" \GRP                  5        G\" \GRR                  5        G\" \GRT                  G\H5        G\" \GRV                  5        G\" \GRX                  5        G\" \GRZ                  5        G\" \GR\                  5        G\" \GR^                  5        G\" \GR`                  GRb                  5        G\" \GRd                  GR                  G\NS3S9  G\" \GRf                  GR                  G\NS3S9  G\" \GRh                  GR                  G\NS3S9  G\" \GRh                  GRj                  S3S9  G\" \GRl                  GR                  G\NS3S9  G\" \GRn                  GR                  G\NS3S9  G\" \GRp                  GR                  G\NS3S9  G\" \GRr                  GR                  G\NS3S9  G\" \GRt                  GR                  G\NS3S9  G\" \GRv                  GR                  G\NS3S9  G\" \GRx                  GR                  G\NS3S9  G\" \GRz                  GR                  G\N5        G\" \GRz                  GRj                  5        G\" \GR|                  GR                  G\N5        G\" \GR~                  GR                  G\N5        G\" \GR                  GR                  G\N5        G\" \GR                  5        G\" \GR                  GR                  SCS9  G\" \GR                  GR                  G\H5        \" \GR                  SSE9GSS j5       Gr\" \GR                  5      SS.S j5       GrS Gr\" \S5      (       a  \" \GR                  5      " G\5        \" \GR                  5      S 5       Gr\" \GR                  GR                  SSE9 GSSSSSS3S.S jj5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  SSE9GSS j5       GrS Gr\" \GR                  \GR                  \GR                  /5      SSSS3S.S j5       r\" \GR                  5      GSS j5       Gr\" \GR                  5      S 5       Gr\" \GR                  5      S 5       Gr\" \GR                  5      S 5       Gr\" \GR                  5       GSSSS.S jj5       GrS GrS GrS Gr\" \GR                  \GR                  /5      SSSSSSS.S j5       GrS GrS Gr\" \GR                  5      " G\" G\5      5      GrG\" G\" S5      5      GrG\" G\" S 5      5      GrS Gr\" \GR                  5      SSSSS.S j5       Gr\" \GR                  5      SSSSS.S j5       Gr\" \GR                  5      SSSSS.S j5       Gr\" \GR                  GR                  5      S 5       Gr\" \GR                  \GR                  /5      S 5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  SSE9GSS j5       GrS Gr GSS jGrS GrGSS jGr\" \GR                  SSE9S 5       Gr\" \GR                  SSE9S 5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  5      GSS j5       GrS GrS Gr\" \GR                  SSE9GSS j5       Gr\" \PGR                  SSE9GSS j5       GrGSS jGrG\" \GR                  GR                  S3S9GrG\" \GR                  GR                  S3S9Gr\" \GR                  SSE9S 5       Gr\" \GR                  SSE9S 5       Gr\S 5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  SSE9GSS j5       GrSSCS.       GSS jjGr\" \GR                  SSE9SS.GSS jj5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                  SSE9GSS j5       Gr\" \GR                   SSE9SCS.GSS jj5       Gr   GS     GSS jjGr\" \GR                  GR                  5      GSGSS jj5       Gr\" \GR                  GR                  5      GSGSS jj5       Gr\" \GR@                  GR                  5       GS   GSS jj5       r\" \GRB                  GR                  5       GS   GSS jj5       r\" \GR                  GR                  5         GS     GSS jj5       Gr\" \GR
                  GR                  5         GS     GSS jj5       GrS Gr\" \GR                  GR                  5      S 5       Gr        GSS jGr        GSS jGr	\" \GR                  SSE9GSS j5       Gr
GSS jGrGSS jGrGSS jGr GSS jGrSS.S jGrGS  GrSGS.GS jGrGS Gr\" \GR&                  SSE9 GSGS j5       Gr          GSGS jGr\" \GR*                  SSE9GS 5       GrGS Gr\" \GR8                  SSE9    GSGS j5       r\" \GR:                  SSE9    GSGS	 j5       rG\" \GR<                  GR                  S3S9Gr\" \GR<                  SSE9GS
 5       rGSGS jGrGS GrGS GrGS GrG\" \GR8                  GR                  S3S9Gr\" \GR8                  5      GS 5       GrG\" \GR<                  GR                  S3S9Gr\" \GR<                  5      GS 5       GrGS Gr \" \GRB                  5      GS 5       Gr!\" \GRD                  5      GS 5       Gr"GS Gr#\" \GRH                  GR                  5       GSGS j5       Gr$\" \GR.                  SSE9     GSGS j5       r\" \GRJ                  SSE9     GSGS j5       Gr%G\" \GRL                  GR                  S3S9G\" \GR.                  GR                  S3S9G\" \GRJ                  GR                  S3S9/Gr'GS Gr(G\" \GR0                  GR                  S3S9Gr)\" \GR0                  SSE9 GSGS j5       rG\" \GRT                  GR                  S3S9Gr+\" \GRT                  SSE9 GSGS j5       Gr*GS Gr,SGS.GS jGr-GSGSGS jjGr.GS Gr/\" \GR`                  5      GSSGS .GS! jj5       Gr0GS" Gr1GS# Gr2GS$ Gr3GS% Gr4\" \GRj                  \GRj                  /5      GSSS3GS&.GS' jj5       Gr6\" \GRn                  5      GSSS3GS&.GS( jj5       Gr7GS) Gr8\GS* 5       Gr9G\" \GRt                  GRv                  S3S9Gr<G\" \GRt                  GRz                  S3S9Gr>G\" \GRt                  GR~                  S3S9Gr@\" \GRt                  SCGS+9GS, 5       Gr:GSGS- jGrA\" \GR                  5      GS. 5       GrB\" \GR                  SSE9GSGS/ j5       GrC\GS0 5       GrD\GS1 5       GrE\GS2 5       GrF\" \GR                  SCGS+9GSGS3 j5       GrH\" \GR                  /SCGS+9GS4 5       GrIGSGS5 jGrJ\" \GR                  /SCGS+9GS6 5       GrK\" \GR                  \GR                  GR                  /SC\7GR                  SQ9GS7 5       GrG\" \GR                  \GR                  /SCGS+9GS8 5       GrM\" \GR                  \GR                  /5      GSSGS .GS9 jj5       GrOG\" \GR                  GR                  5      GrQG\" \GR                  GR                  5      GrSG\" \GR                  GR                  5      GrUG\" \GR                  GR                  5      GrWG\" \GR                  GR                  5      GrY\" \GR                  5      GSGS: j5       GrP\" \GR                  5      GSGS; j5       GrR\" \GR                  5      GS< 5       GrT\" \GR                  SSE9GSGS= j5       GrV\" \GR                  SSE9GSGS> j5       GrX\" \GR                  5      GSSGS .GS? jj5       GrZ\" \GR                  5      GSGS@ j5       Gr\\" \GR                  SSE9GSGSA j5       Gr^\" \GR                  SSE9GSGSB j5       Gr`\" \GR                  5      " G\." GSC5      5        \" \GR                  5      " G\." GSD5      5      Grc\" \GR                  5      " G\." GSE5      5      Gre\" \GR                  5      " G\." GSF\GRN                  GSG95      Grg\" \GR                  5      " G\." GSH\GRN                  GSG95      Gri\" \GR                  SCSCGSIGSJ9GrjG\" \GR                  GR                  S3S9Grm\" \GR                  GR                  SSE9SSS3GSK.GSL j5       Grn\" \GR                  GR                  SSE9GSGSM j5       GrkG\" \GR                  GR                  S3S9GrpG\" \GR                  GR                  S3S9GrrG\" \GR                  GR                  S3S9GrtG\" \GR                  GR                  S3S9GrvG\" \GR                  GR                  S3S9Grx\" \GR                  GR                  SSE9GSN 5       Gry\" \GR                  GR                  SSE9GSGSO j5       Grz\" \GR                  GR                  SSE9GSGSP j5       Gr{\" \GR                  GR                  SSE9GSGSQ j5       Gro\" \GR                  GR                  SSE9GSGSR j5       GrqGSGSS jGr|GSGST jGr}G\|" \GR                  5      Gr~G\}" \GR                  5      GrG\|" \GR                   5      GrG\|" \GR                  5      Gr\" \GR                  5      GrG\}" \GR                  5      GrG\}" \GR                  5      Gr\" \GR
                  5      Gr\" \GR                  SCGSU9Gr\" \GR                  SCGS+9SGSV.GSW j5       Gr\" \GR                  SCGS+9SGSV.GSX j5       Gr\" \GR                  GRz                  G\SCGSYGSZ9Gr\" \GR                  GRz                  G\SCGSYGSZ9GrG\}" \GR                  5        G\}" \GR                  5        \" \GR                  5      Gr\" \GR                   5      Gr\" \GR"                  5      Gr\" \GR$                  GS[GS\9Gr\" \GR&                  5      Gr\" \GR(                  5      Gr\" \GR*                  5      GrG\|" \GR,                  5        G\|" \GR.                  5      Gr\" \GR0                  \7GR                  SE9" G\5        G\|" \GR2                  5        G\|" \GR4                  5        G\|" \GR6                  5        G\}" \R                  5        \" \GR8                  SSC\GRZ                  GS]9Gr\" \GR:                  SSC\GRZ                  GS]9Gr\" \GR<                  SSC\GRZ                  GS]9Gr\" \GR>                  SSC\GRZ                  GS]9Gr\" \GR@                  5      Gr\" \GRB                  5      Gr\" \GRD                  5      " G\5        \" \GRF                  5      " G\5        \" \GRH                  5      Gr\" \GR                  5      GrG\|" \GRJ                  5      Gr\" \GRL                  5        \" \GRN                  GS^GS\9Gr\" \GR                  5        \" \GRP                  \GRZ                  GSG9  \" \GRR                  5      " G\5        \" \GRT                  \GRZ                  GSG9  \" \GRV                  \GRZ                  GSG9  \" \GRX                  \GRZ                  GSG9  \" \GRZ                  \GRZ                  GSG9Gr\" \GR\                  \GRZ                  GSG9  \" \GR^                  \GRZ                  GSG9  G\|" \GR`                  5        G\|" \GRb                  5        G\|" \GRd                  5        G\|" \GRf                  5        G\|" \GRh                  5        G\|" \GRj                  5        G\|" \GRl                  5        G\|" \GRn                  5        G\|" \GRp                  5        G\|" \GRr                  5        G\|" \GRt                  5        G\|" \GRv                  5        G\|" \GRx                  5        G\|" \GRz                  5        G\|" \GR|                  5        G\|" \GR~                  5        SGS_GKGJGrGJGr  GS` GrG\ HR  GrG\" \G\5       H  u  GrGrGr\" G\G\G\G\GSa9  M     G\" \G\5       H  u  GrGrGr\" G\G\G\G\GSa9  M     MT     \" \GR                  GR                  G\jSCGSU9Gr\" \GR                  GRz                  G\jSCGSU9Gr\" \GR                  GR                  G\jSCGSU9  \" \GR                  GR                  G\I5      Gr\" \GR                  GR                  G\I5        \" \GR                  GRz                  G\I5      Gr\" \GR                  GR                  G\5        \" \GR                  GRz                  G\5        \" \GR                  GR                  G\5        \" \GR                  GR                  G\5        \" \GR                  GRz                  G\:5        \" \GR                  GR                  G\:5        \" \GR                  GR                  G\:5        \" \GR                  GR                  G\G5      Gr\" \GR                  GR                  G\G5        \" \GR                  GRz                  G\G5      Gr\" \GR                  G\5        \" \GR                  G\~5        \" \GR                  GR                  G\5        \" \GR                  GRz                  G\5        \" \GR                  GR                  G\5        \" \GR                  GRz                  G\5        \" \GR                  GR                  G\5        \" \GR                  GRz                  G\5        \" \GR                  GR                  G\5        \" \GR                  GRz                  G\5        \" \GR                  G\5        \" \GR                  G\5        \" \GR                  G\5        \" \GR                  G\5      GrGSb GrG\" \GR                  GR                  \GR                  GR                  G\5        G\" \GR                  GRz                  \GR                  GRz                  G\5        G\" \GR                  GR                  \GR                  GR                  G\5        G\" \GR                  GRz                  \GR                  GRz                  G\5        G\" \GR                  GR                  \GR                  GR                  G\5        G\" \GR                  GRz                  \GR                  GRz                  G\5        G\" \GR                  GR                  \GR                  GR                  G\5        G\" \GR                  GRz                  \GR                  GRz                  G\5        G\" \GR                  GRz                  \GR                  GRz                  G\5        GSc GrG\" \GR                  G\j5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\I5        G\" \GR                  GR                  G\G5        G\" \GR                  GR                  G\H5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        G\" \GR                  G\5        \" \GR                  5      " G\5        \" \GR                  5      " G\5        \" \GR                  5      " G\5        \" \GR                  5      " G\5        \" \GR                  5      " G\5        G\" \GR                   \GR                  5        G\" \GR                  \GR                  5        G\" \GR                  \GR                  5        G\" \GR                  \GR                  5        G\" \GR                  \GR                  5        \" \GR
                  5      GSGSd j5       Gr\" \GR                  GR                  5      GSe 5       Gr\" \GR                  GR                  5      GSf 5       Gr\" \GR                  5      GSg 5       Gr	\>GR                  " 5        H  u  GrGr\" \?" G\5      5      " G\5        M      \" \GR                  5      GSh 5       Gr\" \GR<                  5      GSi 5       Gr\" \R                  GR                  GR                   5      GSj 5       Gr\" \R                  GR$                  GR&                  5      GSk 5       Gr\" \R                  GR                  GR(                  GR*                  5      GSl 5       Gr\" \R                  GR.                  GSm5      (       a;  \" \R                  GR.                  GR                  GR                  5      GSn 5       Gr\" \R                  GR                  GR2                  5      SS.GSo j5       GrS GSpGKGJGr  G\" G\5        \" \*5      GSq 5       Gr\" \R                  GR:                  GR<                  SSE9  GSGSr j5       Gr\" \R                  GR:                  GR>                  SSE9GSGSs j5       Gr\" \R                  GR:                  GR@                  SSE9" \GRB                  " G\SCGSt95        \" \R                  GR:                  GRD                  SSE9GSGSu j5       Gr"G\#" 5       Gr$GSGSv jGr%S GSwGK&GJ'Gr'  \" G\'SSE9GSx 5       Gr(\" \GR                  GRR                  SSE9SGSy.GSGSz jj5       Gr*\" \(SSE9    GSGS{ j5       Gr+\" \R                  GR                  GRX                  GR                  5      GS| 5       Gr,\" \R                  GR                  GRZ                  GR                  5      GS} 5       Gr-\" \R                  GR:                  GR\                  SSE9GS~ 5       Gr.SGSGK/GJ0Gr0GJ1Gr1  G\0" 5         G\1" 5         \" \PGRd                  SSE9GS 5       Gr2GS Gr3\" \PGRh                  SSE9GS 5       Gr5\" \GR                  GRl                  SSE9SCSGS.GS j5       Gr7SGSKNGJ8Gr8  \M" G\85        SGSKNGJ9Gr9  G\9GRt                  " 5         G\9GRv                  " 5         SGSKNGJ<Gr<  G\<GRz                  " 5         SGSKNGJ>Gr>  G\>GR~                  " 5         \GR                  GSGS j5       GrAg(      )annotationsN)defaultdict)Callable
CollectionIterableSequence)AnycastTYPE_CHECKING	TypeGuardTypeVar)	ParamSpec)patch)counters)associative_scan_op)triton_kernel_wrapper_mutation)FakeScriptObject)is_opaque_value)get_layout_constraint_tag)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)free_unbacked_symbolshas_free_unbacked_symbolsresolve_unbacked_bindings)
OrderedSet)CeilDivFloorDivIdentityModModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)BaseView	DtypeView
ExpandViewIndexingConstantIRNode	is_triton
MutableBoxOnlineSoftmaxReductionops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivdecode_device
is_dynamicis_gpuis_pointwise_useis_view,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rules#register_op_requires_libdevice_fp64sympy_productuse_scatter_fallback)opsV)ReductionType_T_Pztorchvision::roi_alignzaten::index_add2dict[Callable[..., Any] | str, Callable[..., Any]]	loweringsz/dict[torch._ops.OpOverload, Callable[..., Any]]user_loweringsz6dict[torch._ops.OpOverload, Callable[..., Any] | None]_maybe_layout_constraintsz2dict[torch._ops.OpOverload, torch._ops.OpOverload]inplaceable_foreach_opsc                     [         R                  R                  R                   H=  n U R                   H*  nUR                  S:X  a  UR
                  [        ;   a  M)      g   M?     g)Ncall_functionTF)rR   graphcurrent_nodeusersoptargetforeach_ops)nodeusers     R/home/wildlama/miniconda3/lib/python3.13/site-packages/torch/_inductor/lowering.pycur_node_has_non_foreach_usersrf      sI    $$**JJDGG.DKK;4N  +
     c                   [        [        5      nSn[        U 5       H  u  p4[        U[        5      (       d  SnU4n[        U6 (       + =(       d    [        R                  nS nU H4  n[        U[        5      (       d  M  UR                  R                  5       n  O   Uc   S5       eU(       a  Uu  nXU4   R                  X445        M     U$ )NFTz.foreach op should have at least one tensor arg)r   list	enumerate
isinstancer   rG   r/   #combo_kernel_foreach_dynamic_shapesrB   data
get_deviceappend)	arg_pairsoutunpack_argsiargsuse_foreachdevicets           re   group_foreach_argsrx      s     d
CKY'$))K7DD!!OV%O%O 	 A!Y''**,  !S#SS!GT[!"))1)4 (  Jrg   c                    [        U [        R                  R                  5      (       d  g[	        U SS9=n(       a  [        U5      $ U [        ;   a	  [        U    $ g)zHGet layout constraints. Returns None if there are no layout constraints.NF)with_default)rk   torch_ops
OpOverloadr   tag_to_layout_constraintrY   )fnmaybe_layout_tags     re   maybe_layout_constraintsr      sR    b%**//004ReLLL'(899	&&(,,rg   c                   U [         R                  R                  R                  :X  a  [        $ U [         R                  R                  R
                  :X  a  [        $ U [         R                  R                  R                  :X  a  [        $ U [         R                  R                  R                  :X  a  g [        SU  35      e)NzUnknown layout constraint tag: )r{   _CTagneeds_exact_stridesconstrain_to_fake_tensorsneeds_contiguous_stridesrequire_contiguous_stridesneeds_fixed_stride_orderconstrain_to_fx_stridesflexible_layoutAssertionError)tags    re   r~   r~      s     ehhll...((
ehhll333))
ehhll333&&
ehhll***
:3%@
AArg   c                .    U (       d  [        SU 35      eg )Nzinductor does not support NotImplementedErrorcondmsgs     re   
assert_nyir      s    !$>se"DEE rg   c                  ^  [        T [        [        [        [        45      (       a  T  Vs/ s H  n[        U5      PM     sn$ [        T [        R                  R                  5      (       a  [        R                  T 5        g [        T [        R                  R                  5      (       a-  [        R                  U 4S jT R                  5        5       5        g s  snf )Nc              3  <   >#    U  H  n[        TU5      v   M     g 7fN)getattr).0overloadr   s     re   	<genexpr>,add_needs_realized_inputs.<locals>.<genexpr>   s      %
2@hGB!!.   )rk   ri   settupler&   add_needs_realized_inputsr{   r|   r}   needs_realized_inputsaddOpOverloadPacketupdate	overloads)r   xs   ` re   r   r      s    
 "tS%455689b)!,b99"ejj++,,!!"%
 	 
B

33	4	4$$ %
24,,.%
 	
  :s   Cc                    [        U [        R                  R                  5      (       a*  U R	                  5        H  nU[
        [        X5      '   M     g U[
        U '   g r   )rk   r{   r|   r   r   rY   r   )r   
constraintr   s      re   add_layout_constraintr      sH     "ejj1122H?I%gb&;< ' )3!"%rg   )r   r.   r,                     	   
         c                p    [        U [        5      (       d  U $ U [        ;   d   SU  S35       e[        U    n U $ )Nzid z missing from DTYPE_ID_LOOKUP)rk   intDTYPE_ID_LOOKUPdtypes    re   decode_dtyper     s@    eS!!O#Os5'1N%OO#E"ELrg   c                   [        U [        5      (       a9  [        U R                  5       5      =(       d    [	        U R                  5       5      $ [        U [
        R                  5      (       a  U R                  SL $ [        U [        5      $ NT)	rk   rB   r   	get_dtyper   sympyExpr
is_integerr   r   s    re   is_integer_typer     s^    !Y.Q2B1;;=2QQ	Auzz	"	"||t##!S!!rg   c                ~    [        U [        5      (       a  [        U R                  5       5      $ [        U [        5      $ r   )rk   rB   r   r   boolr   s    re   is_boolean_typer   &  s-    !Y..!T""rg   F)return_compute_dtypec                t    SS jnU Vs/ s H
  oC" U5      PM     nn[        USU 06u  pgU(       a  U$ U$ s  snf )Nc                    [        U [        [        R                  45      (       a  U $ [	        U R                  5       5      n[        R                  " S/U-  U R                  5       S9$ )Nr.   r   )	rk   r    r   Basiclenget_sizer{   zerosr   )inpdims     re   construct_input+get_promoted_dtype.<locals>.construct_input2  sJ    cFEKK011Jclln%C;;sSy@@rg   type_promotion_kind)r   r	   returnr	   )r   )r   r   rt   r   arginpscompute_dtyperesult_dtypes           re   get_promoted_dtyper   -  sO    
A -11DSOC DD1"4	##6#M 1=BlB	 2s   5c                R   [        U [        [        45      (       d  U /n O[        U 5      n [        U 5       Hm  n[        U[        R                  R
                  5      (       d  M.  UR                  5        H+  n[        X5      nU[        ;  d  M  U R                  U5        M-     Mo     U $ r   )
rk   ri   r   r{   r|   r   r   r   rW   ro   )aten_fnr   r   other_fns       re   get_overloadsr   A  s}    ge}--)w-7mb%**5566LLN"209,NN8, +  Nrg   c                    [        U [        R                  R                  5      (       a  XR                  ;   $ [        U [        R                  R
                  5      (       a  XR                  5       ;   $ gNF)rk   r{   r|   r   _qualified_op_namer}   name)r`   	namespaces     re   in_namespacer   Q  sS     "ejj11221111	B

--	.	.GGI%%rg   c           	        [        U R                  [        R                  5      (       a  [	        U R                  5       5      (       a  U $ [        R                  R                  R                  U R                  5       5      nU R                  5       nUby  UR                  S:X  ai  X1:w  ad  [        U5      S:X  d  [        U5      S:X  aF  US   S:X  a=  [        [        R                  " [        R                  R!                  XS5      5      5      $ U $ )z:
Copy cpu scalar if doesn't not match with given `device`
cpur   r.   F)rk   rm   r1   ReinterpretViewr$   r   rR   r]   sizevarsguarding_hints_or_throwrn   typer   rB   
StorageBox
DeviceCopycreate)r   rv   size
cur_devices       re   maybe_copy_cpu_scalarr   [  s     affb00115N	

6 6 7733AJJLADJOOu$ Y!^D	Q47a<r}}';';A5'QRSSHrg   c           	     ~  ^ ^^^ [        T 5       VVs/ s H  u  pV[        U[        5      (       d  M  UPM      nnnTR                  5        VV	s/ s H  u  p[        U	[        5      (       d  M  UPM      n
nn	U(       d  U
(       d  T T4$ U(       d  U(       Ga:  U(       a  [        R
                  mO{T  Vs/ s H=  n[        U[        [        R                  45      (       d  [        US5      (       d  M;  UPM?     nnUR                  S TR                  5        5       5        [        USU06mU(       a  T US      OTU
S      R                  5       mU H  n[        T U   T5      T U'   M     U
 H  n[        TU   T5      TU'   M     S	UU4S jjnT  Vs/ s H
  o" U5      PM     snm TR                  5        VV	s0 s H  u  pX" U	5      _M     sn	nmU(       GaF  [        [!        ["        R$                  " U 4S jU 5       U4S jU
 5       5      5      6 n[!        US   R'                  5       5      n[)        X~S[+        U5       5       H
  u  pVUT U'   M     [)        X[+        U5      S 5       H
  u  pUTU'   M     [-        [+        T 5      5       HD  n[        T U   [.        R0                  5      (       d  M'  [2        R4                  " T U   U5      T U'   MF     T HD  n[        TU   [.        R0                  5      (       d  M'  [2        R4                  " TU   U5      TU'   MF     T T4$ s  snnf s  sn	nf s  snf s  snf s  sn	nf )
z:
Transforms arguments for broadcasting and type promotion
r   c              3  L   #    U  H  n[        US 5      (       d  M  Uv   M     g7f)r   N)hasattr)r   as     re   r   !transform_args.<locals>.<genexpr>  s     !T_7@S!!_s   $	$r   r   c                   > [        U [        5      (       a  [        U T5      $ [        U [        R                  5      (       a   [        R                  " U R
                  TTS9$ U $ )Nvaluer   rv   )rk   rB   to_dtyper1   Constantr   )r   rv   r   s    re   promotetransform_args.<locals>.promote  sJ    #y))U++C--{{%OO
rg   c              3  .   >#    U  H
  nTU   v   M     g 7fr    )r   rs   rt   s     re   r   r     s     3lT!Wl   c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   )r   kkwargss     re   r   r     s     71VAYr   N)r   r	   r   r	   )rj   rk   rB   itemsr{   r   r    r   r   r   extendvaluesr   rn   r   broadcast_tensorsri   	itertoolschainr   zipr   ranger1   r   r7   r   )rt   r   	broadcastr   convert_input_to_boolrs   r   args_indicesr   vkwargs_indicesr   promoting_argsr   broadcastedr   rv   r   s   ``              @@re   transform_argsr  o  s    #,D/N/$!Z95MA/LN$*LLNONDAjI6NaNNOV|3 JJE
 Aa&%++!67771g;N    !!!TV]]_!TT&$7E &2Da!vnQ>O7P
*, 	 A+DGV<DG   A-fQi@F1I  	 	 %))Dq
D),2LLN;NDA!WQZ-N;'3l377
 KN++-.2EC4E&FGDADG HC4E4G(HIDAF1I J s4y!A$q'2;;//$++DGT:Q " A&)R[[11&--fQi>q	  <I OO> *;s.   L#L#L)-L)3:L/1L/L49L9c                   ^ [         R                  " T5      SU4S jj5       n[        U 5      n[        R	                  U5        [
        R	                  [        R                  X25      5        U$ )a  
Add a foreach lowering to lowerings dict.

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                 .   > T" U 0 UD6n[        U5        U$ r   )rC   )rt   r   rq   	decomp_fns      re   wrapped+_register_foreach_lowering.<locals>.wrapped  s    ((C
rg   )rt   r	   r   r	   r   r	   )	functoolswrapsr   rb   r   rW   dictfromkeys)r   r  r  aten_fnss    `  re   _register_foreach_loweringr    sV     __Y  
 W%Hx T]]856Nrg   c                   ^ ^^^^ [         R                  " T5      U UUUU4S j5       n[        T 5      m UR                  [        R                  T U5      5        U$ )a  
Add a lowering to lowerings dict

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                 v  > [        U 5      n [        U5      nSn[        U 5      S:X  a.  [        U S   [         [        45      (       a  Sn[        U S   5      n [        S T 5       5      (       d  [        S U 5       5      (       a   S5       e[        XTTT5      u  pU(       a  U /n T" U 0 UD6n[        U5        U$ )NFr.   r   Tc              3  X   #    U  H   o[         ;   =(       d    [        US 5      v   M"     g7f)_c10d_functionalN)	fallbacksr   )r   r   s     re   r   6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s&      
PW"9_DR1C DDPWs   (*c              3  *   #    U  H	  oS :H  v   M     g7f)rq   Nr   r   r   s     re   r   r    s     6v!:v   zout= ops aren't yet supported)	ri   r  r   rk   r   allanyr  rC   )	rt   r   unpackedrq   r   r  r  r  r   s	       re   r  #_register_lowering.<locals>.wrapped  s    t*!%ft9>ja4-@@HQ=D 
PW
 
 
 6v666W8WW6%)%8:O
 6D((C
rg   )r  r  r   r   r  r  )r   r  r  r   r  lowering_dictr  s   `````  re   _register_loweringr(    sO    & __Y   6 G$Gw89Nrg   c           	     <    [         R                  " [        U UUUUS9$ )z#
Shim to support decorator syntax.
)r  r   r  r'  )r  partialr(  )r   r  r   r  r'  s        re   register_loweringr+    s)     /3# rg   c                >   [        U5      nU (       a  X:X  a  U$ / n[        R                  " [        U 5      [        U5      [        R
                  R                  S9 GH*  u  p4[        R                  R                  R                  U5      (       a  UR                  U5        MG  [        R                  R                  R                  U5      (       a  UR                  U5        M  [        R                  R                  R                  X45        [        [        R                  " U5      R                  5      [        [        R                  " U5      R                  5      :  a  UR                  U5        GM  UR                  U5        GM-     [        [        U5      5      $ )z
Broadcasting logic based on symbolic shapes.

We give the shapes 0 and 1 concrete values, while all other shapes
are symbolic sympy formulas.
)	fillvalue)r   r  zip_longestreversedr   SOnerR   r]   r   is_size_one_or_falsero   check_equalsr   expandfree_symbols)r   boutputr   ys        re   broadcast_symbolic_shapesr9  %  s    	aAF%%hqk8A;%''++V770033MM!WW22155MM!GG))!/5<<?//03u||A7S7S3TTa a  W &!""rg   c                  ^^	 Ub
  Ub   S5       eUc  Uc  [         R                  n[        S U  5       5      (       d  U $ [        S U  5       5      (       a2  U=(       d    [	        U SU06mU4S jnU  Vs/ s H
  oC" U5      PM     sn$ [        S U  5       5      nUR                  5       m	U[        R                  :X  a+  T	[        R                  [        R                  4;   a  U	4S jnOS n/ nU  GH  n[        U[        [        45      (       ag  UR                  [        R                   " ["        R$                  " U" U5      T	UR'                  5       S	9[)        UR+                  5       5      5      5        M  [        U[,        R.                  5      (       aV  UR                  [        R                   " [1        UT	UR'                  5       S
9[)        UR+                  5       5      5      5        M  UR                  U5        GM     U$ s  snf )NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3  l   #    U  H*  n[        U[        R                  [        [        45      v   M,     g 7fr   )rk   r   r   r   floatr!  s     re   r   $promote_constants.<locals>.<genexpr>G  s%     HAz!ekk3677   24c              3  l   #    U  H*  n[        U[        [        [        R                  45      v   M,     g 7fr   )rk   r   r<  r   r   r!  s     re   r   r=  I  s%     
DV:a#uekk233Vr>  r   c                   > [        U [        R                  5      (       a  [        R                  " U T[        S 5      S9$ [        R                  " U T[        S 5      S9$ )Nindexr   rv   r   )rk   r   r   r1   r8   rF   r   r   r   s    re   
const_func%promote_constants.<locals>.const_funcP  sM    !U[[))**5t1D  {{%d@STTrg   c              3  ~   #    U  H3  n[        U[        [        [        R                  45      (       d  M/  Uv   M5     g 7fr   )rk   rB   r7   r1   r   r!  s     re   r   r=  Y  s&     WA:a)Z1U#Vaas   .=	=c                J   > [         R                  " U TS9R                  5       $ Nr   )r{   tensoritem)r
  tensor_dtypes    re   <lambda>#promote_constants.<locals>.<lambda>a  s    %,,q"E"J"J"Lrg   c                    U $ r   r   )r
  s    re   rL  rM  c  s    !rg   r   rA  )r   DEFAULTr$  r#  r   nextr   r{   r   bfloat16float16rk   r   r<  ro   r7   r   r1   r   get_device_or_errorri   r   r   r   r8   )
inputsoverride_return_dtyper   rD  r   ex_round_scalarrq   r   rK  s
           @@re   promote_constantsrX  ?  s    (,?,G OG $)<)D=EEHHHH

DV
DDD% 
);*
 !4*
	U (..v!
1v..	WW	WB<<>L 

*|@ 0 M#
Ca#u&&JJ!!KK+A.*!557
 '	 5;;''JJ!!$|B<R<R<T '	 JJqM- 0 JM /s   4Hc                   ^^^^ [        U U[        R                  S9mU R                  5       mUR                  5       mUUUU4S jn[        R
                  " U R                  5       TUU R                  5       S9$ )zBCompute a + alpha * b using FMA for CUDA floating-point precision.r   c                   > T" U 5      nT" U 5      n[        T[        R                  5      (       a  [        R                  " TT5      nO[        R
                  " TT5      n[        R                  " X#U5      $ r   )rk   r   r   rQ   
index_exprconstantfma)idxa_valb_val
alpha_expra_loaderalphab_loaderr   s       re   inner_fn%_add_with_alpha_fma.<locals>.inner_fn  sW    eU[[))u5JeU3Jwwu%00rg   rv   r   rf  ranges)r   r   rO  make_loaderr?   r   rn   r   )r   r6  rd  rf  rc  re  r   s     ` @@@re   _add_with_alpha_fmark    sk    		;CCE
 }}H}}H1 1 ||~zz|	 rg   c           	     8   ^ ^^^^^^ SS.SUU UUUUU4S jjjnU$ )z_Wraps a pointwise fn and returns a function representing the pointwise in
the define-by-run IR.N)rd  c           	     `  >^^^	^
^^^ Tb%  [        S T 5       5      (       a  T(       a   eT" T6 $ [        TT5      mT(       a  U b  U S:w  a  T(       a  [        TS   [        5      (       az  TS   R	                  5       nTS   R                  5       R                  (       aE  [        R                  R                  (       d&  Ub#  UR                  S:X  a  [        TS   TS   U 5      $ [        T5      m[        TS   U 5      TS'   OU b   eT Vs/ s H  o3R                  5       PM     snmTS   R                  5       mT=(       d    TS   R                  5       mTSS   Hh  n[        U[         R"                  5      (       a  M$  [%        T5      [%        UR                  5       5      :X  a  ML   ST ST SUR                  5        35       e   [        R&                  [        R(                  4m[*        R,                  S L=(       a    [/        [*        R,                  SS 5      S L=(       aa    [*        R,                  R0                  R2                  S L=(       a4    [*        R,                  R0                  R2                  R5                  S	S
5      m
T
=(       a    TT;   m	UU	U
UUUUUU4	S jnT(       d_  S nT H=  n[7        UR	                  5       R                  5      (       d  M-  UR	                  5       n  O   U(       d  TS   R	                  5       nT=(       d    Wn[8        R:                  " UTUTS9$ s  snf )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   rk   r9   r:   r   r   s     re   r   0make_pointwise.<locals>.inner.<locals>.<genexpr>  s&      /
BH3JsF#6	#6&   /1r.   r   cudazndim mismatch  r^   low_precision_pointwise_barrierFc                &  >	 [        U 5      [        T5      :X  d   SU  ST 35       eT[        R                  :X  a  Tb  T" T Vs/ s H
  o" U 5      PM     sn6 $ / n[        T5       Hi  u  p1U" U 5      nTU   R	                  5       nT	(       a1  UT;   a+  [
        R                  " XESS9n[
        R                  " Xe5      nUR                  U5        Mk     T
" U6 nT(       a-  [
        R                  " UTSS9n[
        R                  " UT5      $ U$ s  snf )Nzwrong ndim ru  F)use_compute_types)r   r{   r   rj   r   rQ   r   ro   )rB  loadinputs_loaded	inp_indexrq   	inp_dtypedowncastr   emulate_output_castemulate_precision_castsr   rT  loaders	low_pr_fpoverride_fn_when_input_boolri  s          re   rf  /make_pointwise.<locals>.inner.<locals>.inner_fn  s   u:V,LE7!F8.LL,

"'B'N2W4UWTT%[W4UVV "'0'9OIu+C &y 1 ; ; =I.9	3I#&<<RW#X!ll8?!((- (: -(&  #||C%PH<<%88
# 5Vs   Drh  )r$  rX  rk   r9   rn   r   is_floating_pointr{   versionhipr   rk  ri   mulrj  r   r1   BaseConstantr   rQ  rR  rR   r]   r   r^   metagetrH   r?   r   )rd  rT  
inp_devicer   otherrf  rv   rs   r   r~  r  r  r  ri  allow_alphar   override_devicer  rU  triton_fallbackuse_fma_for_alphas    `      @@@@@@re   innermake_pointwise.<locals>.inner  s   &3 /
BH/
 ,
 ,
 #"?"F++"6+@A UaZ %F1Iv)F)F!'!5!5!7Jq	++-?? % 1 1&2&OOv526!9fQiOO f U3r
= =,23Fq==?F3##%%>)<)<)>ABZEeR__55V I : At1VHAenn.>-?@A    ^^U]]3	GG4 X6dBX$$))5X $$))--.OQVW	 	  6L%9:L	 	, F!,,.--..\\^F  --/ !*F	
 	
o 4s   L+)rT  rB   r   )r   rU  r  r  r  r  r  r  s   ``````` re   make_pointwiser    s"     )- X
 X
 X
t Lrg   c                *   ^ ^^ SSS.SUU U4S jjjnU$ )Nr.   )rd  r   c                  >^	 [        U5      nTS:X  a6  U(       a/  [        US   [         [        45      (       d  UR                  5       m	OTS:X  a  Um	OU m	[	        [
        R                  R                  R                  5      S:H  =(       d<    [
        R                  R                  R                  [        ;   =(       d
    [        5       nS nU H"  n[        U[         [        45      (       d  M   Un  O   Uc   S5       e/ nU HO  n[        U[         [        45      (       d   UR                  U/[	        U5      -  5        M>  UR                  U5        MQ     [        [        U6 5      nU
UUU	4S jn[        U[	        U5      X5      $ )Nr   rt  r   z1at least one input must be a list to a foreach opc                0   > T(       a
  T" U 0 TT0D6$ T" U 6 $ r   r   )rt   r  pw_fnscalar_kwarg
scalar_vals    re   apply_fn7make_foreach_pointwise.<locals>.inner.<locals>.apply_fn2  s%    dA|Z&@AAd|#rg   )ri   rk   r   popr   rR   r]   r^   r_   ra   inplace_foreach_opsrf   ro   rx   r  foreach_group_loop)rd  r   rT  realize_outputsa_list_inputinputbroadcast_inputsgroupsr  r  r  r  r  s            @re   r  %make_foreach_pointwise.<locals>.inner  s[   
 fG#vbzD%=99  JW$JJ $$**+q0 0ww##**.AA0-/ 	 E%$//$  ' 	
?	
'
 EedE]33 ''#l2C(CD !''.  $C)9$:;	$ 	$ "&#l*;XWWrg   )rT  zlist[list[TensorBox]]r   )r  r  r  r  s   ``` re   make_foreach_pointwiser    s    45Q 2X 2Xh Lrg   c                   S/U-  nU R                  5        H  u  u  pVn/ nU H  u  pU" U
5      nXU	'   [        R                  R                  U[        R
                  5      (       d  MF  U(       d  MO  U(       d  MX  UR                  5         UR                  UR                  5       5        M     U(       d  M  [        R                  R                  U5        M     [        S U 5       5      (       d   eU$ )aE  
Common loop over grouped foreach arguments.

Args:
    groups: Result of group_foreach_args - dict mapping (device, use_foreach) to groups
    num_outputs: Number of outputs to produce
    apply_fn: Function to apply to each set of args, returns the output
    realize_outputs: Whether to realize outputs for foreach fusion
Nc              3  (   #    U  H  oS Lv   M
     g 7fr   r   r!  s     re   r   %foreach_group_loop.<locals>.<genexpr>Y       .g}g   )r   rR   r]   has_featureBackendFeatureFOREACHrealizero   get_operation_nameregister_operation_listr#  )r  num_outputsr  r  outputsrv   ru   groupoperation_list
output_indrt   r7  s               re   r  r  =  s     f{"G(.$u$& %Jd^F"(J ##FN,B,BCCK#O %%f&?&?&AB !& >GG++N; )7" .g.....Nrg   Tc                   ^^^ U R                  5       mTT:X  a  U(       a  [        U 5      $ U $ UUU4S jn[        UTS9" U 5      $ )Nc                   > [         R                  " U TTTS9n[        R                  [        R                  4nT(       d  TU;   a  [         R                  " UT5      nU$ )N)	src_dtyperx  )rQ   r   r{   rQ  rR  )r   resultr  r   r  rx  s      re   	_to_dtypeto_dtype.<locals>._to_dtyped  sR    /	
 ^^U]]3	 Ui%7 \\&%0Frg   rU  )r   cloner  )r   r   copyrx  r  r  s    ` ` @re   r   r   ]  sB     IEuQx&Q& )5A!DDrg   rZ  c                   SSK Jn  UnU R                  nU" U[        R                  S9n[        R
                  " U5         UR                  " U6   SSS5        UR                  nU(       d   e[        U5      nS/[        U5      -  n	UR                  5        H  u  u  pn/ nU Hw  u  nnXU'   [        R                  R                  U
[        R                  5      (       d  M?  U(       d  MH  UR                  5         UR                  UR!                  5       5        My     U(       d  M  [        R                  R#                  U5        M     [%        S U	 5       5      (       d   eU	$ ! , (       d  f       GN= f)a)  
This lowers an invocation of foreach_map
The way this works is that an arbitrary N-arg func is provided by the user, looped over by the
polyfill with the same semantics as a foreach op (a loop applying an n-ary function to n args)
and then traced into a subgraph by dynamo.
This code allows us to inline the subgraph into the main graph lowering using the PontwiseSubgraphLowering.
The graph outputs represent the vertically fused sequence of ops, and then register_operation_list
below registers the buffers as horizontally fuseable in the scheduler.
r.   )PointwiseSubgraphLowering)root_graph_loweringNc              3  (   #    U  H  oS Lv   M
     g 7fr   r   r!  s     re   r   _foreach_map.<locals>.<genexpr>  r  r  )subgraph_loweringr  graph_modulerR   r]   set_graph_handlerrungraph_outputsrx   r   r   r  r  r  r  ro   r  r  r#  )subgraphrt   r   r  rT  gmpw_subgraphsub_outputsr  r  rv   ru   r  r  r  r7  s                   re   _foreach_mapr  v  s2    =F			B+BAGGLK	
		[	)  
* ++K;,Ffs;''G(.$u$& 
"(Jww""6>+A+ABB{{ %%f&?&?&AB  >GG++N; )7 .g.....N3 
*	)s    E%%
E4c                   UR                   (       d  U R                  5       R                   (       ag  U R                  5       (       a*  [        XS9n[        R
                  R                  X 5        U$ [        [        R                  R                  SS9" X5      $ U R                  5       n[        R                  [        R                  4n[        R                  =(       a    X4;   =(       d    X;   (       + n[!        XSUS9$ )Nr   Fadd_to_fallback_setT)r  rx  )
is_complexr   r   
empty_liker1   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr{   rQ  rR  r/   r  r   )r   r   dstr  r  rx  s         re   _convert_element_typer    s    1;;=33::<< Q,C""))#1J#**22  I/I&& 	;#9u'9 A4;LMMrg   r  c                  U R                  5       nX1:X  a  U(       a  [        U 5      $ U $ S nU" U5      nU" U5      nXV:w  a)  [        [        R                  R
                  5      " X5      $ [        [        R                  " X5      5      $ )Nc                    U R                   (       a   [        R                  " U 5      R                  $ U [        R                  :X  a  g[        R
                  " U 5      R                  $ )Nr   )r  r{   finfobitsr   iinfor   s    re   _get_primitive_bitwidth1to_dtype_bitcast.<locals>._get_primitive_bitwidth  sD    "";;u%***ejj ;;u%***rg   )	r   r  r  atenviewr   rB   r6   r   )r   r   r  x_dtyper  src_bitsdst_bitss          re   to_dtype_bitcastr    ss    kkmGuQx&Q&+ 'w/H&u-H		0::))!344rg   c                8   UR                   (       d  U R                  5       R                   (       a`  [        R                  " [        R
                  R                  [        R                  R                  R                  R                  X5      5      $ [        X5      $ r   )r  r   rB   r   r1   ComplexViewr{   rQ   r  r  r   r  rC  s     re   _view_dtyper    sa    1;;=33NN!!%)).."5"5";";QF
 	
 A%%rg   r  non_blockingc                   [        U5      nU R                  5       U:X  a  U(       a  [        U 5      $ U $ [        R                  " [
        R                  R	                  XU5      5      $ r   )rF   rn   r  rB   r   r1   r   )r   rv   r  r  s       re   	to_devicer    sM    6"F||~uQx&Q&BMM00LIJJrg   c                    [        XSUS9$ )NTr  )r  )r   rv   r  s      re   _device_putr    s    QTEErg   c
           	     "   U=(       d    U R                   n[        U5      n
[        XU5        Ub  [        U5      n[        U
UUUUU	S9n
[	        U UUUS9" U
5      n
[        [        U5      (       a  [	        [        [        U5      SUS9" U
5        U
$ )z3A pointwise function that maps ops.{name} to inputsN)rU  r  r  r  r  )r  r   r  )r   r  )__name__r=   rM   r  r+  r   r  r   )r   r   r  r   r  rU  r  r  r  r  r   s              re   register_pointwiser    s     #7##D	T	B'#8 #.&12M&N#	
3$?+'
B 
/3	

 	

B udE4  $"7	
 		
 Irg   ldexp)r   rU  )r  r   c                  ^^	 [        S5      mU R                  5       nUR                  5       nUR                  nUR                  (       + =(       a    U[        R                  :g  nU(       a  U(       a  U4S jn[        U5      " X5      $ [        U 5      (       a  [        R                  OUm	U	4S jn[        UT	S9" X5      $ )Nr  c                   > T" X5      $ r   r   )r   nldexp_fns     re   compute_ldexp%ldexp_lowering.<locals>.compute_ldexp'  s    A>!rg   c                   > [         R                  " UT5      n[         R                  " ST5      n[         R                  " X25      n[         R                  " X5      $ )Ng       @)rQ   r   r]  powr  )r   r  
n_out_typetwo
pow_result	out_dtypes        re   compute_fallback(ldexp_lowering.<locals>.compute_fallback/  s@    a3J,,sI.C1J771))rg   r  )r=   r   r  r{   r   r  r   float32)
r   r  r  n_dtype
x_is_floatn_is_intr  r  r  r   s
           @@re   ldexp_loweringr    s    7#HkkmGkkmG**J,,,FEJJ1FHh	" m,Q22 &5Q%7%7EMMW		* "+
  	rg   c                 2  ^^ Sn [        S5      mU4S jnU4S jn[        U5      [        U[        R                  S9/mU4S jn[	        [
        R                  5      " U5      n[        [        U 5      (       a  [	        [        [        U 5      SS9" U5        U$ )z2A pointwise function that maps ops.frexp to inputsfrexpc                    > T" U 0 UD6S   $ Nr   r   rt   r   r	  s     re   frexp0register_frexp.<locals>.frexp0@      d%f%a((rg   c                    > T" U 0 UD6S   $ Nr.   r   r  s     re   frexp1register_frexp.<locals>.frexp1C  r  rg   r  c                 0   > TS   " U 0 UD6TS   " U 0 UD64$ Nr   r.   r   )rt   r   pw_fnss     re   r   register_frexp.<locals>.fnK  s*    ay$)&)6!9d+Ef+EEErg   NrZ  )
r=   r  r{   int32r+  r  r	  r   r  r   )r   r  r  r   r	  r  s       @@re   register_frexpr  ;  s    D E)) 	vvU[[AF
F 





B udE4  $	
 	 Irg   c                0    [        XUS9n[        X5      nU$ )Nr  r  )r  r  )r   pointwise_lowering_fnr  r  r   s        re   register_foreach_pointwiser  ]  s$     
 \
B 
$G	0BIrg   c           
     *   S n[        U[        [        45      (       a  [        U5      " U5      n[        U[        [        45      (       a  [        U5      " U5      nXU/n[	        US   US   [
        R                  S9n[        U5       VVs/ s H  u  pg[        U[        5      (       d  M  UPM      nnn[        U[        U Vs/ s H  odU   PM	     sn6 5       H	  u  pgXtU'   M     [        [        U5      5       H]  n[        XF   [        R                  5      (       d  M&  [        R                   " XF   [#        XHS      R%                  5       5      5      XF'   M_     ['        X5S9" US   [)        US   U5      [)        US   U5      5      $ s  snnf s  snf )Nc                 (    [         R                  " U 6 $ r   )rQ   wherert   s    re   r   where.<locals>.fnl  s    yy$rg   r.   r,   rZ  r   r  )rk   r<  r   constant_liker   r   rO  rj   rB   r  r  r  r   r1   r   r7   r   ri   r   r  r   )	r   r   r6  r   rt   r   rs   r   indicess	            re   r   r   j  sX     !eS\""!Q!eS\""!QQ<DQa.M.U.UE 't_I_TQ
1i0Hq_GIG.'0J'Qa'0JKLQ M3t9dgr{{++ ''d1:6F6O6O6Q1RSDG  ":Q$q'5)8DGU+C  J0Js   F
,F
Fc                    [        U 5      S:X  a+  [        U S   [        [        45      (       a  [	        U S   6 $ U $ [
        R                  " [        S U  5       S5      n/ nU  Hv  n[        UR                  5       5      =oA:X  a  OC[        U5      [        U5      :w  d   [        S [        XA5       5       5      (       a  [        X15      nUR                  U5        Mx     U$ )Nr.   r   c              3  @   #    U  H  oR                  5       v   M     g 7fr   )r   r!  s     re   r   $broadcast_tensors.<locals>.<genexpr>  s     #A&QJJLL&   r   c              3     #    U  HZ  u  p[         R                  R                  R                  U5      [         R                  R                  R                  U5      :g  v   M\     g 7fr   )rR   r]   r   r2  r   r   r6  s      re   r   r'    sM      .
 + GG11!4ww44Q78*s   A"A$)r   rk   ri   r   r  r  reducer9  r   r$  r  r4  ro   )rT  ra   r  r   sizess        re   r  r    s    
6{afQi$//$fQi00(//!#A&#A2 F G1::<((E3Z3v;&# .
 E*.
 +
 +

 q!Aq  Nrg   c                    U $ r   r   r   s    re   nopr.    s    Hrg   
lift_freshc                   [        U [        5      (       d   eUc)  [        [        R                  " U R                  5      5      $ [        U[
        [        R                  45      (       a)  [        R                  R                  R                  U5      O[        S U 5       5      n[        [        U R                  5       5      U5      n[!        [        U[        5      (       d  U4OU5      n/ n[#        U R                  5       5       H`  u  pEXB;   aE  [        R                  R                  R%                  [        R&                  " US5      5      (       a  MO  UR)                  U5        Mb     X0R                  5       :w  a  [+        X5      $ U $ )Nc              3  t   #    U  H.  n[         R                  R                  R                  U5      v   M0     g 7fr   )rR   r]   r   	guard_intr   ds     re   r   squeeze.<locals>.<genexpr>  s(     >#Q177##--a00#s   68r.   )rk   rB   rA   r   rm   r   r   r   rR   r]   r   r2  r   r   r   r   r&   rj   guard_or_falseEqro   r  )r   r   dims	new_shaper4  ss         re   squeezer;    s   a####
{++AFF344 cC,-- 	
""3'>#>> 
 C

-s
3CJsE$:$:sfDDI!**,'	agg..==ehhq!nMMQ (
 "+jjl!:4AArg   c                *    [        [        X5      5      $ r   )r  r;  )r   r   s     re   squeeze_copyr=    s    !!rg   c                    [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ r   )r;  rk   rB   rm   r   r   vals      re   squeeze_rA    sA    
!/Ca####c9%%%%XXAFHrg   c                    [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isinfr  r   	full_liker{   r   r=   r  r   r   s     re   rC  rC    @    qE44	W	B"EJJ?BBrg   c                    [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isnanr  rD  rF  s     re   rI  rI    rG  rg   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nceilr   r  r=   r  rF  s     re   rK  rK    s0    qQx	V	B"a  rg   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )NfloorrL  rF  s     re   rN  rN    0    qQx	W	B"a  rg   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )NroundrL  rF  s     re   rQ  rQ    s0    qQx!b!!$$rg   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )NtruncrL  rF  s     re   rS  rS    rO  rg   c                *   [        U /5      u  n [        U [        R                  5      (       a   [        R
                  " U [        U5      5      $ [        U [        5      (       d   e[        U[        [        45      (       d   e[        U R                  5       5      [        U5      :X  a  U $ [        U R                  5       5      (       d  [        R                  R                  R                  [        U R                  5       5      5      nUS:  aT  [        U5      (       dD  U R!                  [        R                  R                  R                  [        U5      5      U-  5        [        [        R
                  " U R"                  [        U5      5      5      $ r  )rX  rk   r1   r  r7   r   r   rB   ri   r   r#   rR   r]   r   guarding_hint_or_throwrO   
mark_reuserm   )r   r,  x_size_products      re   r4  r4    s   aS!DQ!R__%%  E%L11a####edE]++++QZZ\eEl* ..))@@!**,'
 A&;E&B&BLL  77e8LM!" Z&&qvvuU|<==rg   c                    [        U5      nU H  nSX4'   M	     U n[        U5       H  u  pgUS:w  d  M  [        XV5      nM     [        XQ5      $ Nrt  )ri   rj   	unsqueezer4  )r   shapebroadcast_dimensionsr:  broadcast_dimensionr
  r_  r   s           re   broadcast_in_dimr^    sS    UA3!#  4 	
AA,7!!A  !rg   c                4    [        XR                  5       5      $ r   )r4  r   )r   r8  s     re   	expand_asr`  #  s    !ZZ\""rg   c                  ^^^ [        U R                  5       5      m[        T5      [        T5      :  aK  [        R                  R
                  /[        T5      [        T5      -
  -  T-   m[        U [        T5      5      n [        T5      [        U R                  5       5      :X  d   e[        U R                  5       5      nSn[        [        T5      5       H  nTU   S:X  a  SnX$   TU   -  X$'   M     U(       a&  [        X R                  5       U R                  5       S9$ [        S [        TT5       5       5      (       a  [        [        X5      5      $ UUU4S jn[        T5      (       d  [        U5      (       d|  [         R"                  R$                  R'                  [)        T5      5      nUS:  aD  U R+                  [         R"                  R$                  R'                  [)        U5      5      U-  5        U R-                  5       m[.        R0                  " U R                  5       U R                  5       U[        U5      S9$ )NFr   Tr   rv   c              3  H   #    U  H  u  pUS :H  =(       d    US :H  v   M     g7fr.   Nr   r*  s      re   r   repeat.<locals>.<genexpr>:  s$     
A*@$!AFa1f*@s    "c                   > [        U 5      [        T5      :X  d   e[        U 5      n [        [        T5      5       HI  nTU   S:w  d  M  TU   S:X  a  [        R                  R
                  X'   M5  [        X   STU   5      X'   MK     T" U 5      $ r  )r   ri   r  r   r0  Zeror+   )rB  rs   old_sizerepeatsx_loaders     re   rf  repeat.<locals>.inner_fn?  s}    5zS\)))Us7|$AqzQA;!#$ww||EH.uxHQKHEH % rg   rh  )ri   r   r   r   r0  r1  r  r  emptyr   rn   r#  r  r  r4  r#   rR   r]   r   rU  rO   rV  rj  r?   r   )	r   ri  new_sizezero_tensorrs   rf  old_size_productrh  rj  s	    `     @@re   repeatrp  (  s   AJJL!H
7|c(m#GGKK=CL3x=$@AHLDN#w<3qzz|,,,,AJJL!HK3w< 1:?KkGAJ. !
 X[[]1<<>JJ

A#gx*@
AAAVA())	 !**3H3R3R77++BB(#
 a LL  77h8OP#$
 }}H||~kkmH~	 rg   c                V    [        [        R                  " U R                  U5      5      $ r   )rB   rD   r   rm   )r   r,  s     re   r  r  `  s     T[[/00rg   c                    [        U [        5      (       d   e[        U[        [        45      (       d   e[        [        R
                  " U R                  [        U5      5      5      $ r   )rk   rB   ri   r   r>   r   rm   )r   r8  s     re   permuters  g  sL    a####dT5M****[''d<==rg   c                :
   SSK JnJn  [        U [        5      (       d   e[        XS5      nU R                  5       U   n[        R                  " U5      n[        U[        R                  5      (       d  US:  d   U5       e US:X  a6  [        R                  R                  R                  X5      (       a  US:X  a  U $ SS jn	Su  pUnU(       a_  U	" X(S5      n
Ub@  [        R                  R                  R                  U[         R"                  5      (       a  UnO	U	" X8U5      nU
b  Ub  Xp2SnU(       GdO  [        R                  R$                  nU" [        R                  R                  R&                  Ub  UR(                  R+                  S0 5      O0 5      nU(       a  UR-                  5        H  u  nnUU" S5      [.        R0                  " U5      4:X  a]  [2        R4                  " XX4U5      n[        R                  R7                  U5      Ul        [        R                  R;                  U5        M  UU" S	5      4:X  d  M  [=        S
5      e   [	        [2        R>                  RA                  U RB                  XX4US95      $ U(       d   eU" [        R                  R                  R&                  [        R                  R$                  R(                  S   5      nUc   e[E        U5      S::  d   U5       eSu  nnUR-                  5        H=  u  nnUU" S5      [.        R0                  " U5      4:X  a  UnM,  UU" S	5      4:X  d  M;  UnM?     U
b  Ub   e[2        R4                  " UUUUU R                  5       U   5      n[        R                  R7                  U5      Ul        [        R                  R;                  U5        UnU RG                  5       c  U RI                  5         U
b6  Ub   eU RK                  5       RL                  XRO                  5       U   -  -   nO[2        RP                  " UUU RK                  5       RL                  U RO                  5       U   U R                  5       U   SS9n[        R                  R7                  U5      Ul        [        R                  R;                  U5        Un[S        U R                  5       5      n[S        U RO                  5       5      nUUU'   UU==   U-  ss'   [U        U UUU5      $ ! [         a     GN[f = f)z
Lowers a slice call, creating ExternKernels for the output size & storage offset symbols,
if the indices are unbacked and appropriate semantics aren't known.
If they are known (indices are static/backed/unbacked with info), a SliceView is created.
r   )CallMethodKeyr%   r.   c           	        U c  U$ S n[         R                  " U 5      n [         R                  " U5      nU" [         R                  " [         R                  " U S5      [         R                  " X5      5      5      (       a  U $ U" [         R                  " [         R
                  " U S5      [         R                  " X* 5      5      5      (       a  X-   $ U" [         R                  " X5      5      (       a  U$ U" [         R
                  " X* 5      5      (       a  gU" [         R                  " U S5      5      (       a  [         R                  " X5      $ U" [         R
                  " U S5      5      (       a  [         R                  " X-   S5      $ g )Nc                T    [         R                  R                  R                  U 5      $ r   )rR   r]   r   r6  r   s    re   rL  5slice_.<locals>.compute_slice_index.<locals>.<lambda>  s    qww''66q9rg   r   )	r   r4  AndGeLeLtGtMinMax)rB  r   r  r   s       re   compute_slice_index#slice_.<locals>.compute_slice_index  s   =N9U#||D!eii*EHHU,ABCCL		%((5!,ehhue.DEFF<%&&K&''"##99U))"##99U\1--rg   NNFunbacked_bindingsr   storage_offsetzWUnexpected storage_offset unbacked binding when both start and end indices are resolvedclampr,   Tr   )+%torch.fx.experimental.symbolic_shapesru  r%   rk   rB   _validate_dimr   r   r4  r   rR   r]   r   statically_known_leq	TypeErrorstatically_known_equalssysmaxsizer^   	shape_envr  r  r   pytreeSequenceKeyr1   DynamicSliceSizeregister_bufferr   register_operationr   	SliceViewr   rm   r   maybe_get_layoutr  
get_layoutoffset
get_strideDynamicSelectStorageOffsetri   
as_strided)r   r   startendstepr  ru  r%   r   r  start_index	end_indexambiguous_slicer^   node_unbacked_bindingssymkeypathb_sizer  sym_sizesym_storagerm  new_storage_offset	b_storage	new_sizesnew_stridess                             re   slice_r  p  s   
 a####

"C::<D<<DdEJJ''4!89T93QJ  55d@@	H
. (K
 O)%q9 ?qww//GG 
  
 I+Ct<I"y'<$3#O ww++!:GG&&' !!"5r:	"
 " 6 < < >W}V4f6H6H6MNN00SMF"#''"9"9&"AFKGG..v6/?!@ BB )=  !? LLCUK
 	
 L51	""AGG$8$8$=$=>Q$R ((( !Q&9(99&&Hk)//1W}V,f.@.@.EFFH'78::K	 2 )"333  	

SF ''))&1FKGGv&H#			"""\\^22[<<>RUCV5VV11LLN!!LLN3JJL
	 00;		""9-(QZZ\"Iq||~&KIcNaK1CDDW  s   ;T 
TTc                2   S nS n[        U [        5      (       a_  [        U R                  [        R                  5      (       a6  U R                  5       nU R                  nU R                  R                  5       n U R                  5         [        R                  " U 5      (       d  [        SU  S35      e[        R                  " U 5      u  pg[        R                  " U(       a  UOUR                  U(       a  UOUR                  U Vs/ s H  n[        R                  " U5      PM     snU Vs/ s H  n[        R                  " U5      PM     sn[        R                  " U=(       d    S5      5      n	[        [        R                   " XiS95      $ s  snf s  snf )Nzunrealized as_strided(z, ...)r   rm   layout)rk   rB   rm   r1   r5   rn   r   unwrap_viewr  is_storage_and_layoutr   as_storage_and_layoutFixedLayoutrv   r   r4  r   )
r   r   strider  
new_device	new_dtypestorage
old_layoutr:  
new_layouts
             re   r  r    s$   JI!YJqvvr{{$C$C \\^
GG	FF IIK##A&&!$:1#V"DEE2215G 
j&7&7	J$4$4"&'$Qa$'"()&Qa&)^(q)J R''WHII	 	()s    F- Fc                j    [        U [        5      (       d   e[        XX#5      R                  U l        U $ r   )rk   rB   r  rm   )r   r   r  r  s       re   as_strided_r  6  s.    a####8==AFHrg   c                0    [        XX#5      n[        U5      $ r   )r  r  )r   r   r  r  r  s        re   as_strided_copyr  =  s    8F=rg   c                  ^ ^^^ / mSnT  H1  nTR                  X"UR                  5       T   -   45        TS   S   nM3     T  Vs/ s H  o3R                  5       PM     snmUU UU4S jn[        T S   R                  5       5      nTS   S   UT'   [        R
                  " T S   R                  5       T S   R                  5       UUS9$ s  snf )Nr   rt  c           	       >^
^ [         R                  " U T   [        R                  5      n/ n/ n[	        [        T5      5       GHH  m
T
S:X  a%  [         R                  " S[        R                  5      O*[         R                  " TT
   S   [        R                  5      n[         R                  " TT
   S   [        R                  5      n[         R                  " X5      n[         R                  " X5      nT
S:X  a  UnO+T
[        T5      S-
  :X  a  UnO[         R                  " Xg5      nUR                  U5        [        U 5      m[        TT   TT
   S   -
  5      TT'   UR                  [         R                  " UU
UU4S jS5      5        GMK     US   n	[	        [        T5      S-
  SS5       H!  m
[         R                  " UT
   UT
   U	5      n	M#     U	$ )Nr   r.   c                    > TT    " T5      $ r   r   )rs   idx_loadinputs_loaderss   re   rL  1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>o  s    N1-h7rg           rt  r,   )rQ   r\  r{   int64r  r   r]  geltand_ro   ri   r)   maskedr   )r_  idx_dimmasksmasked_loadsr  r  
start_condend_condmasknext_valrs   r  r   rT  r  inputs_rangess             @@re   rf  pointwise_cat.<locals>.inner_fnM  s   ..S5;;7s6{#A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvg+HAvc&kAo%!xx
5LLCyH %Xc]]15Ea5H%HIHSM

75 $D  #Fq("b1AyyaQH 2 rg   rh  )ro   r   rj  ri   r?   r   rn   r   )rT  r   prev_endr   rf  rm  r  r  s   ``    @@re   pointwise_catr  C  s    9;MHh3<<>#3F(FGH $R(  4::6Coo'6:N. .` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   C	c           	       ^^^^^^^^	^
 [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eU R                  5       mTR                  5       m	TR                  5       m
UUUUUUU	UU
4	S jn[        R                  " U R                  5       TUU R                  5       S9$ )Nr.   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                h  >	 U T
   4nT" U 5      nT" U5      nT" U5      n[        TT[        R                  S9u  pVTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      nTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      n[        R                  " U5      n[        R                  " X'-  5      U-   n[        R                  " U[        R                  " Xh5      5      n	[        R
                  " U	T5      $ rH  )_create_constantsr{   r  r   rQ   r   r  
reciprocalrQ  maximumminimum)r_  channel_idxr  scale
zero_pointqminqmax	inv_scaler@  clampedaxisr   input_loader	quant_max	quant_minscalesscales_loaderzero_pointszero_points_loaders             re   rf  ;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
&y)5==Q
<<5==(LL6E+j%++>JNN5)	ii)*Z7++dCKK$:;||GU++rg   rh  )r   r   r   r{   rQ  r   r  rj  r?   r   rn   )r  r  r  r  r  r  r   rf  r  r  r  s    `````` @@@re   )quantized_decomposed_quantize_per_channelr    s?    v !Q&=(==&{##%&!+G-GG+ENN*.??- 
FuGXFYZ- #enn&'' 
!#enn&6"7!89' $$&L&&(M$002, ," !~~	 rg   c           	        ^ ^ T R                  5         [        T [        R                  5      m U U4S jn[        R
                  " T R                  5       T R                  5       U[        T R                  5       5      S9nUR                  5         U$ )Nc                   > [         R                  R                  5          [        R                  " TR                  5       " U 5      T5      sS S S 5        $ ! , (       d  f       g = fr   )r1   ComputedBufferforce_realizerQ   device_assert_asyncrj  )rB  r   r   s    re   rf  _assert_async.<locals>.inner_fn  s?    ,,.**4+;+;+=e+DcJ /..s   +A
A#rh  )
r  r   r{   r   r?   r   rn   r   ri   r   )r   r   rf  assertion_ops   ``  re   _assert_asyncr    sm    LLND%**%DK ## nnDMMO$	L rg   c                    [        X5      $ r   r  r   s     re   lower_assert_asyncr        ##rg   c                    [        X5      $ r   r  r   s     re   lower_assert_functional_asyncr    r  rg   )r   c               ~  ^^^^^	^
^ [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eTc  [        R                  mU R                  5       m	TR                  5       m
TR                  5       mUU	UUU
UU4S jn[        R                  " U R                  5       TUU R                  5       S9$ )	Nr.   r  r  Expecting input to have dtype , but got dtype: r  c                  > U T   4nT" U 5      nT
" U5      nT" U5      nT	R                   [        R                  :w  a%  [        R                  " U[        R                  5      nTR                   [        R                  :w  a%  [        R                  " U[        R                  5      n[        R
                  " [        R                  " U[        R                  5      U5      U-  n[        R                  " UT5      nU$ r   )r   r{   r  rQ   r   sub)r_  r  r  r  r  r@  r  r  r   r  r  r  r  s         re   rf  =quantized_decomposed_dequantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EMll3	*
rg   rh  	r   r   r   r{   r  rj  r?   r   rn   )r  r  r  r  r  r  r   r   rf  r  r  r  s    ```   ` @@@re   +quantized_decomposed_dequantize_per_channelr    s-    v !Q&=(==&{##%&!+G-GG+??% 
(/@AR@ST% #enn&'' 
!#enn&6"7!89' MM	$$&L&&(M$002  !~~	 rg   c                  ^^^^ U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eU R                  5       mUUUU4S jn[        R                  " U R                  5       T[        R                  " U[        U5      [        U5      S9U R                  5       S9$ )Nr  c                @  > T
" U 5      n[        SU-  U[        R                  S9u  pB[        R                  " X4-  5      U-   n[        TT[        R                  S9u  pg[        R
                  " [        R                  " XV5      U5      n[        R                  " UT	5      $ )N      ?r   )r  r{   r  rQ   rQ  r  r  r   )r_  r  r  r  r  r@  r  r  r  r   r  r  r  s            re   rf  Bquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S! 1%K5==!
	 ii)*Z7&y)5==Q
++ckk#4d;||GU++rg   r  r  rh  )r   r{   rQ  r   r  rj  r?   r   rn   r  r*  r<  r   r   )r  r  r  r  r  r   rf  r  s      ``` @re   0quantized_decomposed_quantize_per_tensor_defaultr
    s     ENN*.??- 
FuGXFYZ- $$&L, , !""E%LS_
 ~~ rg   c               j  ^^ U R                  5       U:X  d   SU SU R                  5        35       eTc  [        R                  mU R                  5       mUU4S jn[        R
                  " U R                  5       T[        R                  " U[        U5      [        U5      S9U R                  5       S9$ )Nr  r  c                   > T" U 5      n[        X[        R                  S9u  p[        R                  " [        R
                  " U[        R                  5      U5      U-  n[        R
                  " UT5      nU$ rH  )r  r{   r  rQ   r  r   )r_  r  r  r  r@  r  r   s        re   rf  Dquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fnI  sY    S!-eu}}Uggcll5%--8*EMll3	*
rg   r	  rh  )r   r{   r  rj  r?   r   rn   r  r*  r<  r   r   )	r  r  r  r  r  r   r   rf  r  s	         ` @re   2quantized_decomposed_dequantize_per_tensor_defaultr  3  s     ??% 
(/@AR@ST% MM	$$&L !""E%LS_
 ~~ rg   c           	     b  ^^^^^^^^	^
 U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       mTR                  5       m	TR                  5       m
U R                  5       mUUUUUUU	UU
4	S jn[        R                  " U R                  5       TUU R                  5       S9$ )Nr  r   r.   expect scale as scalar tensor"expect zero_point as scalar tensorc                  >	 T
" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nT(       aS  TR                  S:X  aC  [
        R                  " U[
        R                  " U5      U5      n[
        R                  " UT	5      $ [
        R                  " U[
        R                  " U5      -  5      U-   n[        TT[        R                  S9u  pV[
        R                  " [
        R                  " XE5      U5      n[
        R                  " UT	5      $ )Nr.   r   r   r   r   )r   r   r   r{   r  rQ   r   r   r^  r  round_to_intrQ  r  r  r  )r_  r  _scale_zero_pointr@  r  r  r  rv   r   r  r  r  r  scale_loaderr  zero_point_loaders           re   rf  Aquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fnw  s1   S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKfkkU*''%!7EC##C//iiv 667+E&y)5==Q
++ckk#4d;||GU++rg   rh  )r   r{   rQ  r   r  r   r   rj  rn   r?   r   )r  r  r  r  r  r   rf  rv   r  r  r  s    ````` @@@@re   /quantized_decomposed_quantize_per_tensor_tensorr  Z  su    ENN*.??- 
FuGXFYZ- u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0F, ,  !~~	 rg   c                 ^^^^^	^
 [        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eTc  [        R                  mU R                  5       mTR                  5       m	TR                  5       m
UUUU	UU
4S jn[        R                  " U R                  5       TUU R                  5       S9$ )	Nr   r.   r  r  r  r  c                h  > T" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT
" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nT	R                  [        R                  :w  a%  [
        R                  " U[        R                  5      n[
        R                  " [
        R                  " U[        R                  5      U5      U-  n[
        R                  " UT5      nU$ )Nr.   r  r   )r   r   r   r{   r  rQ   r   r  )r_  r  r  r  r@  r  r   r  r  r  r  s        re   rf  Cquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FOll3	*
rg   rh  r  )r  r  r  r  r  r   r   rf  r  r  r  s    ``   ` @@@re   1quantized_decomposed_dequantize_per_tensor_tensorr    sU    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  ??% 
(/@AR@ST% MM	$$&L$$&L"..0
 
 !~~	 rg   c                  ^ ^^^^^^^ T S   R                  5       R                  S:H  nU(       a  [        S T  5       5      (       aw  T  H  nUR                  5         M     [        S T  5       5      (       a  [	        [
        R                  /T Q76 u  m n[        [
        R                  R                  5      " T U5      $ [        T 5      S:X  a  [        T S   5      $ [        T S   US5      n[        T S[        R                  06nT  Vs/ s H  n[        Xe5      PM     snm SS jmS mUUU4S	 jm[!        U4S
 jT  5       5      nSUU4S jjm["        R$                  (       a  ['        T U5      $ U(       a)  [)        [*        R,                  R/                  T U5      5      $ UU4S jmSnSmSS jm[        T 5      U::  d8  [        T 5      ["        R0                  ::  a  [        UU4S jT  5       5      (       a  [        U4S j[2        R4                  R6                   5       5      n	[!        U4S jT  5       5      =(       a    U	n
SU U4S jjnU" 5       n[        U4S jT  5       5      =(       a    [!        U4S jT  5       5      (       + nU(       d!  U
(       d  U(       a  U(       d  ['        T U5      $ [)        [*        R,                  R/                  T U5      5      $ s  snf )z@Lower aten.cat, choosing between pointwise_cat and ConcatKernel.r   r   c              3     #    U  H5  oR                  5       [        R                  [        R                  4;   v   M7     g 7fr   )r   r{   int8uint8r   r  s     re   r   cat.<locals>.<genexpr>  s*      DJ5ejj%++66Fs   =?c              3  Z   #    U  H!  n[        UR                  5       5      S :H  v   M#     g7f)r   N)r   r   r#  s     re   r   r$    s!     >ves5>>#$)vs   )+r.   r   c                $   [        U [        5      (       aO  [        U R                  [        R                  5      (       a  U R                  R                  5       $ U R                  $ [        U [        R                  5      (       a  U R                  $ U $ r   )rk   rB   rm   r1   r5   r  r   r   s    re   unwrap_tensorcat.<locals>.unwrap_tensor  s_    a##!&&"++..vv))++vva''66Mrg   c                    [        U [        R                  5      =(       a$    [        U R                  [        R                  5      $ r   )rk   r1   r  rm   r@   )rw   s    re   is_reductioncat.<locals>.is_reduction  s)    !R../TJqvvr||4TTrg   c                  > [        U [        [        R                  45      (       a  T" T" U 5      5      $ T" U 5      =(       dD    [        U [        R                  5      =(       a#    [        U4S jU R                  5        5       5      $ )Nc              3  n   >#    U  H*  nT" [         R                  R                  U5      5      v   M,     g 7fr   )rR   r]   
get_buffer)r   readcan_fuse_reductions     re   r   2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>  s/      .D #177#5#5d#;<<.s   25)rk   rB   r1   r   r?   r$  get_read_names)rw   r0  r*  r'  s    re   r0  cat.<locals>.can_fuse_reduction  sk    a)R]]344%mA&677O !R\\*  ,,. 	
rg   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r   rw   r0  s     re   r   r$    s     B6a.q116   c                X  > [         R                  " U 5      (       a;  [         R                  " U SS9u  p[         R                  R	                  U5      (       + $ [        U [        [         R                  45      (       a  T" T" U 5      5      $ [        U [         R                  5      (       a  gg)NF)freezeT)	r1   r  r  ConcatKernelcan_realize_into_without_copyrk   rB   r   r?   )r   r  _should_lower_cat_inputr'  s      re   r<  #cat.<locals>.should_lower_cat_input  s     ##A&&11!EBJGDDWMMMa)R]]344)-*:;;a&&rg   c                `  > [        U [        [        R                  45      (       a  T" T" U 5      5      $ [        U [        R                  5      (       d  gU R                  5       R                  nU R                  5        H+  nUT" [        R                  R                  U5      5      -  nM-     U$ r  )rk   rB   r1   r   r?   inner_fn_opcountnum_opsr2  rR   r]   r.  )r   countr/  op_countr'  s      re   rB  cat.<locals>.op_count  s    a)R]]344M!,-- !R\\**""$,,$$&DXagg00677E ' rg   r   r,   c                p    U [         R                  R                  [         R                  R                  4;   $ r   )r  catr  constant_pad_ndr`   s    re   additional_pointwise_ops%cat.<locals>.additional_pointwise_ops#  s(    dhh&&(<(<(D(DEEErg   c              3  :   >#    U  H  nT" U5      T:*  v   M     g 7fr   r   )r   rw   MAX_SIMPLE_OP_COUNTrB  s     re   r   r$  (  s     CFq22F   c              3  <   >#    U  H  n[        UT5      v   M     g 7fr   rI   )r   userH  s     re   r   r$  *  s#      
+ S":;;+r   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r   r   r<  s     re   r   r$  1  s     >v&s++vr6  c                   >^^ [         R                  mTc  gTR                  S   n [        U [        [
        45      (       a  U nO.[        U [        R                  R                  5      (       a  U /nOgUU4S jm[        UT5       Hf  u  p#[        US5      (       a  [        UR                  5      S::  a  M1  [        U4S jUR                   5       5      (       a    gT" U5      (       d  Mf    g   g)NFr   c                   > [        U [        [        R                  45      (       a  T" T" U 5      5      $ [        U [        R                  5      $ r   )rk   rB   r1   r   r?   )r   is_unrealized_pointwiser'  s    re   rT  Kcat.<locals>.any_input_has_multi_consumers.<locals>.is_unrealized_pointwiseG  s:    a)R]]!;<<2=3CDD!!R\\22rg   r_   r.   c              3  F   >#    U  H  oTLd  M	  [        U5      v   M     g 7fr   rN  )r   ur^   s     re   r   =cat.<locals>.any_input_has_multi_consumers.<locals>.<genexpr>Q  s"     WIq,AV*'**Is   !!T)rR   r^   rt   rk   ri   r   r{   fxNoder  r   r   r_   r$  )fx_argsinput_nodesr   ir_inputr^   rT  rT  r'  s       @@re   any_input_has_multi_consumers*cat.<locals>.any_input_has_multi_consumers;  s    >>L#"''*G'D%=11%GUXX]]33&i3
 "%[&!9sG,,CII!0C WCIIWWW +844 ": rg   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   rQ  s     re   r   r$  \  s      "
39C"3''6r6  c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r5  s     re   r   r$  ^  s     <V(++Vr6  )r   zTensorBox | ir.StorageBoxr   	ir.IRNoder   r   r`   torch._ops.OpOverload)rn   r   r#  r  require_channels_lastr  rE  r  r  r   r  r  r   r   rO  r   r$  r/   force_pointwise_catr  rB   r1   r9  r   max_pointwise_cat_inputsrR   r^   r_   )rT  r   
cpu_devicer  r;  r   r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_user^  has_multi_consumershorizontal_fuse_catrK  rH  r0  r*  rB  r<  r'  s   `             @@@@@@@re   rE  rE    sh    %%',,5Jc DJ  
 EMMO >v>>>-dhh@@IFA 0 01&#>>
6{aVAY
q	3
*C	%D%L%LE /55fshs"f5F
U

 B6BB  !!VS)) //<==$ !"F 6{//	V77	7CFCCC 
~~++
 
 >v>>Q> 		 	> <=! "
39"
 
 =<V<<< 	 ##6?P --R__++FC899[ 6s   Kc                  ^^^
^^ U R                  5       m[        T5      m[        TTS9m[        TTS9m[        TT:g  UU4S j5        [        R
                  R                  R                  [        R                  " US5      5      nU(       a\  [        R
                  R                  R                  [        R
                  R                  R                  TT   U-   TT   5      S5      nO[[        R
                  R                  R                  [        R
                  R                  R                  TT   TT   U-
  5      S5      nSm
U(       a  U* S4m
OSU4m
[        T5       VVs/ s H  u  pgUTT4;  d  M  UPM     nnnUR                  U5        U
UUUU4S jn	[        [        R                   R#                  XU	5      5      $ s  snnf )N)r_  rankc                    > ST  ST 3$ )Nz(diagonal dimensions cannot be identical z, r   )dim1dim2s   re   rL  diagonal.<locals>.<lambda>o  s     HbQUPVWrg   r   )r   r   c                   > U S   nS/[        T	5      -  nSn[        T5       H2  nUT:X  a  UTS   -   X$'   M  UT:X  a  UTS   -   X$'   M'  X   X$'   US-  nM4     U[        T	5      S-
  :X  d   eU$ )Nrt  r   r.   r,   )r   r  )
r_  diag_idxoriginal_idxcur_dimr4  base_idxrs  rt  num_dimsoriginal_shapes
        re   	reindexerdiagonal.<locals>.reindexer  s    r7sS00xADy"*Xa["8d"*Xa["8"%,1 ! #n-1111rg   )r   r   r   r   rR   r]   r   evaluate_exprr   r|  evaluate_maxevaluate_minrj   ro   rB   r1   GenericViewr   )r  r  rs  rt  offset_negative	diag_sizers   r:  r,  r}  rz  r{  r|  s     ``      @@@re   diagonalr  g  s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-t$ 
	 GG$$11GG))t$t$v- 
	 HGQ<v;$^4N441$8MQ4EN	LL   R^^**5CDD' Os   &G8Gc                ,    [        [        XX#5      5      $ r   )r  r  )r  r  rs  rt  s       re   diagonal_copyr    s    %455rg   c                J    [        U 5      n[        XRX45      n[        Xa5        U$ r   )r  r  	mutate_to)r  srcr  rs  rt  r7  ra   s          re   diagonal_scatterr    s$    5\Ffd1FfMrg   c           	     l   [         R                  " U5      n[         R                  " U R                  5       U   5      nS n[        R                  R
                  R                  [         R                  " US5      5      (       a  X#-   nOE[        R                  R
                  R                  [         R                  " US5      5      (       a  UnUb  [        U5      (       ab  U R                  5         U R                  5       nU R                  5       nU R                  5       R                  Xa   U-  -   nXQ	 Xa	 [        XXg5      $ [        XXDS-   SS9n[!        X5      $ [#        [        R                  R
                  R$                  [        R                  R&                  R(                  S   5      n	U	c   e[+        U	5      S:X  d   U	5       e[-        [/        U	R1                  5       5      5      u  pU R                  5         U R                  5       nU R                  5       nU
n[2        R4                  " U
UU R                  5       R                  Xa   U R                  5       U   SS9n[        R                  R7                  U5      Ul        [        R                  R;                  U5        XQ	 Xa	 [        XXg5      $ )Nr   r.   Fr  r  )r   r4  r   rR   r]   r   r6  r|  rz  r$   r  r  r  r  r  r  r;  r%   r  r^   r  r   rP  iterr   r1   r  r  r   r  )r   r   r_  r   actual_indexrm  
new_strider  slice_resultr  unbacked_offset_symr;  buffers                re   selectr    s.   
,,s
C<<

S)*DLww&&uxxQ'788z	
			(	(#q)9	:	:$S)) IIKzz|HJ!"!6!6<9W!Wa:JJ "!,q8HPUVL<-- 2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&!$'8'>'>'@"AB IIKzz|HJ,**		

SF ''))&1FKGGv&a:BBrg   c                n   [        XS5      nUn[        U[        [        45      (       d^  U R	                  5       U   n[
        R                  R                  R                  [        XA-   S-
  U5      5      nU/U-  nXES-
  U-  -
  US'   / nSnU H#  nXx-   n	UR                  [        XXySS95        U	nM%     U$ )Nr   r.   rt  Fr  )r  rk   ri   r   r   rR   r]   r   r2  r(   ro   r  )
r   r,  r   sizes_x_sizechunksr  r  r   r  s
             re   splitr    s    

"CF edE]++c"!!++HV^a5G,OP6!zU22r
 FEl 	fQUu=>  Mrg   c                    [        XU5      $ r   )r  )r   r,  r   s      re   split_with_sizesr  		  s    3rg   c                    [        XS5      n[        R                  R                  R	                  U R                  5       U   5      n[        U5       Vs/ s H  n[        XU5      PM     nnU$ s  snf r  )r  rR   r]   r   r2  r   r  r  )r   r   r  rs   r  s        re   unbindr  	  s\    

"CWW''

S(9:F).v7AfQQF7M 8s   A.c                6  ^^ U R                  5       n[        U5      n[        XQ5      mUS:X  a  [        [	        U S5      USS9$ UT   n[
        R                  R                  nUR                  X&5        UR                  ST5        [        Xb-
  T5      S-   nUR                  U5      S:  a,  U R                  UR                  [        X-  U5      5      5        / US T QUPUTS-   S  QUPn	UU4S jn
[        [        R                   R#                  X	U
5      5      $ )Nr   F)r  r  r.   c                D   > U S   U T   T-  -   n/ U S T QUPU TS-   S Q7$ )Nrt  r.   r   )r_  dim_idxr   r  s     re   r}  unfold.<locals>.reindexer,	  s?    b'CHtO+8Tc8G8c#'B&788rg   )r   r   r   r  rZ  rR   r]   r   	check_leqcheck_ltr(   rU  rV  r'   rB   r1   r  r   )r   	dimensionr   r  r,  ndimdim_sizer   new_dim_sizeout_sizer}  r   s      `       @re   unfoldr  	  s   JJLEu:D
4
+Cqyi1o4u==SzHwwHt&aHOT2Q6L&&x014	++GL4G,RS	
 EtDlDU379-=DtDH9 R^^**1	BCCrg   c                    [        XS5      n[        U R                  5       5      nUR                  U[        R
                  R                  5        [        X5      $ r  )r  ri   r   insertr   r0  r1  r  )r   r   r9  s      re   rZ  rZ  3	  sA    

"CQZZ\"IS%''++&rg   c                    [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ r   )rZ  rk   rB   rm   r?  s      re   
unsqueeze_r  ;	  sB    
A
Ca####c9%%%%XXAFHrg   c                   [         R                  R                  R                  R	                  [
        R                  " U5      5      n[        U R                  5       5      nUS:  a  XU-   -  nSUs=::  a	  X2-   :  d   e   eU$ r  )	rR   r]   r   r  r  r   sympifyr   r   )r   r   r  r  s       re   r  r  D	  sn    
''


$
$
2
25==3E
FCqzz|D
Qwf}#dm#####Jrg   rt  c                    [        XS5      n[        R                  R                  R	                  U R                  5       U   5      S-  n[        XSUSS9n[        XX"S-  SS9n[        U[        U5      5      $ )Nr   r,   Fr  )	r  rR   r]   r   r2  r   r  r  sigmoid)r   r   new_lenr   r6  s        re   glur  M	  sk    

"Cgg((c):;q@Gqq'/Aqw!59Aq'!*rg   c                Z   ^  U(       a  [         R                  T 5        U 4S jnSUl        U$ )Nc                 |   > S n[         R                  " U[        R                  R                  " T/U Q70 UD65      $ )Nc                d    [        U [        R                  5      (       a  U R                  5       $ U $ r   rk   r1   r9   wrap_for_loweringr   s    re   wrap_tensors7fallback_handler.<locals>.handler.<locals>.wrap_tensors]	  &    ,6q")),D,D1&&(K!Krg   )r  tree_mapr1   FallbackKernelr   )rt   r   r  kernels      re   handler!fallback_handler.<locals>.handler\	  s<    	L "++226KDKFK
 	
rg   T)r  r   _is_fallback_handler)r  r  r  s   `  re   r  r  X	  s&    f
 $(G Nrg   c                 0    [         R                  " S5        g )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   rg   re   _warn_complex_not_supportedr  j	  s    MMtrg   c                   U R                  5       (       a  [        5         gU R                  (       a  gU R                  (       a  gU R                  [
        R                  :X  Ga  U(       d  g[        UR                  [
        R                  R                  5      =(       as    UR                  [        R                  R                  [        R                  R                  [        R                  R                  [        R                   R                  4;   =(       dJ    [        UR                  [
        R                  R                  5      =(       a    [#        UR                  5      (       + $ g)z0Do not support reading or writing to this tensorTF)r  r  is_meta	is_sparser   r{   float8_e8m0fnurk   ra   r|   r}   r  r  rE  r  r  
_scaled_mmrJ   )rw   rc   s     re   unsupported_input_tensorr  s	  s    ||~~#%yy{{ww%&&&
 t{{EJJ$9$9: 		  

""''	Y 4;;

(=(=>W74;;CW

 
	
 rg   c                L   [         R                  R                  [        R                  R
                  R                  R                  4nUb&  UR                  U;   a  U R                  5       (       a  g[        X5      (       a  gU R                  =(       a    [        R                  $ )z2Do not support writing tensor but can read from itFT)r  r  r   r{   rQ   r  r  r  ra   r  r  is_cpur/   disable_cpp_codegen)rw   rc   supported_complex_viewss      re   unsupported_output_tensorr  	  sr     					,,44 DKK+BBq||~~((882222rg   c                ^  ^  T R                   [        R                  R                  L a  gT R                  S:X  a  gT R                   [        R
                  R                  L a  gU 4S jn[        R                  " T R                  0 T R                  D6 H  nU" USS9(       d  M    g   U" T SS9$ )NFplaceholderc                  > [        U [        R                  R                  5      (       d  gSU R                  ;  a  g[
        R                  " U R                  S   5       H]  n[        U[        R                  R                  5      (       d  M.  U(       a  [        UT5      (       a    gMJ  [        UT5      (       d  M]    g   g)NFr@  T)rk   r{   rY  rZ  r  r  tree_leaves_subclasses
FakeTensorr  r  )inp_out_node	is_outputr  rc   s      re   check_skip_conditionCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition	  s    ,66)))&&|'8'8'?@DdE$5$5$@$@AA,T488 9 ,D$77 A rg   )r  T)
ra   r  view_as_complexr  r`   lift_fresh_copyr  arg_tree_leavesrt   r   )rc   allow_cpu_inputsr  r   s   `   re   %fallback_node_due_to_unsupported_typer  	  s    {{d**222ww- {{d**222* %%tyy@DKK@u55 A  55rg   c                  ^ Ub  U" 5       O[         nX;  d  U(       d
   SU  35       eU(       a  [        [        R                  " S5      5      (       a  [	        U /5      (       a  [
        R                  (       a(  U [        R                  R                  R                  ;   ds  U(       dl  [        R                  R
                  R                  (       a4  S[        R                  R
                  l        [        R                  S5        [        SU  S35      eU4S jn[!        U [        R"                  R$                  5      (       a+  U R'                  5        H  n[)        X5      nU" U5        M     g [!        U [        R"                  R*                  [        R"                  R,                  45      (       a	  U" U 5        g [/        SU  S	[1        U 5       35      e)
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                j   > [        U 5        Tb  [        U T5        [        U S S9" [        U 5      5      $ NrZ  )r   r   r+  r  )op_overloadlayout_constraints    re   register_fallback(make_fallback.<locals>.register_fallback	  s8    !+.(!+/@A $G[)
 	
rg   zUnsupported fallback z with type )r3   r   osgetenvr4   r/   fallback_randomr{   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningr   rk   r|   r   r   r   r}   HigherOrderOperatorRuntimeErrorr   )	r`   r  r  override_decompget_decomp_fncheck_decompsr  olr  s	    `       re   make_fallbackr  	  sd    (5'@MOnM"o 
4RD95 	4!!t$$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj1122,,.B!"/Kk* ! 
B..

0N0NO	P	P"22$k$r(LMMrg   c                P    SnU  H  nX-  nM	     [        U[        R                  S9$ )z
TorchInductor offset calculation differs from PyTorch eager offset
calculation for random ops (tl.rand vs torch.rand). In future, we should
strive for same impl for tl.rand and torch.rand.
r.   r   rI  r{   r  )r[  numelr:  s      re   philox_rand_offsetr  
  s+     E	 %u{{++rg   c                @  ^^	^
^ [         R                  " UTU [         R                  R                  U 5      5      R	                  5       m
UR                  5       mUR                  5       m	UU	U
U4S jn[        R                  " UTU[        U 5      S9n[        U 5      nXx4$ )Nc                  > [         R                  " T" / 5      [        R                  5      n[         R                  " T" / 5      [        R                  5      n[         R                  " [         R
                  " T" U 5      [        R                  5      U5      n[         R                  " UU5      n[         R                  " UT5      $ r   )rQ   r   r{   r  r   r\  rand)	rB  seed_index_exproffset_index_exprrand_index_exprr  r   offset_loader
random_posseed_loaders	        re   rf  philox_rand.<locals>.inner_fn
  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**rg   rh  )
r1   r  FlexibleLayoutcontiguous_stridesmake_indexerrj  r?   r   ri   r  )r   seedr  r  rv   r   rf  random_values_nodeoffset_noder  r  r  s        `   @@@re   philox_randr  
  s     
,,T2	
 ln  ""$K&&(M+ + #))Dz	 %T*K**rg   c           	         [         R                  (       a\  [        R                  " [        R
                  [        R                  R                  [        R                  R                  XU5      5      $ [        S5      e)Nz&should be handled in replace_random.py)r/   r  r  r  rB   r   r1   r  r  native_dropoutr  r   )r   ptrains      re   r  r  6
  sW    $$T%8%8%@%@!N
 	

 EFFrg   c                   [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       eU R                  5         [        U5      S:X  d  [        US   [        5      (       a  [        R                  R                  O[        R                  R                  n[        R                  " X /UQ76   U $ )Nr   Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r/   r  rn   r{   rv   r  r   rk   r<  r  
bernoulli_Tensorr1   InplaceBernoulliFallback)r   rt   r  s      re   r  r  A
  s    !!Q\\^u||E7J%J ^J IIK t9>ZQ77 	__## 
 66Hrg   c                    [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       e[        [        U 5      /UQ76 $ )Nr   r  )r/   r  rn   r{   rv   r  r  )r   rt   s     re   bernoulli_pr  P
  sG    !!Q\\^u||E7J%J ^J eAh&&&rg   c                    [         er   r   )r;  s    re   _foobarr  Y
  s    
rg   c                .    [         R                  S5        g )Nz1using triton random, expect difference from eager)r  info)salts    re   _warn_triton_randomr  ^
  s    HH@Arg   c                 J    [        [        R                  R                  5        g r   )r  rR   r]   creation_timer   rg   re   warn_triton_randomr!  c
  s    --.rg   )r  c                     UR                  S5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      eN	generatorz-should have been handled in replace_random.py)r  fallback_rand_generatorr/   r  r  fallback_rand_defaultr   rt   r   s     re   r  r  x
  sT    zz+*&777			

;%$d5f55
H
IIrg   c                     UR                  S5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      er#  )r  fallback_randn_generatorr/   r  r  fallback_randn_defaultr   r'  s     re   randnr+  
  sT    zz+*'888			

;%%t6v66
H
IIrg   c                l    [         R                  " U5      n[         R                  R                  X5      $ r   )r1   get_stride_orderExternKernelrequire_stride_order)input_tensorr  stride_orders      re   inductor_force_stride_orderr2  
  s'    &&v.L??//KKrg   c                    [        S5      e)Nz.should be handled in fuse_seed_creation_pass()r  )rv   s    re   inductor_seedr4  
  s    
I
JJrg   c                ~    [        5         [        R                  " [        R                  " U [        U5      5      5      $ r   )r!  rB   r   r1   RandomSeedsrF   )rA  rv   s     re   inductor_seedsr7  
  s)    BNN5-2GHIIrg   c                z   ^ ^ UU 4S jn[         R                  " T R                  5       T R                  5       U/ S9$ )Nc                N   > [         R                  " TR                  5       T5      $ r   )rQ   	load_seedget_name)r;  rB  seedss    re   rf  &inductor_lookup_seed.<locals>.inner_fn
  s    }}U^^-u55rg   rh  )r?   r   rn   r   )r<  rB  rf  s   `` re   inductor_lookup_seedr>  
  s9    6 !oo	 rg   c                f   [        U [        R                  5      (       d  [        R                  " U 5      n U R                  S:X  ag  U R                  nUc  [        R
                  R                  5       n[        R
                  R                  U5      nUR                  UR                  -  nU$ SnUnU$ )Nrs  i   )
rk   r{   rv   r   rB  rs  current_deviceget_device_propertiesmulti_processor_countmax_threads_per_multi_processor)rv   r_  propthreads_per_round_CPU_GRAIN_SIZEs        re   get_threads_per_roundrG  
  s    fell++f%{{fll;**++-Czz//4&&)M)MM 	   +rg   )r  align_dtypec               *  ^^
^^^ [         R                  (       a   eTS;   d   e/ U Qn [        R                  nUR	                  5       n[
        R                  " XeU [
        R                  R                  U 5      US9R                  5       m
UR                  5       m[         R                  (       a2  UR                  S:X  a"  [        U5      mSS jnU" U5      mU
UUU4S jnOUU
U4S jn[        R                  " UUU/ U QS9n	U	R!                  5         U	$ )	N)r  r+  r  rs  c                N    U [         R                  [         R                  4;   a  gg)Nr   r   )r{   rR  rQ  )dts    re   _vec_from_dtype(inductor_random.<locals>._vec_from_dtype
  s    emmU^^44rg   c           
        > T" S/5      nT" S/5      n[         R                  " UUT[         R                  " T" U 5      [        R                  5      [        T5      S9$ )Nr   r.   )vec)rQ   
rand_eagerr\  r{   r  r   )rB  rng_seedbase_offsetr  r  rE  rP  s      re   rf  !inductor_random.<locals>.inner_fn
  sS    "A3'H%qc*K>>!z%0%++>H rg   c                   > [        [        T5      " T" / 5      [        R                  " T" U 5      [        R                  5      5      $ r   )r   rQ   r\  r{   r  )rB  moder  r  s    re   rf  rT  
  s5    3%Bz%0%++> rg   rh  )rL  torch.dtyper   r   )r/   r  r{   r  rS  r1   r  r  r  r  rj  align_random_eagerr   rG  r?   r   r  )r   r	  rV  r  rH  r   rv   rM  rf  r  r  r  rE  rP  s     `       @@@@re   inductor_randomrY  
  s     %%%%$$$$T7DMME%%'FtR..AA$GPVln  ""$K  V[[F%:1&9	
 k*		 			 w	F NNMrg   rJ  c               h  ^ ^^^	 [         R                  (       a   e/ UQn[        R                  nUR	                  5       n[
        R                  " XeU[
        R                  R                  U5      US9R                  5       mUR                  5       m	UU UU	4S jn[        R                  " UUU/ UQS9$ )NrJ  c           	        > [         R                  " T" / 5      [         R                  " T" U 5      [        R                  5      [         R                  " T[        R
                  5      [         R                  " T[        R
                  5      5      $ r   )rQ   	randint64r\  r{   r  r  )rB  highlowr  r  s    re   rf  "inductor_randint.<locals>.inner_fn  sV    }}ONN:e,ekk:NN3,NN4-	
 	
rg   rh  )r/   r  r{   r  rS  r1   r  r  r  r  rj  r?   r   )
r^  r]  r   r	  r  r   rv   rf  r  r  s
   ``      @@re   inductor_randintr`  
  s     %%%%T7DKKE%%'FtR..AA$GPVln  ""$K
 
 w	 rg   c                    U R                  5       nU R                  5       n[        S [        X5       5       5      US   -   nU R	                  5       US   UUS   4$ )Nc              3  6   #    U  H  u  pUS -
  U-  v   M     g7frd  r   )r   r:  sts      re   r   %_boundaries_helper.<locals>.<genexpr>  s     A/@eaa!er\/@s   rt  )r   r  sumr  r;  )tbr   r  
max_offsets       re   _boundaries_helperrh    s\    
 ;;=D]]_FAs4/@AAF2JNJ
Rr
	 rg   c                H    U R                  5       U R                  5       S   4$ rY  )r;  r  rf  s    re   _sorter_helperrk  (  s    ;;="--/"---rg   	out_int32rightsidesorterc          	     p  ^ ^^^
^ S nU" T 5      (       a  U" U5      (       a  Tb8  U" T5      (       d+  [        [        R                  R                  SS9" T UUTUTS9$ Ub  US:X  a  SmU(       a  [        R
                  O[        R                  m
UR                  5       mT R                  5         Tb  TR                  5         [        T R                  5       5      S:X  a  U
UU UU4S jnO
U
UU UU4S	 jnUR                  5       n[        R                  " UT
UUR                  S
9n	U	R                  5         U	$ )Nc                ^    [         R                  R                  U [        R                  5      $ r   )rR   r]   r  r  	BUCKETIZErj  s    re   rL  searchsorted.<locals>.<lambda>6  s    AGG$7$7
N$$%rg   Fr  rl  rn  Tr.   c           
        > T" U 5      n[         R                  " U[        T5      STTTc  S O
[        T5      Tc  S S9$ SS9$ )Nr   rp  sorter_indicesrQ   	bucketizerh  rk  )r_  r@  index_dtypern  sorted_sequencerp  values_loaders     re   rf  searchsorted.<locals>.inner_fn[  sV    $C=="?3%~t>&3I'-~t  <= rg   c                   >^  T" T 5      nSU U4S jjn[         R                  " U[        T5      U" T5      TTTc  S O
[        T5      Tc  S S9$ U" T5      S9$ )Nc                   > U R                  5       n[        R                  " [        R                  " [
        R                  S [        US S TS S 5       5       5      T5      $ )Nc              3  .   #    U  H  u  pX-  v   M     g 7fr   r   )r   r:  rs   s      re   r   Nsearchsorted.<locals>.inner_fn.<locals>.get_flattened_index.<locals>.<genexpr>r  s     &U9Tqu9T   rt  )r  rQ   r\  r  r+  operatorr   r  )rf  stridesr_  rz  s     re   get_flattened_index;searchsorted.<locals>.inner_fn.<locals>.get_flattened_indexn  sU    --/~~$$ &UWSb\3sPR89T&U  	 rg   rv  )rf  rB   rx  )r_  r@  r  rz  rn  r{  rp  r|  s   `  re   rf  r}  i  sm    $C  =="?3#O4%~t>&3I'-~t  <Ov;V rg   rh  )r  r  searchsortedr  r{   r  r  rj  r  r   r   rn   r?   r   r[  )r{  selfrm  rn  ro  rp  validate_bucketizerf  rv   r  rz  r|  s   `  ` `    @@re   r  r  ,  s    //!$'''9&'A'A 1 1 8 8eT
 	
 DGO!*%++K$$&M 
?##%&!+
	 
		 	0 __Fzz	F NNMrg   rm  rn  c                 ^^^^ [        TR                  5       5      S:X  d   e[        R                  R	                  U [
        R                  5      (       a3  [        R                  R	                  T[
        R                  5      (       d)  [        [        R                  R                  SS9" U TUTS9$ TR                  5         U R                  5       nU R                  5       mU(       a  [        R                  O[        R                   mUUUU4S jn["        R$                  " UTUU R                  5       S9nUR                  5         U$ )Nr.   Fr  r  c                ^   > T" U 5      n[         R                  " U[        T5      STT5      nU$ r  )rQ   ry  rh  )rB  r@  r$  
boundariesrz  r  rn  s      re   rf  bucketize.<locals>.inner_fn  s6    5!--z*
 rg   rh  )r   r   rR   r]   r  r  rs  r  r  ry  r  r  rn   rj  r{   r  r  r?   r   )	r  r  rm  rn  rv   rf  r  rz  r  s	    ` `   @@re   ry  ry    s     z""$%*** 	
E>#;#;<<GG
N,D,DEE 5 55Q:%
 	
 F$$&L!*%++K
 
 ~~	F NNMrg   c                    [        U [        R                  5      =(       a/    [        U [        R                  [        R                  45      (       + $ r   )rk   r1   r9   NonTensorObjOpaqueMultiOutputr   s    re   _is_tensor_irnoder    s:    a# J	BOOR112- ) rg   c                v    [         R                  " [        [        R                  R
                  X45      u  pX4$ r   )r  tree_map_onlyr  r1   r.  require_stride1r;  rt   r   s      re   require_denser    s1    ''2??::TNLD <rg   c                v    [         R                  " [        [        R                  R
                  X45      u  pX4$ r   )r  r  r  r1   r.  require_contiguousr  s      re   r  r    s3    ''
**	LD
 <rg   c                v    [         R                  " [        [        R                  R
                  X45      u  pX4$ r   )r  r  r  r1   r.  r   r  s      re   r   r     s5     ''
22	LD
 <rg   c                v    [         R                  " [        [        R                  R
                  X45      u  pX4$ r   )r  r  r  r1   r.  rf  r  s      re   rf  rf    s3    ''
--	LD
 <rg   c           	        Uc  U $ [        U[        5      (       d  [        U5      (       a  U $ [        U [        R                  5      (       a-  [        R
                  R                  XR                  5       5      $ [        U [        5      (       a!  U  Vs0 s H  o"[        X   X   5      _M     sn$ [        U [        [        45      (       a!  [        U 5      " S [        X5       5       5      $ U $ s  snf )Nc              3  <   #    U  H  u  p[        X5      v   M     g 7fr   constrain_to_fake_tensor)r   r   f_as      re   r   +constrain_to_fake_tensor.<locals>.<genexpr>  s      
=O!$Q,,=O   )rk   r   r   r1   r9   r.  require_exact_stridesr  r  r  r   ri   r   r  )r   fake_argkeys      re   r  r    s    
(,--1J1J
#ryy!!44S//:KLL#tRUVRU3-chFFRUVV	C%	'	'Cy 
=@=O
 
 	
 J Ws   C-c           
         [        S [        X5       5       5      n UR                  5        VVs0 s H  u  pEU[        XSU   5      _M     nnnX4$ s  snnf )Nc              3  <   #    U  H  u  p[        X5      v   M     g 7fr   r  )r   r   r  s      re   r   ,constrain_to_fake_tensors.<locals>.<genexpr>  s!      1MC 	!//1r  )r   r  r   r  )rt   r   	fake_argsfake_kwargsr   r
  s         re   r   r     s[      1 D JPXa)!^<<FX< Ys   Ac           
        ^ U4S jm[        U4S j[        XR                  5       5       5      nUR                  5        VVs0 s H  u  p4UT" X@R                  U   5      _M     nnnX4$ s  snnf )Nc           	       > [        U 5      (       a  UR                  R                  S5      n[        U[        R
                  5      (       d  U $ [        R                  " UR                  5       [        R                  R                  R                  5      n[        R                  R                  X5      $ [        U [        5      (       a  U  Vs0 s H  oDT" X   X   5      _M     sn$ U $ s  snf Nr@  )r  r  r  rk   r{   r  r1   r-  r  rR   r]   r   r  r.  r/  r  )r   fx_argfake_valr1  r  apply_constraints        re   r  1constrain_to_fx_strides.<locals>.apply_constraint  s    S!!{{u-Hh55
..!177#3#3#=#=L ??77JJc4  LOPCS)#(FK@@CPP
 Qs   C)c              3  8   >#    U  H  u  pT" X5      v   M     g 7fr   r   )r   r   r  r  s      re   r   *constrain_to_fx_strides.<locals>.<genexpr>  s       9P+#%%9Ps   )r   r  rt   r   r   )fx_nodert   r   r   r
  r  s        @re   r   r     sl      9<T<<9P D EKLLNSNDAa!!^^A%677NFS< Ts   "A+c                  ^ ^^ UU 4S jmU 4S jm[        U4S j[        [        UT R                  5      5       5       5      nUR	                  5        VVs0 s H  u  p4UT" SUT R
                  U   5      _M     nnnX4$ s  snnf )zGApply stride constraints to SDPA inputs, ensuring dense last dimension.c           
       > [        U5      (       d  U$ UR                  S   nUR                  5        Vs/ s H:  n[        U[        R
                  5      (       a  UR                  R                  OUPM<     nn[        R                  R                  R                  n[        R                  " UR                  5       U5      nU(       a=  US   S:w  a4  [        [        [!        [#        UR%                  5       5      5      5      5      nTR&                  [(        R*                  R,                  :X  a  U S;   a  [#        U5      S:X  d   eSnS n[.        R0                  (       ak  UR3                  5       n	[5        U5      U	U(       a  [7        U5      OS 4nU[        R                  R8                  ;   a  [        R                  R8                  U   $ T" XX5U5      n
Ub  U
[        R                  R8                  U'   U
$ s  snf )Nr@  rt  r   )r   r   r   )r   r.   r,   r   )r  r  r  rk   r{   SymIntrc   exprrR   r]   r   r  r1   r-  ri   r/  r  r   r   ra   r  0_scaled_dot_product_efficient_attention_backwardr  r/   cache_sdpa_constraintmaybe_get_nameidr   sdpa_constraint_cache)r_  r   r  meta_valr:  meta_stride_exprr  r1  	cache_keyarg_namer  _apply_constraint_innerr  s              re   r  )sdpa_constraint.<locals>.apply_constraint,  s    %%J;;u%GOGX
GX!:a66AFFKKA=GX 	 
 GG$$..	**8??+<iHL,1s3<<>/B)C DEL NNDDLLMv|$)))
 (L 	''))+H3'3l#I
 AGG999ww44Y??(h,
  7=AGG)))4Y
s   AG4c                (	  >^ UR                   (       d0  UR                  (       d  [        R                  R	                  X5      $ SmTR
                  [        R                  R                  R                  R                  :H  =(       a    U S:H  n[        U[        5      (       d   e[        UR                  5       5      S;  a  U$ [        R                  " UT5      nU(       a4  [        R                   " [        R                  R#                  U5      U5      $ [        U[$        5      (       aL  UR'                  5       b;  U(       a4  [        R                   " [        R                  R#                  U5      U5      $ U(       Ga  [)        UR                  5       5      n/ nUR'                  5       n	[+        [        UR                  5       5      S-
  5       H}  n
[,        R.                  R0                  R3                  X:   S5      (       d8  U	c  M9  [,        R.                  R0                  R3                  X   S5      (       d  Ml  UR5                  U
5        M     S/[        U5      -  nSUS'   Sn[+        [        U5      S-
  SS5       H  n
XS-      S:w  a
  XU
S-      -  nX;   aD  [,        R.                  R0                  R3                  [7        XS-      T5      S5      (       a  SX'   Ma  [,        R.                  R0                  R3                  [7        UT5      S5      (       d  [9        UT5      T-  nXU
'   M     [        R                  R;                  X5      $ U(       a4  [        R                   " [        R                  R#                  U5      U5      $ [        U[$        5      (       aL  UR'                  5       b;  U(       a4  [        R                   " [        R                  R#                  U5      U5      $ U4S jn[        UR<                  [        R>                  5      (       a\  U" U5      (       dO  U" URA                  5       5      (       a4  [        R                   " [        R                  R#                  U5      U5      $ [        R                  R	                  X5      $ )	Nr   r   r   r   r.   r   rt  r,   c                   > [         R                  R                  R                  [        R
                  " [        U R                  5       S   T5      S5      5      $ )Nrt  r   )rR   r]   r   r6  r   r7  r*   r   )r   	ALIGNMENTs    re   
is_alignedDsdpa_constraint.<locals>._apply_constraint_inner.<locals>.is_aligned  s@    77##22QZZ\"-y91= rg   )!is_cudais_xpur1   r.  r/  ra   r{   rQ   r  '_scaled_dot_product_efficient_attentionr  rk   rB   r   r   is_aligned_realized_tensortry_match_insignificant_stridesrealize_inputr9   maybe_get_strideri   r  rR   r]   r   r  ro   r*   rE   r  rm   r5   r  )r_  r   r  r  r1  effn_attn_fwd_biasis_aligned_tensorr  expanded_dimsmaybe_striders   out_stridesr  r  r  r  s                 @re   r  0sdpa_constraint.<locals>._apply_constraint_inner_  s     HOO??77JJ 	 NNyy~~EEMMN q 	 #y))))s||~f,J99#yI55--c24D 
 sF##$$&2!55--c24D  CLLN+HM //1L3s||~.2377##;;<L<OQRSS ,((@@RSTT!((+ 4 $X.KKOF3x=1,b"51u%*#q1uo5F %ww''??KA.	:A  *+ ww''??	*A  %VY7)CF!'A% 6( ??88JJ55--c24D 
 sF##$$&2!55--c24D 	
 chh,,c??coo/00==55c:<L  33CFFrg   c              3  @   >#    U  H  u  nu  p#T" XU5      v   M     g 7fr   r   )r   r_  r   r  r  s       re   r   "sdpa_constraint.<locals>.<genexpr>  s)      "DC# 	6**"Ds   rt  )r   rj   r  rt   r   r   )r  rt   r   r   r
  r  r  s   `    @@re   sdpa_constraintr  )  s    1fdGL  "+Cgll,C"D D IOWa!"a):;;FW< Xs   $B )r  c                   [        U[        R                  5      (       d&  [        XR	                  5       U R                  5       S9nUnU R                  5       UR                  5       :w  a  [        X0R                  5       5      nU R	                  5       UR	                  5       :w  a  [        X0R	                  5       5      nU R                  5       UR                  5       :w  a$  [        X0R                  5       5      n[        U5      $ [        U5      $ Nrb  )rk   r1   r9   rI  r   rn   r  r   r   r4  r  )r  r  r  r   rq   s        re   r  r    s    c299%%S 09JKACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8Org   )memory_formatc          	         [         R                  " U R                  5       U R                  5       U R	                  5       [        U R                  5       5      S9$ Nrh  )r?   r   rn   r   rj  ri   r   )r   r  s     re   r  r    s@     ||~kkmAJJL!	 rg   c                "   / n[        U [        5      (       a  [        U R                  [        R                  5      (       a  U R                  n [        U [        R                  5      (       aL  UR                  U R                  5       5        U R                  n [        U [        R                  5      (       a  ML  [        U 5      n [        U 5      n U(       a:  U R                  n US S S2    H  n[        R                  " XS9n M     [        U 5      n U $ )Nrt  r  )rk   rB   rm   r1   r   ro   r  r  )r   reinterpret_view_layoutsr  s      re   clone_preserve_reinterpret_viewr    s    !!YJqvvr7I7I$J$JFFB..//$++ALLN;A B..// aLaAFF.tt4F""9A 5aLHrg   r  c               Z   ^^^ UUU4S jn[         R                  " [        U5      TUU /S9$ )Nc                @   > [         R                  " TU S   -  T-   TS9$ )Nr   r   rQ   r\  )rB  r   r  r  s    re   r   iota.<locals>.fn  s!    ~~dU1Xo5UCCrg   rh  )r?   r   rF   )lengthr  r  r   rv   requires_gradr   s    ```   re   iotar    s1    D V$x	 rg   )r   rv   r  
pin_memoryr  c          	     T    Uc   e[        X-
  U5      n[        UU UUUb  UUS9$ SUS9$ )Nr   r  r  r   rv   r  )rE   r  )	r  r  r  r   rv   r  r  r  r  s	            re   arange_start_stepr    sT     S[$'F+v# 
 27# rg   c           	       ^^^^ [        XR                  5       5      nU R                  5       m[        U TS5      m[        R
                  R                  R                  [        R                  " TS5      5      (       a  TU R                  5       T   -   mOo[        R
                  R                  R                  [        R                  " TS5      5      (       a  O+[        [        R                  R                  5      " XTT5      $ [        R
                  R                  R!                  ST5        [        R
                  R                  R#                  TU R                  5       T   5        [%        ['        UT5      U R                  5       5      nUR                  5       mUUUU4S jn[(        R*                  " U R-                  5       U R                  5       U[/        U R                  5       5      S9$ )Nr   c           	       > [         R                  " [         R                  " [         R                  " U T   [        R
                  5      [         R                  " T[        R
                  5      5      T" U 5      T" U 5      5      $ r   )rQ   r   eqr\  r{   r  )r_  r   rB  
src_loaderrj  s    re   rf   select_scatter.<locals>.inner_fn  sX    yyFFs3x5uekk2 sOSM
 	
rg   rh  )r   r   rj  r  rR   r]   r   r6  r   r|  r   rz  r  r  select_scatterr  r  r  r4  rZ  r?   r   rn   ri   )r   r  r   rB  rf  r  rj  s     `` @@re   r  r    sX   
3
&C}}H
3
"Cww&&uxxq'9::

S))	
			(	(%);	<	<   3 3 ; ;<QS%PPGGq%(GGeQZZ\#%67
3$ajjl
3C"J
 
 ||~kkmAJJL!	 rg   c           	     6  ^ ^^^^^^	^
 [        UT R                  5       5      nT R                  5       m
[        T TS5      mT R	                  5       T   m[
        R                  R                  T TTT5      u  mm[        T R	                  5       5      n[        TT-
  TS-
  -   T5      UT'   [        X5      nUR                  5       m	UUUU	UUU U
4S jn[        R                  " T R                  5       T R                  5       U[        T R	                  5       5      S9$ )Nr   r.   c                V  >^ T	S:X  a  TT:X  a  T
S:X  a  T" U 5      $ [         R                  " U T   [        R                  5      n[	        U 5      m[        U T   T	-
  T
5      TT'   / nT	S:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T	5      [        R                  5      5      5        TT:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T5      [        R                  5      5      5        T
S:w  a}  UR                  [         R                  " [         R                  " [        U T   T	-
  ST
5      [        R                  5      [         R                  " S[        R                  5      5      5        U(       d   e[        R                  " [         R                   U5      n[         R"                  " UUU4S j[%        T5      (       a  SOS5      n[         R&                  " UUT" U 5      5      $ )Nr   r.   c                    > T" T 5      $ r   r   )src_idxr  s   re   rL  1slice_scatter.<locals>.inner_fn.<locals>.<lambda>c  s
    Jw'rg   r  )rQ   r\  r{   r  ri   r(   ro   r  r   r4  r  r  r+   r]  r  r+  r  r  r   r   )r_  r  r  src_valr  r   r  r  r  r  r  r   rj  s       @re   rf  slice_scatter.<locals>.inner_fn>  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' ##A

 yySM
 	
rg   rh  )r   r   rj  r  r   r1   r  normalize_start_endri   r(   r4  r?   r   rn   )r   r  r   r  r  r  src_sizerf  r  r  rj  s   ` ````  @@@re   slice_scatterr  /  s    
3
&C}}H
3
"Czz|C H 11!S%EJE3AJJL!HS5[D1H5t<HSM

C"J,
 ,
\ ||~kkmAJJL!	 rg   c                v    [        U [        [        45      (       a  [        U 5      S:  a  [	        U S   5      $ U $ r  )rk   ri   r   r   _unwrapr   s    re   r  r  t  s0    !dE]##A
qt}Hrg   r   rv   r  r  c               :  ^ ^ [        US [        R                  4;   SU 35        [        U(       + S5        [        [	        T 5      [
        5      (       a  T=(       d    [        R                  mOT=(       d    [        R                  " 5       m/ n[        T [        R                  5      (       a  U U4S jnO[        T [        [
        45      (       a  U U4S jnO[        T 5      S:X  d-  [        T S   [        [
        45      (       aE  [        T 5      S::  a6  UR                  [        R                  " [        T 5      5      5        U U4S jnO3[        R                  R!                  [        R"                  " T TUS95      $ [$        R&                  " [)        U5      TUUS	9$ )
Nlayout=r  c                2   > [         R                  " TT5      $ r   r  rB  rm   r   s    re   rf  tensor.<locals>.inner_fn  s    >>$..rg   c                2   > [         R                  " TT5      $ r   rQ   r]  r  s    re   rf  r    s    <<e,,rg   r   r   c                   >^ ^ UUUU 4S jm[        T5      S:X  a  [        R                  " ST5      $ T" S[        T5      5      $ )Nc           	     n  > X:  d   eX-
  S:X  a  [         R                  " TU    T5      $ X-
  S-  U -   n[         R                  " [         R                  " [         R                  " TS   [
        R                  5      [         R                  " U[
        R                  5      5      T" X5      T" X!5      5      $ )Nr.   r,   r   )rQ   r]  r   r  r\  r{   r  )r  r  midbinary_searchrm   r   rB  s      re   r
  /tensor.<locals>.inner_fn.<locals>.binary_search  s    {"{;!#<<UU;;{q(50yyFFuQx=S%++6 "%-!#+ rg   r   )r   rQ   r]  )rB  r
  rm   r   s   `@re   rf  r    s<      4yA~||Au-- CI..rg   rb  rh  )r   r{   stridedrk   r  r   r  get_default_dtyper   r   r<  r   ro   IntegerrR   r]   add_tensor_constantrI  r?   r   rF   )rm   r   rv   r  r  ri  rf  s   ``     re   rI  rI  z  s+   v$..'&0BC:~|,'$-%%$2002!F$$$	/ 
D5#,	'	'	- 
Ta:d1gs|<<TaemmCI./	/( ww**LLU6:
 	
 V$	 rg   c                |    [        U [        5      (       a  Ub  [        X5      n Ub  [        X5      n U $ [	        XUS9$ r  )rk   rB   r   r  rI  )rm   r   rv   s      re   	as_tensorr    sA    $	""D(DT*D$F33rg   c                2    [        U [        R                  S9$ rH  r  )rm   s    re   long_tensorr    s    $ekk**rg   c                   [        [        R                  R                  R                  [        R                  R
                  R                  S   5      nUc   e[        U5      S:X  d   U5       e[        [        UR                  5       5      5      u  p#[        R                  " X#U 5      n[        R                  R                  U5      Ul        [        R                  R                  U5        [        R                  R
                  R                  S   n[!        U["        R$                  ["        R&                  ["        R(                  45      (       a  UR*                  R,                  $ [.        R0                  " U5      $ )Nr  r.   r@  )r%   rR   r]   r   r  r^   r  r   rP  r  r   r1   DynamicScalarr  r   r  rk   r{   r  SymFloatSymBoolrc   r  r   r  )rm   r  binding_symr  r  r@  s         re   _local_scalar_denser    s    2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&   %6%<%<%> ?@KkD9F''))&1FKGGv& ''


#
#E
*C#ennemmDEExx}}}}S!!rg   c                    g r   r   )rm   r   s     re   _assert_scalarr    s     rg   )rv   r  c                   g r   r   )r   r   r  r   rv   r  s         re   _assert_tensor_metadatar    s     rg   c                  ^^^ U m[        U [        [        45      (       d  [        TS5      (       a  TR                  m[        T[        [        45      (       a  UU4S jnO\[        T[
        R                  5      (       a  UU4S jnO5[        TR                  5       5      S:X  d   eTR                  5       mU4S jn[        R                  " UTU[        U5      S9$ )Nr   c                2   > [         R                  " TT5      $ r   r  rB  r   r   s    re   rf  _full.<locals>.inner_fn  s    <<u--rg   c                2   > [         R                  " TT5      $ r   r  r   s    re   rf  r!    s    >>%//rg   r   c                   > T" / 5      $ r   r   )rB  value_loaders    re   rf  r!    s    ##rg   rh  )rk   r   r<  r   r   r   r   r   r   rj  r?   r   ri   )
fill_valuerv   r   r   rf  r   r$  s     `  @@re   _fullr&    s    Ej3,//GE74K4K%#u&&	. 
E5;;	'	'	0 5>>#$)))((*	$ Dz	 rg   c                8    [        [        U5      5      " U 40 UD6$ r   create_tensor_liketensor_constructor)r   r%  r   s      re   rE  rE    s    0<=aJ6JJrg   c                &   ^  S S S S SS S.U 4S jjnU$ )NF)namesr   rv   r  r  r  c                0  > [        U S L S5        [        US [        R                  4;   SU 35        [        U(       + S5        [        U5      nU=(       d    [        R                  " 5       n[        U5      S:X  a;  [        US   [        [        [        R                  45      (       a  [        US   5      nU H$  n[        U[        R                  5      (       d  M$   e   U Vs/ s H  n[        R                  " U5      PM     nn[        TX!U5      $ s  snf )Nnamed tensorsr  r  r.   r   )r   r{   r  rF   r  r   rk   ri   r   Sizer  r   r4  r&  )	r,  r   rv   r  r  r  r   r:  r%  s	           re   r  !tensor_constructor.<locals>.inner%  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLLa>D A!!U\\2222 )-.AQ.Z55 /s   $ Dr   )r%  r  s   ` re   r*  r*  #  s(     6 6. Lrg   )r,  r   r  rv   r  r  c           	         [        U S L S5        [        U5      n[        U5      S:X  a;  [        US   [        [
        [        R                  45      (       a  [        US   5      n[        US XX4S9$ )Nr.  r.   r   r   r  rv   r  )	r   rF   r   rk   ri   r   r{   r/  empty_strided)r,  r   r  rv   r  r  r   s          re   rl  rl  ?  se     u}o.6"F
4yA~*T!WtUEJJ.GHHT!W~d%v rg   c                $   ^  SSSSSS.U 4S jjnU$ )zR
Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
NF)r   rv   r  r  r  c                 > [        U(       + S5        [        US [        R                  4;   SU 35        Uc  U R                  5       nO[	        U5      nU=(       d    U R                  5       n[        U R                  5       5      nT" XaX#US9$ )Nr  r  r  )r   r{   r  r   r   rn   ri   r   )r   r   rv   r  r  r  r   creation_fns          re   _constant_like*create_tensor_like.<locals>._constant_likeW  s~     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!f

 	
rg   r   )r6  r7  s   ` re   r)  r)  R  s#     duTX
 
 rg   c                *    [        [        U 5      5      $ r   r(  )r%  s    re   r#  r#  i  s    0<==rg   c                "   ^  S S S S S.U 4S jjnU$ )Nr2  c                 > [        U[        [        45      (       d   e[        U(       + S5        [        US [        R
                  4;   SU 35        [        U5      =(       d    U R                  5       nU=(       d    U R                  5       nU Vs/ s H  n[        R                  " U5      PM     nn[        T[        U5      X!5      $ s  snf )Nr  r  )rk   ri   r   r   r{   r  r   r   rn   r   r  r&  rF   )r   r   r   r  rv   r  r:  r%  s          re   _new_constant#new_constant.<locals>._new_constants  s     $u....z><06dEMM22gfX4FGU#4q{{})1<<>*./$Qa $/Zv!6DD 0s    C	r   )r%  r<  s   ` re   new_constantr>  r  s     tDT
E 
E rg   r2  c          	     |    Uc  U R                  5       nUc  U R                  5       n[        US UU[        U5      US9$ Nr2  r   rn   r3  rF   )r   r   r   r  rv   r  s         re   	new_emptyrB    sF    }~V$ rg   c                  [        U [        [        45      (       d   e[        U[        [        [        S 5      45      (       d   e[	        US [
        R                  4;   SU 35        [        U5      =(       d    [
        R                  " 5       nU=(       d     [
        R                  " S5      R                  n[        U5      n[        SXBU S9nUR                  5         UR                  R                  n[        R                   " UR                  S/[#        U 5      -  S9Ul        [        U[$        R&                  5      (       d   eU  Vs/ s H  n[(        R*                  " U5      PM     n nU(       a'  U Vs/ s H  n[(        R*                  " U5      PM     snO[$        R,                  R/                  U 5      n[$        R0                  " UUU UU=(       d    SS9Ul        U$ s  snf s  snf )Nr  r  r   )r%  rv   r   r   )ri  F)rv   r   r   r  	is_pinned)rk   ri   r   r   r   r{   r  r   r  rI  rv   rF   r&  r  rm   dataclassesreplacer   r1   r  r   r4  r  r  r  r  )	r   r  r   r  rv   r  	pointwiser  r:  s	            re   r3  r3    s    dT5M****ftUDJ78888v$..'&0BC<5#:#:#<E/u||C(//F6"F6TJI^^  F%%fkk1#D	/JFKfb//0000%)*TELLOTD*  #))&Qa&)11$7 
 NN%FM  +)s   	 G 6 G%c          	     |    Uc  U R                  5       nUc  U R                  5       n[        UUUU[        U5      US9$ r@  rA  )r   r   r  r   r  rv   r  s          re   new_empty_stridedrI    sH     }~V$ rg   c                    [         R                  R                  R                  U5      n[	        [        [        U5      5      UR                  S9n[        R                  R                  X5      $ )N)r  )rR   r]   r   r   sortedr  r   __getitem__r1   r.  r/  )r   r  r1  s      re   copy_stridedrM    sL    WW55f=F%F,&2D2DEL??//@@rg   c                X    UR                  S5      c   S5       e[        U5      " U 40 UD6$ )Nr   z(dtype should be handled by decomposition)r  r*  )r   r%  r   s      re   fullrO    s2    ::g*V,VV*j)$9&99rg   c                  ^^^^ [        U [        5      (       d   eUR                  5       S:X  a  [        XR	                  5       5      $ U R	                  5       m[        T5      S:H  n[        U TU5      mU(       a  [        U S/5      n S/mU R                  5       mUR                  5       mUUUU4S jn[        R                  " U R                  5       U R                  5       UUR	                  5       S9$ )Nr   r.   c                   > [        U 5      n [        R                  " T" U 5      TT   5      n[        U 5      S:X  a  U/n OXT'   T" U 5      $ r  )ri   rQ   indirect_indexingr   )r_  
gather_idxr   index_loaderr   rj  s     re   r   gather.<locals>.fn  sJ    3i**<+<d3iH
s8q=,C!H}rg   rh  )rk   rB   	get_numelrB  r   r   r  r4  rj  r?   r   rn   r   )	r   r   rB  sparse_gradr  r   rT  r   rj  s	    `    @@@re   gatherrX    s     a####ANN,--::<DY!^F
3
'C1qcNs}}H$$&L  ||~kkm~~	 rg   c                V  ^^^^	^
 U(       a+  [        [        R                  R                  5      " XX#U5      $ U(       a   e[	        U [
        5      (       d   e[	        U[
        5      (       d   eS[        UR                  5       5      ;   d   eU R                  5       m	UR                  5       m[        UR                  5       5      mU R                  5       m
/ UR                  5       QT
SS  QmUUUU	U
4S jn[        R                  " U R                  5       U R                  5       UTS9$ )Nr   r.   c                   > [        U 5      [        T5      :X  d   U  ST 35       eT" U S T 5      n[        R                  " UTS   5      // U TS  Q-   nT" U5      $ )Nz != r   )r   rQ   rR  )r_  	var_index
weight_idxindices_loaderindices_ndimrm  weight_loaderweight_sizes      re   r   embedding.<locals>.fn	  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((rg   rh  )r  r  	embeddingr  rk   rB   strr   rj  r   r   r?   r   rn   )weightr$  padding_idxscale_grad_by_freqsparser   r]  r^  rm  r_  r`  s         @@@@@re   rb  rb    s    6 67[f
 	
 :fi((((gy))))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 rg   c           
     L   [        S U  5       5      (       d-   SU  Vs/ s H  o"c  M  UR                  5       PM     sn 35       e[        S U  5       5      (       a  [        S5      e[	        U 5       VVs/ s H  u  p#[        U[        5      (       d  M  UPM      nnn[        U5      S:  d   S5       eS /[        U 5      -  n[        U[        U Vs/ s H  o U   PM	     sn6 5       H(  u  p#UR                  5       U:w  a  [        S5      eX5U'   M*     XT4$ s  snf s  snnf s  snf )Nc              3     #    U  HY  nUc  M  UR                  5       [        R                  [        R                  [        R                  [        R
                  4;   v   M[     g 7fr   )r   r{   r  r  r   r"  r   rs   s     re   r   .check_and_broadcast_indices.<locals>.<genexpr>  sA      A 	M%++u{{EJJLLs   A#AA#z)indices must be int64, byte or bool. Got c              3     #    U  H:  oc  M  UR                  5       [        R                  [        R                  4;   v   M<     g 7fr   )r   r{   r   r"  rj  s     re   r   rk  !  s.      <Cq2%**ekk22Gs
   A8AzFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r#  r   r$  r   rj   rk   rB   r   r  r  rn   )r$  rv   rs   r   
valid_idxsnew_indicess         re   check_and_broadcast_indicesro    s2       
 4G4eGq]Q[[]G4e3fg   <C   ""=>> )' 2O 2jI6N! 2JOz?QD DD&3w<'KJ 1
3S
1AJ
3S TU <<>V#%&VWWA V ""# 5f P 4Ts   DD5DDD!c	           
     l  ^^^^^^^^^^ Sm[         R                  " T5       H  u  pX-
  S:w  d  M  SmM     [        T5       VVs/ s H  u  pUb  M
  X   PM     snnm/ TQU [        T5      [        T5      -   S  QmTS   nT(       a  TT-   mOTS U T-   TUS  -   mUUUUUUUUUU4
S jnTU4$ s  snnf )NFr.   Tr   c                  >
 [        U 5      [        T5      :X  d   e[        T5      [        T
5      :X  d   e[        T5      n/ nTS   nT(       a  SOUnSn[        TS   S-   5       H|  nXd:X  a  XQ-  nTU   c+  U[        U 5      :  d   eUR                  X   5        US-  nM=  TU   nUc   eT
U   nUR                  [        R                  " U" XXA-    5      UT	TS95        M~     / UQXS  QnTc  U$ T" U5      $ )Nr   rt  r.   r   wrap_neg)r   r  ro   rQ   rR  )r_  rq  	new_indexfirst_tensor_indexstart_offsetnext_idxrs   loaderr   r   indexed_sizer$  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizers  rj  s            re   r   *index_output_size_and_inner_fn.<locals>.fnW  s=   3x3{++++?#s<'8888;	+A.3q9K~b)A-.A  qz!#c(***  /A(+)))#A  ))s,2EFG#!)	 /&

^
	 %,yE(92EErg   )r  pairwiserj   r   )r  r$  r}  r~  rz  ry  rj  r   rs  previouscurrentrs   r@  ru  r   r{  r|  s    ````````      @@re   index_output_size_and_inner_fnr  3  s    $ $&//?"&*# @ ,5W+=M+=969+=MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	 F  FD ?_ Ns   	B0B0c                    [        XU5      u  p4n[        R                  " U R                  5       U R	                  5       UUS9$ r  )index_impl_helperr?   r   rn   r   )r   r$  r   r|  rf  r;  s         re   
index_implr  |  s>    0UCK1||~kkm	 rg   c                  ^^ [        U[        [        45      (       d   eU R                  5       m[	        XR                  5       5      u  p[        U5      S:  d   S5       eU Vs/ s H  oUb  UR                  5       OS PM     nn[        XS      R                  5       5      nU R                  5       n[        [        U5      5       Vs/ s H  oQU   c  M
  X   PM     n	nU(       a  SU	;   a  SU;  a  [        S5      e[        [        U5      5       Vs/ s H  oXU   PM	     n	n[        UUUUUU	S UUS9	u  n
mUU4S jnXT4$ s  snf s  snf s  snf )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0rr  c                    > T" T" U 5      5      $ r   r   )r_  index_inner_fnrj  s    re   rf  #index_impl_helper.<locals>.inner_fn  s    s+,,rg   )rk   ri   r   rj  ro  rn   r   r   r  
IndexErrorr  )r   r$  r   rs  r}  rs   rz  r~  r  ry  r|  rf  r  rj  s               @@re   r  r    sL   ge}----}}H9'<<>RG~"F$FF"KRS7a-q}}TA7OS wa01::<=KZZ\F',S\':U':!ajIFI':LUl"q';KLL',S\':;':!1I':L;"@
#K- .009 T V <s   %E		EEEc                     [        XSS9$ ! [         a;    U R                  5         [        [        R
                  R                  SS9" X5      s $ f = f)NTr   Fr  )r  r   r  r  r  rB  r  r   r$  s     re   rB  rB    sM    
!D11 
			

 1 1uM
 	

s   	 AAAc                    [        XSS9$ )NFr  )r  r  s     re   _unsafe_indexr    s    a..rg   c           	     .    [        [        U 5      XUSSS9$ )NTFr   may_realizeindex_put_impl_r  r   r$  r  
accumulates       re   	index_putr    s    a':Tu rg   c           	     .    [        [        U 5      XUSSS9$ )NFr  r  r  s       re   _unsafe_index_putr    s    a':U rg   c                    UR                  5       U R                  5       :w  a  [        X R                  5       5      nU(       a  [        X5      n[        U [	        US   X 5      5      $ r  )rn   r  r   r  r   )r  r$  r   r  s       re   index_put_as_masked_fillr    sP    T__..%!23D T5U9::rg   c                   SSK Jn  [        [        R                  [
        R                  R                  R                  R                  5      n[
        R                  R                  n[        R                  (       dQ  UbN  U" U5      (       aA  SnUR                  R                  SS 5      =n(       a  U SU 3nU[
        R                  l        [        R                   " XPXU5        U $ )Nr.   ),_fx_node_is_input_dependent_cudagraph_unsafezLindex_put_ fallback with boolean indexing is not compatible with CUDA graphsstack_trace Found from : 
 )utilsr  r   r  
index_put_rR   r]   r^   ra   _overloadnamer/   graph_partitionr  r  disable_cudagraphs_reasonr1   IndexPutFallback)	r  r$  r  r  r  r  r  r   r  s	            re   index_put_fallbackr    s    C$//177+?+?+F+F+T+TUK
 gg""G""8AA\!,,**=$??;?E*;-8C,/)7JGKrg   c           	         [        XX#SSS9$ )NTr  r  r  r$  r  r  s       re   r  r    s    v4 rg   c           	         [        XX#SSS9$ )NFTr  r  r  s       re   _unsafe_index_put_r    s    vD rg   c                  ^ U(       aU  S m[         R                  " U 5      UR                  5       ;   a*  [        U4S jU 5       5      (       d  UR	                  5         UR                  5       S:X  a  [        U5      S:X  a  US   R                  5       [        R                  [        R                  4;   a_  US   n[        [        UR                  5       5      [        U R                  5       5      5       H  n[        US5      nM     [        X/X#5      $ [        R                  " 5       (       a  [!        XX#5      $ U HH  nUc  M  UR                  5       [        R                  [        R                  4;   d  M<  [!        XX#5      s  $    U R                  5       n	[        U	5      n
U(       aQ  [#        U R                  5       5      (       a3  U
S:X  a  [%        U S/5      n [!        XX#5      n U
S:X  a  [%        U / 5      n U $ ['        X R                  5       5      n [)        XR+                  5       5      u  pU Vs/ s H  ob  UR/                  5       OS PM     nn[1        U [2        5      (       d   eU R	                  5         U
S:X  a  [%        U S/5      n [5        XS      R                  5       5      n[        [        U5      5       Vs/ s H  oU   PM	     nn[7        U	UUUUUS US9u  nn[9        UU5      nU R+                  5       nUc   e[         R:                  " UU R                  5       UR/                  5       UUU(       a  SOS S9n[         R<                  " S [         R>                  " U 5      US	9n[@        RB                  RE                  U5      Ul#        [@        RB                  RI                  U5        U
S:X  a  [%        U / 5      n U $ ! [,         a    [!        XX#5      s $ f = fs  snf s  snf )
Nc                $   [        U [        5      (       a  [        U R                  [        R                  5      (       a  U R                  R                  5       n [        U [        R                  5      =(       a    [        U R                  [        R                  5      =(       al    [        U R                  SS 5      =(       aN    U R                  R                  R                  [        R                  R                  R                  R                  L $ g)Nr  F)rk   rB   rm   r1   r5   r  r   r.  r   r  ra   r{   rQ   r  randpermr  )indices    re   indice_slice_from_randperm3index_put_impl_.<locals>.indice_slice_from_randperm  s     &),,FKK1U1U002vr}}5 V"6;;@VY=V ++22eiinn6M6M6U6UU	 rg   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   )r   r  r  s     re   r   "index_put_impl_.<locals>.<genexpr>  s      H
=D6&v..Wr6  r.   r   rt  r  
atomic_addrv   r   rf  ri  output_indexerscatter_moder   r  rm   )%r1   try_get_namer2  r#  r  rV  r   r   r{   r   r"  r  r   rZ  r  $are_deterministic_algorithms_enabledr  rK   r  r   ro  rn   r   rj  rk   rB   ri   r  r4  Scatterr  MutationLayoutSHOULDREMOVErR   r]   r  r   r  )r  r$  r  r  r   r  r  r;  rB  r  x_ndimr}  rs   rz  r~  ry  expected_vals_sizerf  rv   scatterr  r  s                        @re   r  r     sq   	 ??4 F$9$9$;;C H
=DH
 E
 E
 NN 	aLAAJ  "uzz5;;&??qzs4==?+S-ABAT2&D C'ffII 1133!$DD !2uzz5;;6O!O%dVHH  ]]_F[FB4>>CSTTQ;qc?D!$DQ;b>Dfnn./FE"=__&#
 LSS7a-q}}TA7OSdI&&&&LLN {D1# wa01::<=K',S\':;':!1I':L;#A	$  F./F __Fjjnn##%!%/\TG ,,T2F
 ''))&1FKGGv&{D"~Kc  E!$DDE T <s   O 5O"O'OOr  c                   ^^^^^	 [        XSSS9u  pEmTR                  5       mU R                  5       m	UUUUU	4S jn[        R                  " U R	                  5       U R                  5       UUS9$ )NFrr  c                   >^  TR                   [        R                  :w  a,  [        R                  " T" T 5      [        R                  5      nOT" T 5      n[        R
                  " UUU U4S jT5      $ )Nc                     > T" T " T5      5      $ r   r   )_unsafe_index_fnr_  self_loaders   re   rL  8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,Nrg   )r   r{   r   rQ   r   r  )r_  mask_valr  fillr  mask_loaderr  s   ` re   rf  &_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUrg   rh  )r  rj  r?   r   rn   r   )
r  r  r$  r  ri  r;  rf  r  r  r  s
    ` `   @@@re   _unsafe_masked_indexr    ss    "3UU#F ""$K""$KV V  nn	 rg   c           	         [        XS5      nU R                  5       n[        [        U5      5       Vs/ s H%  nX&   (       a  [	        X&   XV   * XV   S-
  5      OS PM'     nn[        XUSS9$ s  snf )Nr   r.   T)r  )r   r   r  r   r  r  )r   r  r$  r  masked_valuer[  rs   clamped_indicess           re   #_unsafe_masked_index_put_accumulater    sx    q)LJJLE s7|$$A 7>jgj58)UX\2dJ$   Q$OOs   ,A,c                X    [         R                  " U[         R                  " X 5      5      $ r   )rQ   r  r  r   minmaxs      re   r  r    s    ;;sCKK/00rg   c                J    [        U 5      n[        XRX45      n[        Xa5        U$ r   )r  r  copy_)r  r  r   r  r  r7  output_views          re   as_strided_scatterr    s$    4[FV6BK	+Mrg   c                0    [        [        U 5      XU40 UD6$ r   )scatter_r  )r   r   rB  r  r   s        re   r  r    s    E!Hc#888rg   r+  include_selfc               T   [        U[        5      n[        U UUR                  5       [	        [
        R                  U(       a  UR                  5       O
[        U5      5      U(       a  UR                  5       R                  OSU5      (       a  [        R                  " U UUUUUUS9  U$ g )Nznot implr  )rk   rB   rP   r   r
   r{   r   r   rn   r1   ScatterFallback)r  r  r   rB  r  r+  r  src_is_tensors           re   scatter_fallbackr    s     sI.MU[[]#--/S	J!.J  	%	
 rg   r+  c          	         US;   d   eUcV  [        [        R                  [        R                  R
                  R                  R                  5      n[        XPXX4S9nUb  U$ US:X  a  SnOUS:X  a  Sn[        XX#U5      $ )N)Nr   multiplyr  r   re  r  prod)
r   r  r  rR   r]   r^   ra   r  r  scatter_reduce_)r  r   rB  r  r+  r  fallback_results          re   r  r    s    ....~dmmQWW-A-A-H-H-V-VW*s3
 &""	:	4e&99rg   c                .    [        [        U 5      XU5      $ r   )scatter_add_r  r   r   rB  r  s       re   scatter_addr    s    a#c22rg   c                    [        XX#S5      $ )Nre  )r  r  s       re   r  r    s    15u55rg   c                0    [        [        U 5      XX440 UD6$ r   )r  r  )r   r   rB  r  reduction_typer   s         re   scatter_reducer    s    58SOOOrg   )r  c          
       ^ ^^^^ US;   d   e[        [        R                  R                  5       5      S:X  a"  S[        R                  R                  5       ;   d   S5       e[	        T[
        5      (       a  [        T T5      m[        [        R                  R                  T TUTUUS9nU(       a  U$ [	        T [        5      (       d   eS[        UR                  5       5      ;   d   e[        T R                  5       5      nUS:X  a  [        T S/5      m [	        T[        5      (       a*  [        TR                  5       5      S:X  a  [        TS/5      m[	        U[        5      (       a*  [        UR                  5       5      S:X  a  [        US/5      nUR                  5       S:X  a  T $ [        T T5      mT R!                  5         UR#                  5       m[	        T[        5      (       a  TR#                  5       OS mUUU 4S jnU UU4S	 jn	S
 n
T R%                  5       nUc   eU(       d  [&        R(                  " UT R                  5       U 4S jUR                  5       US S9n[&        R*                  " S [&        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        [&        R(                  " UT R                  5       U	UR                  5       UU
" U5      S9n[&        R*                  " S [&        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        US:X  a  [        T / 5      m T $ )N)Nre  r  meanamaxaminr.   r  zKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                   > TR                  5       n[        U5      n[        U 5      n[        R                  " T" U 5      US:X  a  SOUT   SS9UT'   U$ )Nr   r.   F)rs  )r   r   ri   rQ   rR  )r_  r[  r  indirect_idxr   rT  r  s       re   r  'scatter_reduce_.<locals>.output_indexer&  sU    5zCy11DAIq5:
S rg   c                l   > T(       a  T" U 5      $ [         R                  " TTR                  5       5      $ r   rQ   r]  r   )r_  r  r  r  s    re   r   scatter_reduce_.<locals>.fn0  s*    c?" <<T^^%566rg   c                    U S:X  a  gU b   eg )Nre  r  r   r  s    re   backend_reduce_str+scatter_reduce_.<locals>.backend_reduce_str8  s    U? >!>rg   c                N   > [         R                  " STR                  5       5      $ r  r  )rB  r  s    re   rL  !scatter_reduce_.<locals>.<lambda>H  s    3<<4>>3C#Drg   r  r  )r   r  r  r   rk   r    rE  r  r  rB   rc  r   r   r  rV  r  r  rj  rn   r1   r  r  r  rR   r]   r  r   r  )r  r   rB  r  r+  r  r  r  r  r   r  rv   zero_outr  r  rT  r  s   `` `           @@re   r  r    s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&&C)****t}}DqyD1##y!!c#,,.&9Q&>3n%##ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@&@"dJ7 __F::.."D>>#)
 ""006

 gg--f5	""6*
 jjnn~~%'/G ,,T2F
 ''))&1FKGGv&qyD"~Krg   c                d  ^^^^^^ U R                  5         U R                  5       mU R                  5       T* S  mU R                  5       S T*  nT Vs/ s H,  n[        R                  R
                  R                  U5      PM.     snm[        U5      T:X  d   eUn[        TU5       VVs/ s H	  u  phXh-  PM     snnm[        U5       H  u  piU	c  M
  SU	-  TU'   M     U4S jmUUUUU4S jn
[        R                  " U R                  5       U R                  5       U
/ UQUQS9$ s  snf s  snnf )Nr  c                  > [         R                  " U [        R                  5      n T(       a:  [         R                  " U [         R
                  " S[        R                  5      5      n [         R                  " U [         R
                  " U[        R                  5      5      n [         R                  " U [        R                  5      n [         R                  " XSS9$ )N      ?Fr  )
rQ   r\  r{   r  r   r]  r  r   r  rR  )r   r  r   exacts      re   scale_fn$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::rg   c                   > U T* S  nU S T*  nT
" / UQ[        UTT5       VVVs/ s H  u  p4nT	" X4U5      PM     snnnQ5      $ s  snnnf r   )r  )r_  r   r6  rs   r:  r   i_sizes
inv_scalesr  r  rj  s         re   r   upsample_nearestnd.<locals>.fn  s_    H!HWaW#aW:UV:UJA$8A$':UVW
 	
Vs   Arh  )realize_hintrj  r   rR   r]   r   r2  r   r  rj   r?   r   rn   r   )r   r|  scales_xr  r  batchrs   o_sizesor  r   r  r  r  rj  s      ``      @@@@re   upsample_nearestndr  m  s    NN}}HjjlA23GJJL1"E6=>gqww))!,g>Gx=AG$'$9:$9DA!%$9:Jh'%KJqM (	;
 
 ||~kkm!!!	 9 ?
 ;s   3D'+D,c                    [        XU4SS9$ )Nr.   r  r  r   r|  r  s      re   upsample_nearest1dr    s    avi1==rg   c                    [        XU4SSS9$ )Nr.   Tr  r  r  r  s      re   _upsample_nearest_exact1dr    s    avi1DIIrg   c                    [        XX#4SS9$ )Nr,   r  r  r   r|  scales_hscales_ws       re   upsample_nearest2dr    s     ax.BaHHrg   c                    [        XX#4SSS9$ )Nr,   Tr  r  r  s       re   _upsample_nearest_exact2dr    s     ax.BatTTrg   c                    [        XX#U4SS9$ )Nr   r  r  r   r|  scales_dr  r  s        re   upsample_nearest3dr     s     ax8.LPQRRrg   c                    [        XX#U4SSS9$ )Nr   Tr  r  r  s        re   _upsample_nearest_exact3dr"    s     	X6!4 rg   c                .   ^  [        U 4S jU 5       5      $ )Nc              3  R   >#    U  H  n[         R                  " UT5      v   M     g 7fr   r  )r   r   r   s     re   r   $_create_constants.<locals>.<genexpr>  s     6Aa''s   $')r   )r   rt   s   ` re   r  r    s    6666rg   c                   ^^^ U R                  5       mU R                  5       mUUU4S jn[        R                  " U R	                  5       U R                  5       UTS9$ )Nc                   > [        U 5      n [        U 5      [        T5      :X  d   eT H  nTU   S-
  X   -
  X'   M     T" U 5      $ r  )ri   r   )r_  r   r8  r,  rj  s     re   rx  rev.<locals>.loader  sN    3i3x3u:%%%Cc
Q#(2CH  }rg   rh  )rj  r   r?   r   rn   r   )r   r8  rx  r,  rj  s    ` @@re   revr)    sM     }}HJJLE ||~kkm	 rg   c                   S nU" 5       (       a  g[        U5      S:w  d  [        U R                  5       5      S:w  a  gU R                  5         [        U [        R
                  5      (       a  [        U R                  [        R                  5      (       a  [        U R                  R                  [        R                  5      (       dH  [        R                  (       aX  [        U R                  R                  [        R                  5      (       a%  U R                  R                  R                  (       d  gU R                  5         [        R                  " U 5      u  pEUR                  nUS   S:w  a  gUS   S:w  d  US   S:w  d	  US   S:w  a  gUS   nUS:X  a  gUS   nUR                   S   n	XU-   :  a  gU R                  R                  R                  n
UR                   S   UR                   S   U-   /nU["        R$                  R&                  U
'   [)        U UUR                  UR*                  5      n[-        USXU-   SS	9n[/        X5        [0        S
   S==   S-  ss'   U$ )z
This optimization changes the semantics of padding from 'clone'
style to 'view' style.

Thanks to functionalization, this change can still maintain numerical
correctness.
c                    [         R                  R                  n U c  g[        U R                  5      n[        U5      S:X  aF  US   R                  [        R                  R                  [        R                  R                  4;   a  gg)z
Conservatively check if padding can be fused with downstream op.
1. if the downstream op is a sum, then there is little benefit to
   do inplace padding
2. if the downstream op is a matmul, doing inplace padding can
   save membw.
Tr.   r   F)rR   r]   r^   r   r_   r   ra   r  mmr  addmm)r^   r_   s     re   _padding_can_be_fused6inplace_constant_pad_nd.<locals>._padding_can_be_fused  sk     ww++l(()u:?uQxGGOOJJ3
  
 rg   Nr   r,   r.   r   r   F)r   r  r  r  inductorinplace_padding)r   r   r  rk   r1   rB   rm   r   r  r/   can_inplace_pad_graph_inputInputBufferr   freeze_layoutr  r  r   rR   r]   buffer_to_padded_sizer  r  r  fill_r   )r   paddingr%  r.  r;  r  r  npadstride0rowsizebufnamepadded_size	resized_xsliced_xs                 re   inplace_constant_pad_ndr?    s   (  7|qC

-2
 IIK
 q",,''!&&"--00qvv{{B$5$56622qvv{{BNN;; vv{{OO((+IAmmGqzQqzQ'!*/WQZ1_1:DqyajGkk!nG4ffkkG;;q>6;;q>D#89K-8AGG!!'*		I iQgT>QVWH	(Z*+q0+rg   c           	     @   [        S U 5       5      (       d  gU R                  5       n[        U5      n[        [	        USSS2   USSS2   5      5      nSnSn[        U5       H/  u  nu  pU	S:w  a    gU
S:  a  Ub    gUS-
  U-
  nU
nM'  U
S:  d  M/    g   Uc  gU R                  5       nUb  UR                  S:X  a  g[        U5      nX|U'   U R                  5       n[        U5      " U5      n[        U5      " XUS9n[        S   S	==   S-  ss'   [        X/U5      $ )
a'  Decompose right-pad into cat([x, fill], dim) and delegate to cat lowering.

The cat lowering already has heuristics for choosing between pointwise_cat
(fusion) and ConcatKernel (memory planning / zero-copy).  By routing through
cat() we reuse those heuristics rather than duplicating them here.
c              3  B   #    U  H  n[        U[        5      v   M     g 7fr   )rk   r   r   r  s     re   r   _pad_as_cat.<locals>.<genexpr>O  s     37az!S!!7s   Nr,   r.   r   r   rb  r0  pad_rewritten_as_cat)r#  r   r   ri   r  rj   rn   r   r   r   r*  r   rE  )r   r7  r%  r,  r  	pad_pairspad_dim
pad_amountrs   leftrn  rv   	pad_shaper   fill_value_typed
pad_tensors                   re   _pad_as_catrL  E  s<    37333JJLEu:DS1wqt!t}56I GJ%i0=D1919"QhlGJQY 1  \\^FfkkU2 UI#gKKME$U+J7#$45vJ Z/0A50((rg   c                  ^^^^^^^ [        U5      S-  S:X  d   e[        S U 5       5      (       a  [        U 5      $ [        R                  (       a  [        XT5      nU(       a  U$ [        XT5      nUb  U$ U R                  5       n[        [        [        [        US S S2   USS S2   5      5      5      5      m[        U5      [        T5      -
  m/ mT H?  u  pVTR                  [        R                  R                  R                  U5      U45        MA     [        US T 5      n/ m[        TUTS  5       HC  u  u  pn
TR                  U
5        UR                  [         R"                  " X-   U	-   5      5        ME     [        U5      [        U5      :X  d   e[%        U R'                  5       5      " T5      mUUUUU4S jmUUU4S jnU R)                  5       m[*        R,                  " U R/                  5       U R'                  5       UUS9$ )Nr,   r   c              3  *   #    U  H	  oS :H  v   M     g7fr   Nr   rB  s     re   r   "constant_pad_nd.<locals>.<genexpr>}  s     
#7a67r"  r.   c                N  >^  / n[        T T	S  TT5       HL  u  nu  p4nUS:w  a  UR                  [        US5      5        US:w  d  M2  UR                  [        X%5      5        MN     [        R
                  " [        R                  U5      n[        R                  " UU U
4S jT5      $ )Nr   c                    > T" T 5      $ r   r   )rB  rj  s   re   rL  /constant_pad_nd.<locals>.mask.<locals>.<lambda>  s	    rg   )	r  ro   range_mask_lowrange_mask_highr  r+  rQ   r  r  )rB  r  r_  r^  r]  r  boundsr%  
mask_sizesr  rj  s   `     re   r  constant_pad_nd.<locals>.mask  s    (+E!"Ivz(J$C#faxN323qyOC89	 )K
 $/zz$ 7DDrg   c                   > [        U S T 5      n[        U TS  T5       H  u  nu  p4UR                  X#-
  5        M     [        U5      [        U 5      :X  d   eT" U5      $ r   )ri   r  ro   r   )rB  rt  r_  r^  _highbounds_precompr  r  s        re   	offset_fn"constant_pad_nd.<locals>.offset_fn  sa    rO	!$U12Y!?C#SY' "@9~U+++Irg   rh  )r   r#  r  r/   r1  r?  rL  r   ri   r/  r  ro   rR   r]   r   lookup_precomputed_sizer   r4  r   r   rj  r?   r   rn   )r   r7  r%  rq   r,  lhr|  r^  r]  r   r\  rV  r[  r  rW  r  rj  s     `         @@@@@@re   rF  rF  z  s   L1"""

#7
###Qx%a*=J a*
-C

JJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68Nqww//GGJANO  uRay/KJ qr3T$5<<
T(9:; 4 {s5z)))q{{}-j9JE E }}H||~kkm	 rg   c                    [         R                  " [         R                  " U [        R                  5      [         R                  " [
        R                  " U5      [        R                  5      5      $ r   )rQ   r  r\  r{   r  r   r  )rs   r^  s     re   rT  rT    s@    66q%++&u}}S)5;;7 rg   c                    [         R                  " [         R                  " U [        R                  5      [         R                  " U[        R                  5      5      $ r   )rQ   r  r\  r{   r  )rs   r]  s     re   rU  rU    s7    66q%++&tU[[) rg   c                T    [         R                  " [        X5      [        X5      5      $ r   )rQ   r  rT  rU  )rs   r]  r^  s      re   
range_maskrd    s#    88q  rg   c                   ^ ^^^^^^^ T R                  5       T* S  mT R                  5       mT=(       d    S/T-  mUUUUUUU U4S jnU$ )Nr   c                h  >^^ U S T*  mU T* S  m[         R                  " [        R                  [	        T5       Vs/ s H   n[        TU   TU   T
U   -   T
U   * 5      PM"     sn5      nT	(       a   [        R                  " UUUUUU4S jT5      $ [        R                  " UUUU4S jT5      $ s  snf )Nc                 .   > [        TTT S9" / TQTQ5      $ )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixr   s   re   rL  ;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    3A~3O"fNrNrg   c                    > T" / TQT Q5      $ r   r   )rj  rl  rj  s   re   rL  rm    s    (>V>b>*Brg   )r  r+  rQ   r  r  rd  r  )rB  rs   r  rj  rl  r   r%  r`  rk  r7  	padding_hr   rj  s      @@re   ry  )constant_boundary_condition.<locals>.load  s    uC45\HHLQRUJWJqZ1qtil2Yq\MBJW
  JJ  
	
 D"BJO
	
 Xs   'B/
)r   rj  )	r   r%  r7  rk  r   ry  r`  ro  rj  s	   ````` @@@re   ri  ri    sE     	


cTUA}}H$A39I
 
* Krg   dilationc               .   Uc  S/[        U5      -  n[        U SXA   -  -   Xa   X!   S-
  -  -
  X1   S-
  -   X1   5      nU(       a  [        U SXA   -  -   Xa   X!   S-
  -  -
  SX1   S-
  -  -   X1   5      n[        R                  R                  R                  [        R                  " US-
  X1   -  U -
  XA   -
  S5      5      (       a  US-  n[        R                  R                  R                  [        R                  " Xx5      5      (       a  SnXu4$ UnXu4$ )Nr.   r,   r   F)	r   r(   rR   r]   r   r6  r   rz  r7  )	r   rs   kernel_sizer  r7  	ceil_moderr  x_outx_alts	            re   pooling_sizerx    s8   3W%	A
NX[KNQ,>??69q=Q	E
 '*nk[^a/01 69q=!" I
 77**HHeai69,q07:=qA
 
 QJE77**588E+ABBI  Erg   c               l    [        X5      n [        R                  " [        R                  U 5      nUS:  $ )N   )rL   r  r+  r  r  )rt  n_dimwindow_sizes      re   %should_fallback_max_pool_with_indicesr}  	  s-    {2K""8<<=Krg   assert_fallbackc                  US:X  a  S/U-  nUS:X  a  S/U-  nU(       d  Un[        X5      n[        X%5      n[        X55      n[        XE5      n[        U [        5      (       d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U R	                  5       5      US-   US-   4;   d   e[        XS9nUb  Xv:X  d   eXX4U4$ )Nr   r.   r,   r{  )rL   rk   rB   r   r   r}  )r   rt  r  r7  rr  r{  r  use_fallbacks           re   max_pool_checksr    s    !|#+1}3;{2K&(F7*GH,Ha####{u$$$v;%w<5   x=E!!!qzz|EAI 66668RL"...<??rg   c               &  ^^^^^ U R                  5         U R                  S T*  nU R                  T* S  n[        [        T5       V	s/ s H  n	[	        X   XTTUTS9PM     sn	6 u  pU R
                  nU[        R                  L a  SO;UR                  (       a  [        S5      O[        R                  " U5      R                  n[        U5      [        U
5      -   n[        T5      (       d'  [        U5      (       d  [        S T 5       5      (       a  [        XTS9mOU R                  5       mUUUUU4S jn[         R"                  " SU U R%                  5       UUUUUS9n[         R"                  " S	U U R%                  5       [        R&                  UUUUS9n[)        UR*                  R*                  [         5      (       a  UR-                  5         [)        UR*                  R*                  [         5      (       a  UR-                  5         UU4$ s  sn	f )
Nrq  F-infc              3  *   #    U  H	  oS :  v   M     g7frd  r   r3  s     re   r   )_max_pool_with_offsets.<locals>.<genexpr>K  s     ,EHqUHr"  rh  c                   > U S T*  nU T* S  n[        T5       Vs/ s H  nX4   T	U   -  X   TU   -  -   TU   -
  PM      nnT
" / UQUQ5      $ s  snf r   r  )r_  reduction_idxrl  bhrs   rj  rr  r{  r7  r  rj  s         re   fn_inner(_max_pool_with_offsets.<locals>.fn_innerP  s    Wuf%\ 5\
! UVAY=#3hqk#ABWQZO! 	 
 &2''	
s   %Ar  r  
input_noderv   	dst_dtyper  rf  ri  reduction_rangesargmax)r  r[  r  r  rx  r   r{   r   r  r<  r  r  ri   r$  ri  rj  r@   r   rn   r  rk   rm   r  )r   rt  r  r7  rr  ru  r{  r
  dhwr4  dhw_outr   	min_valuerm  r  r  offsetsrj  s     ``` `          @re   _max_pool_with_offsetsr  ,  s    NNGGGeVE
''5&'
C
 5\	

 " X "	

G GGE EJJ 	$66eFmEKK<N<R<R  E{T']*H
7||s9~~,EH,E)E)E.qG==?( ( ||~$	F ||~++$	G &++""I..',,##Y//7?s

s   Hc                    [        U5      n[        U UUUUUSS9u  pp4n[        R                  " SS9   [	        U UUUUUUS9u  pU[        U	[        R                  5      4sS S S 5        $ ! , (       d  f       g = f)NFr~  rz  unroll_reductions_thresholdr  )r   r  r/   r   r  r   r{   r!  )
r   rt  r  r7  rr  ru  r{  r;  r  r  s
             re   !_low_memory_max_pool_with_offsetsr  w  s     E 1@	1-KA 
"	50
 x44 
6	5	5s   -A**
A8c                R  ^^^^^^ [        T5      mU R                  5       m[        R                  " [        R
                  " [        R                  T5      5      mUUUUUU4S jn[        R                  " U R                  5       [        R                  UU R                  5       S9nU$ )Nc                   > T	" U 5      n[         R                  " UT
5      n[        R                  " UT5      nT" X5      n[         R                  " [        R
                  " UTT* S  5      [        R                  5      $ r   )rQ   rR  r0   _flattened_index_to_ndr\  _flatten_indexr{   r  )r_  r  offset_sympyr  idhwincrements_to_index
input_sizert  r{  offsets_loaderr|  s        re   offsets_to_indices4_pool_offsets_to_indices.<locals>.offsets_to_indices  sk    $,,V[A&==lKX"36~~))$
E670CDekk
 	
rg   rh  )r   rj  r   r  r  r+  r  r  r?   r   rn   r{   r  r   )	r  rt  r  r  r  r$  r{  r  r|  s	    ```  @@@re   _pool_offsets_to_indicesr    s     E((*N--	 0 0{ KLK
 
 !!#kk#!	G Nrg   c                J   ^^^^ [        U5      mUUUU4S jn[        XX&5      $ )Nc                   > U T* S  n[        T5       Vs/ s H  nX#   TU   -  X   TU   -  -   TU   -
  PM      sn$ s  snf r   r  )r_  r  r  rs   rr  r{  r7  r  s       re   r  D_low_memory_max_pool_offsets_to_indices.<locals>.increments_to_index  s`    %\ 5\
! UVAY=#3hqk#ABWQZO!
 	
 
s   %=)r   r  )r  rt  r  r  r7  rr  r  r{  s      ``` @re   '_low_memory_max_pool_offsets_to_indicesr    s-     E
 
 $j rg   c           
     ~    [        XX#XFS9u  pp4n[        XX#XEUS9u  p[        U	UU R                  U* S  UUU5      n
X4$ )Nr  )r  r  r  r[  )r   rt  r  r7  rr  ru  r{  r;  rq   r  r$  s              re   _max_pool_with_indicesr    si     1@	1-KA *	ELC 6	G <rg   c           
         [        XX#XESS9$ Nr,   r  r  r   rt  r  r7  rr  ru  s         re   max_pool2d_with_indicesr         "	A rg   c           
         [        XX#XESS9$ Nr   r  r  r  s         re   max_pool3d_with_indicesr    r  rg   c                n  ^^^^^^^^^^^ TS:X  a  SS/mUS:X  a  SS/nT(       d  Tm[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        U5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         U R                  5       n[        U[        5      (       a  [        UR                  R                  [        5      (       a  UR                  R                  n	U	R                  5       n
U
c   e[        R                  " S [        R                  " U
U	R                  5       U	R                  5       S9U	S9nUR                  5         UR                  5       nOUR                  5       nUS L=(       a    US   S:H  =(       d    US L=(       a    US   S:H  n[        S U 5       5      (       a  [!        XTTTXVU5      $ UR                  5       Gt pmU R                  5       Gt nmmUR#                  5       mU R#                  5       m[%        UR                  5       5      n['        UU4S j[)        TS   S-  5       5       5      m['        UU4S	 j[)        TS   S-  5       5       5      mTT-  nUS
:  a  [!        XTTTXVU5      $ UR                  5       mUUUUUUUUUUU4S jn[        R*                  " U R                  5       U R                  5       UUS9nU(       a  [        R,                  R/                  U5      $ U$ )Nr   r.   r,   r  )rv   r   r   r  c              3  *   #    U  H	  oS :g  v   M     g7frd  r   r3  s     re   r   3max_pool2d_with_indices_backward.<locals>.<genexpr>:  s     
$8a68r"  c              3     >#    U  H>  n[        [        UTS    5      [        S [        UTS    -
  TS    5      5      -
  S5      v   M@     g7fr   r.   Nr  r(   r   r`  rt  r  s     re   r   r  G  O      *A 	HQq	"SHQQ5GPQ,S%TTVWXX*   AA	c              3     >#    U  H>  n[        [        UTS    5      [        S[        UTS    -
  TS    5      5      -
  S 5      v   M@     g7fr.   r   Nr  r   wrt  r  s     re   r   r  K  r  r  rz  c                  > U Gt pn[         R                  " UT-  U-   [        R                  5      nUTS   -   nUTS   -   n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n	[        T5       GHD  n
[        T5       GH0  n[         R                  " U[         R                  " U
[        R                  5      5      n[         R                  " U[         R                  " U[        R                  5      5      n/ UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9PnT" U5      nT" U5      n[         R                  " X5      nU	c>  [         R                  " UU[         R                  " S[        R                  5      5      n	GM  [         R                  " [         R                  " [         R                   " X5      [         R                   " X5      5      U5      n[         R                  " U[         R                  " U	U5      U	5      n	GM3     GMG     U	c   eU	$ )Nr   r.   Fr  rt  r  )rQ   r\  r{   r  r(   r  r]  r  r  r   rR  r  r  r   r  r  r  )r_  rl  r`  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   r  grad_loaderh_window_sizer]  indices_sizert  r7  pooled_heightpooled_widthr  w_window_sizewidths                      re   r   ,max_pool2d_with_indices_backward.<locals>.fnZ  s+   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|8#"yyy#,,sEMM*J H 88FF2-FF2- D  #yyswwx/KXVHE , (H ###rg   rh  )rk   rB   r   r   r  r  rm   r?   rn   r1   r  r  r   decide_layoutr  r$  )fallback_max_pool2d_with_indices_backwardrj  ri   r  r  r   r.  rf  )grad_outputr   rt  r  r7  rr  ru  r$  	gO_striderm   rv   x_bufferx_strideis_channels_last_batch_heightr;  rm  r|  r   rq   r  r  r]  r  r  r  r  r  s     ```                @@@@@@@@re    max_pool2d_with_indices_backwardr    s    !|a&1}q6a####{q   v;!w<1x=Aqzz|&&& ,,.I!YJqvv{{I$F$Fvv{{"!!!$$$$nn&]]_
 
 	 &&(%%' ,A!1A 3)A,!"3  
$8
$$$8K(w
 	
  jjlVe&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(w
 	
 ##%L9 9v 

%%'##%	C 44S99
rg   c                8   ^^ U R                  5       mUU4S jnU$ )Nc           
       >^ ^^^	^
 Uu  mm	Uu  mm
Uu  pE[         R                  " [         R                  " [         R                  " TT-   [        R
                  5      [         R                  " U[        R
                  5      5      [         R                  " [         R                  " T
T	-   [        R
                  5      [         R                  " U[        R
                  5      5      5      n[         R                  " UUUU	U U
U4S jT5      $ )Nc                 ,   > T" / TQT T-   PTT-   P5      $ r   r   )h_start_indexrj  iwrl  w_start_indexrj  s   re   rL  3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s$    HNvN}r'9N=2;MNOrg   )rQ   r  r  r\  r{   r  r  )rl  
incrementsstart_indicesend_indicesh_end_indexw_end_indexr  r  rj  r  r  pad_valrj  s   `      @@@@re   ry  !pad_adaptive_loader.<locals>.load  s    B'4$}#. xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
rg   rj  )r   r  ry  rj  s    ` @re   pad_adaptive_loaderr    s    }}H
, Krg   c                    [         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n	XgX4$ )N)out_diminp_dim)r  r*  )
r  r  h_inw_inh_outw_outr  r  r  r  s
             re    compute_indices_adaptive_poolingr    sU    %%k$OM##IdKK%%k$OM##IdKK}AArg   c                \   ^^^^^^ Uu  pgUu  p[        XXgX5      u  mmmmUUUUUU4S jn
U
$ )Nc                   > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	[         R                  " [        TS   5      [        TS   5      5       H"  u  pU" UX/XW/Xh/5      nU	c  Un	M  T" X5      n	M$     U	$ r  )r  productr  )r_  rx  rl  r  bwr  r  r  r  r  rj  r  r@  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                re   r    _adaptive_pooling_fn.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C ~#C0 X rg   r  )r  r  r  in_sizes	out_sizesr  r  r  r  r  r   r  r  r  r  s     `  `     @@@@re   _adaptive_pooling_fnr    sF     JDLE 	)E	
 . Irg   c                d   ^^^
^^^^ Uu  nmUu  px[        XUTXx5      u  mm
mmU
UUUUUU4S jn	U	$ )Nc                  > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	S n
[         R                  " [        TS   5      [        TS   5      5       H  u  pU" UX/XW/Xh/5      n[        R                  " X[-   T-  U-   U-   [
        R                  5      nU
c  Un
O+[        R                  " [        R                  " X5      X5      n
U	c  Un	M{  T" X5      n	M     U
$ r  )	r  r  r  rQ   r\  r{   r  r   gt)r_  rx  rl  r  r   r  r  r  r  maxvalmaxindexrj  r  r@  rB  r  r  r  r  r  r  r  s                  re   r   )_adaptive_pooling_fn_with_idx.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C NN#t+m;b@%++E  99SVVC%8%J~#C0) X, rg   r  )r  r  r  r	  r
  r  r  r  r  r   r  r  r  r  r  s     `  `    @@@@@re   _adaptive_pooling_fn_with_idxr    sK     JD$LE 	)dE	
! !F Irg   c           	     &  ^ ^^ T R                  5       [        R                  :X  a  [        S5      e[	        T [
        5      (       d   e[        U5      S:X  d   eT R                  5         T R                  5       Gt p#n[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUu  pVX5:X  a  XF:X  a  [        T 5      $ US:X  d  US:X  a/  / UQUPUPn[        UT R                  5       T R                  5       S9$ X5-  S:X  a*  XF-  S:X  a"  [!        X55      [!        XF5      /n[#        T U5      $ [%        X5-   S-
  U5      n	[%        XF-   S-
  U5      n
['        U5      XV/-   nT R                  5       nX-  nUS:  a  [)        T U5      $ S nS n[+        UUX/X4/XV/[,        R.                  S	9m[1        [3        T 5      5      mUUU 4S
 jn[4        R6                  " T R                  5       UUUS9nU$ )Nz0'adaptive_avg_pool2d' not implemented for 'Long'r,   r   rb  r.   rz  c                    [        X-  U5      $ r   r(   rB  r  r  s      re   r  )_adaptive_avg_pool2d.<locals>.start_indexP      733rg   c                2    [        U S-   U-  U-   S-
  U5      $ r  r  r  s      re   r  '_adaptive_avg_pool2d.<locals>.end_indexS  "    g-7!;WEErg   r  r  r  r	  r
  r  c                `   > [         R                  " T" U [        T5      5      T" U T5      5      $ r   )rQ   truedivr  )r_  fn_sumones_loaderr   s    re   r    _adaptive_avg_pool2d.<locals>.fna  s-    {{3+A./[1I
 	
rg   rh  )r   r{   r  r  rk   rB   r   r  r   rR   r]   r   r2  r  rl  rn   r(   
avg_pool2drE   ri   fallback_adaptive_avg_pool2dr  rQ   r   r  	ones_liker?   r   )r   r|  r
  r  r  r  r  o_sizert  h_kernel_maxw_kernel_maxrm  r   r|  r  r  r   rvr  r   s   `                 @@re   _adaptive_avg_pool2dr)  *  s   {{}#MNNa####{q   NNU$77%%d+D77%%d+DLE }QxzUaZ'5'%''V1;;=HH|qT\Q.,ht.CD![))DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ""1.77F &il3K

 
		||~	
B Irg   c           	       ^ ^^ T R                  5       [        R                  :X  a  [        S5      e[	        T [
        5      (       d   e[        U5      S:X  d   eT R                  5         T R                  5       Gt p#n[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUu  pVUS:X  d  US:X  aV  / UQUPUPn[        UT R                  5       T R                  5       S9[        U[        R                  T R                  5       S94$ X5-  S:X  a  XF-  S:X  a  [        e[!        X5-   S-
  U5      n[!        XF-   S-
  U5      n	[#        U5      XV/-   n
T R                  5       nX-  nUS:  a  [%        T U5      $ S nS n['        UUX/X4/XV/[(        R*                  S	9m[-        UUX/X4/XV/[(        R*                  S	9mUU 4S
 jnUU 4S jn[.        R0                  " T R                  5       UUU
S9n[.        R0                  " T R                  5       [        R                  UU
S9nUU4$ )Nz,adaptive_max_pool2d not implemented for Longr,   r   rb  r.   rz  c                    [        X-  U5      $ r   r  r  s      re   r  (adaptive_max_pool2d.<locals>.start_index  r  rg   c                2    [        U S-   U-  U-   S-
  U5      $ r  r  r  s      re   r  &adaptive_max_pool2d.<locals>.end_index  r  rg   r  c           	     <   > T" U [        T[        S5      5      5      $ Nr  r  r<  )r_  inner_func_max_valr   s    re   inner_fn_max_val-adaptive_max_pool2d.<locals>.inner_fn_max_val      !#':1eFm'LMMrg   c           	     <   > T" U [        T[        S5      5      5      $ r0  r1  )r_  inner_func_max_idxr   s    re   inner_fn_max_idx-adaptive_max_pool2d.<locals>.inner_fn_max_idx  r5  rg   rh  )r   r{   r  r  rk   rB   r   r  r   rR   r]   r   r2  rl  rn   
ValueErrorrE   ri   fallback_adaptive_max_pool2dr  rQ   r  r  r?   r   )r   r|  r
  r  r  r  r  r%  r&  r'  rm  r   r|  r  r  r3  r8  r(  rir7  r2  s   `                  @@re   adaptive_max_pool2dr=  u  s3   {{}#IJJa####{q   NNU$77%%d+D77%%d+DLEzUaZ'5'%''V1;;=H%%++allnK
 
 	
 |qT\Q.DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ."1.;; 7"1.;;NN 
		||~!	
B 
		||~kk!	
B r6Mrg   c                j   ^ ^^^^^^ TT   mTT   mTT   mT R                  5       mUUUUUU U4S jnU$ )Nc                |  > TR                  5       n[        U5      S:X  a]  [        U 5      S:X  a  T" SU S   TS-
  T
-
  /5      nOK[        U 5      S:  a  T" U S   U S   TS-
  T
-
  /5      nO$T" SSTS-
  T
-
  /5      nOT" / U QTS-
  T
-
  P5      n[        R                  " UTR	                  5       5      n[        R                  " TT-
  [
        R                  5      n[        R                  " TS-
  [
        R                  5      n[        R                  " [        R                  " U[
        R                  5      [        R                  " U[
        R                  5      5      n[        R                  " [        R                  " US5      SU5      n[        R                  " XC-   U-  5      [        R                  " X7-  5      -
  n[        R                  " U[
        R                  5      n[        R                  " XF5      n	[        R                  " [        R                  " XU5      [        R                   " T5      5      $ )Nr   r.   r   r,   )r   r   rQ   r\  r   r{   r  r  r   float64r   r  rS  r  rR  r   r  )rl  rs   samples_shapesamplei_exprdiffout_sz_exprrd  seq_ir  r   in_sz	kernel_szndimsout_szsamplessamples_loaders             re   ry  )_fractional_pooling_offsets.<locals>.load  s     ((*}"6{a (F1Iuqy3(GHV! (F1Iuqy3(OP (Auqy3(?@ $$>f$>eai#o$>?F7#4#4#67~~ei/=nnVaZ=LLu}}-s||K/W
 		#&&a0!U;		6?e34syy7PPUEKK0vvf*$$SYYtD%A5==QVCWXXrg   r  )rK  rG  rJ  rH  r   rI  ry  rL  s   `````` @re   _fractional_pooling_offsetsrN    s@    C[F#JE#I((*N!Y !YF Krg   c                    [        XX#SS9$ r  _fractional_max_poolr   rt  r|  random_sampless       re   fractional_max_pool2drT        STUUrg   c                    [        XX#SS9$ r  rP  rR  s       re   fractional_max_pool3drW    rU  rg   c                  ^^^^ U R                  5         U R                  S T*  U R                  T* S  pe[        R                  " SS9   [	        T5       Vs/ s H  n[        UUUUTUS9PM     snmU R                  5       mUUU4S jnUU4S jm[        U5      [        U5      -   n	U R                  5       n
[        R                  " SU U R                  5       U
U
UU	US9n[        R                  " SU U R                  5       [        R                  U
UU	US9n[        U[        5      (       d   U5       e[        UR                   R                   [        5      (       a  UR#                  5         [        U[        5      (       d   U5       e[        UR                   R                   [        5      (       a  UR#                  5         [%        XU R                  T5      nX4sS S S 5        $ s  snf ! , (       d  f       g = f)	Nrz  r  )rK  rG  rJ  rH  rI  r   c                4   > U S T*  nT" / UQT" X5      Q5      $ r   r   )r_  r  rl  r  r{  rj  s      re   r  &_fractional_max_pool.<locals>.fn_inner  s,    5&\FOfO':3'NOPPrg   c                   > U S T*  nU T* S  n[        T5       Vs/ s H  nTU   " X#U   5      X   -   PM     sn$ s  snf r   r  )r_  r  rl  bdhwr4  dhw_index_fnr{  s        re   r  1_fractional_max_pool.<locals>.increments_to_index  s]    5&\Fvw<D u%A QQ0=3CC%  s   ;r  r  r  )r  r[  r/   r   r  rN  rj  ri   r   r@   r   rn   r{   r  rk   rB   rm   r  r  )r   rt  r|  rS  r{  r
  inp_dhwr4  r  rm  r   r  r  r$  r]  r  rj  s       `         @@@re   rQ  rQ    s   NNWWWuf%qwwvw'77	"	5 5\

 " (&"% "

 ==?	Q	 ;k!22!! <<>(	
 ""#<<>kk(	
 &),,4f4,fkk&&	22NN'9--6w6-gll''33OO*!''+>
 { 
6	5

 
6	5s   G)G$.E,G)$G))
G7c           	     `  ^ ^^ T R                  5         T R                  5       Gt pVn[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUGt pXn	Xh-  S:X  a'  Xy-  S:X  a  [        T [        Xh5      [        Xy5      /SS9$ [        Xh5      n
[        Xy5      nS mU4S jn[        TUX/Xg/X/[        R                  S9mUU 4S jn[        R                  " T R                  5       T R                  5       U[!        U5      S9nU$ )	Nr   r.   )divisor_overridec                F    [        X-  [        R                  " U5      5      $ r   )r'   r   r  r  s      re   r  0upsample_nearest2d_backward.<locals>.start_indexV  s    ug(>??rg   c                   > T" U S-   X5      $ r  r   )rB  r  r  r  s      re   r  .upsample_nearest2d_backward.<locals>.end_indexY  s    EAI99rg   r  c                (   > T" U [        T5      5      $ r   )r  )r_  r  r   s    re   r   'upsample_nearest2d_backward.<locals>.fne  s    c.q122rg   rh  )r  r   rR   r]   r   r2  r"  r(   rE   r  rQ   r   r?   r   rn   r   ri   )r   r|  r  r  r  r  inp_hinp_wout_hout_wr&  r'  r  r   r(  r  r  s   `              @@re   upsample_nearest2d_backwardrl  A  s    NNJJLVEGG&&u-EGG&&u-E 'VE}emq0&(>?RS
 	
 5(L5(L@: ""1.77F3 
		||~kkmJ
B Irg   c                "    [        U UUUUUUSS9$ )Nr,   rh  _avg_poolndr   rt  r  r7  ru  count_include_padra  s          re   r"  r"  s  )     		 	rg   c                "    [        U UUUUUUSS9$ )Nr   rh  rn  rp  s          re   
avg_pool3drt    rr  rg   c                  ^^^^^^^ T(       d  TmT(       d  S/T-  m[        TT5      m[        TT5      m[        TT5      m[        U [        5      (       d   e[        T5      T:X  d   e[        T5      T:X  d   e[        T5      T:X  d   e[        U R	                  5       5      TS-   TS-   4;   d   eU R                  5         U R	                  5       S T*  nU R	                  5       T* S  m[        [        T5       V	s/ s H  n	[        TU	   U	TTTU5      PM     sn	6 u  p[        T5      (       d  [        U5      (       a  [        U STS9mSnOU R                  5       mSn[        U5      [        U
5      -   nU R                  5       n[        U [        R                   SS9nUUUU4S	 jn["        R$                  " [&        R(                  T5      nUS
:  a;  [        S [        TT5       5       5      (       a  [*        TS-
     nU" U TTTUTU5      $ US
::  a  [,        R.                  " S
S9O[0        R2                  " 5       nU R5                  5       nUc   eU   [6        R8                  " SU UUUUUTS9nS S S 5        [;        WR<                  S5      (       a9  [        UR<                  R<                  [6        5      (       a  UR?                  5         U(       a  U(       a  U(       a  UOUn[A        UU5      nO<UUUUUU4S jn[B        R8                  " U R5                  5       UUUS9n[A        UU5      n[E        UU5      $ s  sn	f ! , (       d  f       N= f)Nr   r.   r,   r  rh  TF)r   r   c                   > U S T*  nU T* S  nUn[        T5       Vs/ s H  oSU   TU   -  XE   -   TU   -
  PM     nnT	" / UQUQ5      $ s  snf r   r  )
r_  r  rl  r  rj  rs   r   r7  r  rj  s
         re   r  _avg_poolnd.<locals>.fn_inner  sn    Usd#Z>CCjIjefQi"%''!*4jI&2'' Js   A
rz  c              3     #    U  HD  u  p[         R                  R                  R                  [        R
                  " X5      5      v   MF     g 7fr   )rR   r]   r   statically_known_truer   Ne)r   r   r:  s      re   r   _avg_poolnd.<locals>.<genexpr>  s9       ,DA 	
..uxx~>>,s   AAr  re  r  rm   c                  > U T* S  n/ n[        T5       H  nX   TU   -  TU   -
  n[        R                  " UT
U   -   T	U   TU   -   5      nT(       d1  [        R                  " US5      n[        R                  " UT	U   5      n[        R
                  " XT-
  [        R                  5      nUR                  U5        M     [        R                  " [        R                  U5      $ r  )r  r   r~  r  rQ   r\  r{   r  ro   r  r+  r  )r_  r  divide_factorsrs   hstarthendfactorrq  r   r`  rt  r7  r  s          re   fn_count_avg_poolnd.<locals>.fn_count  s    cTUBN3Z*WQZ7yy+a.!8!A$:KL("YYvq1F 99T1Q40Du{{C%%f-   ##CGG^<<rg   rh  )#rL   rk   rB   r   r   r  r  r  rx  r$  ri  rj  ri   r   r   r   rO  r  r+  r  r  fallbacks_avg_poolndr/   r   
contextlibnullcontextrn   r@   r   r   rm   r  div_primr?   r   )r   rt  r  r7  ru  rq  ra  r   r
  rs   r  
ceil_modeshad_paddingrm  r   output_dtyper  r|  fallbackcontextrv   r(  divisorr  r  divide_factorr`  rj  s    ``` ` `                  @@re   ro  ro    s     #){C0K&#&F7C(Ga####{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E 7||s:.q#3?==?E{T%[(HKKME%	;CC!L( ( ""8<<=KRC  V,    (a0
 	
 " 	4##%  \\^F	 "(	
 
 rwwJrww||Y$G$G


*&6"K"g&	= 	= "((<<>	
 "m,FE""I

t 
s   =L;M  
Mc                (  ^^^^^^^^^^^^^^ Tb  TS:w  d   S5       eT(       d  TmT(       d  SS/m[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmm[        TSTTTU5      u  p[        TSTTTU5      u  pU R                  5       mTS   =(       d    TS   =(       d    U
=(       d    UmU R                  5       Gt nmm[        UR                  5       5      nUR                  5       n[        UU4S j[        TS   S-  5       5       5      m[        UU4S j[        TS   S-  5       5       5      mTT-  nUS:  a  [        U UTTTUTT5      $ UUUUU4S	 jmUUUUUUUUUUUU4S
 jn[        R                  " U R                  5       UUUS9nU$ )Nr   divisor must be not zeror,   r  r.   c              3     >#    U  H>  n[        [        UTS    5      [        S [        UTS    -
  TS    5      5      -
  S5      v   M@     g7fr  r  r  s     re   r   &avg_pool2d_backward.<locals>.<genexpr>X  r  r  c              3     >#    U  H>  n[        [        UTS    5      [        S[        UTS    -
  TS    5      5      -
  S 5      v   M@     g7fr  r  r  s     re   r   r  \  r  r  rz  c           	     n  > [         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " [         R
                  " X5      U5      n[         R                  " [         R
                  " X5      U5      n	[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n
[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n[         R                  " U[         R                  " S[        R                  5      5      n[         R                  " U	[         R                  " S[        R                  5      5      n	[         R                  " U
[         R                  " T[        R                  5      5      n
[         R                  " U[         R                  " T[        R                  5      5      n[         R
                  " [         R                  " X5      [         R                  " X5      5      nU$ )zc
This computes the scaling factor that we will divide an element
by when `count_include_pad=False`
r   r.   )
rQ   r]  r{   r  r  r  r  r   r\  r  )r  r  stride_hstride_wpad_hpad_wkernel_hkernel_wr~  wstartr  wendr  heightrt  r7  r  r  s                re   !compute_pool_size_without_padding>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingo  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<.6.6{{GGF%GGCNN65;;7?
 {{GGF%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CD 5swwt7LMrg   c                  > U Gt pnUTS   -   nUTS   -   n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n[        T5       GHD  n	[        T5       GH0  n
[         R                  " U[         R                  " U	[        R                  5      5      n[         R                  " U[         R                  " U
[        R                  5      5      nTb  TnO"T(       d  T(       d  TS   TS   -  nOT" X5      n[         R                  " T" / UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P5      U5      n[         R                  " [         R                  " X5      [         R                  " X5      5      nUc=  [         R                  " X[         R                  " S[        R                   5      5      nGM  [         R                  " U[         R                  " X5      U5      nGM3     GMG     Uc   eU$ )Nr   r.   Fr  r  )rQ   r\  r(   r{   r  r  r]  r  r  r   r  rR  r  r  r  r   r  )r_  rl  r`  r  r  r  r  r  r  r  r  r  r  r  partr  r  rq  ra  r  r  r  rt  r7  r  r  r  r  s                   re   r   avg_pool2d_backward.<locals>.fn  s   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=bEE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2%FF2% #"yyS\\#u}}5UVH"yyswwx/FQHS , (V ###rg   rh  )rk   rB   r   r   r  rx  rj  ri   r   r  r  fallback_avg_pool2d_backwardr?   r   rn   )r  r   rt  r  r7  ru  rq  ra  r;  _h_out
ceil_mode1_w_out
ceil_mode2rm  r   r|  r   r(  r  r  r  r  r  r  r  r  r  s     ``` ``          @@@@@@@@@re   avg_pool2d_backwardr  /  s"    #'71'<X>XX<a&k9----a####{q   v;!w<1qzz|&&&

Q%;F &eQVWiXF))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ? ?B 
		%%'	
B Irg   c                  ^^^^^^^^^^^^^^^^^  Tb  TS:w  d   S5       eT(       d  TmT(       d  / SQm[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmmm [        TSTTTU5      u  p[        TSTTTU5      u  p[        T STTTU5      u  pU R                  5       m[        T5      =(       d    U
=(       d    U=(       d    UmU R                  5       Gt nmmm[        UR                  5       5      nUR                  5       nUU4S j[        S5       5       u  mmmTT-  T-  nUS	:  a  [        U UTTTUTT5      $ UUUUUU 4S
 jmUUUUUUUUUUUUUU4S jn[        R                  " U R                  5       UUUS9nU$ )Nr   r  )r   r   r   r   )r   r   r.   r,   c              3  r   >^#    U  H+  m[        UUU4S  j[        TT   S-  5       5       5      v   M-     g7f)c           
   3  z   >#    U  H0  n[        UTT   -  [        S UTT   -
  TT   -  5      -
  S5      v   M2     g7fr  )r  )r   r4  rs   rt  r  s     re   r   0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sH      
. VAYQ[^);q	(I!JJANN.s   8;r,   N)r  r  )r   rs   rt  r  s    @re   r   &avg_pool3d_backward.<locals>.<genexpr>  sA      3
 A	 	 
;q>A-.
 	
 	
 s   37}   c           	       > S T 5       u  p4nS T 5       u  pgnS T 5       u  pnS [        XU/X4U/XgU/5       5       u  pnS [        XU/XU/TTT/XgU/5       5       u  nnnS XU4 5       u  pnS [        UUU/TTT/5       5       u  nnn[        R                  " [        R                  " [        R                  " X5      [        R                  " UU5      5      [        R                  " UU5      5      nU$ )Nc              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   rQ   r]  r{   r  r   r:  s     re   r   Qavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s"     'UfQ(D(Dfr>  c              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   r  rB  s     re   r   r    s"     MWs||Au{{;;Wr>  c              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   r  )r   r   s     re   r   r     s%      (
2=QCLLEKK((+r>  c              3  ~   #    U  H3  u  pn[         R                  " [         R                  " X5      U5      v   M5     g 7fr   )rQ   r  r  )r   r  r:  pads       re   r   r  $  s5      "
	c GGCGGAM3''s   ;=c           
   3     #    U  Hk  u  pp4[         R                  " [         R                  " X5      [         R                  " [         R                  " U[        R
                  5      U5      5      v   Mm     g 7fr   )rQ   r  r   r\  r{   r  )r   r  r   r   r  s        re   r   r  *  sV      

'"# KK!3773>>#u{{+KS#Q 's   A3A5c              3     #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7frO  rQ   r  r]  r{   r  )r   r  s     re   r   r  5  s4      "
1 KKs||Au{{;<<1   AA	c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   rQ   r  r\  r{   r  )r   r  r   s      re   r   r  9  s6      
K KKS^^C=>>K   A	A)r  rQ   r  r  )pdr  r  stride_dr  r  pad_dr  r  kernel_dr  r  dstartr~  r  dendr  r  r  depthr  rt  r7  r  r  s                      re   r  >avg_pool3d_backward.<locals>.compute_pool_size_without_padding  s.   'Uf'U$HMWMe(
2=(
$H"
 x8<uU>S"


 '*(X.&u%	'	

dD"
 &1"

tT 2UFE4JK
dD GGCGGD)3774+@A3774QWCX
 rg   c                R  > U Gt pp4S [        X#U/T5       5       u  p#nS [        X#U/TT!5       5       u  pVnS [        X#U/T!5       5       u  pn
S XVU4 5       u  pVnS [        XU
/TTT /5       5       u  pn
S n[        T5       GH  n[        T5       GH  n[        T"5       GHo  nS [        XVU/XU/5       5       u  nnnTb  TnO*T(       d  T(       d  TS   TS   -  TS	   -  nO
T" UUU5      n[        R                  " T" / UQ[        R                  " [        R
                  " U[        R                  " U[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U	[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U
[        R                  " S[        R                  5      5      5      T S
S9P5      U5      n[        R                  " [        R                  " [        R                  " X5      [        R                  " UU	5      5      [        R                  " UU
5      5      nUc>  [        R                  " UU[        R                  " S[        R                  5      5      nGMB  [        R                  " U[        R                  " UU5      U5      nGMr     GM     GM     Uc   eU$ )Nc              3  .   #    U  H  u  pX-   v   M     g 7fr   r   )r   r
  r  s      re   r   2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>D  s     A)@vq17)@r  c              3     #    U  H<  u  pn[         R                  " [        X-
  U-   U5      [        R                  5      v   M>     g 7fr   rQ   r\  r(   r{   r  )r   r
  r   r:  s       re   r   r  F  s:      %
>a NN8AEAIq15;;??>s   AAc              3     #    U  H8  u  p[         R                  " [        X5      S -   [        R                  5      v   M:     g7frd  r  )r   r
  r:  s      re   r   r  K  s4      
. NN8A>A-u{{;;.s   A Ac              3     #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7frO  r  )r   pstarts     re   r   r  P  s4      %
5 KKQ <==5r  c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   r  )r   pend
pooled_dims      re   r   r  T  s;      
%  KKcnnZEFF%r  c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   )rQ   r   r]  r{   r  )r   r  p_s      re   r   r  `  s:      "+JF R(EFF+r  r   r.   r,   Fr  r  )r  r  rQ   r  rR  r  r  r]  r{   r  r  r  r   r  r   )#r_  rl  r4  r`  r  pdstartr  r  pdendr  r  r  pd_r  r  r  r  r  r  r  r  r  rq  d_window_sizera  r  r  r  rt  r7  pooled_depthr  r  r  r  s#                        re   r   avg_pool3d_backward.<locals>.fnB  s   AAaAY)@Aa%
ay+v>%
!'

Q1Iv.
e
%
"W5%
!'
$'u%m\'R%
e 'C]+ /C"*-$w7#C+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um 0 , (r ###rg   rh  )rk   rB   r   r   r  rx  rj  r$  ri   r   r  fallback_avg_pool3d_backwardr?   r   rn   )!r  r   rt  r  r7  ru  rq  ra  r  _d_outceil_mode_dr  ceil_mode_hr  ceil_mode_wr;  rm  r   r|  r   r(  r  r  r  r  r  r  r  r  r  r  r  r  s!     ``` ``             @@@@@@@@@@@@re   avg_pool3d_backwardr    s    #'71'<X>XX<k9----a####{q   v;!w<1qzz|&&&$%JJL!VUFE&q+vw	F ';F 'q+vw	F ))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B Irg   c                F   U R                  5       n[        U[        5      (       a  U/nOU(       d  [        [	        U5      5      n[	        U5      S:X  a  [        U5      S;   d
   SU 35       e/ $ [        U5      n[        [	        U5      5       Hg  nX   S:  a'  X==   [	        U5      (       a  [	        U5      OS-  ss'   SX   s=::  a  [	        U5      :  a  ML  O  [	        U5      S:X  a
  X   S:X  a  Mg   e   [	        [        U5      5      [	        U5      :X  d   S5       eU$ )Nr   )r   r  rt  zinvalid axis: r.   zreduction axis not unique)r   rk   r   r  r   r   ri   r&   )r   r  r   rs   s       re   _validate_reduction_axisr    s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t97Q;GCIIs4y14GDG'c$i'CINtw!|LL  z$ CI-J/JJ-Krg   )r  c          	       ^^^^^^^ Ub  [        X5      n U R                  5       m[        [           " [	        X5      5      n/ n/ m/ m/ m[        [        T5      5       HT  nXq;   a'  TR                  U5        TR                  TU   5        M/  TR                  U5        UR                  TU   5        MV     SmUS;   Ga  [        T5      S:  a  [        U 5      (       a  [        U R                  [        5      (       a  SmO[        U R                  [        R                  5      (       d\  [        U R                  [        R                  5      (       ao  [        U R                  R                  [        R                  5      (       a<  U R!                  5       nUR#                  5       =(       d    UR%                  5       (       + mUUUUUUU4S jn	T(       a1  ['        T5      n
T H  n[(        R*                  R,                  X'   M!     OUn
U R/                  5       m[1        U R3                  5       U=(       d    U R5                  5       U R5                  5       U	U
TS9$ )NFr  argminr.   Tc                  > [        U5      [        T5      :X  d   eT
(       a.  [        U 5      [        T5      :X  d   eT Vs/ s H  o U   PM	     n n[        U 5      [        T5      :X  d   eS /[        U 5      [        U5      -   -  n[        R                  " [        TU 5      [        TU5      5       H	  u  pEXSU'   M     T	" U5      nT(       a{  U Vs/ s H  n[        R
                  " U5      PM     nnUS   n[        S[        U5      5       H  nUTU   -  Xr   -   nM     U[        R                  " U[        R                  5      4$ U$ s  snf s  snf r  )r   r  r  r  r   r4  r  rQ   r\  r{   r  )rB  reduction_indexrs   rt  r_  varr   rindex
linear_idxinner_loaderkeepdimskept_idxreduced_idxreduced_sizesshould_compute_logical_indexr   s            re   rx  %_make_reduction_inner.<locals>.loader  s:   ?#s;'7777u:T***'/0x!1XxE05zS]***Fc%j3+??@	!% #k?"C
HC !cN
 Y' (/>?!ell1oF?  J1c&k*'-*::VYF
 + 3>>*ekkBCC+ 1 @s   E
 E)rv   r  r  rf  ri  r  )r   r   r&   r   r  r  r   ro   r:   rk   rm   r>   r1   r   r   Bufferr  is_transposedis_contiguousri   r   r0  r1  rj  r  rn   r   )r   r  r  r   rU  r  
kept_sizesrs   r  rx  rm  r  r  r  r  r  r   s     `        @@@@@@re   _make_reduction_innerr    s    Q::<Dc?3A<=DJHMK3t99q!  a)OOAd1g&  $) .."aLLaffk**+/( 2 233qvvr}}--*QVV[["))2T2T\\^F$$&Df.B.B.D*D ) 6 :A''++HK  ==?L||~'81;;=++-& rg   c                $   ^ ^ SS S.UU 4S jjjnU$ )Nr   c          	     l  > TS;   aL  U R                  5       [        R                  :X  a*  [        U 5      (       a  [	        U [        R
                  5      n [        U UUUTTS9n[        R                  " STU S.UD6n[        UR                  R                  [        5      (       a  UR                  5         U$ )Nr  )r  r  r   rU  r  )r  r  r   )r   r{   r   r:   r   r  r  r@   r   rk   rm   r  )r   r  r  r   r   r  rU  r  s         re   r  make_reduction.<locals>.inner  s    
 22+!EKK(A&"7)
 !!XAXQWXKK
 
 NNrg   r   r   )r  rU  r  s   `` re   make_reductionr    s    T  4 Lrg   c                   Ub  [        X5      n [        X5      n[        U R                  5       U R	                  5       4U R                  5       4U R                  5       US9$ )N)rv   dtypes	inner_fnsr   r  )r   r  r  rn   r   rj  r   )r   r  r   s      re   _make_scan_innerr  $  sV    Q!D||~==?$ZZ\ rg   r   c                 ^ Ub  [        X5      n U R                  5       m[        X5      nU R                  5       nU[        R
                  [        R                  4;   a  [        U [        R                  5      n [        XU5      n[        U4S jU 5       5      n[        R                  " X`R                  5       U R                  5       S9n[        R                  " U[        UR                  5       5      5      n[        [!        XV5      U5      $ )Nc              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r   rs   r   s     re   r   mean.<locals>.<genexpr>=       04a$q'4r   rA  )r   r   r  r   r{   rR  rQ  r<  sum_rO   r1   r8   rn   r7   r   ri   div)r   r  keepdimr   r  
sum_resultdenomr   s          @re   r  r  2  s    Q::<D#A,D;;=Lu~~66Q$aw'J0400Ee;;=XEeT**=*=*?%@AEC
*L99rg   c                F  ^
 Uc  SnU R                  5       m
[        X5      n[        XSS9nU(       a  UR                  5         [	        [        X5      5      n[        XaU5      n[        U
4S jU 5       5      nU(       a  [        R                  " X-
  S5      n[        R                  " XR                  5       U R                  5       S9n[        R                  " U[!        UR                  5       5      5      n[#        Xx5      n	U(       d  U	4$ U(       a  UO
[%        XQ5      nX4$ )Nr.   T)r  c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r   s     re   r    var_mean_sum_.<locals>.<genexpr>P  r  r   r   rA  )r   r  r  r  squarer  r  rO   r   r  r1   r8   r   rn   r7   r   ri   r  r;  )r   r  
correctionr  return_meanx_meandiffsr  r  x_varr   s             @re   var_mean_sum_r  C  s    
::<D#A,D!4(F3q>"Ee7+J0400E		%,a0e;;=XEeT**=*=*?%@AE
"ExVGF$9F=rg   c                   [        X5      n[        XUS S S9nUS   n[        US   5      nU R                  5       nU(       a  UR                  S:X  d  [
        R                  nO[
        R                  R                  n[        U[        R                  5      =(       a#    [        U5      U:*  =(       a    [        U5      S:g  $ )Nr  r  r   rU  ri  r  r   r.   )r  r  rO   rn   r   r/   r  cppuse_two_step_variance_thresholdrk   r   r  r   )r   r  r  r   ri  reduction_numelrv   	thresholds           re   use_two_step_variancer  ]  s     $A,D"	wd$F HF#F+=$>?O\\^Fv{{e+66	 JJ>>	?EMM2 	' I-	'&!Q&rg   c                 ^^^^^ Tc  Sm[        XUS S S9nUR                  S5      nUR                  S5        UR                  S5        [        R                  R                  " SU4SU R                  5       S.UD6u  pxn	UR                  5         U R                  5       mU R                  5       m[        X5      n[        U4S jU 5       5      mS	 mUUUU4S
 jn
[        U
5      " U5      nU(       a  UR                  5         X4$ U4$ )Nr.   r  rf  r  r  welford_reduce)r  r  r   c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r   s     re   r   $var_mean_welford_.<locals>.<genexpr>  s     1Dq47Dr   c                   [        U [        R                  5      (       aK  U R                  (       d:  [        R
                  " [        R                  " U [        R                  5      U5      $ [        R                  " X5      $ r   )
rk   r   r   	is_numberrQ   r   r\  r{   r  r]  rC  s     re   get_constant_or_index_expr5var_mean_welford_.<locals>.get_constant_or_index_expr  sH    a$$Q[[<<q%++ >FF||A%%rg   c                   > T" TT5      nT" TT5      n[         R                  " ST5      nU [         R                  " X2U-
  5      -  $ r  )rQ   r]  r  )rm   cNzeror  r   r  rnumels       re   r  #var_mean_welford_.<locals>.scale_fn  sC    &z59&vu5||Au%ckk$A...rg   r   )r  r  r1   WelfordReductionr   r   r  r   r  rO   r  )r   r  r  r  r  r   rx  r  m2r;  r  r  r   r  r%  r   s     `         @@@@re   var_mean_welford_r)  u  s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KDa JJLKKME::<D#A,D1D11F&
/ / 
"2
&Cy6Mrg   c               6  ^ U R                  5       m[        T5      n[        XSS9n [        U UUUUS9n[        R
                  R                  (       d  [        XUS9(       a  [        S0 UD6O
[        S0 UD6n[        U4S jU 5       5      nU(       d  US   $ U$ )NFr  )r   r  r  r  r  )r  r  c              3  :   >#    U  H  n[        UTS S9v   M     g7f)Fr  N)r   )r   r   r   s     re   r   #var_mean_helper_.<locals>.<genexpr>  s     Fv!8Ayu5vrL  r   r   )r   r   r   r  r/   mtiadisable_welford_reductionr  r  r)  r   )	r   r  r  r  r  r   r   r7  r   s	           @re   var_mean_helper_r/    s    I))4M.A
F KK11$Q7C 	
 ((  FvFFF'6!93V3rg   )r  r  c                   [        XX#SS9$ )NFr  r  r  r  r/  r   r  r  r  s       re   var_r4    s    	% rg   c                   [        XX#SS9$ )NTr1  r2  r3  s       re   var_meanr6    s    	$ rg   c                (   US:  a"  [        [        R                  " U 5      U* U5      $ US:X  a  [        R                  " SU5      $ US:X  a  U $ [        XS-  U5      n[        R                  " X35      nUS-  S:X  a  [        R                  " X05      nU$ )Nr   r.   r,   )pow_recursiverQ   r  r]  r  )r   r8  r   r  s       re   r8  r8    s    1uS^^A.E::Av||Au%%Av11fe,FWWV$F	A!|#Mrg   c                .    [         R                  " X5      $ r   )rQ   r  r   r6  s     re   
pow_nativer;    s    771=rg   )r  c                  ^ ^^ [        T[        5      (       a*  TR                  5       (       a  [        T [	        T5      5      $ [        T[        5      (       a  TS:X  a  [        T 5      $ [        T[        5      (       a  TS:X  a  [        T 5      $ [        S T T4 5       5      n[        U5      n[        T[        5      =(       a*    STs=:  =(       a    S:  Os  =(       d    U=(       a    TS:  nU(       aY  T R                  5       mU UU4S jn[        R                  " T R                  5       T R                  5       UT R                  5       S9$ [        T [        5      (       aA  T S:X  a  [!        TS5      $ T S	:X  a)  [#        TR                  5       5      (       a  [%        T5      $ U(       aN  [        T [        5      (       a  ['        T T5      $ [        T[        5      (       a  [)        T T5      $ [+        T T5      $ [-        T T5      $ )
Nr   r.   c              3     #    U  H6  n[        U[        R                  5      (       d  M$  UR                  5       v   M8     g 7fr   )rk   r1   rB   r   r!  s     re   r   pow.<locals>.<genexpr>  s'     N1*Q2Ms
   #A A i    r   c                F   > [        T" U 5      TTR                  5       5      $ r   )r8  r   )r_  r   r6  rx  s    re   r   pow.<locals>.fn  s     a??rg   rh  r,   )rk   r<  r   r  r   sqrtr  rP  r   rj  r?   r   rn   r   r   r    rE  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr;  )r   r6  r   is_integer_powembed_exponentr   rx  s   ``    @re   r  r    s   !U1c!f~	Au		!s(Aw	As		QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]337Na  &q!,,6""-a33-a33arg   c                f   [        U [        5      (       a  U R                  nOU n[        U[        5      (       a  UR                  n[        U[        R                  5      (       d  [
        R                  " U R                  5       U R                  5       UR                  5       U R                  5       S9n[        U[        [        45      (       d   eUR                  n[        U[        R                  5      (       d   e[        U[        R                  5      (       av  UR                  5       (       da  UR                  5       (       dL  [        UR                  [        R                  5      (       d#  UR!                  5         UR                  Ul        U $ [        R"                  R%                  XUS9  U $ )Nrh  unsafe_alias)rk   rB   rm   r1   r   r?   r   rn   r   rj  r   r5   r;   is_input_bufferis_module_buffer	NopKernelr  r  realize_into)changedr@  rK  changed_datarc   s        re   r  r    sC   '9%%||#y!!hhc2==))%%'##%__&##%	
 $: 67777ii#r}}----,..$$&&((**l''66 	HH!!.. /  Nrg   c                ,    [        U [        X5      5      $ r   )r  rE  )r   r%  s     re   r6  r6  8  s    Q	!011rg   c                    XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ r   r  rn   r   r   r4  r   r  )r  r  r  s      re   r  r  =  sH    
z

C)
*C
3
(C
lln
%CSrg   c                .    [         R                  " X5      $ r   )rQ   floordivr:  s     re   rV  rV  H      <<rg   c                .    [         R                  " X5      $ r   )rQ   truncdivr:  s     re   rY  rY  M  rW  rg   c                .    [         R                  " X5      $ r   )rQ   div_rnr:  s     re   _div_rnr\  R  s    ::arg   c                t   [        U 5      =(       a    [        U5      n[        U 5      =(       a    [        U5      nUS:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [	        X5      5      $ US:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [        X5      5      $ [        X5      $ )NrN  z5floordiv operands can not be boolean at the same timerS  z5truncdiv operands can not be boolean at the same time)r   r   rV  rN  r\  rY  rS  r  )r   r6  rounding_modeboth_integerboth_booleans        re   div_modera  W  s    "1%</!*<L"1%</!*<L X!XX
 ".x~G53GGX!XX!-x~C5Q3CCq9rg   c                    [        U 5      =(       a    [        U5      nU(       a  [        X5      $ [        [        R                  R
                  5      n[        U5      " X5      $ r   )r   logical_andr=   r  r  r  r  )r   r6  	both_boolr   s       re   r  r  k  sH    "9q'9I1  **+b!!''rg   c                   [        U [        R                  5      (       a  [        U R                  5      $ [        U [        R
                  5      (       a  [        U R                  5       5      $ [        U [        R                  5      (       a  U $ [        U [        R                  5      (       d  g[        R                  R                  R                  U R                  5       5      n[        R                  " U5         [         R"                  " [        R$                  SS5         U R&                  " U R)                  5       6 nSSS5        SSS5        [        W[        R                  R*                  R,                  5      (       d   e[        UR.                  [        R                  5      (       a  UR.                  $ g! , (       d  f       N= f! , (       d  f       N= f)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)rk   r1   r;   get_constant_valuerm   r5   r  r   Loopsr{   	_inductorops_handlerExtractConstantsHandlerrn   rR   set_ops_handlerr   objectr  rf  inner_fn_argsvirtualizedOpsValuer   )r   r  rq   s      re   rg  rg  u  s1   
 !R]]##!!&&))!R[[!!!!--/22!R[[!! a""oo))AA!,,.QG	'"R&&(8$?jj!//+, 	@ 	#
 c5??66??@@@@#))R[[))yy 	@? 	#"s$   ;'G"F; G;
G		G
Gc                v   [        S X4 5       5      nU(       a  [        X5      $ [        U5      =nbs  U R                  5       R                  S:w  aU  UR
                  S:X  a+  [        R                  " [        S5      UR
                  5      nOSUR
                  -  n[        X5      $ S n[        U5      " X5      $ )Nc              3  \   #    U  H"  n[        U5      =(       d    [        U5      v   M$     g 7fr   )r   r   r!  s     re   r   div_prim.<locals>.<genexpr>  s"     O1oa(>OA,>>s   *,r   r   infr  c                 (    [         R                  " U 6 $ r   )rQ   r  r!  s    re   r   div_prim.<locals>.fn  s    {{D!!rg   )r#  rY  rg  rn   r   r   mathcopysignr<  r  r  )r   r6  is_integralr  r  r   s         re   r  r    s    OOOK~ &a((5!,,.:M:MQV:V ==AuU|W]]CJw}},J1!!" "a##rg   c                N    [        X4[        R                  S9u  p[        X5      $ r  )rX  r   INT_TO_FLOATr  r:  s     re   r  r    s*     	
$C$P$PDA A>rg   c                z    [        U 5      =(       d    [        U 5      nU(       a  S nOS n[        U5      " X5      $ )Nc                .    [         R                  " X5      $ r   )rQ   modr:  s     re   r   fmod.<locals>.fn  s    771= rg   c                .    [         R                  " X5      $ r   )rQ   fmodr:  s     re   r   r    s    88A>!rg   )r   r   r  )r   r6  ry  r   s       re   r  r    s5    !!$:(:K	!
	" "a##rg   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nre  r  r   r   r   r   r{   r  r  r   r  r  r   r   s        re   r  r    sM     	''+;AKKM+J+J
-	U	;Bax--rg   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   r   rt  Tr  c                @    U u  nUu  n[         R                  " X#5      4$ r   )rQ   r   a_tupleb_tupler   r6  s       re   
combine_fncumsum.<locals>.combine_fn  !    rg   r  r   r  r   r   r   )r   r   r   r{   r  r   r   r   r  r1   Scanr   fallback_cumsumr   r  r   r  r   r  s         re   cumsumr    s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~q%88Mrg   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   r  Tr  c                @    U u  nUu  n[         R                  " X#5      4$ r   )rQ   r  r  s       re   r  cumprod.<locals>.combine_fn   r  rg   r  r  r  r   )r   r   r   r{   r  r   r   r   r  r1   r  r   fallback_cumprodr  s         re   cumprodr    s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~599Mrg   c                   S nU R                  5       n[        U R                  5       5      S:X  a  US;   d   e[        U 5      $ [	        XUS9n[
        R                  R                  " S0 UDSU0D6u  nUc	  [        XS9$ U$ )Nc           	     .   U u  nUu  n[         R                  " X#5      n[         R                  " X#5      nXE:g  [         R                  " U5      ) -  n[         R                  " U[         R
                  " [         R                  " XE-
  5      5      U-   U5      4$ r   )rQ   r  r  rC  r   log1pexp)r  r  r   r6  min_vmax_vr  s          re   log_add_exp_helper(logcumsumexp.<locals>.log_add_exp_helper  ss    A!A!CIIe$4#45		$		#''%-*@ AE I1MOOrg   r   r  r  r  rh  r   )	r   r   r   r  r  r1   r  r   fallback_logcumsumexp)r   r   r  r   r   r  s         re   logcumsumexpr    s{    P KKME
1::<Ag~~Qxa7FGG4FGIV~$Q00Mrg   c                  ^ [        U R                  5       5      S:X  a+  TS;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9n[        U TUS9nU[        R
                  4US'   U R                  5       U4S	 j4US
'   [        R                  R                  " S0 UDSU0D6u  pVUc
  [        U TS9$ XV4$ )Nr   r  r   r  Fr   arg_break_ties_leftr  r  c                T   > [         R                  " U T   [        R                  5      $ r   rQ   r\  r{   r  r_  r  s    re   rL  cummax.<locals>.<lambda>1      CNN3t9ekk:rg   r  r  rh  r   )r   r   r  r  r{   r  r   r1   get_reduction_combine_fnr  rj  r  r   fallback_cummaxr   r  r   r  r   r  r$  s    `     re   cummaxr  "      
1::<AwQxAU[[999KKME,,5J ad%8Fu{{+F8	:F; ggnnEvE*EOF~qd++?rg   c                  ^ [        U R                  5       5      S:X  a+  TS;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9n[        U TUS9nU[        R
                  4US'   U R                  5       U4S	 j4US
'   [        R                  R                  " S0 UDSU0D6u  pVUc
  [        U TS9$ XV4$ )Nr   r  r   r  Fr  r  r  c                T   > [         R                  " U T   [        R                  5      $ r   r  r  s    re   rL  cummin.<locals>.<lambda>H  r  rg   r  r  rh  r   )r   r   r  r  r{   r  r   r1   r  r  rj  r  r   fallback_cumminr  s    `     re   cumminr  9  r  rg   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nr  r  r   r  r  s        re   r  r  P  sM     	''+;AKKM+J+J
-	e	<Bax--rg   c                V    [        U [        R                  5      n [        S5      " XUS9$ )Nr$  r  r  )r   r{   r   r  r   r   r  s      re   
reduce_anyr  [  s$    EJJA% w??rg   c                F    Ub  [        XUS9[        XUS94$ [        U S US9$ Nr  )reduce_amaxreduce_argmaxr  s      re   
reduce_maxr  a  6    
g6!8
 	

 qtg66rg   c                F    Ub  [        XUS9[        XUS94$ [        U S US9$ r  )reduce_aminreduce_argminr  s      re   
reduce_minr  l  r  rg   xor_sumr  r  r  r  r  
logical_or)r  r  r  stabler   
descendingc          
        Uc  SnU R                  5       nU R                  5       n[        [        U5      U5      n[        U5      S:X  a'  [	        U 5      [        SU[        R                  U5      4$ [        U5      (       a  XB   OSn[        R                  R                  (       a  [        R                  nO[        R                  n[        R                  R                  R!                  U[        R"                  " U5      R$                  5      (       d
  ['        XX#S9$ [)        USSXuSS9nS/[        U5      -  n	[        U5      (       a  XiU'   [+        X5      n[-        X5      n[.        R0                  R3                  UU R4                  UR4                  4U R7                  5       UR7                  5       4UUUUS9u  pU
c
  ['        XX#S9$ Uc   eU
[9        U[        R                  5      4$ )NFr   r.   r  r  )rv   r  r  r   r  r  r  )r   rn   r   r   r  r&  r{   r  r/   tritondecompose_sort_opsr  int16rR   r]   r   statically_known_ltr  r  sort_fallbackr  r  r4  r1   Sortr   r   rj  r   )r   r  r   r  r[  rv   r  	idx_dtyper$  
view_shaper  s              re   sort_stabler    s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZZuzQH }}''KK	KK	77//%++i:P:T:TUUQ3NNQVG s5z!J
5zz"37'GW$Gggnn'==?G$7$7$9: % OF ~Q3NN8GU[[111rg   c                    [        U SXS9$ )NFr  )r  )r   r   r  s      re   sortr    s    qCGGrg   c                D   [         R                  R                  (       d  [        U 5      $ U R	                  5       n[
        R                  " [        R                  U[        R                  " S5      5      n[        X/5      n[        USS9u  pEUS-
  S-  n[        USU5      $ )Nr.   r   rh  r,   )r/   r  r  median_fallbackr   r  r+  r  r  r   r  r  r  r  )r  r   r  flatsorted_valsr;  r   s          re   median_defaultr    s}    ==++t$$==?DX\\4q1ABEgD 1-NK	qA+q!$$rg   c                   [         R                  R                  (       d  [        XU5      $ U R	                  5       n[        U5      nUS:X  a5  [        U 5      [        SU R                  5       [        R                  U5      4$ [        XA5      n[        U SUS9u  pVX1   nUS-
  S-  n[        XQU5      n	[        XaU5      n
U(       a  [        X5      n	[        X5      n
X4$ )Nr   Tr  r   r.   r,   )r/   r  r  median_dim_fallbackr   r   r  r&  rn   r{   r  r   r  r  rZ  )r  r   r  r[  r  r  sorted_idxsr  r   r  r$  s              re   
median_dimr    s    ==++"4g66MMOEu:DqyT{E!T__%6UKKK
4
%C*4#FK
A	
Q1AKa(F[q)G6'G)?rg   c           	     Z  ^^^^^^^^  [         R                  R                  (       d  [        XU5      $ U R	                  5       n[        U5      nU R                  5       nUS:X  a'  [        U 5      [        SU[        R                  U5      4$ [        XA5      n[        U SUS9u  pgX1   n[        USS[        R                  USS9n	[        R                  " S5      /U-  n
XU'   [!        X5      n	[#        X5      n	U	R%                  5       mU4S jn[&        R(                  " [+        U5      [        R                  UUS9n[-        XaU5      nUR%                  5       m UR%                  5       mU	R%                  5       mUUU 4S	 jn[&        R(                  " [+        U5      [        R.                  UUS9nUR%                  5       mU	R%                  5       mUU4S
 jn[&        R(                  " [+        U5      [        R                  UUS9n[1        UU5      u  nnU	R%                  5       mUR%                  5       mUU4S jn[&        R(                  " [+        U5      [        R                  UUS9n[3        UUSS9n[-        XaU5      n[-        XqU5      nU(       d  [5        UU5      n[5        UU5      nUU4$ )z9Lower aten.mode via sort-based decomposition or fallback.r   Tr  r.   Fr  c           	        > [         R                  " [         R                  " T" U 5      [         R                  " S[        R
                  5      5      [         R                  " S[        R
                  5      5      $ Nr.   r   )rQ   r  r  r]  r{   r  )r_  positions_loader0s    re   prev_pos_fn!mode_default.<locals>.prev_pos_fn  sI    {{GG%c*CLLEKK,HILLEKK(
 	
rg   rh  c           
        > [         R                  " [         R                  " T" U 5      T" U 5      5      [         R                  " T" U 5      [         R                  " S[
        R                  5      5      5      $ r  )rQ   or_ner  r]  r{   r  )r_  positions_loadershifted_loadersorted_loaders    re   is_boundary_fn$mode_default.<locals>.is_boundary_fn  sO    wwFF=%~c':;FF#C(#,,q%++*FG
 	
rg   c                   > [         R                  " T" U 5      T" U 5      [         R                  " S[        R                  5      5      $ rY  )rQ   r   r]  r{   r  )r_  is_boundary_loaderpositions_loader2s    re   boundary_pos_fn%mode_default.<locals>.boundary_pos_fn.  s6    yys#c"LLU[[)
 	
rg   c                   > [         R                  " [         R                  " T" U 5      T" U 5      5      [         R                  " S[        R
                  5      5      $ r  )rQ   r   r  r]  r{   r  )r_  last_boundary_loaderpositions_loader3s    re   
run_len_fn mode_default.<locals>.run_len_fnC  s@    wwGG%c*,@,EFLLEKK(
 	
rg   r  )r/   r  r  mode_fallbackr   r   rn   r  r&  r{   r  r   r  r  r   r  r  r4  rj  r?   r   rF   rX  r   r  r  r;  )!r  r   r  r[  r  rv   r  r  r  	positionspos_view_shaper  prev_positionsshifted_valsr  is_boundaryr  boundary_poslast_boundaryr;  r  run_lenmax_pos	mode_vals	mode_idxsr  r  r  r  r  r  r  r  s!                            @@@@@@@@re   mode_defaultr	    sr    ==++T00MMOEu:D__FqyT{E!VU[[%@@@
4
%C*4#FK
A 	%++fEI mmA&'$.N3Y/Iy(I "--/
 %%V$kk	N +N;L++-M!--/N ,,.
 ""V$jj	K %002!--/
 ##V$kk 	L lC0M1 "--/(446
 V$kk	G G#=G{1I{1IIs+	Is+	irg   c                p   [         R                  R                  (       d  [        XX#U5      $ U R	                  5       n[        U5      nUS:X  a5  [        U 5      [        SU R                  5       [        R                  U5      4$ [        Xb5      n[        U SX#S9u  px[        XrSU5      n	[        XSU5      n
X4$ )Nr   Tr  )r/   r  r  topk_fallbackr   r   r  r&  rn   r{   r  r   r  r  )r  r   r   largestrK  r[  r  r  r  r  r$  s              re   topkr	  \  s    ==++TcF;;MMOEu:DqyT{E!T__%6UKKK
4
%C*Ts K Ka+F[q!,G?rg   c                   [         R                  R                  (       d  [        XX#5      $ U R	                  5       n[        U5      nUS:X  a5  [        U 5      [        SU R                  5       [        R                  U5      4$ [        XR5      n[        U SUS9u  pg[        XbUS-
  5      n[        XrUS-
  5      n	U(       a  [        X5      n[        X5      n	X4$ )Nr   Tr  r.   )r/   r  r  kthvalue_fallbackr   r   r  r&  rn   r{   r  r   r  r  rZ  )
r  r   r   r  r[  r  r  r  r  r$  s
             re   kthvaluer
	  m  s    ==++ #77MMOEu:DqyT{E!T__%6UKKK
4
%C*4#FKKa!e,F[q1u-G6'G)?rg   c                6    [        U U[        R                  US9$ )Nr   r   r  )r  r   r{  )r`   r   r  s      re   register_pointwise_numericr	    s!    
;HH'	 rg   c                \    [        U R                  5        [        U [        R                  S9$ r  )rN   r  r  r   r{  rG  s    re    register_pointwise_numeric_ldf64r	    s'    '4
;HH rg   )r  )r   c                 ^^^^^	^
 [        U UU[        R                  S9mU R                  5       mUR                  5       mUR                  5       m	U R	                  5       nTR
                  =(       a@    [        R                  R                  (       + =(       a    USL=(       a    UR                  S;   m
UUUU	U
U4S jn[        R                  " U R	                  5       TUU R                  5       S9$ )aR  
Computes self + value * tensor1 * tensor2 using FMA for better precision.

Matches eager CUDA kernel order: self + value * (tensor1 * tensor2)
This is computed as: fma(value, tensor1 * tensor2, self)

Note: FMA is only used for floating-point types on non-AMD GPUs. For integer types,
we fall back to regular arithmetic since FMA doesn't support integers.

For floating-point types, we use mul_rn (round-to-nearest multiplication)
to force rounding of the product before the FMA. This prevents Triton's
compiler from fusing the multiplication with the FMA, matching eager's
rounding behavior.
rZ  Nrs  xpuc                  > T" U 5      nT" U 5      nT	" U 5      nTS:X  a  T
(       a  [         R                  " X#U5      $ T
(       a  [         R                  " X#5      nO[         R                  " X#5      n[	        T[
        R                  5      (       a  [         R                  " TT5      nO[         R                  " TT5      nT
(       a  [         R                  " XTU5      $ [         R                  " U[         R                  " XT5      5      $ r  )
rQ   r^  mul_rnr  rk   r   r   r\  r]  r   )r_  self_valt1_valt2_valt1_times_t2
value_exprr   r  	t1_loader	t2_loaderuse_fmar   s         re   rf  addcmul.<locals>.inner_fn  s    s#33A:'776844  **V4K''&1K eU[[))u5JeU3J77:H== 778SWWZ%EFFrg   rh  )r   r   rO  rj  rn   r  r{   r  r  r   r?   r   r   r  tensor1tensor2r   rv   rf  r   r  r	  r	  r	  s      `  @@@@@re   addcmulr!	    s      ;CC	E ""$K##%I##%I __F 	+!!!	+$	+ KK?*	 G G<  }}	 rg   c                 ^^^^^	^
 [        U UU[        R                  S9mU R                  5       mUR                  5       mUR                  5       m	U R	                  5       nTR
                  =(       a@    [        R                  R                  (       + =(       a    USL=(       a    UR                  S;   m
UUUU	U
U4S jn[        R                  " U R	                  5       TUU R                  5       S9$ )a  
Computes self + value * (tensor1 / tensor2) using FMA for better precision.

Matches eager CUDA kernel order: self + value * (tensor1 / tensor2)
This is computed as: fma(value, tensor1 / tensor2, self)

For value=1: self + tensor1 / tensor2 (no FMA needed, just add the division)
For value!=1: fma(value, div_rn(tensor1, tensor2), self)

Note: FMA is only used for floating-point types on non-AMD GPUs. For integer types,
we fall back to regular arithmetic since FMA doesn't support integers.

We use div_rn (round-to-nearest division) to force proper rounding, preventing
Triton from fusing operations in ways that change the rounding behavior.
rZ  Nr	  c                  > T" U 5      nT" U 5      nT	" U 5      nT
(       a  [         R                  " X#5      nO[         R                  " X#5      nTS:X  a  [         R                  " X5      $ [	        T[
        R                  5      (       a  [         R                  " TT5      nO[         R                  " TT5      nT
(       a  [         R                  " XTU5      $ [         R                  " U[         R                  " XT5      5      $ r  )rQ   r[  r  r   rk   r   r   r\  r]  r^  r  )r_  r	  r	  r	  	t1_div_t2r	  r   r  r	  r	  r	  r   s         re   rf  addcdiv.<locals>.inner_fn  s    s#33 

62IF3IA:778// eU[[))u5JeU3J77:(;; 778SWWZ%CDDrg   rh  )r   r   r{  rj  rn   r  r{   r  r  r   r?   r   r   r	  s      `  @@@@@re   addcdivr&	    s    " ;HH	E ""$K##%I##%I __F 	+!!!	+$	+ KK?*	 E E:  }}	 rg   r   r  logical_not)r  )r   r  rU  identity)r  pointwise_overrides_datac              #  V  ^#    [         U   m[        U TR                  S 5      nUc  g U4S jn[        U[        R
                  R                  5      (       a:  UR                  5        H%  n[        X$5      nUTR                  U" U5      4v   M'     g UTR                  U" U5      4v   g 7f)Nc                6   > TR                   c  [        U 5      $ g r   )r  r  )r`   rm   s    re   make_triton_fallback6_get_pointwise_overrides.<locals>.make_triton_fallback  s    ;;#B'' rg   )	r)	  r   r   rk   r{   r|   r   r   r   )nsr   r`   r,	  olnamer  rm   s         @re   _get_pointwise_overridesr0	    s     #D)D	TYY	%B	z( "ejj1122llnF$Bd..0DR0HHH % $**,@,DDDs   B&B)r	  c                d   ^ U [         U'   [        R                  U 5        U4S jn[        X5        g )Nc            	     ~   > T" U 0 UD6n/ n[        U S   U5       H  u  pEUR                  [        XESS95        M      U$ )Nr   TrJ  )r  ro   r  )rt   r   resultsmut_resultsr   r  outplace_ops         re   r   $register_foreach_inplace.<locals>.fn  sK    t.v.tAw0KCy4HI 1 rg   )rZ   r  r   r  )aten_opoutplace_aten_opr5	  r   s     ` re   register_foreach_inplacer9	    s,    07,-G$ w+rg   c                .   ^ [        U S S9U4S j5       nU$ )NrZ  c                 j   > T" U 0 UD6n[        X S   R                  5       5      n[        U S   U5      $ r  )r   r   r  )rt   r   r  r5	  s      re   r   register_inplace.<locals>.fn  s:    d-f-&q'"3"3"56a&))rg   )r+  )r7	  r5	  r   s    ` re   register_inplacer=	    s#    wD9* :*
 Irg   c                    g r   r   r  s      re   sym_constrain_ranger?	  7      rg   c                    [         R                  R                  R                  S   n[	        U[
        R                  5      (       a  UR                  R                  $ [        U5      $ r  
rR   r]   r^   r  rk   r{   r  rc   r  r   r   r   r@  s      re   r  r  <  D    
''


#
#E
*C#u||$$xx}}3xrg   c                    [         R                  R                  R                  S   n[	        U[
        R                  5      (       a  UR                  R                  $ [        U5      $ r  rB	  rC	  s      re   
sym_striderF	  E  rD	  rg   c                "    U R                  5       $ r   )rV  r   s    re   	sym_numelrI	  N  s    ;;=rg   c                     [        U 5      S:X  a#  [        U S   [        [        45      (       a  U S   n [        R
                  " U 6 $ r  )r   rk   ri   r   r   Addr!  s    re   sym_sumrL	  W  s;     4yA~*T!WtUm<<Aw99drg   c                    [        S5      e)NzHelpful for debuggingr   )r  rt   r   s      re   foobarrN	  `  s    
5
66rg   c                8    U R                  5         [        U 5      $ r   )r  r  r   s    re   _realizerP	  e  s    IIK8Org   c                R    U R                  5         [        R                  " X5        U $ r   )r  r1   ResizeStorageBytes)variablerm  s     re   resize_storage_bytes_rT	  k  s!    (-Org   c                    U R                  5         UR                  5         [        R                  " [        R                  " X5      5      $ r   )r  rB   r   r1   SetSourceTensorKernel)r  source_tensors     re   set__source_tensorrX	  r  s3    LLNB44TIJJrg   r  c                    XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ r   rT  )r  r  s     re   
fsdp_copy_rZ	  {  sH    :J^^-.sMMO,S,,.)""rg   c                 ^^^^ [        U [        5      (       d   e[        U[        [        45      (       d   eUc  [        R
                  nU[        R                  :X  a  [        SU 35      eU[        R                  :X  a  [        U5      S:X  d   eU[        R                  :X  a  [        U5      S:X  d   eU R                  5       mU R                  5       nU R                  5       n[        R                  " 5       (       ay  [        R                  R                   R"                  (       aP  [%        U5      (       a  ['        S5      mO6[)        U5      (       a!  [        R*                  " U5      R,                  mOSmOSm[.        R0                  R2                  R5                  TS5      (       a  [7        UTX4S9$ U R9                  5       nUS L=(       a    [;        S	 U 5       5      nU(       a  [=        U T/5      nOW[        U R>                  [@        RB                  5      (       a  U R>                  RE                  5       U l        [G        U T/S
/5      nURI                  5       m[@        RJ                  RM                  X5      n[@        RN                  " XCX5      RQ                  5       mUUUU4S jn	[R        RT                  " XCU	[        U5      S9n
U
$ )Nzunsupported memory format: r   r   nanTr  r   rb  c              3  v   #    U  H/  n[         R                  R                  R                  US 5      v   M1     g7frO  )rR   r]   r   r  r  s     re   r   resize.<locals>.<genexpr>  s-      2@G100A66s   79r.   c                  >^ T" U 5      m[         R                  " T[        R                  5      n[         R                  " T[        R                  5      n[         R                  " X5      n[         R
                  " UUU4S jT5      $ )Nc                    > T" T /5      $ r   r   )
flat_indexflat_loaders   re   rL  *resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(Arg   )rQ   r\  r{   r  r  r  )	r_  flat_index_exprlimitr  ra	  rb	  	old_numelout_indexeruninitialized_vals	       @re   rf  resize.<locals>.inner_fn  sX     %
..U[[Ay%++6vvo-zz$ ACTUUrg   rh  )+rk   rB   ri   r   r{   contiguous_formatpreserve_formatr  channels_lastr   channels_last_3drV  r   rS  r  r  deterministicfill_uninitialized_memoryr   r<  r   r  r  rR   r]   r   r  rO  r  r$  r  rm   r1   r5   r  r  rj  r   stride_ordered_for_memory_formatr  r  r?   r   )r   r   r  r   rv   r  has_overlappingx_flat
out_striderf  rq   rb	  rf	  rg	  rh	  s              @@@@re   resizert	    sA   a####dT5M****//---8HII+++4yA~~...4yA~~IKKME""$F 	2244KK%%??%   %ee$$ %E 2 6 6 $  ww//	1==D+5HH  "GT) c 2@G2 /O a)% affbkk**VV'')AFA	{QC0$$&K""CCDXJ..ANNPKV V 

Xd4jC Jrg   )auto_functionalizedc                    SSK Jn  UR                  U5      n[        R                  " U UU0 UEUES9  UR                  5        VVs0 s H  u  px[        U[        5      (       d  M  Xx_M      snn$ s  snnf )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_args)*torch._higher_order_ops.triton_kernel_wraprw	  get_constant_argsr1   UserDefinedTritonKernelr   rk   rB   )	rx	  constant_args_idxry	  rz	  r   rw	  constant_argsr  r@  s	            re   triton_kernel_wrap_r	    sl     M%778IJM 7/v//	 &,\\^R^z#y7QHCH^RRRs   A/%A/c                n   [        S U /UQ 5       5      (       aY  Sn[        R                  R                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  R                  XX#5      n[        [        [        R                  U5      5      $ )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   ro  r!  s     re   r   cond.<locals>.<genexpr>  s%     
M;La:a 1Yq\1;Lrr  z"control flow operator: torch.cond.r  r  )r$  rR   r]   r^   r  r  r  r1   Conditionalr   ri   maprB   )predtrue_fnfalse_fnoperandsr   r  r  s          re   r   r     s     
MD;L8;L
MMM2''..3377tLL;LE*;-8C,/)^^""4(EFI$$f-..rg   c                   [         R                  (       dr  [        S X#-    5       5      (       aY  Sn[        R                  R
                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  R                  XX#U5      n[        U[        5      (       d   e[        [        [        R                  R                   U5      5      $ )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   ro  r!  s     re   r   while_loop.<locals>.<genexpr>  s*      *3A 	1f.)A,.3rr  z(control flow operator: torch.while_loop.r  r  )r/   r  r$  rR   r]   r^   r  r  r  r1   	WhileLoopr   rk   r   ri   r	  _maybe_wrap_as_tensor_box)cond_fnbody_fncarried_inputsadditional_inputsstack_outputr   r  r  s           re   
while_loopr	    s     !!c *3* ' ' 9''..3377tLL;LE*;-8C,/)\\  .\F fh''''BLL::FCDDrg   )r	  c                    [         R                  R                  " U /UQ76 n[        [	        [
        R                  U5      5      $ r   )r1   InvokeSubgraphr   ri   r	  rB   )subgraph_fn
identifierr	  r  s       re   invoke_subgraphr	     s5    %%k=H=FI$$f-..rg   c                z   [         n[        U R                  R                  5       GHb  u  p4UR                  S:X  aA  U[
        R                  R                  ;  d   eX   [
        R                  R                  U'   MW  UR                  S:X  a\  [
        R                  R                  U5      u  pV[        R                  R                  R                  [
        R                  XEU5      nM  U[
        R                  R                  ;  d   e[
        R                  R                  n U[
        R                  l        [
        R                  R                  U5      [
        R                  R                  U'   U[
        R                  l        GMe     U[         L a  [        S5      eU$ ! U[
        R                  l        f = f)a  Process nodes from a FX graph by executing them through V.graph.

This is a common pattern for executing a subgraph's nodes:
- Placeholder nodes are mapped to the provided args
- Output nodes return their result
- Other nodes are executed via V.graph.run_node

r  r7  zNo output node found in graph)_MISSINGrj   r]   nodesr`   rR   envfetch_args_kwargs_from_envr{   rY  Interpreterr7  r^   run_noder  )r  rt   r7  rs   rc   output_argsr   saved_current_nodes           re   process_subgraph_nodesr	     s3    F\//55677m#qww{{*** $AGGKKWW "#''"D"DT"JKXX))00$VTFqww{{***!"!5!5:'+$$%GG$4$4T$:D!'9$! 7$ :;;M (:$s   $AF##F:)control_depsc                   / nU  HI  n[        U[        5      (       d  M  UR                  5         UR                  UR	                  5       5        MK     [
        R                  R                  R                  nSn[        U5      U-   [        U5      :X  d   e[        [
        R                  R                  5      n[        UR                  R                  R                  SS95      [        U5      :X  d   e[        UR                  [        U5      5      nU (       d   e[
        R                  R                  US  HI  n	U H@  n
U	R                  nUc   e[
        R                  R                   U   R#                  U
5        MB     MK     U$ )a;  
Lower control_deps_op by ensuring dependencies are realized and tracking them.

The control_deps_op HOP makes dependencies explicit in the graph. During lowering:
1. Realize all additional dependencies to ensure they're computed
2. Execute the target operation normally
3. Track the dependencies for the scheduler
r,   r  rG  N)rk   r9   r  ro   r;  rR   r]   r^   rt   r   
operationsr  
find_nodesr	  ri   operation_nameadditional_buffer_depsr   )additional_depsr	  rt   	dep_namesdeporiginal_args
arg_offsetoperation_lenr7  r`   dep_nameop_names               re   control_deps_op_loweringr	  ;   s@    I#v&&(  GG((--MJt9z!S%7777**+M{''--88M8JKsSWyXXX $K$<$<d4jIF? gg  0!H''G&&&GG**7377A " 1 Mrg   )schemec               
   S n[         R                  R                  R                  R	                  SS 5      nUc   e[        U R                  R                  R                  5       GH  u  pVUR                  S:X  a!  X%   [         R                  R                  U'   M7  UR                  S:X  Ga  [         R                  R                  U5      u  px[        R                  " XxR                  5       5       H  n	U	R                  5         UR                  (       a7  [         R                  R                   R#                  U	R%                  5       5        [         R                  R&                  R#                  U	R%                  5       5        M     [(        R*                  R,                  R/                  [         R                  XgU5      nGM_  [         R                  R1                  U5      [         R                  R                  U'   GM     U$ )Nquant_optionsr  r7  )rR   r]   r^   r  r  rj   r  r	  r`   r	  r	  r  r  r  r  codegen_low_precisionlow_precision_codegen_opsr   r  invoke_quant_opsr{   rY  r	  r7  r	  )
r	  r	  r	  r7  r	  rs   rc   rt   r   r
  s
             re   invoke_quant_tracerr	  j   sW   FGG((--11/4HM$$$[55;;AAB77m# (AGGKKWW 77==dCLD__T==?;		 66GG5599!:N:N:PQ((,,Q-A-A-CD < XX))00$fMF ! 0 0 6AGGKK% C( Mrg   c                  ^
 SSK JnJn  [        U5      S:  a  [	        S5      e[
        R                  " X5       Vs/ s H&  nU" UR                  5       UR                  5       S9PM(     nnU" X5      m
U
4S jn[        US   SS S9n[        S U 5       5      US	'   [        S
 U 5       5      US'   [        R                  R                  " SUSS.UD6n	U	S   c  [	        S5      eU	$ s  snf )Nr.   )InputDescriptorlower_pointwise_subgraphr   zSUnable to generate code for associative_scan op, because there are lifted argumentsrb  c                h   > T" / [         R                  " U 5      Q[         R                  " U5      Q76 $ r   )r  r  )lhsrhslowered_combine_fns     re   wrapped_combine_fn,associative_scan.<locals>.wrapped_combine_fn   s6    ! 
$
$
 	
rg   r  c              3  @   #    U  H  oR                  5       v   M     g 7fr   r   r!  s     re   r   #associative_scan.<locals>.<genexpr>   s     7Bq[[]]Br(  r  c              3  @   #    U  H  oR                  5       v   M     g 7fr   r  r!  s     re   r   r	     s     <Ar(  r  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )r  r	  r	  r   r  r  r  r   rn   r  r   r1   r  r   )r  xsr	  r	  r	  r   subgraph_inputsr	  r   r  r	  s             @re   associative_scanr	     s     M
!a
 	
 ((A 	akkmALLNC(   2*N
 be!48F7B77F8<<<F;WW^^ %" F
 ayLMMM-s   -C'c                    g r   r   )tokenss    re   _sink_tokensr	     r@	  rg   c                     g r   r   r   rg   re   _make_tokenr	     r@	  rg   c                   SSK JnJn  U" U5      nUc  U[        R                  R
                  R                  L a  SSKJnJ	n  UR                  5       n	U	(       a  U	R                  R                  [        R                  R
                  R                  5      n
U
(       aW  [        X5      (       d   eU
R                  US   5      nU(       a*  [        U5      S:X  d   S5       e[!        [#        U5      5      n[        [$        R&                  R(                  5      nU[*        ;   a-  [*        U   " U0 UD6n[,        R.                  " [0        S U5        O<S n[,        R2                  " U[4        R6                  R8                  " U/UQ70 UD65      n[        [$        R&                  R(                  US 5      S:  d   S	U S
35       eU(       a  [$        R&                  R:                  R=                  U5      n[$        R&                  R(                  US  H^  nS Ul        U(       d  M  URA                  5       n[$        R&                  RB                  U   RE                  URA                  5       5        M`     W[$        R&                  R:                  U'    S n[,        R2                  " UX#45      u  nnU" UUU5      n[        URR                  5      S:X  a  X4$ [        URR                  5      S:X  a  X4$ U /UQ7$ ! [F         aU  n[I        U5      n[J        RM                  SUU5        [        U[N        [P        45      (       a  U /UQ7s SnA$ X4s SnA$ SnAff = f)zs
We lower the operator directly, and then we add StarDep dependencies to all
the newly created nodes in the graph.
r   )_get_effect_get_schemaN)InvokeSubgraphCacheTracingContextr.   zMultiple effects NYIc                "    U R                  5       $ r   )r  rH	  s    re   rL  with_effects.<locals>.<lambda>   s
    !))+rg   c                d    [        U [        R                  5      (       a  U R                  5       $ U $ r   r  r   s    re   r  "with_effects.<locals>.wrap_tensors   r  rg   zCNo operation nodes were generated when lowering effectful operator .c                     gr   r   r   rg   re   rL  r	     s    drg   c                f   [        U [        R                  5      (       a  U R                  5       $ [        U [        5      (       aU   U R
                  n[        US5      (       a5  [        UR
                  S5      (       a  UR
                  R                  5       $ U $ U $ ! [        [        4 a     U $ f = f)Nrm   get_example)
rk   r1   TorchBindObject	get_valuerB   rm   r   r	  AttributeErrorr   )r   r  s     re   convert_ir_to_value)with_effects.<locals>.convert_ir_to_value   s    !R//00{{}$Ay))ffGw//Gm5 5  '||7799 H	 '(;< s   AB B0/B0z5Failed to get schema for %s: %s. Assuming list output)*torch._higher_order_ops.effectsr	  r	  r{   rQ   higher_orderr	  torch._guardsr	  r	  try_gethop_dispatch_set_cache	get_cacherk   get_effectsr   rP  r  rR   r]   r	  rW   r  r  rB   r  r1   r  r   effectful_opsr  has_side_effectsr;  additional_star_depsr   r  rc  r  r  r   ri   returns)tokenr`   rt   r   r	  r	  effect_typer	  r	  tracing_ctxinvoke_subgraph_cacheeffectsr	  r  r  prev_effect_buffernew_opr	  r	  schema_argsschema_kwargsschemae	error_msgs                           re   with_effectsr	     s    I b/KrUYY%;%;%K%KKE$,,.$/$F$F$P$P		&&66%! %!"7MMMM/;;DGDw<1,D.DD,"&tG}"5K **+M 
Y2//Y(=vF	L "++222GGG
 qww!!-.12Q6 
MbTQRS6 WW2266{Cgg((8F&2F#!! //+,,W599:L:U:U:WX 9  	
k* #	$ &,__$&
"] Rm< 6>>a	V^^		!  #F	CR	
 fudm,,#F##?"#s+   (K3 3
M=AM?MMMM)register_comm_loweringsregister_symm_mem_loweringsc                >   [        XSSSS9nUS   n[        R                  R                  R	                  [        U5      5      n[        R                  R                  " S0 UDSUS.D6u  pVUS:X  a\  [        R                  R                  R                  U[        R                  5      (       a  [        R                  " SU SUS	.UD6u  pxXx4$ [        R                  S
5        [!        XSS9n	["        [$        R&                     " [)        X	5      5      n
[+        XSS9nX4$ )zf
Lowering inductor_prims.prepare_softmax_online to compute max/sum in one pass if no split is needed.
TNr  r  online_softmax_reduce)r  r  r.   r,   )r  
num_outputreduction_hintzTOnline softmax is disabled on the fly since Inductor decides to split the reduction.)r  r   )r  rR   r]   r   simplifyrO   r1   r@   
num_splitsstatically_known_geqr/   r  r<   r   r  debugr  rW   r  r  r  r  )r   r   r   r  r%  hint	num_split
max_tensor
sum_tensorr  r  xsums               re   prepare_softmax_onliner

  #!  s   
 #	d$dF 01WW&&}5E'FGFll-- 
.OD A~!''**??22  "8!>!> "
Qt"
?E"

 %% 			b	
 1D1!#a,/Ct,zrg   c                     [         R                  R                  5       =(       a!    [         R                  R                  5       S:  $ )z.Check if we're on SM100+ hardware (Blackwell).)r   r   )r{   rs  is_availableget_device_capabilityr   rg   re   _is_sm100_or_laterr
  Q!  s,    ::""$V)I)I)Kw)VVrg   c           	        [        5       (       d  [        S5      eU R                  5       nU[        R                  [        R
                  [        R                  4;  a  [        SU 35      eU[        R                  :w  a  [        U [        R                  5      n [        R                  " [        R                  SS[        R                  SSS9n[        U5      " U 5      n[        U[        R                  5      $ )z
Lowering for cvt_e8m0_rceil. Uses PTX cvt.rp.satfinite.ue8m0x2.f32 on SM100+.

The PTX instruction takes 2 float32 and outputs 2 e8m0 packed in uint16.
Currently we pass 0.0 as the second input and only use the low byte result.
zFcvt_e8m0_rceil requires SM100+ (Blackwell) for PTX instruction supportzAcvt_e8m0_rceil requires float32, float16, or bfloat16 input, got z)cvt.rp.satfinite.ue8m0x2.f32 $0, 0.0, $1;z=h,rTr.   )asmconstraintsr   is_purepack)r
  r   r   r{   r  rR  rQ  r:  r   r  r*  rQ   inline_asm_elementwiseuint16r  r"  )r   r   r   r  s       re   cvt_e8m0_rceil_loweringr
  V!  s     !T
 	
 MMOEU]]EMM5>>BBOPUwW
 	

 sEMM*			""7ll
B B$FFEKK((rg   )r
  r
  c           	     <  ^ ^^^^^^	 [        U6 n[        S U 5       5      mU Vs/ s H  ofR                  5       PM     snm	U UUUUU	U4S jn[        R                  R                  US   R                  5       TU[        US   R                  5       5      S9$ s  snf )Nc              3  @   #    U  H  oR                  5       v   M     g 7fr   r	  rp  s     re   r   /lower_inline_asm_elementwise.<locals>.<genexpr>!  s     ;FSFr(  c           
        >^  [        U 4S jT 5       5      n[        R                  " UTTTTT	TS.6nT[        R                  [        R
                  4;   a%  [        R                  " U[        R                  5      nU$ )Nc              3  2   >#    U  H  o" T5      v   M     g 7fr   r   )r   rx  r_  s     re   r   Alower_inline_asm_elementwise.<locals>.inner_fn.<locals>.<genexpr>!  s     7wVVC[[ws   )r
  r
  r   r
  r
  input_dtypes)r   rQ   r
  r{   rR  rQ  r   r  )
r_  valsr  asm_strr
  r   r
  r
  r  r
  s
   `  re   rf  .lower_inline_asm_elementwise.<locals>.inner_fn!  sh    7w77++#%
 U]]ENN33\\&%--8Frg   r   rh  )	r  r   rj  r1   r?   r   rn   ri   r   )
r
  r
  r   r
  r
  rT  r   rf  r
  r  s
   `````   @@re   lower_inline_asm_elementwiser!
  |!  s     'F;F;;L,23FS F3G $ <<ay##%F1I&&()	   ) 4s   B)r  )quantized_lowerings)mkldnn_lowerings)jagged_loweringsc              #  v  #    [        U [        R                  R                  5      (       d   S5       e[        R                  U 5      n [        U 5      " [        U 5      5        Sv   U(       a
  U[        U '   g[        R                  U 5        g! U(       a
  U[        U '   f [        R                  U 5        f = f7f)zR
A context manager to force fallback an op. Used in unit test
for FallbackKernel.
z+Only OpOverload to make the clean up easierN)	rk   r{   r|   r}   rW   r  r+  r  r  )r`   old_handlers     re   force_fallbackr'
  !  s      b%**//00 50 --#K".r23'IbMMM" 'IbMMM"s   AB9	B ''B9(B66B9rc  )rp   zIterable[Any]r   z4defaultdict[tuple[Any, bool], list[tuple[int, Any]]])r   Callable[..., Any]r   zCallable[..., Any] | None)r   ztorch._C.Tagr   z%Callable[..., tuple[Any, Any]] | None)r   r   r   rc  r   None)r   zuCollection[torch._ops.OpOverload | torch._ops.OpOverloadPacket] | torch._ops.OpOverload | torch._ops.OpOverloadPacketr   zlist[Any] | None)r   z3torch._ops.OpOverloadPacket | torch._ops.OpOverloadr   zCallable[..., tuple[Any, Any]]r   r)
  )r   zint | torch.dtyper   rW  )r   r	   r   z'TypeGuard[TensorBox | sympy.Expr | int])r   r	   r   zTypeGuard[TensorBox | bool])rt   r	   r   r   r   r   r   rW  )r`   z9Any | torch._ops.OpOverloadPacket | torch._ops.OpOverloadr   rc  r   r   )r   rB   rv   torch.devicer   rB   )rt   	list[Any]r   zdict[str, Any]r  r   r   &ELEMENTWISE_TYPE_PROMOTION_KIND | Noner  r   r   z tuple[list[Any], dict[str, Any]])r   re  r  r(
  r   r(
  )
r  r(
  r  r   r   r,
  r  r   r'  rV   )r   r,
  r   z.Callable[[Callable[_P, _T]], Callable[_P, _T]]r  )NNNFFN)Frd  )FT)r   rB   r   rW  r  r   rx  r   )r   rB   r   rW  )r   rB   rv   r*
  )F)r   rB   r  rB   r   )r   rB   r,  zSequence[sympy.Expr]r   rB   r  )r  rB   r  rB   r  rB   r  r   r  r   r  r   r   rW  r   rB   )r  rB   r  rB   r  rB   r  r   r  r   r  r   r   rW  r   torch.dtype | Noner   rB   )r  rB   r  r<  r  r   r  r   r  r   r   rW  r   rB   )r  rB   r  r<  r  r   r  r   r  r   r   rW  r   r-
  r   rB   )r  rB   r  rB   r  rB   r  r   r  r   r   rW  r   rB   )r  rB   r  rB   r  rB   r  r   r  r   r   rW  r   r-
  r   rB   )r   r   r.   )r  r   rs  r   rt  r   r  )T)rw   ztorch.Tensor)rc   ztorch.fx.Node)NTFN)rv   r*
  )
r   	list[int]r	  rB   rV  rc  r  r   rH  rW  )
r^  r   r]  r   r   r.
  r	  rB   r  r   )rf  rB   r   z.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr])rf  rB   r   ztuple[str, sympy.Expr])r{  rB   r  rB   rm  r   rn  r   ro  
str | Nonerp  TensorBox | Noner   rB   )r  rB   r  rB   rm  r   rn  r   )r.   )r   r   rB  r   )r   NNr.   )NNN)rt  FF)r   r   )r  re  r   r   r+  r/
  r  r   )r   r   r+  r/
  )r   r   r  r   )r,   F)r	  ztuple[float | None, ...]r  r   r  r   )r  float | None)r  r1
  r  r1
  )r  r1
  r  r1
  r  r1
  )r   rB   r7  zSequence[int]r%  r<  r   r0
  )rs   
sympy.Exprr^  zsympy.Expr | int)rs   r2
  r]  r2
  )rs   r2
  r]  r2
  r^  r2
  )Nr  N)
r  rB   rt  Sequence[int | torch.SymInt]r  r3
  r  zlCallable[[Sequence[int | torch.SymInt], Sequence[int | torch.SymInt]], torch._inductor.virtualized.OpsValue]r   rB   )Nr   r.   F)r  )NNNN)r   r   FTN)r  rS   r   )r   rb  r   zir.Constant | None)rt  F)rt  TT)r`   ztorch._ops.OpOverloadPacket)r   z-list[ir.TensorBox | ir.ShapeAsConstantBuffer])r	  ir.Subgraphr	  rc  )r  ztorch.fx.GraphModulert   r+
  )r	  r4
  )r  r4
  r	  ztuple[torch.Tensor]rd  (B  
__future__r   r  rE  r  r  loggingrw  r  r  r  r  collectionsr   collections.abcr   r   r   r   typingr	   r
   r   r   r   typing_extensionsr   unittest.mockr   r   r{   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreer  _pytreer  torch._dynamo.utilsr   (torch._higher_order_ops.associative_scanr   r|	  r   "torch._library.fake_class_registryr   torch._library.opaque_objectr   torch._library.utilsr   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r    torch.fx.experimental.sym_noder!   r"   r  r#   r$   r%   torch.utils._ordered_setr&   torch.utils._sympy.functionsr'   r(   r)   r*   r+   _dynamo.utilsr-    r/   r0   r1   r2   decompositionr3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   ro  rQ   rR   rj  rS   rT   rU   FALLBACK_ALLOW_LIST	getLoggerr  r  rW   __annotations__rX   rY   r|   r}   r  r  tr_c10dr  r   _higher_order_opsr  rb   r  rZ   quantized_decomposedrf   rx   r   r~   r   r   r   r  r  r"  r  bmmconvolutionconvolution_backwardr  r  r  r,  r  r  _int_mmr"  r!  r  r  r  rR  r  r@  	complex32	complex64r   rQ  r   r   r   r   r   r   r   r   r  r  r(  rO  r+  r9  rX  rk  r  r  r  r   r  r  r  r  r   r  r  
device_putr  r  r{  r  r  r  r  r   r  aliasdetachdetach_liftview_ofr.  r   r/  r;  r=  rA  rC  rI  rK  rN  rQ  r  rS  r4  r^  r`  rp  _unsafe_viewreshapers  slicer  r  r  r  quantize_per_channelr  r  r   r  _functional_assert_asyncr  dequantize_per_channelr  quantize_per_tensorr
  dequantize_per_tensorr  rI  r  r  rE  r  r  r  r  r  r  r  r  rZ  r  r  r  r  cacher  r  r  r  r  r  rngprimsr  r  r  	bernoullir  r  r  	lru_cacher  r!  r  r&  r$  r%  r+  r*  r)  randint	rand_like
randn_likerandint_likestreamsrecord_event
wait_eventsynchronize_eventsynchronize_deviceforce_stride_orderr2  r	  r4  r<  r7  lookup_seedr>  rG  randomrY  rand_eager_offsetrand_eager_offsetsr`  rh  rk  r  r  ry  	NO_OPMATHr  r  r  r   rf  r  r   r   r  _adaptive_avg_pool3dadaptive_max_pool3d*_scaled_dot_product_attention_math_for_mpsuniformexponential_pdist_forwardsoft_margin_loss_backward_fused_rms_normr	  _is_compiledembedding_dense_backwardr-  native_layer_norm_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation_grouped_mm
_cudnn_rnn_cudnn_rnn_backward
miopen_rnnmiopen_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backward	nanmedianr  resize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_stategraphsafe_run_with_rng_staterun_dtensor_rng_opmasked_scattermasked_scatter_backwardr  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   r,  r  r  #_scaled_dot_product_flash_attention	quantized,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward0_scaled_dot_product_fused_attention_overrideable9_scaled_dot_product_fused_attention_overrideable_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerepeat_interleave_weight_norm_interface_backwardr  r  r  r  r  arange
start_stepr  r  r  r  scalar_tensorr  
LongTensorr  r  r  r  r&  rE  r*  rl  r)  r#  r  r$  
zeros_liker>  rB  r3  rI  rM  rO  rX  rb  ro  r  r  r  rB  r  r  r  r  r  r  r  r  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater  r  r  r  r  r  r  r  r  r  r  r  r   r"  r  r)  r?  rL  rF  rT  rU  rd  ri  rx  r}  r  r  r  r  r  r  r  r  r  r  r  r)  r#  r=  r;  rN  rT  rW  rQ  rl  rt  
avg_pool1dr  ro  r  r  r  r  r  r  r  r  r  r  r)  r/  r  r4  r6  r8  r;  r  Tensor_TensorrF  ScalarrD  Tensor_ScalarrE  r  r6  r  rV  rY  r\  r  ra  r  rg  r  true_divider  re  r  r  r  r  r  r  r  r  r  r  r  r  r$  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r	  r	  r
	  r		  medianr  r   r  rV  r  r  r  r	  r	  r	  rsqrtr  rC  expm1relur  rB  r  r  r!	  r&	  _foreach_addcmul_foreach_addcmul_scalar_foreach_addcdiv_foreach_addcdiv_scalarcossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr  tantanhrc  r'	  r  logical_xorr  r  	clamp_min	clamp_maxnegr  	remaindersignsignbit	_neg_viewler  r  r  r  r  coshsinhacosacoshasinasinhatan2atanatanhrx  erfcerfinvhypotlog10log2	nextaftercodegen.commonr  r)	  r0	  r   r`   r   r  _foreach_addListforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_clone_foreach_copyforeach_copyr9	  _foreach_add__foreach_mul__foreach_div__foreach_copy__foreach_addcmul__foreach_addcdiv_r=	  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r?	  r  r   rF	  rI	  r   methodfuncrL	  rN	  _inductor_testr  rP	  r0  rT	  set_source_TensorrX	  fsdprZ	  rt	  *torch._higher_order_ops.auto_functionalizeru	  r	  r	  r   r	  while_loop_stack_outputr*  r	  rm  r	  r	  .torch._inductor.fx_passes.control_dependenciesr	  r	  invoke_quantr	  r	  r	  r	  r	  comm_loweringr	  r	  r

  r
  cvt_e8m0_rceilr
  r
  r!
  r  r"
  register_quantized_opsregister_woq_mm_opsr#
  register_onednn_fusion_opsr$
  register_jagged_opscontextmanagerr'
  r   rg   re   <module>rm     s+=   "        	 
  # D D ? ? '    +  $ $ ( H U ? 8 :    M 
 0  - 8 8 =    $      * T]t_ !   !@B	= BBD? DTV Q Vuzz,,-/	yy~~
))

		"5::#8#89; ../
))*
 !!6!679 NP K Pyy55 90B	B*BF
	" 	"3;3.3 
3   !!$$$$--&&, {{zz{{{{{{}}}}}}

."# "'CC8C C 	C( ANQ	(O
OO O @	O
  O &Od"/A42!2 2 @	2
  2 F2n ,44 4*#4?D:  $fR5pB UYEE$E,0EMQE2 5**77TR) S)X 5--4HN IN* @E 5 5. 499??=& >& ;@e K K 5##>F ?F 
7?? $,^ $7DD 4::4H I>>   	
 4::DI J0 4))UPTU V. DJJT\\499emmTU V 4doo&s+ 4<<T:B ;B, 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>4 5))tD
 E
 4>>t<# =# 4;;4  4n 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::481#++AT eE 9eEP 4??=J >J8 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / / / X/d$ 4%%))*$ +$ 400445$ 6$ //T %)/// / 	/
 / / / "/ //d ,,44$!!! ! 	!
 ! ! !!H ..66D %)!!! ! 	!
 ! ! "! !!H ,,33/// / 	/
 / / //d ..554 %),,, , 	,
 , , ", ,,^ 488b: b:J 4==d;5E <5Ep 4%%4@6 A6 4((dC D 4;;D9=C :=C@ 4::48 94 4((dC  D  4;;D9 : 4;;D9D :D8 4>>t< = 4??= > 488 $   D
3 
3%6 %6T 	6Nr	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  dnnd 3 doot 4 d 6 eii,,44 5 eii**22 3 eii1199 : eii22:: ; 499J J 4::J J >44$OL PL
 >&&DAK BK >''TBJ CJ
 >--4H	 I	( >((dC $}}5
5
5 5
 5 5 D5p n.. / n// 0 >))tDLM	(09FI E8 . 4$$++F
 #^^
^ 	^
 ^ ^ ^ ^ G^B NN/N/X/X 777 	7
 77t 0 d))113J KaJ d'' ( d&& ' d== > dll ' d&&U 3 d!!#5 6 d,,5 9 d"" /99%%E 	::U
 d!! " d"" # doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d . d'')@ A doo} - d&&(: ; doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++ , d""$6 7 dnn  dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dhhU + d((- 8 ell$$;; < ell$$77 8 ell$$AA B ell$$77 8
 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  djj  0088	
 99AA	
 ,,44	
 ,,66	 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 99AA	
 BBJJ	
 d++33_ E d++55 6 d,,44o F d//77 I d0088/ J d   d$$++T B d22::<N O
 499$7 8 4::"  * 4"##d**+E2 5:: ( 4;;))tD 

  E. 4&&DA  B F 4%%4@A AAH ELL$"4"4e6I6IJKtDU 5 L5p 5??#4 $4 5##$+ %+ 4++,)" -)"X 4&&' ( 4//0%)59$ 1<K8 EKK,- 
 .$.> t/0B50IJ
1!45	 21 56
  4>>" $T$4  # 4%%&Td '@ 4))*"4 +" 5%%--.A /A EJJ		*+: ,:
 4;;D9! :!H 4>>t< =B#F FR"1J 4::48
 9
 4%%4@/ A/ 4>>t< = 4))* +;. 4??= > >44$O Psl !1%%5!  0@,,44%0 ,
 4,,$G H, 4;;QUV	P W	P 1 1 4**E F 4<<T:9 ;9 & 
  D 4==d;AE : <:" 4##>3 ?3 4$$$?6 @6 4&&DAP BP 4''TBPT q Cqp , ', 	,
 ,^ 4**223> 4> 41199:J ;J 4**223LPI*I=II 4I 41199:LPU*U=IU ;U 4**223 "!!S S 	S
 S 4S 41199: "!!	 	 		
 	 ;	7 599$$% &*\\(\6;\\~2)2)(2)6;2)2)j 4''TB8 C8v :>@ MQ < JN@ @:HV 5::PTU 5 V5D- -	 > 	11t$< 4//TJ 
 K
 4//TJ 
 K
 -=))11- ) 488dSN TNb8B&R2j  0%%5  
 4,,-B .BJ  0$$%  
 4++,N -Nb)X 4--.V /V 4--.V /VAH 433;;<BF. =.b 4??=  >* 4??=  >, T__,,%HT__,,%HT__,,%H A#H  0$$%  
 4++F b GbJ  0$$%  
 4++F D GDN& HLO Od < 499:t : : 40(V4. DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,- --`"J 4::2 2 4::48 9       488t, -& DHH:.( /(@ EII;$/$ 0$. 	txx'7DD

 DIIuzz*d;$ <$  DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :, 4;;D9 :, 499. . . 488@ @
 48867 77 48867 77 %--  	!: ;		*>%+@A		*>%+@A!$++.85;;? "$++.85;;? HH ,	 !!1!1uM 499##>!re +2 ?+2\ 499$$$?H @H !!2!2N$T]]%:%:PUV "4;;#6#6ER&t{{ER  !2!2N 4;;&&DA% B% 4;;??= >& 499$$$?j  @j Z 499$$$? @  4==((dC D$ 	#4::.&txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4 4<<40-. F 1FR 4<<40-. F 1FR 5  't'  5  't' 
 ! *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* $.. !' * $.. !' *""'8
 4>> "$))L 499  4<<uzz B $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE$ %D4Ld50 	 3+		
5 5Mt50 	 3+		
5 %, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,113 7 4,,<<c B-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4--t 4 4.. 6 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4.. 6)$*<*<dC, T..335E t00779K T..335E t00779K T..335E t00779K !3!3!;!;\ !!  
 !!   C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  ( $,,  , $// "#5 6 $++ z * $// "#6 7 $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% & 4??&&' ( 4>>" # "'')LFD(01$7 * 5==! " 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>7##uyy~~++334# 5# 599>>(()%) = *=@ K J ! " 12S 3S( 599))..DI/2/ J/ 599))44$OE PE& 	II22JT24
 599))99tT/ U/
 8 H H G <T:+ ;+\ 5**77TRDH   S8 &DA  4G  B F 599??//778 9 599??..667 8 599))66DQc  Rc L P O     >88dS* T*ZW
 >00dK") L")J 	22 37QF     ! !  * * ,  ' ' )    + + -    $ $ &  rg   