3jHqSSKrSSKrSSKJs Jr SSKJr SSKJ r SSK J r J r SSK Jr \R"\5rS\ R$S\ R&S \4S jrS\ R$S\ R&S S S S4S jrS\ R$S \ R.4Sjr\ \\\4"5rS\ R.S S4SjrS\ R.S \4SjrS\ R$S\S S 4SjrS\ R$4Sjr Sr!Sr"g)N) is_symbolic) OrderedSet)configir)Vxcomm_buffer_typereturnc[U5n[U[R5(ag[U[R5(aUR 5(dgUR 5n[U[R5(ag[U[R5(ag[U[R5(a[UR55(dgg)zY Check if an input can be realized as a comm buffer of the specified `comm_buffer_type`. TF) _get_data isinstancerLoopsBuffershould_allocateget_output_specCommBufferLayout FixedLayoutFlexibleLayoutr get_numel)r r datalayouts W/home/wildlama/miniconda3/lib/python3.13/site-packages/torch/_inductor/comm_lowering.pycan_realize_as_comm_bufferr7s Q5U4Sj5nU"TR@5UU4Sj5nU"TRB5U4Sj5nU"TRD5U4Sj5ng![a [ R S5 gf=f)z, Register lowerings for the comm subsystem. zRInductor support for distributed collectives depends on building torch.distributedNr)add_layout_constraintcloneconstrain_to_fx_stridescopy_register_loweringc&>T"UT5 T"U5$r3)fnrQrSrUs rregister_comm_lowering7register_comm_lowerings..register_comm_loweringsb"9: $$rr5r6rrr c>[XU5(a [XU5$T"U5n[R(aGUR 5 [ R RRUR55 [RRU5n[RRTRR UUU5 U$r3)rCrOr reorder_for_compute_comm_overlapr rr.no_fuse_buffer_namesr,r/r ExternKernelrequire_contiguous_AllReduce_Kernelcreate_inplace all_reduce_rN)r5r6rc10drRs r _all_reduce,register_comm_lowerings.._all_reduces 0 K K' C CCj  2 2 KKM GG ( ( , ,S\\^ <oo005 ++    $ $      rc>[XU5(a T"U[XU55n[U5 U$[RR U5n[R RTRRUUU5 U$r3) rCrOr1rr^r_r`rarbrN)r5r6rretrcrTs r _all_reduce_-register_comm_lowerings.._all_reduce_s 0 K K$SZ@C c "Joo005 ++    $ $      rc>UVs/sH nT"U5PM nn[RRTRRUUU5 U$s snfr3r_CollectiveKernelraall_reduce_coalesced_rN)inputsr6rr5rcrRs r_all_reduce_coalesced6register_comm_lowerings.._all_reduce_coalesced sS(./%*/ ++  & & . .      0sAct>[RRTRRUUU5 U$r3rk)rnr6rrcs r_all_reduce_coalesced_7register_comm_lowerings.._all_reduce_coalesced_s6 ++  & & . .      rc[RR"X/UQ76n[U[R5(de[R R U5$r3)rrlcreate_out_of_placerIRNoderGrH)kernelrnargsr)s r_create_out_of_place5register_comm_lowerings.._create_out_of_place"sI##77NN$ ****||""4((rcB>T"TRRUUU5$r3)all_gather_into_tensorrN)r5 group_sizerryrcs r_all_gather_into_tensor8register_comm_lowerings.._all_gather_into_tensor's(#  ' ' / /      rc >[R"[RR[R R TRRUUU55$r3) rErFrrGrHrlru all_gather_into_tensor_coalescedrN)rnr}rrcs r!_all_gather_into_tensor_coalescedBregister_comm_lowerings.._all_gather_into_tensor_coalesced0sM LL    4 455==    rcr>[RRTRRUUUUS9 U$N)out)rrlraall_gather_into_tensor_outrN)r5r}rrrcs r_all_gather_into_tensor_out._all_gather_into_tensor_out<s> ++  + + 3 3    ,  rcD>T"TRRUUUU5$r3)reduce_scatter_tensorrN)r5r6r}rryrcs r_reduce_scatter_tensor7register_comm_lowerings.._reduce_scatter_tensorGs+#  & & . .       rc t>[RRTRRUUUUUS9 U$r)rrlrareduce_scatter_tensor_outrN)r5r6r}rrrcs r_reduce_scatter_tensor_out;register_comm_lowerings.._reduce_scatter_tensor_outQsA ++  * * 2 2     ,  rc >[R"[RR[R R TRRUUUU55$r3) rErFrrGrHrlrureduce_scatter_tensor_coalescedrN)rnr6r}rrcs r _reduce_scatter_tensor_coalescedAregister_comm_lowerings.._reduce_scatter_tensor_coalesced]sP LL    4 444<<    rcD>T"TRRUUUU5$r3)all_to_all_singlerN)r5output_split_sizesinput_split_sizesrryrcs r_all_to_all_single3register_comm_lowerings.._all_to_all_singlejs+#  " " * *       rc>T"U5n[RRTRRXU5 U$r3rrlra broadcast_rN)r5srcrrcrRs r _broadcast+register_comm_lowerings.._broadcastts8Cj ++ OO # #Sz  rcr>[RRTRRXU5 U$r3r)r5rrrcs r _broadcast_,register_comm_lowerings.._broadcast_|s/ ++ OO # #Sz  rct>T"[RRRRUUUU5$r3)rJrK_dtensorshard_dim_alltoallrN)r5 gather_dim shard_dimrrys r_shard_dim_alltoall4register_comm_lowerings.._shard_dim_alltoalls5# II   1 1 9 9       rc>[U5(aU$[RRTRR U5 U$r3)r4r _WaitKernel create_wait wait_tensorrN)r5rcs r _wait_tensor-register_comm_lowerings.._wait_tensors8 C J ""4#3#3#;#;SA rc~>[RRU5nT"TRRXX#5$r3)rr^r_isendrN)r5dsttagrryrcs r_isend'register_comm_lowerings.._isends0oo005#DJJ$6$6#RRrc>[RRU5n[RR TR R XX#5 U$r3)rr^r_rlrairecvrN)r5rrrrcs r_irecv'register_comm_lowerings.._irecvsCoo005 ++ JJ  #  rc >UVs/sH"n[RRU5PM$ nnTRRn[ R R [RRUUUUUU5unnn n n SSS5 W (a USU 35e[UW5HJupU R5 U S:XdM[ R RU R55 ML USR5n[R"[R"US9UUW W 5n/n[![XW55HunupnU S:Xa~UR"R%[R&"[R("US9X_55 UR*R%UR55 UR%U5 M[R,"[RR/U5U[0U4/5nUR%[R2R5U55 M U$s snf!,(df  GN=f)N rr)device)rr^r_ batch_p2p_opsrNrr. fake_moderlprocess_kernelzipr mark_buffer_mutatedr/ get_deviceMultiOutputLayout enumeratemutation_outputsappendMutationOutput NoneLayout alias_names MultiOutputtensor_to_layoutlistrGrH)op_list peer_listtag_listtensorsrtrwexample_output tensor_argsnon_tensor_argsunflatten_argsunbacked_bindingsop tensor_argrpackedresultsiex_out placeholderrcs r_batch_p2p_ops/register_comm_lowerings.._batch_p2p_opssBIJ'Q2??55a8'J##++ WW  $$33   ! %E2C1D&EE$!';7NB    W}++J,?,?,AB8 Q**,%%   /      "+C.,Q"R AvW}''..%%bmm6&BAN""))!**,7q!!nn((99&AAYK r||22;?@#SaK  s)I!+I I()#rJrK_c10d_functional all_reduceAttributeErrorloginfoloweringrQrRrSrTrUrrGstrrball_reduce_coalescedrmrvr|rrrrrr broadcastrrrrrrr)rYrdrhrorrr~rrrrrrrrrrrrrryrQrcrRrSrTrUs @@@@@@@rregister_comm_loweringsrs ""--% 99 % %DDOO, \\C  -<D,,- \\C  .0D5567D6678)ryy) D778 9 DAAB  C  D;;<=D667 8 D::; < D@@A  B  D223 4 DNN+,DOO,- EII..AAB C D,,-.DJJ'S(SDJJ'(D../101G      s$K K/.K/c V ^^^[RRmTR SSKJn U"TRRTRR5 U"TRRTRR5 SSK Jn S[R S[R"S S S [R 4S jmS[R S [$S [R 4U4S jjmU"TR5S[R S[$S [$4UU4Sjj5nU"TR5S[R S[$S [$S[R 4UU4Sjj5nU"TR5S[R S[R S[$S [$4UU4Sjj5nU"TR5S[R S[R S[$S [$S[R 4 UU4Sjj5nU"TR&5S[R S[$S [$4UU4Sjj5nU"TR(5S[R S[$S [$S[R 4UU4Sjj5nU"TR*5S[R S[$S [$4UU4Sjj5nU"TR,5S[R S[$S [$4UU4Sjj5n U"TR.5S[R S[$S [$S[R 4UU4Sjj5n U"TR05S[R S[$S[2S [$S[R 4 UU4Sjj5n U"TR45S[R S [$S[R 4UU4Sjj5n U"TR65S[R S [$S[8S[R 4UU4S jj5n U"TR:5S[R S[R S![R S"[R S [$4 UU4S#jj5nU"TR<5S+S[R S[R S![R S"[R S [$4 UU4S$jjj5nU"TR>5S[R S[R S%[R S"[R S [$4 UU4S&jj5nU"TR@5S,S'[R S([R S[2S [$S[$4 UU4S)jjj5nU"TRB5S,S([R S [$S[$4UU4S*jjj5ng![a [ R S5 gf=f)-z@ Register lowerings for symmetric memory (symm_mem) operations. z7symm_mem ops not available, skipping symm_mem loweringsNr)register_out_variantr)rUr5r rrr cUR5 [RRUR 5UR 5UR 5UR5S9n[X1U5 U$)z Fallback: insert a Pointwise identity copy allocated in P2P via CommBufferLayout. Used when we don't control the input's allocation. )rdtypeinner_fnranges) r r PointwiserHrr; make_loaderget_sizer#)r5r rcopys r_copy_input_to_comm_buffer?register_symm_mem_lowerings.._copy_input_to_comm_buffers] ||"">>#--/__&<<> #  tzB rc>[U[RR5(a'[ U[RRU5 U$T"U[RRU5$)a3 Ensure inp is in P2P memory for a symm_mem collective. If inductor controls the buffer's allocation (ComputedBuffer, or any buffer with FlexibleLayout/FixedLayout), switch its layout to CommBufferLayout in-place, zero-copy. If inductor does not control allocation (e.g. InputBuffer), insert a Pointwise identity copy into a new CommBufferLayout buffer. This adds an extra Triton kernel. Returns the possibly new TensorBox. TODO(tianrengao): eliminate the extra kernel for static-shape InputBuffers by pre-allocating P2P memory in the wrapper and DMA .copy_() )rrr?r@r#)r5rrs r_maybe_realize_symm_mem._maybe_realize_symm_mems]$ &c2+<+<+E+E F F "3(9(9(B(BJ OJ-!!** rr6c >T"X5n[R"[RR[R R TR RUUU55$r3)rErFrrGrHrIrMrNr5r6rrrLs r_symm_mem_one_shot_all_reduceBregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce#sY &c6 LL      $ $,,44    rrc >T"X5n[R"[RR[R R TR RUUUU55$r3)rErFrrGrHrIone_shot_all_reduce_outrNr5r6rrrrLs r!_symm_mem_one_shot_all_reduce_outFregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_out4s\&c6 LL      $ $0088    r symm_buffer local_inputc >T"X5n[R"[RR[R R TR RUUUU55$r3)rErFrrGrHrIone_shot_all_reduce_copyrN)rrr6rrrLs r"_symm_mem_one_shot_all_reduce_copyGregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copyGs\.kF  LL      $ $1199    rc >T"X5n[R"[RR[R R TR RUUUUU55$r3)rErFrrGrHrIone_shot_all_reduce_copy_outrN)rrr6rrrrLs r&_symm_mem_one_shot_all_reduce_copy_outKregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copy_outZs_.kF  LL      $ $55==    rc>T"X5n[RRTRRUUU5 U$r3)rrIrHtwo_shot_all_reduce_rNrs r_symm_mem_two_shot_all_reduce_Cregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_oB &c6    ) ) 1 1      routputc >T"X5n[R"[RR[R R TR RUUUU55$r3)rErFrrGrHrItwo_shot_all_reduce_outrN)r5r6rrrrLs r!_symm_mem_two_shot_all_reduce_outFregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_out~s\&c6 LL      $ $0088    rc>T"X5n[RRTRRUUU5 U$r3)rrIrHmultimem_all_reduce_rNrs r_symm_mem_multimem_all_reduce_Cregister_symm_mem_lowerings.._symm_mem_multimem_all_reduce_rrc >T"X5n[R"[RR[R R TR RUUU55$r3)rErFrrGrHrImultimem_one_shot_all_reducerNrs r&_symm_mem_multimem_one_shot_all_reduceKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reducesY &c6 LL      $ $55==    rc >T"X5n[R"[RR[R R TR RUUUU55$r3)rErFrrGrHrI multimem_one_shot_all_reduce_outrNrs r*_symm_mem_multimem_one_shot_all_reduce_outOregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reduce_outs\&c6 LL      $ $99AA    rrootc >T"X5n[R"[RR[R R TR RUUUUU55$r3)rErFrrGrHrImultimem_one_shot_reduce_outrN)r5r6r$rrrrLs r&_symm_mem_multimem_one_shot_reduce_outKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_reduce_outs_&c6 LL      $ $55==    rc >T"X5n[R"[RR[R R TR RUUU55$r3)rErFrrGrHrImultimem_all_gather_outrN)r5rrrrLs r!_symm_mem_multimem_all_gather_outFregister_symm_mem_lowerings.._symm_mem_multimem_all_gather_outsY &c6 LL      $ $0088    rsplit_last_dimc >T"X5n[R"[RR[R R TR RUUUU55$r3)rErFrrGrHrIreduce_scatter_outrN)r5rr-rrrLs r_symm_mem_reduce_scatter_outAregister_symm_mem_lowerings.._symm_mem_reduce_scatter_outs\&c6 LL      $ $++33    r in_splitsout_splits_offsetsc>T"X5nT"X5n[RRTRRUUUUU5 gr3)rrIrHall_to_all_vdevrN)r5rr2r3rrrLs r_symm_mem_all_to_all_vdev>register_symm_mem_lowerings.._symm_mem_all_to_all_vdevsP&c6%c6    $ $ , ,       rc >T"X5nT"X5n[RRTRRUUUUUU5 gr3)rrIrHall_to_all_vdev_2drN)r5rr2r3r major_alignrrLs r_symm_mem_all_to_all_vdev_2dAregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2dsS&c6%c6    ' ' / /       rin_splits_offsetsc>T"X5nT"X5n[RRTRRUUUUU5 gr3)rrIrHall_to_all_vdev_2d_offsetrN)r5rr=r3rrrLs r#_symm_mem_all_to_all_vdev_2d_offsetHregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2d_offset'sP&c6%c6    . . 6 6       rin_tileout_tilec>T"X5nT"X5n[RRTRRUUUUU5 gr3)rrIrH tile_reducerN)rBrCr$rr6rrLs r_symm_mem_tile_reduce:register_symm_mem_lowerings.._symm_mem_tile_reduce;sP*'>*8@    ( (       rc>[U5HupVT"Xc5X'M T"X5n[RRTRR UUUUU5 gr3)rrrIrHmulti_root_tile_reducerN) in_tilesrCrootsrr6rrBrrLs r _symm_mem_multi_root_tile_reduceEregister_symm_mem_lowerings.._symm_mem_multi_root_tile_reduceOsc$H-JA1'FHK.*8@    + + 3 3       rr3)r9)"rJrKrLrMrrrtorch._library._out_variantrrNrrr rrUrrGr?rrrrrr!r&intr*r/boolr5r9r?rErI)rrUrrrr rrrrr"r'r+r0r6r;r@rFrLrrrLs @@@rregister_symm_mem_loweringsrQs" 99%% $$ A$$,,((00))11--55 , \\++C  & \\ 8x334 \\   5 x778 \\   \\  9 $x889 \\ \\    : $x<<= \\ \\    \\  > (x445  \\   6 x778 \\     9 $x445  \\   6 x<<= \\   > x@@A \\   \\  B $x<<= \\     \\  > (x778 \\  \\ 9 x223 \\     4 $x//0 \\ \\<<LL   1&x223  \\ \\<<LL   4*x99: \\ \\<<LL   ;&x++,  ,,   -&x667  ,,   8]  JKs&XX('X()#loggingrJtorch.utils._pytreeutils_pytreerEtorch._inductor.utilsrtorch.utils._ordered_setrrr virtualizedr getLogger__name__rrGr?rPrr#rvr tuplerOrr+r1r4rCrOrrQrWrrr]s, $$-/!T ||')'8'8 <% ||%''%?% %P    "))   c3h0288t8= =d= ?" bll ZzKr