3j_f%SSKJr SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK r SSK r SSK r SSK r SSK r SSKrSSKrSSKJrJr SSKJrJr SSKJrJrJrJrJr SSKJr SSKJr SS KJrJ r \(aSS K!J"r"J#r#J$r$ SS K%J&r& SS K'J(r( SS K)J*r* SSK+r+SSK,r,SSK-r,SSK.J/s J0r1 SSK2J3r3J4r4 SSK5J6r6 SSK7J8r8J9r9 SSK:J;r; SSKr> SSK?J@r@ SSKAJBrB SSKCJDrDJErEJFrF SSKGJHrH SSKIJJrJJKrKJLrLJMrMJrJNrN SSKOJPrP SSKQJRrRJSrSJTrT SSKUJVrVJWrW SSKMJXrXJYrYJZrZJ[r[ SSK\J]r]J^r^ SSK_J`r` SSKJaraJbrbJcrcJdrdJereJfrf SSKgJhrh SS KiJjrjJkrk SS!KlJmrmJnrn SS"KoJprpJqrq SS#KrJsrs SS$K/JtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{J|r|J}r}J~r~JrJrJrJrJrJrJr SS%KJr \GR"\5r\,GRGR\S&5r\,GRGR\S'5r\,GRGR\S(5r\,GRGR\S)5r\S*rS+\S,'\"S-5r\"S.5r\GR,"S/S055r\GR,"S1S255r"S3S45r\GR,"S5S655r\GR,"S7S8\55r"S9S*5r\GR:SlS:j5rSmS;jrSnS<jrSoS=jr\GR,"S>S?9"S@SA55rSpSBjr"SCSD5rSqSEjr"SFSG\5r"SHSI\5r"SJSK\5rSrSLjrSsSNjr"SOSP\5r"SQSR\5r"SSST\5r"SUSV\5r"SWSX\5rStSuSYjjrSvSZjrSwS[jrSxS\jrSyS]jr\GR,"S^S_55r\GRl"5rSzS`jrS{SajrS|SbjrS}ScjrS}SdjrS~SejrS~SfjrS~SgjrS~Shjr"SiSM5r"SjSk5rg)) annotationsN)Counter defaultdict) as_completedFuture)AnyGeneric TYPE_CHECKING TypeAliasTypeVar) ParamSpec OrderedSet)ComputedBuffer Pointwise)CallableIteratorSequence) ModuleType)EnterCudaStreamContextLine)PythonWrapperCodegen)counters dynamo_timed)use_pipelined_autotuning) LambdaFuture PyCodeCache)TritonTemplateCallerBase)get_metric_tableis_metric_table_enabled)get_stream_name) free_symbols)free_symbol_is_typesymbol_is_typeSymT) has_triton)commsconfig config_comms dependenciesirmetrics)can_codegen_without_upcasts)BackendFeatureget_scheduling_for_deviceKernel) estimate_nccl_collective_runtime/estimate_nccl_collective_runtime_nccl_estimator)Dep MemoryDepStarDepWeakDep)GPUTooOldForTriton TritonMissing)count_flops_fx)assign_origin_nodeget_device_typeGraphPartitionSignature MultiOutputMultiOutputLayout NoneLayout)LoopBody)MemoryPlanningInfoForBufferMemoryPlanningInfoForNode)DeviceProperties ReductionHint) green_textred_text)SimplifyIndexing)&_unstable_customized_partition_wrapper cache_on_selfcmpdevice_need_guardget_current_backendget_device_tflopsget_dtype_sizeget_gpu_dram_gbps get_op_namesGraphPartitionMapIndentedBuffer is_collectiveis_cudagraph_unsafe_opis_gpuis_multi_outputs_template#is_output_of_multi_outputs_templateis_wait sympy_product)Vfusion loop_orderingcompute_dependencies cudagraphsBaseSchedulerNoder PartitionType_T_Pcz\rSrSr%SrS\S'SrS\S'SrS\S'S r\ S S j5r \ SSS jj5r S r g) FusionResultmNz bool | None should_fusezCallable[[], bool] | None callable_fnLambdaFuture | NonefuturecVURSLURSL- (dS5eg)NzLFusion result should contain either fusion decision or callable_fn, not both)rfrgselfs S/home/wildlama/miniconda3/lib/python3.13/site-packages/torch/_inductor/scheduler.py __post_init__FusionResult.__post_init__ss0  ,1A1A1MN Z Nc[US9$)N)rfrd)clsrfs rmfuseFusionResult.fusexs  44rpc[XS9$)Nrgrirr)rsrgris rm from_callableFusionResult.from_callable|s CCrp)rfboolN)rgCallable[[], bool]rirh) __name__ __module__ __qualname____firstlineno__rf__annotations__rgrirn classmethodrtrx__static_attributes__rzrprmrdrdmsf#K#-1K*1"&F & 55LPD,D6IDDrprdcL\rSrSr%S\S'S\S'S\S'SrS\S 'S S jrS rg) PendingFusionr}rgr_node1node2Nrhric2URUR4$r|rrrks rmget_fusion_nodesPendingFusion.get_fusion_nodess DJJ''rprz)return+tuple[BaseSchedulerNode, BaseSchedulerNode])r~rrrrrirrrzrprmrrs$##  "&F &(rprcD\rSrSrSr\SSj5r\SSj5r\SSj5r \SSj5r \SSj5r \SSj5r \SS j5r \SS j5r\SS j5r\SS j5r\SS j5r\SSj5rSrg)MixOrderReductionz This class contains utility functions to decide if we should fuse reductions reducing across different dimensions of the same input tensor. cpUR5=(a [SUR555$)Nc3# UHln[U[5(dMUR5(dM1[UR[5(dMRURR SLv Mn g7fr|) isinstance SchedulerNode is_reductionnoder _split_size.0subnodes rm 7MixOrderReduction.is_split_reduction..sZ+ +'=1 1$$& 17<<8 1GLL $ $D 0+sA6A6A6A6)rall get_nodesrs rmis_split_reduction$MixOrderReduction.is_split_reductions3  " s+ >>++ (  rpcURU5(GaSnSnUR5GHn[U[5(a4UR 5(a[UR [ 5(dMOUR Rce[RRR[UR R55nUR Rce[RRR[UR R55nUcUnUnGM[RRRX%5(d USU35e[RRRX65(aGMUSU35e UceX#4$URS$)N v.s. r)rrrrrrr_original_rangesrZgraphsizevarssimplifyrY_original_reduction_rangesstatically_known_equalsgroup)rsrxnumelrnumelr curxnumel currnumels rmget_numel_rnumel"MixOrderReduction.get_numel_rnumels  ! !$ ' 'FF>>+w 66,,.."7<<@@||44@@@GG,,55!',,"?"?@ ||>>JJJGG,,55!',,"I"IJ >&F&F77++CC4  {3477++CC4  {341,8% %%# #::a= rpcURU5nURU5n[U5S:wd[U5S:wdX4:Xag[U5[[U55:H$)NF)rlentuplereversed)rsrrg1g2s rmhas_mix_reduction_orders*MixOrderReduction.has_mix_reduction_orderssX ! !% (  ! !% ( r7a<3r7a<28RyE(2,///rpcSnURRH.n[U[5(dMURU:XdM,Un O U(dgUR nURR nU(dI[U[5(d[U55eURSRR nU(de[U5[UR5- (dg[RRR[!UR"5[!UR%555(agg)z0 The access to 'buf' is not a broadcast access. NFrT) read_writesreadsrr4nameindex var_rangesFusedSchedulerNodetypesnodesrr"rZrrrrYsizevalues)rsbufr found_depdeprrs rm_is_full_access!MixOrderReduction._is_full_accesss  ##))C#y))chh#o * %%00 d$677 HDJ< H7Q33>>Jz:&E4F4F)GG 77   3 3 ).. )=9J9J9L+M  rpc/nUR5UR5-nUHDnURXQ5(dMURXR5(dM3URU5 MF U$r|)used_buffer_namesrappend)rsrrout common_readsrs rmget_common_read!MixOrderReduction.get_common_readsb..053J3J3LL C""3..33F3Fs3R3R 3  rpc<[URX55S:$Nr)rrrsrrs rmhas_common_read!MixOrderReduction.has_common_reads3&&u4599rpcURU5n[RRR USUS-SS9$)Nrrfallback)rrZrroptimization_hint)rsrrs rm get_numelMixOrderReduction.get_numels>  ! !$ 'ww11"Q%"Q%-!1LLrpc$URU5$r|)rrs rmget_fusion_score"MixOrderReduction.get_fusion_score s }}U##rpc([RR(dg[RR (agUR 5(aUR 5(dgUR5RnUS;d[U5S:wagUR5(aUR5(dgURUR5-(d"URUR5-(agURX5(dg[RX5n[!U5S:XagUR#U5(aXpeOUR#U5(aX!peOgUR%U5nUup[RR&(dSn [RR(R+[,R."X-U 55(dg[RR(R+[,R."XS-55(dg[RR(R+[,R."US55(dg[1SUR3555(ag[RR(R5U S 5(dg[R7U5(ag[9S UR3555n U $) z@ Check whether we can fuse two reductions with mix loop orders. F)cudaxputritonriPric3# UH]nUR5(dMURRR[R [R 4;v M_ g7fr|)rrdatareduction_hintrDINNERDEFAULTrs rmr-MixOrderReduction.can_fuse..[sU 7##%  GLL   , ,##%%  7s A'AA'i@c3# UH9nUR5(dMURR5S;v M; g7f)>sumprodN)rrget_reduction_typers rmrrqs@ 2##%  GLL + + -  2s A$A)r(rmix_order_reductionrZr cpp_wrapperrU get_devicerrLr ancestorsget_operation_namesrrrris_contiguous_noder#mix_order_reduction_non_strict_moder guard_or_truesympyGeanyrstatically_known_leqrr) rsrr device_typercontiguous_node other_nodernrowncol size_thresrs rmcan_fuseMixOrderReduction.can_fuses_ }}00 77  ||~~U\\^^&&(--  .";/8;!!##5+=+=+?+? OOe779 9 OOe779 9++E99)88F |  !  ! !% ( (*/Z  # #E * **/Z  ! !/ 2 }}@@#J 77##11%((4; 2STT 77##11%((42JKK 77##11%((42FGG  +446     ww44T9EE  / / @ @ &//1    rpc$URX5$r|)rrs rmare_mix_order_reductions*MixOrderReduction.are_mix_order_reductions|s||E))rpch^^[UU4SjTRR55(dgg)Nc3\># UH!nTRURT5v M# g7fr|)is_contiguous_loadr)rrrsrs rmr7MixOrderReduction.is_contiguous_node..s) >TsC " "388T 2 2>Ts),FT)rrr)rsrs``rmr$MixOrderReduction.is_contiguous_nodes1 >B>N>N>T>T   rpc:SSKJn UR5Hn[U[5(deUR nUR URnUVs/sH owRU:XdMURPM" nn[U5S:XaM}UHyn URU n URn [U R55n [R R"R%U U U 5n U SS:XaMmU SS:XaMx g M gs snf)Nr)MemoryUsageTyperFT)torch._inductor.loop_bodyr rrr_body memory_usageLOAD buffer_name index_namerindexing_exprsrlistkeysrZrr stride_vars)rsr parent_noder r loop_bodyentriese index_namesr index_exprr var_symbolsrs rmr$MixOrderReduction.is_contiguous_loads=))+DdM22 22 I,,_-A-ABG18QAMMS?D6DII,<,<+=>?        v[9I9I9K1L0MN O        v]74;M;M;O3P2QR S tzz?a    vYtzzl; <!!##   vZ0 1q! JJD$$vQZ0'"   S !!!## "!s *(F;; G c6URR5$r|rr3rks rmr3SchedulerBuffer.get_nameyy!!##rpc0URceURR5(dgURR5(dVURR5(d7[ URR 5[ R5(a4[RRRUR5 g[[RS5(aUR5[RR;a[RRUR5nXR R";a$UR R"URnO#UR R$URn[RRR'UUR5 g[RRRUR5 g)Nargs)rshould_allocateget_inputs_that_alias_outputget_mutation_namesrget_output_specr+CommBufferLayoutrZr wrapper_codecodegen_allocationhasattrkernelr3inplace_update_buffersr+name_to_donated_buffer name_to_bufcodegen_inplace_reuse)rlinput_buffer_name input_buffers rmallocateSchedulerBuffer.allocatescyy$$$yy((**  II 2 2 4 4yy++--$))335r7J7JKK GG 3 3DII >  AHHf % % 188#B#BB!" ? ?  P  NN$I$II#~~DD% $ $~~99:KLQQ GG 6 6   GG 3 3DII >rpc&URce[URR[R5(d[ UR5(agUR H$n[UR[5(dM$ g gNFT)rrr@r+r?rVr0 OutputNode)rluses rmcan_freeSchedulerBuffer.can_freesmyy$$$ dii&& 6 6:S II; ; ::C#((J//rpc40nUHrn[UR5U;a?URU[UR55U[UR5'M[X2[UR5'Mt [UR 55Ulgr|)idrmergerrr0)rlr0rFrds rm set_usersSchedulerBuffer.set_userssm&(C#((|v%'*yy388 1E'Fr#((|$'*r#((|$  &--/* rpcTURceURR5$r|)rrQrks rmrASchedulerBuffer.get_aliases s%yy$$$yy5577rpcTURceURR5$r|)rrRrks rmrCSchedulerBuffer.get_mutations%yy$$$yy++--rpcRURR5R5$r|)rrSrrks rmrSchedulerBuffer.get_devicesyy((*5577rp)r0Nrr"rr$rNonerr{)r0r/rrvrz Sequence[str]rtorch.device | None)r~rrrr dataclassesfieldrr0rAr1r5r9rHr3r_rerjrArCrrrzrprmr)r)sv O))'--dCE>C.9.?.?3/J+ $$($?B +8.8rpr)c$\rSrSr%SrS\S'Srg)SchedulerDonatedBufferiNr,r-rz)r~rrrr-rrrzrprmr~r~s ,0K)0rpr~c\rSrSr%S\S'S\S'S\S'S\S'S\S 'S\S 'S\S 'S \S 'S\S'SrS\S'S\S'S\S'SrS\S'S\S'S\S'SrS\S'SVS jrSWS!jr SXS"jr SXS#jr SXS$jr SYS%jr SXS&jrSZS'jrS[S(jrS\S)jrS]S*jrS^S+jrS_S,jrS`S-jrSZS.jrSaS/jrSaS0jrSZS1jrSZS2jrSbS3jrSXS4jrSXS5jr\SaS6j5r\SaS7j5r \S^S8j5r!\S^S9j5r"ScS:jr#SdS;jr$SeS<jr%SfS=jr&S^S>jr'S^S?jr(S^S@jr)S^SAjr*S^SBjr+S^SCjr,S^SDjr-S^SEjr.SgSFjr/S^SGjr0SZSHjr1ShSiSIjjr2\SjSJj5r3\SjSKj5r4\SjSLj5r5SkSMjr6SlSNjr7\SmSOj5r8SnSPjr9\SnSQj5r:SoSRjr;SpSSjr<\=SqSTj5r>SUr?g)rr_iOrderedSet[str]rz7tuple[torch.device, tuple[tuple[sympy.Expr, ...], ...]]r last_usager$min_input_distancemax_input_distance min_order max_orderrBmpi_nodedict[str, str]mutation_renamesNir.Operation | Nonerlist[SchedulerBuffer]outputsdict[str, SchedulerBuffer]outputs_by_name float | Noneoverride_estimated_runtimedependencies.ReadWritesrOrderedSet[Dep]unmet_dependenciesFr{writtenc XlSUlg)Nc/$r|rz)rOkwargss rm,BaseSchedulerNode.__init__..5sBrp)r+debug_device_str)rlr+s rm__init__BaseSchedulerNode.__init__2s$- & rpc zXl[5UlSUlSUl[[ "5UlSUlUR5Vs/sHn[URUUS9PM snUl URVs0sHo3R5U_M snUl 0Ulgs snfs snf)NrF)r+rr-)rrrrrr"rr get_outputsr)r+rr3rr)rlroutputrs rm_init_from_node!BaseSchedulerNode._init_from_node8s #"#"#$    **,  - ..   -   @D||L|  3|L!#  Ms B3 B8cV[U5RSUR5<S3$)Nz(name=)rr~r3rks rm__repr__BaseSchedulerNode.__repr__Rs't*%%&fT]]_,?qAArpcUR5n[5nURUS[U5RS[[ USS55RSUS[ URR5SUS[ UR5SUS [ URRUR- 5SUS URSUS URSUS 35 UR5 UR5H"nURUR55 M$ SSS5 UR!S 5 URUR#55 UR+5R-5$!,(df  N]=f![$a [&R)SSS9 NOf=f)#Longer form printout for trace logsr<(rN) .writes =  .unmet_dependencies = .met_dependencies = .min_input_distance = .max_input_distance = z.outputs = [ r>Ignoring error in debug_str()Texc_info)r3rRsplicerr~getattrrBrwritesrrrrrDrrHr?debug_str_extra ExceptionlogwarningrErstrip)rlrrrs rmrHBaseSchedulerNode.debug_strUs}} bd  QtGD&$$?@IIJKj))00123WT%<%<=>?74#3#3#9#9D"> ? ?j33$$&'%4H EOO$6$6$@$@ A Aj33..0*2O2O2QR4H z""rpcp[RSUURURR5 g)Nz(%s: unmet_dependencies = %s, writes = %s)rinforrrrks rm log_detailsBaseSchedulerNode.log_detailss,  6   # #    # #  rpcgNFrz)rlself_dep other_deps rmreorder_loops_by_dep_pair+BaseSchedulerNode.reorder_loops_by_dep_pairrpcSURR55Vs0sHnX!;dM X!U_M snUlURURR UR55 gs snf)Nc38# UHoRv M g7fr|rrrs rmr9BaseSchedulerNode.update_mutated_names..sQ-Pc-P)rreads_and_writesrset_read_writesrename)rlrenamesrs rmupdate_mutated_names&BaseSchedulerNode.update_mutated_namesssRT-=-=-N-N-PQ! Q D$- Q!  T--44T5J5JKL ! s A7 A7cXURURRU55 gr|)rr with_readrlrs rm add_fake_depBaseSchedulerNode.add_fake_deps! T--77<=rpcB[SUR555$)Nc3n# UH+oR5=(d UR5v M- g7fr|)rArCrrs rmr=BaseSchedulerNode.has_aliasing_or_mutation..s* @ROO  4!2!2!4 4@Rs35)rrrks rmhas_aliasing_or_mutation*BaseSchedulerNode.has_aliasing_or_mutations% @D@P@P@R   rpcfXlURRUlUR5 gr|)rrr prune_deps)rlrws rmr!BaseSchedulerNode.set_read_writess&"&"2"2"8"8 rpcb^UR5n[U4SjU55nX1- Ulg)Nc3F># UHnTRX5v M g7fr|)get)rkmutation_real_names rmr3BaseSchedulerNode.set_last_usage..s !U 1"4"8"8">"> !)used_or_aliased_buffer_namesrr)rlfuture_used_buffersr used_bufferss ` rmset_last_usage BaseSchedulerNode.set_last_usages-88: !!U !UU &<rpcJURHnUR5 M gr|)rr_)rlrs rmmark_runBaseSchedulerNode.mark_runs<.s W HHW)r itertoolschainrrrrks rmr#BaseSchedulerNode.used_buffer_namess?  t'7'7'='=t?O?O?V?VW   rpc^[5m[R"URRURR 5Vs/sH7n[ U[5(aUR(aM+URPM9 nn[U5S:aUR5nTRU5 [RRR!U5(aDUR#U4Sj[RRUR%555 [U5S:aMT$s snf)z Returns buffer names used by this node, including aliases. Note: is_fake WeakDeps are excluded since they are purely for ordering and should not affect buffer lifetime. rc38># UHnUT;dM Uv M g7fr|rz)ralias used_namess rmrABaseSchedulerNode.used_or_aliased_buffer_names..s("5J. E"5s  )rr r rrrrr6is_fakerrpopaddrZrname_to_bufferrextendrQ)rlrdepsrs @rmr.BaseSchedulerNode.used_or_aliased_buffer_namess'1l !t'7'7'='=t?O?O?V?VW WsG,, CHHW  $i!m((*C NN3 ww%%))#.. !"!7!7"224"5 $i!m! s *E;EcN^[U4SjTR55Tlg)Nc3t># UH-nURTRR;dM)Uv M/ g7fr|)rr+available_buffer_namesrrrls rmr/BaseSchedulerNode.prune_deps..s0- .xxt~~DDD C.s(8 8rrrks`rmrBaseSchedulerNode.prune_depss#",- ..- # rpc^^SU4Sjjm[U4SjTRR55nTRTRR U55 g)Nc>[U[5(dgURTRR;agTRRURR 5nU[ RR;$r) rr6rr+r[r5rZrremoved_operations)rop_namerls rm should_prune7BaseSchedulerNode.prune_weak_deps..should_prunesbc7++xxt~~999nn00:KKMGagg888 8rpc3F># UHnT"U5(dMUv M g7fr|rzrrr$s rmr4BaseSchedulerNode.prune_weak_deps..s 1C\#5FCC1! !rr3rr{)rrrr remove_reads)rl to_remover$s` @rmprune_weak_deps!BaseSchedulerNode.prune_weak_depssN 9 ++11   T--::9EFrpcD[XURR5 gr|)_prune_redundant_depsr+r[)rlname_to_fused_nodes rmprune_redundant_deps&BaseSchedulerNode.prune_redundant_depss d8R8RSrpcTURceURR5$r|)rget_operation_namerks rmr3BaseSchedulerNode.get_namerprpc"UR5$r|r3rks rmget_first_name BaseSchedulerNode.get_first_names}}rpcB[SUR555$)Nc3@# UHoR5v M g7fr|r8rrs rmr8BaseSchedulerNode.get_operation_names..sG6Fd--//6F)rrrks rmr%BaseSchedulerNode.get_operation_namessGdnn6FGGGrpc:[SUR55$)Nc3@# UHoR5v M g7fr|r8rrs rmr5BaseSchedulerNode.get_buffer_names..sALS,,..Lr?)rrrks rmget_buffer_names"BaseSchedulerNode.get_buffer_namessADLLAAArpcB[SUR555$)Nc3d# UH&n[U[5=(a [USS9v M( g7f)T)disallow_fp32_opsNrrr-rns rmrABaseSchedulerNode.can_codegen_in_low_precision..s6 & q- ( G+AF G%s.0rrrks rmcan_codegen_in_low_precision.BaseSchedulerNode.can_codegen_in_low_precisions% ^^%   rpcB[SUR555$)Nc3f# UH'n[U[5=(a [U5v M) g7fr|rJrKs rmr@BaseSchedulerNode.can_codegen_without_upcasts..s, % q- ( K-H-K K%s/1rNrks rmr--BaseSchedulerNode.can_codegen_without_upcastss# ^^%   rpcU/$r|rzrks rmrBaseSchedulerNode.get_nodes s v rpcUR$r|)rrks rmrBaseSchedulerNode.get_outputss ||rpc URU$r|)r)rlbuf_names rm get_outputBaseSchedulerNode.get_outputs##H--rpcTURceURR5$r|)rrrks rmrBaseSchedulerNode.get_devices%yy$$$yy##%%rpcVUR5nUSL=(a URS:H$Ncpu)rrrldevices rmis_cpuBaseSchedulerNode.is_cpus'"T!:fkkU&::rpcbUR5nUSL=(a [UR5$r|)rrUrrbs rmrUBaseSchedulerNode.is_gpus'"T!9fV[[&99rpcgrrzrks rmrBaseSchedulerNode.is_reduction"rpcgrrzrks rmis_native_matmul"BaseSchedulerNode.is_native_matmul%rjrpcgrrzrks rm is_split_scanBaseSchedulerNode.is_split_scan(rjrpcgrrzrks rm is_templateBaseSchedulerNode.is_template+rjrpcgrrzrks rm is_externBaseSchedulerNode.is_extern.rjrpcgrrzrks rm is_foreachBaseSchedulerNode.is_foreach1rjrpcgrrzrlread_deps rm can_inplaceBaseSchedulerNode.can_inplace4rjrpcgrrzrks rmhas_side_effects"BaseSchedulerNode.has_side_effects7rjrpc 0 ^SSKJn [T[5(a[R (a[ RRTR5[R5(a[[ R[RRR R"5(a[%[ RSS5b['[ RS5(dgTR([ RR*-TR,R.-nSU4SjjnTR15GHnUR2nUceUR55(aUR75(dnUR95(dYUR;5[ RR<;d-[UR?5[@RB5(aMTRDRFGHnURHTR,RJ;a$TR,RJURHnO/TR,RLROURH5nU(dM[ RRPRSUT5(dM[URT[V5(aMURXceURXVs/sH%nUR2R;5U;dM#UPM' n nTR,R[URHT5n U (aGMQ[]U 5S:XdGMcU SR^(dGMzU SR2TLdGMUR2cGM[UR2R?5[@R`[@Rb[@Rd[@RB45(aGMURT(am[URTR2[@Rf[@Rh45(a*[]UR2R755S:aGMU"UR2UR25(dGMU"U5(dGM[ RRjRmUR;5UR;55 [[ R[RRR R"5(an[ RRnRqUR;55 [ RRnRqUR;55 UR;5[ RRrUR;5' GM GM gs snf) zf Decide if there should be inplace updates for the node and record the decision in the active kernel. r)can_match_buffer_size mutationsNrOc>^URRT5nUR5m[5nURHnUR n[ U[5(dM&UR5URR;dURRU5ULaMnUU4SjURR55-n[U5S:dM g g)Nc3L># UHnURT:XdMUv M g7fr|r)rorZs rmr^BaseSchedulerNode.decide_inplace_update..single_index_in_fused_node..ms&Evv)AEs$ $rFT) r+get_fused_noder3rr0rrr_r9r1rrr)buf_to_be_inplaced fused_noderrG user_noderZrls @rmsingle_index_in_fused_nodeKBaseSchedulerNode.decide_inplace_update..single_index_in_fused_nodeUs ,55DDTJJ)224H%/LD*00 II !)->??,,.-77JJK)33BB9M%&&22CCE t9q= '1*rpr)rr)rr{):codegen.wrapperrrrr(inplace_buffersrZr has_featurerr.INPLACE_BUFFERSrXrrcodegensimd SIMDKernelrrWrr"r+completed_operationsrrrPrQrRr3removed_buffersrSr+rTrrrrZr[rrU can_reuser-NopKernelSchedulerNoder0has_cross_stream_hazardrr}r?r>MutationLayoutSHOULDREMOVEFallbackKernelr=rO make_inplacerrrY) rlrinconsequential_nodesrrbuf_noderead input_bufxremaining_usesrs ` rmdecide_inplace_update'BaseSchedulerNode.decide_inplace_update:s& ; t] + +&&##DOO$5~7U7UVVqxx)@)@)E)E)P)PQQ188[$7C&))  NNgg(( )nn11 2  D##%CxxH' '',,..88::..00<<>QWW%<%<<h668":M:MNN((..99 E EE $ E Edii PI $ : : > >tyy III,,66y$GG&y'<'<>TUU$??666"+&!0A66??,4II!0#& /3nn.T.T 4/+ 43/14*1-999*1-22d:%NN6 *%NN::< " " 4 4 " = = " 3 3 !!&11 * ) 5 5 : :!#!2!2BNN C!!!$INN$O$O$Q RUV V1)..#((KK6yAA  2293E3E3GX%HHeoo&=&=&B&B&M&MHH..2293E3E3GHHH..223<<>B&..077 G}/&6&s "X.Xc[R(dgU(aUR(agURceURR 5n/nUGH5nUR S:XaMUR S5 UR S5 SUR SUR3nSUR;aUSURS3-nUR U5 SUR;dMURSnURS S S 9S nUR SURS S5RSS5RSS5RSS5-5 UR S5 UR S5 GM8 [U5S:XagURU5 SUlg)Nrrz#pragma CMT ORIGIN:z #pragma CMT  seq_nrz seq_nr: stack_trace|r)maxsplitr {z{{}z}}r\z\\z#pragma CMT END ORIGINrT) r(comment_originrr get_originsr4rtargetmetarsplitreplacer writelines) rlbuffer only_onceorigins out_linesr op_info_strrstack_trace_last_lines rmcodegen_originating_info*BaseSchedulerNode.codegen_originating_infos$$   yy$$$))'') Attx   R   2 3(az:K166!)hqvvh7G6H,II   [ )&!" !6 7 (3(:(:3(:(KB(O%  "+33C>WS$'WT4(Wf   !9:  $36 y>Q   )$ rpc"URSSS9$)NT include_readsinclude_writes!get_read_write_buffers_sizes_implrks rmget_read_write_buffers_sizes.BaseSchedulerNode.get_read_write_buffers_sizess55t6  rpc"URSSS9$)NTFrrrks rmget_read_buffer_sizes'BaseSchedulerNode.get_read_buffer_sizess55u6  rpc"URSSS9$)NFTrrrks rmget_write_buffer_sizes(BaseSchedulerNode.get_write_buffer_sizess556  rpcL[URXS9R5SS9$)Nrr)start)rget_read_write_buffer_accessesr)rlrrs rmr3BaseSchedulerNode.get_read_write_buffers_sizes_impls1  / /+ 0 fh   rpc^^ ^ ^^^[T[5(a0$[T[5(a![TR[5(a0$[T[5(af[TR[ R 5(a=TRR[RRRLa0$S Sjm[T[5(a@T"[TR5S5[TR5S5-5mO [S5m[ R""[$5nU(a:TR&R(H nX4R*R-U5 M" U(a:TR&R.H nX4R*R-U5 M" U(a&[1STR&R(55O [15nU(a&[1STR&R.55O [15nS U4Sjjm[T[25(a[1UU4SjU55nXg- nXW- n0nXV-Hn [5U4S jX955m U [6R8R:;a[6R8R:U n O>U [6R8R<;a[6R8R<U n OMS U U UU4S jjm T "U 5n X;aXU 'MX==U - ss'M U$)a Counting the number of bytes accessed for a kernel is surprisingly tricky. In particular, there is a differentiation between 'theoretical' memory accesses and practical memory accesses. For example, a layernorm kernel may actually access an input 3 times, but in theory, it only needs to access its input once (and may be optimized to do so through say, persistent reductions) Another example is that even though a buffer is passed in, we may not access the entire buffer. This may occur if we are accessing a slice of the buffer. Another tricky case is for indirect indexing, where the amount of bytes accessed depends on the values of the input. What this function aims to compute is the memory accesses for worst-case inputs, best-case optimization. What this means is that for each buffer we compute the amount of potential accesses in two ways and take the minimum. 1. Numel in ranges multiplied by number of deps the buffer has 2. The buffer size Returns memory accesses per buffer. cR[RRRUSS9$)Nrr)rZrrr)ss rm try_size_hintGBaseSchedulerNode.get_read_write_buffer_accesses..try_size_hint:s"77##55a!5D DrprreAc38# UHoRv M g7fr|rrs rmrCBaseSchedulerNode.get_read_write_buffer_accesses..OsB+ACxx+Arc38# UHoRv M g7fr|rrs rmrrTC+BCxx+Brc>TRRURn[SU55n[ U[U5- 5S:$)Nc38# UHoRv M g7fr|r)rrGs rmr\BaseSchedulerNode.get_read_write_buffer_accesses..is_materialized..[s!>))rr)r+r[r0rr)rrr0buf_usesrls rmis_materializedIBaseSchedulerNode.get_read_write_buffer_accesses..is_materializedYsGNN..s399E!!>!>>Hx*V"4459 9rpc3\># UH!nT"UTR5(aMUv M# g7fr|r)rrrrls rmrr_s#)%_S$++-Nvs, ,c3(># UHnTv M g7fr|rz)rr node_numels rmrrhs$R;QCZ;Qsc>U(dg[U[R5(aUR5$[UR[ 5(aTR RUR5RnSnUHn[UR[5(aM$[UR[5(de[URR[5(a8URR5HnUT"UR5- nM M g U$[UR[R5(a#[!U4SjUR#555$T "[%UR'555n[)UR+55[-TU5-$)Nrc3n># UH*nT"[RRU55v M, g7fr|)rZr get_buffer)rmut_name get_buf_bytess rmrZBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytes..s/(@H&agg&8&8&BCC(@25)rr+TorchBindObjectrr@r>r+r[r3r0rrcr_r=rr?rrRrYrrN get_dtypemin) rr0totrG sched_buf buf_elemsbuf_accessed_elemsrrlrs rmrGBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytesqs[c2#5#566,,.. ,=>>!NN66s||~FLLEC %%dii<<$)$))5FGGGG%diinnkBB-1YY-B-B-D #}Y^^'D D.E $%!&J BMM::(+(>(>(@ !.mCLLN.K LI)#--/:S*I>rp)rz sympy.Exprrr$)rr"rSequence[BaseSchedulerNode]rr{)rz4ir.Buffer | ir.TensorBox | ir.TorchBindObject | Nonerr$)rrExternKernelSchedulerNoderr=r+r op_overloadr_prims rng_primsgraphsafe_run_with_rng_staterrY get_rangesr$ collectionsrrrrrrrrrrrZrr graph_inputs)rlrr buf_accessesrrrrbuf_byte_accessesrZr buf_bytesrrrrrs` @@@@@rmr0BaseSchedulerNode.get_read_write_buffer_accessess6 d2 3 3I d5 6 6: II{< < I t6 7 7499b&7&788 %%||%%BBCI E dM * *&doo/23 1! 456J SJ"..t4 ''--XX&--c2. ''..XX&--c2/  B4+;+;+A+AB B  C4+;+;+B+BC C   : d. / /()%)O-F+E,.H!$$R<;Q$R!R 177111gg,,X6QWW111gg**84# I# # # J&c*I0.7(+!+y8+g'j! rpc^URcgURR5nUcg[U5nUcg[U[R 5(aURR n[RRRUSS9n[SS==U- ss'U$)Nrrinductor flop_count) rget_origin_noder9rrSymIntexprrZrrrr)rlfx_nodeflopsresolved_flopss rmestimate_flops BaseSchedulerNode.estimate_flopss 99 ))++- ?w' = eU\\ * *JJOOE));;EA;N\*n<*rpcTURb UR$UR5$r|)r_get_estimated_runtimerks rmget_estimated_runtime'BaseSchedulerNode.get_estimated_runtimes)  * * 622 2**,,rpc,UR5SR5SnURR5n[ [ U55(dg[ UR5(a[UR[R5(de[R(av[U5n[5nURU5nUb[U[5(deU$[!U5nUc[#UR5nUR%X6S9 U$[#UR5$[/UR5(ag[1U5nUbU$URR35n [55n [7U 5S-n U S::a[9SU 35eU S::a[9SU 35eUR=5n U S:XdU cUR?5U - n U S- nU$SnUR?5nUcSOUnX-U - S -nX- n[AUU5n U S- nU$![&an[(R+U5 SnAgSnAf[,an[(R+U5 SnAgSnAff=f![:a gf=f) z3 Returns estimated op runtime in milliseconds (ms) rNvaluelJ)z-gpu_memory_bandwidth cannot be <= 0, but got z"gpu_flops cannot be <= 0, but got g.Ag?r)!rrrrSrUr;rSrr+IRNoder),runtime_estimations_use_nccl_lib_estimations)get_estimate_runtime_cache_key_from_snodeget_estimate_runtime_cachelookupfloatr2r1 set_value ValueErrorrr TypeErrorrX maybe_estimate_runtime_benchmarkmaybe_get_dtyperOrMAssertionErrorrr rmax)rlrr@ cache_keycache cache_valmsrretdtypegpu_memory_bandwidth gpu_flops flops_estnsfactor counted_bytes compute_time transfer_times rmr (BaseSchedulerNode._get_estimated_runtimes nnq!--/2))+of-..  # #dii33 33 LL I$ OI68E % Y 7I ,))U;;;;((HNBz=diiHOOIO8I7 BBTYY   .t4 ?J((* #4#6 )%069I$q($CDXCYZA~$'I)%UVV '') >Y.2247KKBcBI99; *2 *Y6#= %< } - #X o     :  sDAH363H3*H3AJ3 J=I J$I>>J JJcgr|rzrks rmget_template_node#BaseSchedulerNode.get_template_noderpc0UR5nUceU$r|r/)rltemplates rmget_template_node_or_throw,BaseSchedulerNode.get_template_node_or_throws!))+###rpc`[S[U555nUSUnXnXS-SnX#U4$)zA For the list of nodes, get the prologue, template, and epilogue c3X# UH upUR5(dMUv M" g7fr|rr)rirLs rmrCBaseSchedulerNode.get_prologue_template_epilogue..sP,list[BaseSchedulerNode]rzJtuple[list[BaseSchedulerNode], BaseSchedulerNode, list[BaseSchedulerNode]])@r~rrrrrrrrrrrHrrrrrrrrrrrrrrr-r2r3r9rIrrErOr-rrr[rrdrUrrlrorrrurxr}rrrrrrrrr r r r/r5r'rCrrzrprmr_r_s BB NN''$$ $D $ ""///3 3((''GT #4B*6+ # !.7  M>  =#2=HV= = 6 G T">T T .HHBB    .&;:IX9=-$-15- -^       ! 37  L!!L!37L! L!\$- UUn  1& 1 S 1 1rpcR[RRR5$r|)rr codecache LocalCacherzrprmrr&s ?? $ $ / / 11rpc^[URSS5nURRnURR/UQURRQURR 5nURR n[ R"X#45upESSjm[U4[U4SjU55-5nU$)Npython_kernel_namerc[U[R5=(a/ [U[R[R45(+$r|)rr+rGeneratorStateOpaqueObjectStaters rm _is_tensor_ir@get_estimate_runtime_cache_key_from_snode.._is_tensor_ir5s<!RYY' !!2#7#781 -  rpc3t># UH-nT"U5(a[UR55OSv M/ g7fr|)rr)rar]s rmr.<s+U9a}Q'7'7ajjl#TA9s58rw) rrinputsfill_non_provided_args constant_argsrpytree tree_flattenr"r)snoderXrOr flat_argsflat_args_pytree_specrr]s @rmrr+s -A2F ::  D :: , ,*$*))*  DZZ  F'-':':D>'J$I   U9U U VI rpc[U[5(dg[RRR [RRR [RRRS.n[URSS5nX!;ag[UR[R5(dgX$)N)zextern_kernels.mmzextern_kernels.bmmzextern_kernels.addmmrXr) rrropsatenmmbmmaddmmrrr+ ExternKernel)rgmms_fnsrXs rm_get_mm_like_fnrrAs e6 7 7"YY^^..#iinn00 %  4 4G !-A2F( ejj"// 2 2  &&rpc d^^ SnSn[R(a[T5nUcgUnUU 4SjnOg[T5n[ 5nUR U5nUb[ U[5(deU$SSKJ m U"5upxSSK J n U RUUUSSSS9n URXJS 9 U $) Nc>T"T5$r|rz)rgsnode_args_kwargssrmr2maybe_estimate_runtime_benchmark..Zs !25!9rpr)rur) benchmarker )memory_warmup_itersbenchmark_itersmax_benchmark_durationr)r(!runtime_estimations_mms_benchmarkrrrrrrrutilsru$torch._inductor.runtime.benchmarkingrw benchmarkr) rgbench_fnargs_kwargs_fnmm_fnrr r!rOrrwr"rus ` @rmrrQsHN //& =99%@I & (E Y'I)U++++(!#LD@    !  B OOIO( IrpT)slotsc\\rSrSr%S\S'S\S'S\S'S\S'S SjrSS jrSS jrS rg ) WhyNoFuseiwr"name1name2reasonztuple[Any, ...]rOcXUR5UlUR5Ulgr|)r3rrrlrrs rmrWhyNoFuse.__init__~s^^% ^^% rpcFXlX l[RU5 gr|)rrO fusion_logdebug)rlrrOs rm__call__WhyNoFuse.__call__s  rpcpSURSURS3URUR--$)Nz cannot fuse z with r<)rrrrOrks rm__str__WhyNoFuse.__str__s6djj\ |2> KK$)) #  rp)rOrrrNrr_rr_rrv)rr"rOrrrvrs) r~rrrrrrrrrzrprmrrws& J J K &  rprc[U[[45(a[U[S9n[ R "USS9nSU;aS[R"US53$U$)Nkey)rDr ) rrsetsortedr"pprintrBtextwraprD)objrFs rmrBrBsU# C())Sc" ^^C *F v~HOOFG4566 Mrpc@\rSrSrSSjrS SjrS SjrS Sjr\rSr g) rcic&[U/5Ulgr|rrs rmrOutputNode.__init__s",cU"3rpcgrrzrks rmrOutputNode.is_reductionrjrpcg)Nrzrzrks rmrQ'OutputNode.get_inputs_that_alias_outputrrpcg)NOUTPUTrzrks rmr3OutputNode.get_namesrp)rN)rr5rrvrwrxrs) r~rrrrrrQr3rrrzrprmrcrcs4Hrprcc^^^^^[R"5mTRHUn[U[5(aMTUR R 5nTTUR5==S- ss'MW SUUUU4Sjjm[U4SjTR55nU(a?TRU- TlTRTRRU55 gg)aU Prunes weakdeps intended for mutation ordering on an upstream fused node if after fusion there is another dependency on the fused upstream node, making the weakdep redundant In essence this enforces an ordering on fusions. As fusions occur, weakdeps will be incrementally removed, enabling other fusions, ensuring they are fused in order. rc>[U[5(apTURR5nTTUR 5S:=(a TR R UTUT5nTUT:HnU=(d U$g)NrF)rr6rr5r3r+fusable_weak_dep)rr# is_redundant is_self_depr[name_to_dep_countr1rs rmr$+_prune_redundant_deps..should_prunes c7 # #!#((+<<>G,"7+446nn55'0$ -W5=K.; .rpc3F># UHnT"U5(dMUv M g7fr|rzr's rmr(_prune_redundant_deps..s.,s2C.r)Nr*) rrrrr6rr5r3rrrr+)rr1r[rr# deps_to_prunerr$s``` @@rmr0r0s'2&9&9&;&&#w''!#((+<<>G 09BBD E J E'  ..M"&"9"9M"I T--::=IJrpc^^\rSrSrS U4SjjrS SjrS SjrS SjrS SjrS Sjr Sr U=r $)ric>[TU]U5 URU5 URUR 55 [ U[ R5(a`UR5(aJ[R"URSR5nSnUR5nXSU44UlgggNrr)superrrrget_read_writesrr+UserDefinedTritonKernelcan_fuse_epiloguemathr mutable_argsshapeget_device_or_errorr)rlr+rnumelrrc __class__s rmr"ExternKernelSchedulerNode.__init__s # T" T1134 dB66 7 7DEGR= Bxrpc[UR[R5(deURR U5$r|)rrr+rpr)rlwrappers rmr!ExternKernelSchedulerNode.codegens2$))R__5555yy  ))rprr+r*rrErrvrsrwrSequence[Sequence[sympy.Expr]]rrrrv) r~rrrrrrurrrr __classcell__rs@rmrrs( 3cW**rprc,^\rSrSrSU4SjjrSrU=r$)ric>[TU]U5 URU5 URUR 55 gr|)rrrrrrlr+rrs rmrNopKernelSchedulerNode.__init__s5 # T" T1134rprzr)r~rrrrrrrs@rmrrs 55rprc^\rSrSr%SrS\S'S\S'S#U4SjjrS$S%SjjrS$S&S jjrS'S jr S(S jr S)S jr S*S jr S)Sjr S+SjrS)SjrS,SjrS-SjrS.SjrS/SjrS/SjrS/SjrS/SjrS0SjrS1SjrS2SjrS3SjrS4S5Sjjr\S6Sj5r\S6Sj5rS7Sjr\S8S j5r \S/U4S!jj5r!S"r"U=r#$)9rizi A SchedulerNode is a node for scheduling that encapsulates either a ComputedBuffer or a TemplateBuffer. z tuple[Sequence[sympy.Expr], ...]_sizesr@rcf>[TU]U5 URU5 UR5 gr|)rrr_compute_attrsrs rmrSchedulerNode.__init__ s, # T" rpc[UR[R[R45(deURR UUS9uUlnX0lURR5nURRU5RnXE"UR 54Ul [R(+=(d [UR 5(+n[UR[R5(a)UR#URR%US95 gUR#[&R$"UR/UR Q7SU065 g)Nextra_indexing_constraintsrecompute_sizes_body_func) normalizer)rrr+rTemplateBuffersimplify_and_reorderrrrr+ get_backendgroup_fnrr(loop_ordering_after_fusionrUrrextract_read_writesr*)rlrrbodyrcrshould_normalizes rmrSchedulerNode._compute_attrss; $))b&7&79J9J%KLLLL II::'A&?;  T ..0>>--f5>>ht{{34  &@@@  KKI E  dii!2!2 3 3   --8H-I   00JJ!%8H rpc[SURR55nURUUS9 U(aDUR URR U5R UR55 gg)Nc3`# UH$n[U[[45(dM Uv M& g7fr|rr6r5rs rmr8SchedulerNode.recompute_size_and_body..8$0 1CZgwEW5XCC1. .r)rrrrrrrr)rlrr fake_depss rmrecompute_size_and_body%SchedulerNode.recompute_size_and_body3s &00 ++110 &  'A&?       **95<.IrrrrSIMDScheduling)rrrrr*rrrrrrpointwise_read_writes clear_cache codegen.simdrcandidate_tilings cache_clear)rlrneed_clear_tiling_cacherrs rmrefresh_dependencies"SchedulerNode.refresh_dependenciesDs &00 ++110 &   , ,  ![[ 4= Yy ! VD)) *   ""..t4 " 4  , , 8 8 : #rpcURRU5UlURRUlUR SSS9 g)NFTrr)rreorder_iter_loopssizesrr)rl new_orders rmapply_new_loop_order"SchedulerNode.apply_new_loop_orderbsAZZ22   jj&&  !!E4!PrpcURR5n[URR5U- n[ [ U55n[ [ X"U-55nUR XC-5 [URS5S:XdeURSURSSURSS44Ulg)Nrrr)rget_original_num_rdimsr iter_varsrrangerr)rl num_rdims num_pwdimspwdimsrdimss rmswap_pw_red_dimension#SchedulerNode.swap_pw_red_dimensionjsJJ557 --.: uZ()eJY(>?@ !!%.14::a=!Q&&&ZZ]TZZ]1%5tzz!}Q7G$HH rpcDURR5UlU$r|)rextract_pw_from_reductionrks rmr'SchedulerNode.extract_pw_from_reductiontsZZ99;  rpc[RU5(dg[UR[R 5(deURR 5 UR5 SSS5 g!,(df  g=fr|)rrrrr+rwith_original_inner_fnrrks rmcancel_reduction_split$SchedulerNode.cancel_reduction_splitxs[ 33D99 $))R%6%67777 YY - - /    !0 / /s !A;; B c[UR[R[R45(deUR R X5UlUR RUlURR5nURRU5RnX4"UR54Ul URSSS9 g)NTr)rrr+rrr#expand_dimension_for_pointwise_noderrrr+rrrr)rl dimension new_rangercrs rmr1SchedulerNode.expand_dimension_for_pointwise_nodes$))b&7&79J9J%KLLLLZZCC   jj&& ..0>>--f5>>ht{{34  !!D$!OrpcURR5UlURRUlUR SSS9 g)NTFr)r merge_loopsrrrrks rmr SchedulerNode.merge_loopss<ZZ++- jj&&  !!D%!PrpcSnURSn[U5URs=:XaUR:XaO OURU5nU(aP[=R S- sl[ RSUR5U5 URU5 g[ RSUR55 g)Nrrz"Reorder loops for %s with order %sTzEDon't reordering %s because we can not decide the suitable loop orderF) rrnum_varsdecide_loop_order_to_matchr,num_loop_reorderingloop_ordering_logrr3r)rlrrr self_sizess rmr'SchedulerNode.reorder_loops_by_dep_pairs [[^ z?h// E93E3E E ;;IFI   ' '1 , '  # #4dmmoy   % %i 0  # #W  rpcNUR5nUSURS3USURS3USUR3/nURR 5Hn[ U[ 5(aMURn[RRU5n[ U[R5(aMfURUS[UR535 M [ UR ["5(aSURSUS35 UR[$R&"UR R)5S 55 UR*ceUR-UR/55 S R1U5$) Nz.group.device = rz.group.iteration = rz .sizes = z _layout = zclass z _loop_body:rr)r3rrrrrr6rrZrrr+rrrBr@rr@rrDrHrrrjoin)rlrlinesrrZrs rmrSchedulerNode.debug_str_extrasM}}f$TZZ]O 4f' 1  7fIdkk] +  ##446Cc7++88gg((2!#r'9'9::LLH:Z 8K7L!MN 7 djj( + + LL6${3 4 LL)=)=)?H Iyy$$$ T//12yyrpcUR$r|)rrks rmrSchedulerNode.get_ranges {{rpcd[UR[R[R45(dS[ UR5<35e[ URR55=(a0 URSL=(d URR(+$Ntype(self.node)=) rrr+rrrr{rrhas_partial_accumulaterks rmrSchedulerNode.is_reductions$))b&7&79J9J%KLL tDII ! LDII0023 JJ$  Gdjj&G&G"G rpc[UR[R5(dS[ UR5<35eURR 5S:H$)Nr2dot)rrr+rrrrks rmrlSchedulerNode.is_native_matmulsM$))R%6%677NB) B B B&"B)&B)),CcU(a URO[UR5up#[R"URU[ R R/[U5-/S9$)zL Get the memory dependencies in either the pointwise or the reduction axes. ) hidden_args) rrr*rrrSZeror)rl pointwise keep_sizes ignore_sizess rm"pointwise_or_reduction_read_writes0SchedulerNode.pointwise_or_reduction_read_writessR 3<4;;$++AV // JJ %'',,#lBS1S0T  rpc URSS9$)z8 Get the memory dependencies in the non-reduction axes. TrUrXrks rmr#SchedulerNode.pointwise_read_writess 666FFrpc URSS9$)z4 Get the memory dependencies in the reduction axes. Fr[r\rks rmreduction_read_writes#SchedulerNode.reduction_read_writes&s 666GGrpc(UR5(ag[SUR555(ag[URR 5S:Xa[ U[R5(a[[URR 55n[ U[R5(dS[U5<35eURUR:H=(a URUR:H$g)NFc3@# UHoR5v M g7fr|)rArCs rmr,SchedulerNode.can_inplace..0s?,>S  ,>r?rztype(write_dep)=)rrrrrrrrr*r4r<iterrrr)rlr| write_deps rmr}SchedulerNode.can_inplace-s      ?D,<,<,>? ? ? t&& '1 , l,,2 2 T$"2"2"9"9:;Ii)?)?@@ WEUT)_DVBW W@>>Y__4X)..9X Xrpc8[5n[UR[5(aURR 5HnUR S:XdMUR S:XdM'SUR;aURSS:Xd0[UR5S:XdMeURSS:XdMzURSUR;aURSO)[UR5S:aURS OS 5 M U$) N call_methodstoremode atomic_addrxrrrrr) rrrr@rr4rrrrOr)rlbuffers_store_as_atomic_addrs rm_get_atomic_add_buffers%SchedulerNode._get_atomic_add_buffers:s7A|# djj( + + ,,.GG}, w.4;;.4;;v3F,3V Na/DIIaLL4P033!T[[0 F+.1$))n.Adiilr/+*rpc|>URb!URRS5(ag[TU] 5$)Ndevice_assert_asyncT)rhas_oprrrlrs rmrSchedulerNode.has_side_effectsNs5 :: !djj&7&78M&N&Nw'))rp)rrr)r+r*rz%ir.ComputedBuffer | ir.TemplateBufferrrvNN)r'tuple[dict[Any, Any], list[Any]] | NonerzCallable[_P, _T] | Nonerrv)rrurzCallable[..., Any] | Nonerrv)rr{rr{rrv)r Sequence[int]rrvrurr_)rr$rr$rrvrFrsrrwrR)rASequence[sympy.Expr]rrv)rArrzdict[sympy.Expr, sympy.Expr])rArrrvrN)rUr{rr)rrrMrI)$r~rrrr&rrrrrrrrrrr rrrrrlrorrr/rBrIrrXrIrr_r}rmrrrrs@rmrrs -, O4  OS=A$K$;  FOS?C$K$=  ";;8<; ;<QI"PP),P P" Q!.7 . ,  7 8O! 8  % 0!%      GG HH ++&**rprc z^TRnTR[RR UVs/sHo"R PM sn55 [ U4Sj[ R"UVs/sHo"RPM sn655TR R- Tlgs snfs snf)Nc3h># UH'nURTR5;dM#Uv M) g7fr|rrE)rr group_snodes rmr2refresh_group_node_dependencies.._s/ Pxx{;;== CP"2 2) rrr* ReadWrites merge_listrrunionrr)r|rrs` rmrefresh_group_node_dependenciesrVs  F**6+J6aMM6+JK  !'')O1*>*>)OP   ! ! ( (  )",K *Ps B3 4B8r*c[U[[45(deX lXlSUl[ R"UVs/sHo3RcMURPM sn6Ul[U5 [SUR55Ul [SUR55Ul [SUR55Ul[SUR55UlUR!5Vs0sHoDR#5U_M snUlgs snfs snf)Nc38# UHoRv M g7fr|rrrs rmr"init_group_node..wH5G 5Grc38# UHoRv M g7fr|)rrs rmrrxrrc38# UHoRv M g7fr|)rrs rmrry)&8&8rc38# UHoRv M g7fr|)rrs rmrr|rr)rrGroupedSchedulerNoderr+rrrrrrrrrrrrr3r)r|r+rrrs rminit_group_noderhs k$68L#M N NN N%K&,,% Av!+!++v AK$K0H[5G5GHHKH[5G5GHHK%()&1&8&8)&K"&))&1&8&8)&K"(3'>'>'@#'@ '@#K B#sD6D6D;c^\rSrSr%SrS\S'\S#Sj5rS$SjrS%Sjr \ S&Sj5r S'S jr S(U4S jjr \ S)S j5rS)S jr\ S*S j5rS+SjrS)SjrS)SjrS,U4Sjjr\ S*Sj5r\ S*Sj5rS-SjrS)Sjr\ S.Sj5r\ S.Sj5r\ S.Sj5r\ S.Sj5r\ S/Sj5rS0Sjr\ S.Sj5rS1Sjr S2Sjr!S3Sjr"S)S jr#\ S.U4S!jj5r$S"r%U=r&$)4riz This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be fused together. The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. rSrcURURLde[U[[45(deUR 5(aA[U[ 5(a,[UR [R5(deO[U[[45(de[[R"UR5UR555nU"URU5$r|) r+rrrrrrrr+r=rr r r)rsrrr>s rmrtFusedSchedulerNode.fuses%//111%-1C!DEEEE     :e5N#O#Oejj"..99 99em5G%HII IIY__U__%68IJK5??E**rpcURHAn[U[5(deUR5(deUR 5 MC U$r|)rrrrrrlrs rmr,FusedSchedulerNode.extract_pw_from_reductionsK{{Gg}55 55'')) ))  - - /# rpcxURH*n[U[5(deUR5 M, gr|)rrrrrs rmr(FusedSchedulerNode.swap_pw_red_dimensions/{{Gg}55 55  ) ) +#rpc[[SSUR5555n[U5S:Xag[ U5nU$)Nc3# UHAnUR5(dUR5(dM/UR5v MC g7fr|rrrur r=s rmr4FusedSchedulerNode.estimate_flops..= 0''))T^^-=*D'')) 0 .A A rrfilterrrrrlfpsr#s rmr !FusedSchedulerNode.estimate_flopsK  $ 0    s8q=#h rpcUR5(agSnURHhn[U[5(d gUb<[ U5[ UR S5:wa[ RS5 gUR SnMj SnUce[U5URs=:XaUR:XaO OURU5nU(d%[ RSUR55 g[=RS- sl [ RSUR5U5 URH+n[U[5(deURU5 M- [U5 g) z0 Return true if a loop reordering is performed. FNrz1Can not reorder fused node due to different sizeszODont reordering fused node %s because we can not decide the suitable loop orderrz-Reorder loops for fused node %s with order %sT)rrrrrrrr&rrr#r$r3r,r%rr)rlrrr'rgrs rmr,FusedSchedulerNode.reorder_loops_by_dep_pairsH      [[Ee]33%% *;uU\\RS_?U*U!''GaJ! %%% z?h// E93E3E E ;;IFI  # #a  ##q(# ;T]]_i [[Ee]33 33  & &y 1! (-rpc~>[TU]U5 [XU5 /Ul[ USS9R Ulg)Nc4[UR55$r|)r$rr\s rmr-FusedSchedulerNode.__init__..ss1>>3C/Drpr)rrrr0rr)rlr+rrs rmrFusedSchedulerNode.__init__s6 #0%' %DEKK rpc~SRURVs/sHoR5PM sn5$s snfN_r*rr3rlrs rmr3FusedSchedulerNode.get_name+xxt{{;{!{;<<;:c<URSR5$rrr3rks rmr9!FusedSchedulerNode.get_first_name{{1~&&((rpc[R"URVs/sHoR5PM sn6$s snfr|rrrrErs rmrE#FusedSchedulerNode.get_buffer_names0 !L 1"4"4"6 !LMM!L<cn/nURH"nURUR55 M$ U$r|rrrrlrFrs rmrFusedSchedulerNode.get_outputs/(*KKD MM$**, -  rpc [UR5VVs/sH+upUR5SUSUR53PM- nnnURSRnUbUR UR 55 [R"SRU5R5S5$s snnf)Nz.snodes[z] = rrr) r=rr3rHrrrrrDr*r)rlr:rr+s rmr"FusedSchedulerNode.debug_str_extras%T[[1 1}}xs%0@/A B1  {{1~""   LL335 6tyy/668&AA s2B=clURVs/sHoR5PM nnUSU3$s snf)Nz , snodes: )rr)rlr snodes_strs rmr"FusedSchedulerNode.debug_str_shorts89=E**, Ez*..Fs1c>[TU]X5 [5n[UR5H/nURX5 UR UR 5 M1 gr|)rrrrrupdater)rlrrrrs rmr!FusedSchedulerNode.set_last_usage sQ 2G0:|T[[)D    3 H  & &t 7*rpc[R"URVs/sHoR5PM sn6$s snfr|)rrrrrs rmr$FusedSchedulerNode.used_buffer_namess0!MA"5"5"7!MNN!Mrc[R"URVs/sHoR5PM sn6$s snfr|)rrrrrs rmr/FusedSchedulerNode.used_or_aliased_buffer_namess58< D 1,,. D  DrcUR$r|rrks rmrFusedSchedulerNode.get_nodes r/rpcT[U5RSUR5S3$)Nz(nodes=rrrks rmrFusedSchedulerNode.__repr__#s't*%%&gdmmo->a@@rpc:[SUR55$)Nc3@# UHoR5v M g7fr|)rrs rmr2FusedSchedulerNode.is_reduction..(s9[>>##[r?rrrks rmrFusedSchedulerNode.is_reduction&s9T[[999rpc:[SUR55$)Nc3@# UHoR5v M g7fr|)rlrs rmr6FusedSchedulerNode.is_native_matmul..,s=A%%''r?rrks rmrl#FusedSchedulerNode.is_native_matmul*s====rpc:[SUR55$)Nc3@# UHoR5v M g7fr|)rors rmr3FusedSchedulerNode.is_split_scan..0s:k??$$kr?rrks rmro FusedSchedulerNode.is_split_scan.s:dkk:::rpc:[SUR55$)Nc3@# UHoR5v M g7fr|r9rs rmr1FusedSchedulerNode.is_template..4s8Kq==??Kr?rrks rmrrFusedSchedulerNode.is_template2s8DKK888rpcxURH*nUR5(dMUR5s $ gr|)rrrr/rlrs rmr/$FusedSchedulerNode.get_template_node6s3KKD!!--// rpc URS$rrrks rmrFusedSchedulerNode.get_device=szz!}rpc:[SUR55$)Nc3@# UHoR5v M g7fr|)rrs rmr>FusedSchedulerNode.has_aliasing_or_mutation..BsEA--//r?rrks rmr+FusedSchedulerNode.has_aliasing_or_mutation@sEEEErpc[er|NotImplementedError)rlrs rmr'FusedSchedulerNode.update_mutated_namesF!!rpc[er|r)rlrs rmrFusedSchedulerNode.add_fake_depIrrpc[er|rr{s rmr}FusedSchedulerNode.can_inplaceLrrpcUR5nSRSUR55n[5nUR US[ U5R SUSUS[URR5SUS[UR5SUS [URRUR- 5SUS URSUS URSUS 35 UR5 UR5H"nUR UR!55 M$ S S S 5 UR#S5 UR UR%55 UR-5R/5$!,(df  N]=f![&a [(R+SSS9 NOf=f)rr=c3L# UHn[U5Rv M g7fr|)rr~rKs rmr/FusedSchedulerNode.debug_str..RsF+QQ 0 0+s"$r<rrrrrrrrz.outputs = [ Nr>rTr)r3r*rrRrrr~rBrrrrrrrDrrHr?rrrrrEr)rlr node_typestrrrs rmrHFusedSchedulerNode.debug_strOs}}xxF$++FF  bd  Q|n-j))00123WT%<%<=>?74#3#3#9#9DURb[SUR55$[TU] 5$)Nc3@# UHoR5v M g7fr|)rr=s rmr6FusedSchedulerNode.has_side_effects..nsG;4,,..;r?)rrrrrrs rmr#FusedSchedulerNode.has_side_effectsks0 ;; "G4;;GG Gw'))rp)rr0rr_rr_rrrwrurOrF)r+r*rrSrrvrsrIrrrHrLrwrR)r torch.devicerG)rr3rrvrM)'r~rrrr&rrrtrrrIr rrr3r9rErrrrrrrrrrlrorrr/rrrrr}rHrrrrs@rmrrs $# +% +.? +  + +, ")!).7) )VL ==)NN B/ 8#2 8HV 8  8OO  A::>>;;99 FF """*8**rprcV^\rSrSrSU4SjjrSSjrS SjrS SjrSrU=r $) FusedMixOrderReductionsircl>[RU5(d[RU5(deX!p!XlX l[TU]UR [UR55[UR55-5 [RUR5Ul gr|) rrrrrrr+rrrr)rlrrrs rmr FusedMixOrderReductions.__init__ss 33E::$77>> >> 5    OOT%//"34tEOO._get_ancestors..s:Eq{{Errrr>rs rm_get_ancestorsAFusedMixOrderReductions.sub_node_can_fuse.._get_ancestorss,C99:E:; ;rpcB[5nUR"SU56$)Nc3@# UHoR5v M g7fr|)rrKs rmrZFusedMixOrderReductions.sub_node_can_fuse.._get_operation_names..sF14466r?rr s rm_get_operation_namesGFusedMixOrderReductions.sub_node_can_fuse.._get_operation_namess ,C99FFG Grp) count_bytes)r>tuple[BaseSchedulerNode, ...]rr) rrr+rrrrtypingcastr$score_fusion_memoryr)rlrr other_nodesr rs rmsub_node_can_fuse)FusedMixOrderReductions.sub_node_can_fusese%<====e%<==== ~~&&uu&U  / /   #66u== < H0 H  H ~.1Ek1RR{+.BE>.RR""$ $ {{T^^77RW7Xzz rpc[U[5(dUURURXR45=(d' URURXR45$URURURURUR45=(a/ URURUR[ 55$r|)rrrrrrrlothers rm can_fuse_with%FusedMixOrderReductions.can_fuse_withs%!899)) EJJ=J'' EJJ=I J )) EKK$**ekk)BK((U[[%'J KrpcbURR5nURRU5n[ U[ 5(aWUR URUR5nUR URUR5n[ XE5$URURXR45(a1UR URU5n[ X`R5$UR URU5n[ URU5$r|) rrr+rrrrtrr)rlrrcbackend fused_node1 fused_node2rs rm fuse_with!FusedMixOrderReductions.fuse_withs&&(..,,V4 e4 5 5!,,tzz5;;?K!,,tzz5;;?K*;D D%%djj%**GG$\\$**e< .z::FF$\\$**e< .tzz:FFrp)rrrr)rr_rr_rr)rr_) r~rrrrrrr"rrrs@rmrrrs< =2  2 !2 3 2 h KGGrprcz^\rSrSrSU4Sjjr\S Sj5rS SjrS SjrS Sjr Sr U=r $) $FusedExternTritonKernelSchedulerNodeic0>[UR[R5(de[R "[ [X#/5n[TU]%X5 X l X0l URRUl URUl gr|)rrr+rrrrr_rr kernel_nodefused_epiloguerr)rlr+r'r(rrs rmr-FusedExternTritonKernelSchedulerNode.__init__sv +**B,F,FGGGGT"34{6ST +&,))33%-- rpcURn[UR5S:XdeUR[ [ UR55R nURR[U55 [X1U5$Nr) r+rrr[r<rdrr0removeNodeUserr%)rsrrr+original_mutated_buffers rm epilogue_fuse2FusedExternTritonKernelSchedulerNode.epilogue_fusesy OO 5++,111"+"7"7 e../ 0 5 5#  %%,,Xe_=3IeLLrpc[URR[R5(de[UR R[R 5(deUR RR5(de[R"UR RRSR5nSSK J n URUR/U5upESSKJn U"UR/U5nSSKJn U"XGU5n U R'5n UR RR)XRRU 45$)Nrr)SIMDKernelFeatures)FusedUserDefinedTritonKernel)rr(rr+rr'rrrrrrtorch._inductor.codegen.simdrget_tiling_and_scores,torch._inductor.codegen.simd_kernel_featuresr2torch._inductor.codegen.tritonr3rcodegen_with_epilogue_fusion) rlrrrtilingrr2kernel_featuresr3fused_user_kernelnew_kernel_srcs rmr,FusedExternTritonKernelSchedulerNode.codegens#$--22B4E4EFFFF$**//1K1KLLLL$$668888 $**//<Q Y( (rpc^[TU5nTR5(aUR5(a[R"[T5m[R"[U5n[ TR 5[ UR 5:HnU(dU"S5 U=(a3 [U4Sj[TR UR 555$UR5(arTR5(a U"S5 g[R"[U5nURT5nUbURRTU5$U"S5 gTR5(aqUR5(a U"S5 g[R"[T5mTRU5nUbTRRXb5$U"S5 g[S5e) Nzforeach do not have same lengthc3^># UH"upTRRX5v M$ g7fr|)r+r)rlrrFs rmr6ForeachKernelSchedulerNode.can_fuse..8 s.)ADA""++A11A*-zXcandidate producer is a reduction, foreach ops cannot be fused with reductions currentlyFz5candidate producer is not dep of any foreach consumerzXcandidate consumer is a reduction, foreach ops cannot be fused with reductions currentlyz5candidate consumer has no dep in any foreach producerzXAt least one node passed to ForeachKernelSchedulerNode.can_fuse should be a foreach node)rrxrrrCrrrrGrrGr+rrOr)rsrFrKwhy foreach_matchconsumer_subnodeproducer_subnodes ` rmr#ForeachKernelSchedulerNode.can_fuse/ s(+    X%8%8%:%:{{#=xHH{{#=xHH0C4HHM 56 S)A)&  " "$$&&n{{#=xHH'@@J +))228=MNN G H  " "$$&&n{{#=xHH'@@J +))223CNN G H f  rpc `UR5(dUR5(deUR5(a4[R"[U5nURnUR nO3[R"[U5nURnUR nSnSnUR5(aUR5(a[R"[U5n[R"[U5n[ URUR5VVs/sHupx[RXx5PM n nnGO?UR5(a[R"[U5nURU5n /n UnSnURHBn XLa*[RX5n U nU RU 5 M1U RU 5 MD OUR5(a[R"[U5nURU5n /n UnSnURHBn XLa*[RX5n U nU RU 5 M1U RU 5 MD O [S5eU"URU UUUUS9$s snnf)NzTAt least one node passed to ForeachKernelSchedulerNode.fuse should be a foreach node)use_custom_partition_algo prev_node_1 prev_node_2enable_autotune)rxrrrCr]r`rGrrrtrOrrGrr+)rsrFrKr]r`r^r_rSrT fused_nodesrZrnew_noderYs rmrtForeachKernelSchedulerNode.fuse^ s\""$$(;(;(=(===    {{#=xHH(0(J(J %&66O{{#=xHH(0(J(J %&66O     X%8%8%:%:{{#=xHH{{#=xHH AADA#''-A K " "{{#=xHH'@@J K"KK +166tFH"*K&&x0&&t, ( " "{{#=xHH'@@J K"KK +166xFH"*K&&x0&&t, (!f     &?##+   Ks0!J*c>^0Tl0TlUbUcv[TT] X5 UH_nURR HnUTRUR 'M UR5Hn UTRU 'M Ma GONUTlUTl STl /Tl TR[RRURUR/55 [!U4Sj[ R""UR$UR$555TRR&- Tl[)UR*UR*/5Tl[-UR.UR./5Tl[)UR0UR05Tl[-UR2UR25TlUR55(a[7U[85(deXEpO[7U[85(deXTpU R:TlTR:R=U R:5 U RTlU R5Hn U TRU 'M TRV V Vs0sH(oR>RA5HupX_M M* snn n TlUTl!USRE5nU(deU[FRH"S5444Tl%[ [LRNRP"5Tl)UTl*gs snn n f)Nc3h># UH'nURTR5;dM#Uv M) g7fr|r{rs rmr6ForeachKernelSchedulerNode.__init__.. s5 xxt'<'<'>> C r~r combo_kernel)+rErJrrrrrrr+rrr0rr*rrrrrrrrrrrrrxrrCrrritemsr]rrExprrrfxNoderr`)rlr+rr]r^r_r`rrr foreach_noderrgrvrcrs` rmr#ForeachKernelSchedulerNode.__init__ s  +"5 G Y / ,,22D37D%%dii03!446D.2D%%d+7 'DN DKDI)+DJ  ''22 ,,k.E.EF )//#66 8V8V ""))*  #!+"7"79N9N!OPDN +"7"79N9N!OPDN&).. 0N0N'D #'*.. 0N0N'D #%%''!+/IJJJJ+6j!+/IJJJJ+6j)33DN NN ! !*"6"6 7 , 9 9D "668*4!!$'9#'++@"-:O:O:U:U:W$!:W+@D *C&%%' v > :<>? !%((--02 .@s"/Mc UVs/sHn[U[5(dMUPM nnU(aW[RS[ U5UVs/sH+oDR cMUR R 5PM- sn5 UVs/sHn[U[5(dMUPM nnU(a[RS[ U55 UVs/sHn[U[5(dMUPM nnU(a[RS[ U55 UVs/sH,n[U[[[[45(aM*UPM. nnUVs/sHn[U[5(dMUPM nnU(a[RS[ U55 UVs/sHn[U[5(aMUPM nnUVs/sHo"R5(dMUPM n nU (a [RS[ U 5U 5 UVs/sH o"U ;dM UPM nn[R(avUVs/sHo"R5(dMUPM n nU (a[RS[ U 55 UVs/sHo"R5(aMUPM nnU$s snfs snfs snfs snfs snfs snfs snfs snfs snfs snfs snf)Nz/ComboKernels: %d external nodes are filtered %sz+ComboKernels: %d grouped nodes are filteredz;ComboKernels: %d FusedMixOrderReductions nodes are filteredz+ComboKernels: %d foreach nodes are filteredz0ComboKernels: %d template nodes are filtered: %szCComboKernels: %d reduction nodes are filtered (pointwise_only mode))rrrrrrrrrrrCrrr(combo_kernels_pointwise_onlyr) rsr>rexternrgrouped mix_orderfiltered_nodes foreach_nodestemplate_nodesreduction_nodess rmcombinable_nodes+ForeachKernelSchedulerNode.combinable_nodes sl#OUj4M&N!UO  IIAF 5;UVTyy(&&(VU  $Kez!5I'J1eK  II=G  !&P1A7N)OQ P  IIMI   *-(+     & %!A7Q)RA~    IICSEW X% %!Z;U-VA~  &4G^}}!^G  IIBN#  &4O^7N!^O  . .*8M.QNNK>c dUR5n/nSn[UVVVs/sH>nUH4n[U[5(dMUR 5HnUPM M6 M@ snnn5nUGH/n[ [ 5n UHinUR5n U (a"U RS:XdU RS:XaM<UR5U-(aMVXRU5 Mk U R5Hn [ [ 5n U H0nXRRUS5RU5 M2 U R5H=n UR[S[!U 5U5Vs/sH nXX-PM sn5 M? M GM2 U$s snnnfs snf)zC Returns a list of lists of nodes that are to be grouped together. mpsrar)_topological_sort_nodesrrrrErrrrrrrnode_to_streamrrr r)r+ sorted_nodes grouped_nodes max_num_nodesrrrZexcluded_buffer_namesr> device_groupsrc device_nodes stream_groups stream_nodesr:s rm&_default_group_nodes_for_combo_kernelsAForeachKernelSchedulerNode._default_group_nodes_for_combo_kernels/ s!88:   1;* )E!Dd$;<!% 5 5 7H !8 !) 2 "ED! *v{{e3v{{e7K))+.CC%,,T2!. 4 4 6 DOPTDU (D!":":">">tQ"GHOOPTU)$1$8$8$:L!((&+1c,.?%O%O)Q->?%O%; !7%":K @sF& "F& F-4Callable[[Scheduler], list[list[BaseSchedulerNode]]]!group_algorithm_for_combo_kernelscU[lgr|rCr)custom_group_algorithms rm%set_group_algorithm_for_combo_kernels@ForeachKernelSchedulerNode.set_group_algorithm_for_combo_kernelsf s # #Drpc,[RU5$r|rr+s rmgroup_nodes_for_combo_kernels8ForeachKernelSchedulerNode.group_nodes_for_combo_kernelsn s*KKIVVrpc[er|rrks rmr#ForeachKernelSchedulerNode.mark_runt rrpc[er|rrks rmr"ForeachKernelSchedulerNode.codegenw rrpcgrrzrks rmrx%ForeachKernelSchedulerNode.is_foreachz r1rpc,[UR5$)z]Returns a list of nodes which comprise the combo kernel. These nodes may be vertically fused.)rrrks rmget_subkernel_nodes.ForeachKernelSchedulerNode.get_subkernel_nodes} sDKK  rpct[[RRSUR555$)ziReturns all nodes contained in this kernel, unpacking fused nodes into their constituent scheduler nodes.c3@# UHoR5v M g7fr|)rrs rmr7ForeachKernelSchedulerNode.get_nodes.. s1UA++--r?)rr r rHrrks rmr$ForeachKernelSchedulerNode.get_nodes s(IOO111U1UUVVrpc<URSR5$r)rr9rks rmr9)ForeachKernelSchedulerNode.get_first_name s{{1~,,..rpc[XURR5 URHnUR U5 M gr|)r0r+r[rr2)rlr1rs rmr2/ForeachKernelSchedulerNode.prune_redundant_deps s5 d8R8RSKKD  % %&8 9 rp)rr`rrrrrrJrrrrEr+rrr]r0)rFr_rr,)rKr_rr,rFr_rKr_rr{)rFr_rKr_rrC)NNF)r+r*rrSr]r{r^r,r_r,r`r{rrvr>rSrrS)r+r*rlist[list[BaseSchedulerNode]])rrrrvrurwrrSrLrsrJ)r~rrrr&rGrOrrrtrrxr'rrrrrrrrxrrr9r2rrrs@rmrCrC s ) !) !&, , \> (> 4E> #> > J1504 %L/L/(L/$( L/ . L/ . L/L/ L/L/\?+? ??B00 &00h /&(/ T   WW &WW ""! W /:">: ::rprCc^\rSrSr%SrS\S'\SSj5rSSU4SjjjrSSjr SSjr \ SS j5r SS jr \ SS j5rSS jr\ SS j5rSSjrSSjr\SSj5rSrU=r$)ri a' This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be *grouped* together (it does not allow another node to be scheduled in between its constituent nodes, nor does it allow another node to fuse into any of its constituent nodes). The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. Fusion will still happen among the nodes within each GroupedSchedulerNode. At codegen time, this scheduler node will be unpacked and codegen is called on each constituent node. rSrc^USRm[U4SjU55(deU"TU5nUH nUTRUR5'M" UTRUR5'U$)Nrc3>># UHoRTLv M g7fr|r)rrr+s rmr.GroupedSchedulerNode.create.. sB64>>Y.6s)r+rr1r3)rsr grouped_snodergr+s @rmcreateGroupedSchedulerNode.create su1I'' B6BBBBBIv. E=JI ( ()9 :AN $$]%;%;%=>rpcH>[TU]U5 [XU5 X0lgr|)rrr temp_grouping)rlr+rrrs rmrGroupedSchedulerNode.__init__ s$ #0 +rpcBUR(a UR$URH)nXRRUR 5'M+ URRUR 5 URR UR5$)zw Do fusion among nodes within this GroupedSchedulerNode, and then unpack this GroupedSchedulerNode into regular nodes. )rrr+r1r3 fuse_nodes)rlrgs rmunpackGroupedSchedulerNode.unpack so   ;; [[EBGNN - -enn.> ?! NN - -dmmo >~~((55rpcURURRU55 URR U5 gr|)rrrrr)rlfake_deps rmr!GroupedSchedulerNode.add_fake_dep s5 T--77AB ##H-rpc~SRURVs/sHoR5PM sn5$s snfrrrs rmr3GroupedSchedulerNode.get_name rrc<URSR5$rrrks rmr9#GroupedSchedulerNode.get_first_name rrpc[R"URVs/sHoR5PM sn6$s snfr|rrs rmrE%GroupedSchedulerNode.get_buffer_names rrcn/nURH"nURUR55 M$ U$r|rrs rmr GroupedSchedulerNode.get_outputs rrpc[[SSUR5555n[U5S:Xag[ U5nU$)Nc3# UHAnUR5(dUR5(dM/UR5v MC g7fr|rr=s rmr6GroupedSchedulerNode.estimate_flops.. rrrrrs rmr #GroupedSchedulerNode.estimate_flops rrpcUR$r|rrks rmrGroupedSchedulerNode.get_nodes r/rpcbUR(aURSR5$S$r)rrrks rmrGroupedSchedulerNode.get_device s$.2kkt{{1~((*CtCrpcgrrz)rsrFrKs rmrGroupedSchedulerNode.can_fuse rrp)r)rrSrr)F)r+r*rrSrr{rrvr)rr3rrvrsrIrrOrLryr)r~rrrr&rrrrrrrIr3r9rErr rrrrrrs@rmrr s $#$ + +( + +  + + 6.==)NN "Drprc 0^^[RSUU4Sjj5n[[[ [ TS5555n[ U5S:aUVs/sHnTUPM snm[ R(aURUS9 U$s snf)zu A heuristic to decide loop iteration orders. This has not been well tuned and may be something we should autotune. cz>TUS:Xd TUS:Xa[TUS:HTUS:H5$TVs/sHn[X 5PM nnTVs/sHn[X!5PM nn[S[X4555n[S[X4555nXV:agXe:ag[X5$s snfs snf)Nrc3F# UHupUS:H=(d X:v M g7frNrzrsl_asl_bs rmr5pick_loop_order..index_cmp.. $ 7VDAI $ $7V!c3F# UHupUS:H=(d X!:v M g7frrzrs rmrr rrr )rJabsrrG) r`bsl stride_len_a stride_len_ba_firstb_firstrstride_lengthss rm index_cmp"pick_loop_order..index_cmp s 8q=E!HMuQx1}eAh!m4 4.<<^rBE ^ <-;<^rBE ^ < 7:<7V   7:<7V      1y#=>#33 .  E$5$55 .  -  rpc6URR5$r|rKrks rmr3NodeUser.get_name rMrpcURURLde[URUR=(a URUR=(a UR5$r|)rr-r}rrs rmriNodeUser.merge sPyyEJJ&&& II    2!2!2 LL *U]]  rprzNrt)robjectrr{rs)rr-rr-) r~rrrrr}rr9rr3rirrzrprmr-r- s3 ((KGTL $ rpr-c"[R$r|)r(r}rzrprm*used_non_deterministic_runtime_estimationsr" s  3 33rpc[5nUR5n[U[R5(aUR [ UR5[ UR5-[ UR5-5 [U[R5(a$UR [UR55 U$Ub SU35eU$)z=Get free symbols from a node's layout (size, stride, offset).z*Expect layout to be None but found layout=) rmaybe_get_layoutrr+Layoutrr"rstrideoffsetrget_layout_symintsr)rfree_symbol_usesr@s rmr(r( s1;  " " $F&"))$$  %6==) *6==) * fb;; < <  # #$6v}}$E F ~T!KF8TT~ rpc.[U[5(a([5R"SUR56$UR ceUR R 5nUR"SUR R556 U$)z{ Gets symbols used in a scheduler node, including free symbols from the node's operations and layout symints from outputs. c38# UHn[U5v M g7fr|get_scheduler_node_symbol_uses)rrgs rmr1get_scheduler_node_symbol_uses.. s M,U33rc38# UHn[U5v M g7fr|)r()rir_nodes rmrr. s M5L' W % %5Lr) rrrrrrget_free_symbol_usesrr)rr)s rmr-r- s$*++|!! M M   99  yy557 MTYY5J5J5L M rpczUR5nUbURb UR$[R$z4Check per-template flag, fall back to global config.)r/allow_epilogue_fusionr(epilogue_fusionrAtbs rm_is_epilogue_fusion_enabledr8 8  ( ( *B ~"22>'''  ! !!rpczUR5nUbURb UR$[R$r3)r/allow_prologue_fusionr(prologue_fusionr6s rm_is_prologue_fusion_enabledr= r9rpc~UR5=(a' UR5(+=(a [U5$r|)rrr8rs rmis_epilogue_fusionr? 4  /!!# # / ' .rpc~UR5=(a' UR5(+=(a [U5$r|)rrr=rs rmis_prologue_fusionrB r@rpc<[X5=(d [X5$r|)r?rBrs rmis_template_fusionrD s e + O/A%/OOrpc*[X5(aU$U$r|)r?rs rmtemplate_fusion_pw_noderF s&u445?%?rpc^\rSrSrSrSlSjrSlU4SjjrSmSjrSnSjrSoSjr SpSjr SqS jr \ SrS j5r \ RSsS j5r SnS jrStS jrSuSjrSnSjrSnSjrSnSjrSnSjrSvSjrSwSjrSnSjrSxSjrSySjrSnSjrSnSjrSnSjrSwSjrSnSjrSzSjr S{S|Sjjr!S}Sjr"S~S jr#SnS!jr$SS"jr%SS#jr&S{SS$jjr'SS%jr(SS&jr)SS'jr*SS(jr+SS)jr,SS*jr-SS+jr.SS,jr/SS-jr0S{SS.jjr1SS/jr2SS0jr3SS1jr4SS2jr5SS3jr6SS4jr7SS5jr8SS6jr9SS7jr:SS8jr;SS9jr<SS:jr=SSS;jjr>SS<jr?SS=jr@SS>jrASS?jrBSSS@jjrCSSSAjjrDSSBjrESSCjrFSSDjrGSSEjrHSnSFjrISnSGjrJSnSHjrKSSIjrLSSJjrMSSKjrNSSLjrOSSMjrPSSNjrQ\RSSOj5rSSSPjrTSSQjrUSSRjrVSSSjrWSSTjrXSSUjrYSwSVjrZSwSWjr[SwSXjr\SSYjr]SSZjr^SS[jr_SnS\jr`SS]jraSS^jrbSS_jrcSnS`jrdSSajreSSbjrfSScjrgSSdjrhSnSejri\ SSfj5rj\ SSgj5rkSShjrlSnSijrmSSjjrnSkroU=rp$)r*i z A Scheduler is a graph of BaseSchedulerNodes. It is responsible for optimizations such as fusion, reorder, and graph partition. cp[S5 URU5 SSS5 g!,(df  g=f)NzScheduler.__init__)r_initrlr>s rmrScheduler.__init__ s# . / JJu 0 / /s' 5c  >^[T T]5 T[Rl0Tl[ [5Tl[R"5Tl [5Tl [/[RRR5Q[RR R5Q[RR"R5Q5TlUVs/sHnTR'U5PM snTlSTlSTlTR/5 TR$R1[RR R55 TR(HnUR35 M STlTR75TlTR(Vs0sHo"R;5U_M snTlTR(VVs0sH*o3R?5HoDR;5U_M M, snnTl TR<RC5Tl"0Tl#0Tl$[5Tl%[LRN"TR(TR@TRD5TlTRQ5 TRSTR(5TlTRU5 TR(Vs0sHo"R;5U_M snTl"TRW5 TRY5 [Z=R\[_TR(5- sl.SSK0J1nJ2n U"TR(5 [_TR(5Tl3TRi5 TRSTR(5Tl[[j[l[l4"5Tl7[pRrb%[pRr"TR(5Tl[pRt(a'SSK;J0Tl?STl@0TlATR5 TRTR(5Tl[pRb%[pR"TR(5Tl[STR(55(aTRU5 TR5 TR5 [pR(d[pR(aA[5(a2[RRRR5 [pR(a#[SSSS 9 TRSS 9 SSS5 TR5 [pR(aSS KUJTn U"TR(TR@TRD[[RRR55[[RR555Tl[pR(Gd;[pR(Ga%[pR(d#SS KUJYn U "TR(TR@5 [5(a[R(a|[pR(d[R(aRSn TR(H!n[UR5(dMSn O U (aSS K&Jan U "TR(5 [R(aSSKcJdn U "SSU4SjS9 [LR"TR(5TlTR5 [pR(a~[pRR(a_[pRR(a@TRTR(5TlTRTR(5TlTR5 [RRpRR(aTR5 U"TR(5 [RRTR(5 TR5 [5Tlt0TluSTlv[S5RU4Sj5 [5Tlygs snfs snfs snnfs snf!,(df  GN=f)Nr)log_ir_post_fusionlog_ir_pre_fusionr)distributed_autotuneFc3B# UHn[U[5v M g7fr|)rr%r=s rmr"Scheduler._init..q s" " tA B B"z#Scheduler.create_combo_kernel_nodesTlog_pt2_compile_eventlog_waitcounter) num_ck_nodes)reorder_for_peak_memory)1assign_memory_planning_info_for_scheduler_buffers)6align_runtime_estimations_across_all_distributed_ranks)trace_structuredartifactc SSS.$)N#scheduler_nodes_before_comm_overlapstring)rencodingrzrzrprmr!Scheduler._init.. s E$,)rpc >SR[TR5VVs/sH0upSUS3UR5-SUR 53-PM2 snn5$s snnf)Nz zsnode[r>z buffer_names:)r*r=r>rHrE)r:rLrls rmrr` sov{{ )2$**(= )>%QCqMkkm, .q/A/A/C.DEF)> (s7A$ ) metadata_fn payload_fn graph_statsc^>TRTR[TR5S.$)N)graph_idnum_nodes_before_fusionnum_nodes_after_fusion)post_grad_graph_idnum_orig_nodesrr>rksrmrr` s% 33+/+>+>*-djj/rp)zrrrZrr+backendsr<_post_grad_graph_counterrir count_graph_partition_counterrrrr constantstorchbind_constantsrcreate_scheduler_noder> previous_node current_nodeupdate_zero_dim_cpu_tensorrrdefault_device_contextget_donated_buffersrZr3rJrr[copyr1rrseen_template_fusionsr'decide_global_ordering_of_commsr]topological_sort_scheduledead_node_eliminationcompute_ancestorscompute_input_distancesr,ir_nodes_pre_fusionrtorch._inductor.debugrMrNrjcreate_foreach_nodesrr"logged_slow_fusionr(_pre_fusion_custom_passdistributed_max_autotune_gemmrrOscheduler~buff_to_stream_multi_stream_nodesstream_idx_to_user_obj_idx_populate_stream_assignmentsr_post_fusion_custom_passrr finalize_multi_template_buffersmax_autotune_gemm max_autotunerrrselect_algorithmPrecompileThreadPoolshutdown_instance combo_kernelsrcreate_combo_kernel_nodes_enforce_conditional_orderingrWmemoryget_output_names deterministic reorder_for_compute_comm_overlaprXr"r)6runtime_estimations_align_across_all_distributed_ranksr}rrSrrYreorder_sink_verbose_loggingtorch._loggingrZ$reorder_compute_and_comm_for_overlapprocess_grouped_nodesgraph_partitionrr^%reorder_for_reducing_graph_partitions&maybe_reorder_for_minimizing_partition,reorder_for_partition_with_simple_dependencycompute_last_usage test_configstrack_memory_lifecycleinsert_memory_check_nodesr graph_diagramdebug_draw_graphbuffer_names_to_freeorigin_to_index_current_stream_ctxradd_row removed_ops)rlr>rLrrrMrNrOrWrXhas_collectivesrYrZrs` rmrIScheduler._init s  <> "&'?"@(1(9%5?\!&0 %%**, ""'') ,,113 ' #>CCUd003UC 7;6: '') ##**177+<+<+A+A+CDJJD OO <@#  $ $ & #&*ZZ; %/JJL!OZ;  -1JJ8 ,6DBRBRBT3LLNC BTNJ8 AE@Q@Q@V@V@X3513 L " :: JJ     # #  !!#33DJJ?  ""$<@JJ"GJq::<?J"G   $$& ##s4::6#O$**%!$**o !!#33DJJ? ",U38_"="?  ) ) 577 CDJ  / / . ) )$ /  " " $=?.0). :<' ))+__TZZ0  * * 688DDJ        & & (  ,,.  $ $(;(;&(( OO , , A A S S U   5&* $ ..D.A  **,  ) ) 70   ''177//44671773356 DJ###(O(O(O11UAJJ 0 0 ;<< WW<<#PP#( JJD$TYY//*.'#K4::V88; !  CCDJJODJ ""$  " " (( CCDDTZZPDJJJ4::VDJ ! ?? ! ! . . E E  * * ,4::& djj) 6@\!:<GK '//  -7LAD; 8 H#Hhs$6ee#61e(5e.;e33 fc0n[RRHgn[[RRU[R 5(dM?[ U[RRUSS9X'Mi U$)N)r-)rZrgraph_inputs_originalrr+ DonatedBufferr~)rlname_to_donated_bufrs rmrvScheduler.get_donated_buffers sl GG11D!''77=r?O?OPP,BGG11$7 $-#)2#"rpc^ SSKJm 0n[R"S5nURHnT nUR bCUR R 5nUb&XQ;a[U5nXaU'XPRU'XnX@RU'UR5HnX@RU'M M [U 4SjURR555(a[SUR5S5nUbURHnUR n UR5bM"[U [ R"5(dMC[U R$[ R&5(dMn[ R&"US9U lM [U 4SjURR555Ulg)aPopulate node_to_stream and buff_to_stream from IR node stream_idx. Reads the stream_idx field set on IR nodes during lowering to determine which stream each scheduler node should run on. This field is propagated from 'custom.stream' FX node metadata via IRNode.current_stream_idx(). r)DEFAULT_STREAM_IDXNc3,># UH oT:gv M g7fr|rz)rrrs rmr9Scheduler._populate_stream_assignments..' sM0L1&&0Lsc3f# UH'oR5cMUR5v M) g7fr|rrKs rmrr) s RA||~s11rcc3.># UH nUT:gv M g7fr|rz)r stream_idxrs rmrr7 s' :  , ,:s)stream_constantsrr rmr>rget_stream_idxr<rr~rErrrrrr+Bufferr@r?r) rluser_obj_to_stream_idxstream_idx_counterrr user_obj_idxnew_stream_idxrrcr0rs @rmr&Scheduler._populate_stream_assignments s~ 924&__Q/JJD+Jyy$#yy779 +#A)-.@)A?M|)rlrs rmrScheduler.debug_draw_graphW s1 ::>>:D AS H +  6 Irpc[R[R5(a:[R SU5 UR HnUR 5 M gg)Nz%s:)r isEnabledForloggingINFOrr>r)rllabelrs rmdebug_print_nodesScheduler.debug_print_nodes^ sD   GLL ) ) HHUE "   "# *rpcPUR5cS5eUR5(a [X5$[U[R [R 45(a [X5$[U[R5(a [X5$[U5e)Nz2All nodes passed to scheduling must have an origin) ris_no_oprrr+rrrrprrrs rmrqScheduler.create_scheduler_noded s!- @ - ==??)$5 5 r00"2C2CD E E , , boo . .,T8 8%d+ +rpc[5n/nURR5n[RR R 5HnUVs/sH0nXS;dM [URU[5(aM.UPM2 nnU(dMIURU5 UVs/sHoPRUPM nn[RS:n[UUSUS9nURU5 UHnXRU'M M URV s/sHoR!5U;dMU PM sn [#U5-Ulgs snfs snfs sn f)NrFr]r`)rr1rrZrlistsrrrJrrr(combo_kernels_autotunerCrr>r3r) rlremoved_node_namesfe_nodeskept_node_namesnamesrrr`fe_noders rmrScheduler.create_foreach_nodesq sK.8l11668WW]]))+E"!D*#4#4#4T#: UR_TR`5 UR 5HnURM5HznUR 5TR`T)"U5'UR 5TR`U'TRbReUU5TRbUR 5'M| M GMg [RRg5H4n[4R7SU5 U "U[i[?U555 M6 U(a[RRjHnUR9S S 9Hn X;dU SU Rm535eX=n(dM/TR:URo5H5n[4R7SUU 5 U "U[i[?U555 M7 M M TR`HnU[RR ;aFU "U[i[?U555 [RRpRsU5 MgU[RRt;dMU "U[i[?U555 M [w[RR Rm55V Vs0sH un nUU _M n!n n[RRpVs/sHnU!UPM sn[Rl<TRHFnUR 5H/nUR{T(UR 5RN5 M1 MH TR|H.nTR|UR{T(URN5 M0 [5n"U"RS5 T(RO5Haunn#U"R5 U#RNV$s/sHn$U$R 5PM n%n$U"RSUSU%S35 SSS5 Mc U"RS5 U"R5R5n&[R7S5 [R7SU&5 gs sn fs snn fs snfs sn$f!,(df  M=f)zQ Create dependency edges between nodes, handling aliasing and mutation properly. cP>\rSrSrSrSS SjjrS SjrS U4SjjrSrg) 1Scheduler.compute_dependencies..DedupListi a This data structure behaves like a list except it makes sure the elements remain unique. Normally one could use a OrderedSet/dict for this purpose however the list in question gets elements appended as it is being iterated over which means that we need to keep the list semantics. NcTU=(d /UlU=(d [5Ulgr|)rhr membership)rlrhrs rmr:Scheduler.compute_dependencies..DedupList.__init__ s #[b ","< rpcXR;agURRU5 URRU5 gr|)rrhrr)rl node_users rmr8Scheduler.compute_dependencies..DedupList.append s3/ !!),##I.rpc>[R"URUR5nURURVs/sHo3UR;dMUPM sn-nT"XB5$s snfr|)rrrrh)rlrnew_membershipr new_items DedupLists rm__add__9Scheduler.compute_dependencies..DedupList.__add__ sc!+!1!1$//5CSCS!T JJ${{**!t.FA{* !;;*s A0A0)rhrrt)rhzlist[_T] | NonerzOrderedSet[_T] | Nonerrv)rrarrv)r DedupList[_T]rr) r~rrrr&rrrr)rsrmrr s@ *.48 =& =2 =  = /  < UTR;aT"TRU5$U$r|)r)rLrrls rmr.Scheduler.compute_dependencies..rename s,D)))d33A677HrpFcN>TT"U5R[XU55 gr|)rr-) used_by_namerr}r name_to_usersrs rmadd_user0Scheduler.compute_dependencies..add_user s' &. / 6 69 rpNcUR$r|rr\s rmr0Scheduler.compute_dependencies..sAFFrprTz scheduling %s) unbacked_onlycUR$r|rr\s rmrrs!&&rpz not in )rj mutating_bufr)r)rzscheduling output %sz+scheduling output %s for unbacked symint %sr'z': r=rzBUFFER USER LIST z===== AFTER SCHEDULING ===== %s)rLr"rr")FF) rr"rrr}r{rr{rrv)Er rarrr>rr3rrr@r+r?rrArZrrrrrir" TensorBoxrrget_unbacked_symbol_defsSymbolrrr1rJrr5rrr<rdr4rjrCrhr_r6additional_buffer_depsadditional_star_depsrrr}rrrrrrc graph_outputsrrEmutated_inputsrror=mutated_input_idxsrjrZrRrrDrErcompute_dependencies_log)*rlrbuf1 buf1_name buf2_namelist1list2combinedrrunbacked_symbol_to_origin_nodevalfsrsym_sizehas_non_input_unbacked_defsunbacked_symbol_defsunbacked_symbol_usesrTrr node_modealt_namerGout_buf other_nameis_aliasadd_deprrZrrr inp_nameslogbufrrmr0r"rrrs*` @@@rmr]Scheduler.compute_dependencies s  <  <@@K?V?V @ JJD((* MMO tyy//??D,,./!3!%!1!1!3I M1i=6P -i 8 -i 8#(=#0C -c 2e ;#0#5#>5= c 2 $1 #m33@3K i03@3K i0"4+<  !&!  6       JL& 77''..0C#uzz****B9=26+C..(+||~S~!Auzz9RA~S!Ann=A6:-"1',#JJD99( (($* 224:J$ *!!U\\2222/3+:8< 25* JJD IIotyy 1*yy,,,'-II222F(($ .A>#X&D%EF><>>K#'#4#4Q#7#C#C#EC --gclln.EF$F .D$$++,1 d&6&6&=&=!>??S?sI..HH  '')3,,./1444 # 1 1 3H%h/HXt,%%ghY&GH -h 7 = ===?dmmo=$)$))5FGGGG'+yy'<'<'>G)0)9)9);J)/ );J(073F3F3H'HH -- '$.1408L!"%ZtD%(? !> !4*B7799$--/J$5!!''4==?D"QR K 7777 H$6!!''"23I ((..!$00TYY.>.>t.DE/  % %d&;&; <'') # 1 1 3H>AllnD))&*:;69llnD))(3//33HhG++CLLN;!4*ar002H II,h 7 Xz'(*;< =3 'ww,,111EA>#X&D&I&I&K%LM>;==q=(,(9(9!(<(M(M(OHII M ( ! %Xz'(:K/LM )P F-))Dqww+++z'$-89&&**40***z'$-89 *,5QWW5I5I5N5N5P+Q +QKE4D%K+Q  )*(>(>& (>IdO(>& " JJD'') mCLLN;AAB*//D  ' ' - 7 7 d8K8Q8Q R0 ! c'--/JC/4{{;{!{; #c%23!0  c  "))+ &&';< &&'I3OKTX & "<!s6"#l l2l !ll*lll l) c 2^^SSKJnJnJnJn [ [ RRR55nU"TRU5n[RRR(dU"TRTR5 [ [ RR!55nU"TRUU5un n [#[%TR55V s/sHn //4PM sn mUHn U R&S:XaU R(S:XaM%U R*R-5n TU R.SR1U 5 TU R2SR1U 5 M SSKJn U "5 SUU4Sjjn /n[9TR5HEunnUR1U5 UR1U "X[%TR5S- :HS95 MG UTl gs sn f)Nr)rXcompute_memory_timelineFreeableInputBufferget_freeable_input_bufr)register_check_mem_opcN>TUSnTUSnX#U/n[R"[[R"S5S9[R R RR/USS9nSTRUR53Ul [TU5$)Nrrrarc$UUSUSUSS.4$)Nrrr)alivedead is_final_steprz) tensor_argsrds rmrWScheduler.insert_memory_check_nodes..construct_mem_check_node..s(!.q!1 -a 0)6q)9Crp)r@rXr4nontensor_argsunflatten_args mem_check_) r+MemoryCheckKernelr?rrcrk_inductor_debugcheck_memory_stepdefaultr>r3rr)step_idxr3expected_newly_aliveexpected_newly_deadr6rrlstep_allocs_deallocss rmconstruct_mem_check_nodeEScheduler.insert_memory_check_nodes..construct_mem_check_nodes$8#A!#D "6x"@"C 2WN''!e)<=yy00BBJJ-  D%/tzz(/C/L/L/N.O"PD ,T48 8rp)r3)r=r$r3r{rr)rrXr+r,r-rrZrrrr>rrr(rWr[rr r size_alloc size_freerr3 start_steprend_step#torch._inductor.runtime.debug_utilsr.r=)rlrXr+r,r-rname_to_freeable_input_bufr buf_info_listrbuf_inforZr.rA new_nodesr:rr@s` @rmr#Scheduler.insert_memory_check_nodess  )31773G3G3L3L3N(O "4::| < #%%== = D,, *4AGG4L4L4N)O 5 JJ &   q!$C O4C 4RH4C &H""a'H,>,>!,C//1H !4!4 5a 8 ? ? I !2!2 3A 6 = =h G& N 9 9*. 9 & 9 92  ,GAt   T "   (DJJRS@S;SU -  eC s8Hc^ [R(dg/n[UR5GHnSSjm SnUR 5Hn[ U 4SjUR 55nU(a][RSUR55 [RRRUR55 MSnM UR5(+=(a U(+nU(dURU5 M[RSUR55 [RR RUR55 UR"R$HnUR&UR(;dMUR(UR&R nUVs/sH2oR*R5UR5:wdM0UPM4 snUR(UR&lM GM [-[U55UlURHnUR/5 M gs snf) z Remove any nodes without users Nc~UR=(d+ UR5[RR;$r|)rr3rZrr")rGs rmcan_eliminate_user;Scheduler.dead_node_elimination..can_eliminate_users&||Tt}}!'':T:T'TTrpFc34># UH nT"U5v M g7fr|rz)rurOs rmr2Scheduler.dead_node_elimination..s#M9a$6q$9$99zremoved dead buffer: %sTzremoved dead operation: %s)rGr-rr{)r(use_dcerr>rrr0rrr3rZrrrrrr"rrrr[rrr-) rl updated_nodesractive_buffersr can_eliminaterr0rRrOs @rmr{Scheduler.dead_node_eliminations~~   TZZ(D U#N'') ##M399#M M  II7HGG++// ?%)N *!% 5 5 77NUT;afTRU5 [URSS9H*nURT;aMT"TUR5 M, TR U5 gg)NcUR$r|r)ds rmrDScheduler.topological_sort_schedule..visit../saffrpr)rrrrr)rLrrJrFseenvisits rmrc2Scheduler.topological_sort_schedule..visit,sa} !!"6"6rrrJrFrbrcs @@@@rmrz#Scheduler.topological_sort_schedule"sj +,.59V *, ! !D--/%) T"0D $K rpc URVs/sH0n[UR[R5(dM.UPM2 nn[ S[ U55Hin[[X#R555n[[X#S- R555nX#R[XTSS95 Mk gs snf)NrTr) r>rrr+ Conditionalr rr<rdrErr6)rlrLconditional_nodesr:r prev_bufs rmr'Scheduler._enforce_conditional_ordering=szz !!Z%GAz  q#/01A%6%9%J%J%L MNLD!2q5!9!J!J!LMNH  - -TJ 2 s -C C cN^[5n[U[[[[ [ 45(a/URHnURUR5 M O[S[U5S35eU4SjU5n[[U4SjU555$)Nz+get_unmet_dep_nodes is not implemented for .c3^># UH"nTRUR5v M$ g7fr|)r[r5rs rmr1Scheduler._get_unmet_dep_nodes..Zs(XZc))#.??AAZrVc3B># UHnTRUv M g7fr|r1)rrLrls rmrrn[sQ=at66q9=s) rrrrrrrrrr RuntimeErrorrr)rlrg unmet_depsr unmet_dep_opss` rm_get_unmet_dep_nodesScheduler._get_unmet_dep_nodesHs&0l  )&"$   //sxx(0=d5k]!L YZX JQ=QQRRrpc/n[RURS5n0nURHQnURU5n[ U5X$'UH*nUR U/5nUR U5 XsU'M, MS UR5VV s/sHupU S:XdM UPM n nn U (aUR U 5 U H9nUR U/5Hn X+==S-ss'M URU5 M; UR5VV s/sHupU S:XdM UPM n nn U (aMU(aS5eU$s sn nfs sn nf)zE Sort nodes by their topological order, return a list of node lists. rrzTopological sort failed!) rFfromkeysr>rtrrrrhr) rlrr>childrenrrrcrLrmzero_deg_nodesrGs rmr}!Scheduler._topological_sort_nodes]s, djj!,#%JJD,,T2Dd)EKLLb) ! ). @ a! @ LL (#$LLB/DK1$K0 ! $-2KKMDMDAQ!VaMND n444y AEs E)E E,Ecj0nURHwn[5nURHBnURURR 5nUR U5 X1U-nMD X1UR5'X2lMy [UR5HupbXbl Xbl M g)z Populate each node.ancestors N) r>rrr[rr5rr3rr=rr)rlname_to_ancestorsrrr dep_node_namers rmr|Scheduler.compute_ancestorsws 9;JJD)3I.. $ 0 0 : K K M  m,}== /2;dmmo .&N%TZZ0KE"N"N1rpc0n0nURHnUR(dSnSnOURVs/sH/nXRURR 5S-PM1 nnURVs/sH/nX RURR 5S-PM1 nn[ U5n[ U5nXAUR5'XRUR5'XClXSl M gs snfs snf)z Populate each node's min/max_input_distance with the depth from graph inputs, measured as dependency hops before fusion. Nodes whose dependencies are all satisfied by graph inputs/constants have depth 0. rrN) r>rr[rr5rrr3rr) rlname_to_min_distancename_to_max_distancermin_distmax_distr dep_min_dists dep_max_distss rmr}!Scheduler.compute_input_distancess 02/1JJD** $66! 7))9)9#(()C)T)T)VW6! $66! 7))9)9#(()C)T)T)VW6! }-}-4< 14< 1&. #&. #) ! !s 6D>6Dc[R(dgURHn[U[[ 45(a)UR 5(d[RS:waMIUR5H?n[U[5(aUR5(aM/UR5 MA M g)Nhalide) r(rr>rrrrU cpu_backendrrrr )rlrrgs rmr Scheduler.merge_loopss00 JJDd]4F$GHHKKMMf&8&8H&D)!%775;L;L;N;N!!# *rpc[SSSS9 [S5Hn[U5n[R SUS-U5 UR USS9n[U5n[R S US-UU5 XC:XdUS:XdMk[R S US-5 O [ R(d[ R(aUR USS9nUsS S S 5 $!,(df  g =f) z2 Combine eligible nodes into FusedSchedulerNodes. zScheduler.fused_nodesTrSryz/===== attempting fusion (%d/10): %d nodes =====rF)is_reorder_roundz=completed fusion round (%d/10): fused %d nodes into %d nodes z+===== fusion complete (%d iterations) =====N) rr rrrfuse_nodes_oncer(rloop_index_inversion_in_fusion)rlr>r:old_lennew_lens rmrScheduler.fuse_nodess #4QU 2Ye*  EE ,,UU,Ke*  TE  %A$$Eq1u',1188,,UT,J;   sA3C%AC%% C3c/nURH:nUR[U[5(aUR 5OU/5 M< Xlg)z1 Unpack GroupedSchedulerNode into regular nodes. N)r>rrrr)rlrKrs rmrScheduler.process_grouped_nodessF.0 JJD   !+D2F!G!G dV  rpc[U5S:deUSR5nX lURU5n[ SSSS9 UR U5sSSS5 $!,(df  g=f)k Benchmark fused list of nodes and return the execution time in milliseconds on randomly generated inputs. rbenchmark_fused_nodesTcompile_time_autotune_time_us)rTdynamo_compile_column_usN)rrrrrr)rlr>rcrs rmrScheduler.benchmark_fused_nodessm5zA~~q$$&$""6*  #"&%D 007    s A"" A0c[U5S:deUSR5nX@lURU5n[ S5 UR XUS9sSSS5 $!,(df  g=f)rrgenerate_kernel_code_from_nodes hint_overrideN)rrrrrr)rlr>benchmark_kernelrrcrs rmr)Scheduler.generate_kernel_code_from_nodessi5zA~~q$$&$""6* ; <::};= <  A cV[RRnU(dg[R SX5 UR HnUR 5n[USS5(a$XB;d[U[R5(aMKURnX$n[U[R5(a'URUR5 URn[U[R5(dMXe:wdM[R!SUUU5 g g)z Check if selecting a Triton template would cause layout conflicts. Returns True if there's a conflict and we should fall back to ATen. FzNode %s has constraints %sr@NzOLayout conflict detected for %s: template expects %s but layout is frozen to %sT)rZrbuffer_layout_constraintsrrrbr3rrr+ReinterpretViewr@FlexibleLayout freeze_layout_with_exact_stridesr& FixedLayoutr)rl multi_node constraintsinpinp_namer@expected_layouts rm!_has_layout_conflict_for_template+Scheduler._has_layout_conflict_for_template"sgg77  . H$$C||~HC400.c2#5#566ZZF)3O&""3"34444_5K5KL&"..11o6O e#  7%:rpc 0[UR5GHeup[U[5(dM[UR[ R 5(dMHURn[RR(dUR5upEO [SUR555n[U[RR R5(aUR!U5(aUR5H:n[U[RR"R$5(dM8Un O [W[RR"R$5(dS5e[U[RR R5(a[R&(a0nXGS'[R&HmnURUS9n U R)5V V s0sHup[U [5(dMX_M n n n [+U R)5SS9SnXgU'Mo URR-U5 OURR/U5 GM[ R0R3UR45 UR75n SSS5 W R8n[U[ R:5(deUR8n[U[ R<5(deUR>(a[AXR>5 URBUl!UREXX5 GMh gs sn n f!,(df  N=f)aP Finalize a backing choice for MultiTemplateBuffers which did not already have a choice finalized through fusion. In the case of an extern choice, this will result in replacing the SchedulerNode. If a MultiTemplateBuffer did not have any fusion opportunities, finalizing a choice will force completion of compilation and benchmarking. c3# UH<n[U[RRR5(dM8Uv M> g7fr|)rrrrExternKernelCaller)rtimings rmr.`s7*E) & % @ @ S S #F*Es 7A AzZNo extern kernel detected to fallback to when layout constraints fail for Triton templatesNrc US$r+rzr\s rmr;Scheduler.finalize_multi_template_buffers..s qQRtrprr)#r=r>rrrr+MultiTemplateBufferr(r%force_extern_kernel_in_multi_templateget_min_choicer<choice_timingsrrrrrrmulti_kernel_hintsrhrfinalize_as_triton_callersfinalize_as_triton_callerrcurrent_originsr output_noder StorageBoxOperationBuffer origin_noder:r@ _replace_node)rlr:rrmin_node_unfusedrchoicecallershinttimingsrrmtriton_timings out_tensorbox out_storage out_buffers rmr)Scheduler.finalize_multi_template_buffersMs!,GA$ ..: 21144"YY **PP*4*C*C*E'$a'+*4*C*C*E ($$OO&&?? ==jII&0&?&?&AF) & % @ @ S S  4: 0 % 'B *"EOO$D$D$W$W  y  $OO&&?? 00NP(8 %+$=$=D&0&?&?d&?&SG-4MMO.,;DA#-a1I#J!%,;+. &))=)=)?^%TUV%WF,2DM%> <[U4SjU55$)Nc3D># UHoRT5v M g7fr|)r)rrrs rmr?Scheduler._replace_node..rename_deps..sKdsjj)9::ds r)rrs rm rename_deps,Scheduler._replace_node..rename_depssKdKK Krp)rrrr)rrqr>rJr3r1r r rrrrrrrGrr[r0rrrr) rlrrr:rnew_scheduler_noder real_namernew_outold_outrs @rmrScheduler._replace_nodes{ "*9!77 C* 1 -?$--/*3E 0??4#3#3#9#94;R;RSC 3377$GGyG.1hh +T L1<  1 11 -0;  * * 0 00 &&,!$  * * ,d.>.>.@! G4;  W--/ 0#MMGM ! (,~~$'+~~$'+~~$(,%rpc&[SU55$)Nc3# UHn[URS5=(a_ URSL=(aJ [URRS5=(a# URRRS:Hv M g7f)rN scatter_moderk)rWrrrrKs rmr,Scheduler._any_atomic_add..sp  AFFF # 9d" 9 ^4 9 ((L8 9sB B )r)rl node_lists rm_any_atomic_addScheduler._any_atomic_adds     rpc(URUSUS9n[R"U5n[RR R 5nUR5(dSnXd4$URSUS9n[U[5(deXd4$)NT)rrtriton_) kernel_name source_code) rrloadrr async_compile AsyncCompileuse_process_poolrrr)rlr>rsrc_codemodrfuts rmcompile_kernelScheduler.compile_kernels77 D 8 x(55BBD --//C z &&9(&SCc<00 00zrpc^^^^^^^^^^ ^!^"^#^$^%^&^'^(^)^*^+[STT455n[R(dU(d[R S5$TR 5(a-[ TR5[R5(a*TR5(dTR5(a[R S5$TR5nUSR5mT(deTRS:Xa)[RS:wa[R S5$TR5n[[ R""XE55nTR%U5(a[R S5$SSKJm [+TT5m+USR5mTceSUU4Sjjm%U(Ga}[S TT455(GacTR5SLmT(aTR5OTR5m*[ T*[R,5(deTR/T*5(a[R S 5$0m$/m"[R0GHInT*R3U5m[5TR75S S 9Hup[ U[8R:R<R>5(dM:T*RAU5 T"RCU/TREXhRFS 9Q75 SSS5 M [IS5n Sn 0n T"HWupnU bU RK5 T*RAU5 TRYUT5unnUX'UU :aUn Un SSS5 MY U T*RZU'[ U [\5(deU T$U'GML [R^mST*Rb55n[e5=(a! T(+=(a U[Rf:*m#[IS5[IS5sm'm(Sm&T#(dPT*R35mT*Ri5um&m'[5TR75[jRl"S5S 9nOT*RbVs/sHnUS4PM nnT(a-T(aTRoU5OTRoU5um(nO9T(d[R S 5$TRq5m([sTTT(5m)SSK:J;n /m"SnUHunn[ U[\5(dMT(d-[yUS5(aURzT*Rz:waMQT(a UT'T(-:a O_US- nU[Rf:a ODT*RAU5 T"RCU/TREU5Q75 SSS5 M [}T"5S:Xa[R S 5$SUUUUU"U#U$U%U&U'U(U)U*U4Sjjn[RUT"SS5$TREU5mTREU5m!TREU5m SUUUU U!U%UU+4Sjjn[RUT SS9$!,(df  GM=f![LaTn[NRQ[RRT5(a [NRWST(dSOSU5 SnAGMSnAff=f!,(df  GM=fs snf!Ua SSS5 GMf=f!,(df  GM"=f)o If config.benchmark_fusion is False, always return True. Otherwise, return True if fusion can brings speedup. c3# UHDnUR5=(a( [UR5[R5v MF g7fr|)rrrr/r+rrKs rmr.Scheduler.speedup_by_fusion..sD $ MMO J1..0"2H2HI J#sA ATrrarCompilationErrorNc z>[R[R5(aXU-:aE[R STR 5TR 5[ X-U- S55 g[R STR 5TR 5[XU-- S55 gg)Nz9can fuse (benchmark): fusing %s with %s cause %sx speedup.3fz=cannot fuse (benchmark): fusing %s with %s cause %sx slowdown)rrrDEBUGrrErErF)ms_fusedrrrrs rm log_fusion/Scheduler.speedup_by_fusion..log_fusions&&w}}55Ci'$$S..0..0"syH&.=s aPQdrprrinfException in compiling %s: %sr@rBc3B# UHn[U[5v M g7fr|)rr)rrys rmrrhs%ASA 1677ASrRr) CantSplitallowed_prologue_inpsc >[S5nSn0nT(awT(a[T[R5(deTR 5mTR 5ummTVs/sHnUST;dMUPM snm[ TU4SjS9mTGH\upEnUbUR5nO'T(dURnUR5 OSnT(a=TR!U5 TR#UT5upXU'X:aU nUnSSS5 MTU:H=(d TT-TUT-:n U(dMU (dMUR5 UR$(aUR&(deUR$Sn U R&n U R(n[+TTUR&U UUR,R.[0R2"T55nU(dGM[Un O T(a T"UTT5 T(a UTT-:aWUbT[4R6(aUTS'TR9T5 OTR;U5 T(aUTR<S'gg s snf![aTn[R[R5(a [RST(dSOSU5 SnAGM8SnAff=f!,(df  GMP=f) Nrrc>TUS$rrz)rrs rmrKScheduler.speedup_by_fusion..benchmark_when_ready..snQqT&:rprrr@rBTF)rrr+rrrrrFr precompilerrrrrrswap_as_triton_callerr launchersn_regsn_spillsrbmreqrrCrr(rrr_choice_timings) min_ms_fusedms_fused_choice new_timings fut_choicerri mod_fusedresrrpathfusible_choicecompiled_kernelrrshould_fuse_epiloguebench_epiloguerrcr5future_choicesget_choice_timings_async hint_override_best_fusion_choicer min_choicerr ms2_fusedrrls rmbenchmark_when_ready9Scheduler.speedup_by_fusion..benchmark_when_readys$U| "& +%*ZAWAW*X*XXX%/%>%>%@N&0&?&?&AOJ +9&*8J%a=N:#*8&N &,&:&N 2@-FI!!-"(--/C!/"+"3"3CNN,"&C&'==fE-1-L-L ) &.NH 3;/'6/7 28FE'&0N"Sy>&+AI+MM' 3>>"--/#&==V]]B#B.1mmA.>O+:+A+AL-<-E-EN3A # # & , . & 6 6 0 7 7 ?40 4328 %y2@|"|S#6',#)*D%100AP8>"==< #<<_M%;F 2248 }&.%!%227==AA&,, ?2A z ! !!FEs=' I+8I+I0-#I0I0,"K0 K:AK  K K! c(>^^^^^^SSKJn T ST ST S4HnUcMUR5 M TRT ST 5umm[R "T5(a T"S5 gTRT ST 5umm[R "T5(a T"S5 gTRT ST 5umm[R "T5(a T"S5 gT"TTT5 [ S5(a[TTT-:aRTT4TR;a@TRRTT45 [S5RUUUUUU4S j5 TTT-:$!Ua gT anS [U5;aSnAg eSnAff=f) Nr)NoTritonConfigsErrorrz%register spilling of the first kernelFz®ister spilling of the second kernelz%register spilling of the fused kernel slow_fusionc $>TTTTTTTTT-- S.$)N) kernel1_pathkernel1_latency kernel2_pathkernel2_latencyfused_kernel_pathfused_kernel_latencyslow_down_ratiorz)rrrpath1path2 path_fusedsrmrr`s&053605365?8@3;sSy3I%rpLoop-carried variableT) )torch._inductor.runtime.triton_heuristicsr rFrrisinfr rrrrr")r rrrrrr*r+r,rrcfuture_and_mod_l1future_and_mod_l1_fusedfuture_and_mod_l2rrlrWs @@@@@@rmrr-sA*!,)!,/2  ?JJL  "&!@!@)!,"JC zz#CD$!%!@!@)!,"JC zz#DE$+/+J+J/2,(Hj zz(++CD$xc20 >>$c 1"EN$2I2II//33UENC(7?? $cCi//+! '.#a&8#s<E*AE*5;E*1;E*-AN!SIF% @ @ U U!#99&A&-- &!%!4!4$3CWCW"5""BA "T %U| CG 1?-FI !!-"MMO$99&A)-)H)H%v*$/7 +#l2+3L.4OBA2@(=H **=9!/3KLLLLBQ0?U";X$==N!$%AKASAS%"  )*R&&R&&*Q*QQ % U|U5\HC15J+!+!:!:!<",";";"= C&,"((*0C0CA0F'# 8B7I7I&J7I!1v7I#&J'..{;33K@U'',,U33224=/)"a'#((//j !j !j !X --$nQ&7&:  !% 3 3K @  $ 3 3K @ &*&9&9/&J #F F P --09PQR9S. g BA"%!%227==AA&,, ?2A z ! !!BAF'Kb%!!>= ! >=sa-Z,Z?7$\ 3\3 ] $\8, Z< ? \ A\\ \0 8] >] ]  ] ] c<URUR5$)z0Look up the node in Scheduler name_to_fused_node)r1r9rs rmrScheduler.get_fused_nodeys&&t':':'<==rpcP[RSUR5UR55 UR5nUR5U:XdeUR U5R X5nUR U5 UR U5 URU5 URRUR5Vs0sHofR5U_M sn5 URRU5nUbXpRU'U$s snf)Nzfusing %s with %s) rrr3rrrtr,rr1rrr~r)rlrrrarcnode3rLstream1s rmfuse_two_nodesScheduler.fuse_two_nodes}s ,enn.>@PQ!!#!V+++  (--e;5!5! &&U__EV'WEV e(;EV'WX%%))%0  )0   & (XsD#cURX5(a5URX5(dU"5(aURXU5 ggNTF)rwill_fusion_create_cyclerK)rlrr speedup_fnras rmfuse_if_speedupScheduler.fuse_if_speedupsA MM% ' '11%??   k :rpcU(Ga/n0n[5nUGH(nXa;a[X5S:deXRS5n[X5S:XaURU5 UR 5upX:Xa[ X5(deUn OX:Xde[ X5(deU n URU 5U LaMUR(a3URRn U ceURU 5 Xv4XK'MURXURU5(dGMURU5 GM+ [U5Hqn XKup|URURUR5URUR5URU5(dM`URU 5 Ms UHn URU 5 M U(aGMgg)z Evaluate pending template fusions for a set of fusion candidate nodes. The fusion candidate nodes are pointwise nodes as potential epilogue or prologue fusions rrN)rrrrrr?rBrrirrQrgrrr) rltemplate_fusion_candidatesratemplate_futuresfuture_to_pending_fusionfusions_to_remove candidatepending_fusionrrrAfcands rm"_evaluate_pending_template_fusions,Scheduler._evaluate_pending_template_fusionss)-/  %@J| 7 ;6ABaGH">@ %-e;;;;$)M ----e;;;;$)M &&}5]J!((&--44A=(=$++A.3A2M,/++n&@&@+*--i8K8P""23'?'B$''''(<(<=''(<(<="..  &))$/4'*..q1'q)(rpc^^^SUUU4SjjnUGH\upxU"Xx5 TRU5nTRU5n[Xx5(aXx4TR;aMSTRXxU5(dMlTR Xx5(aMTR Xx5n U R b[U R UUU RS9n [Xx5(aWXx4TR;deTRRXx45 [Xx5n X;a/X;'X;RU 5 O U TU'U TU'GM6U R(dGMJTRXxT5 GM_ g)Nc>TRU5T;dTRU5T;GaTRTRU5TRTRU555nUceUR5up4URnTR US5 TR US5 TRU5ULdeTRU5ULdeU"5(aTR X5(aGMTR X4T5 TRU5T;aGMTRU5T;aGMggr|)rrrrgrrOrK) rrrY node_key1 node_key2 is_speeduprapending_fusionsrls rmresolve_pending_fusions.resolve_pending_fusionss3 ##E*o=&&u-@!0!4!4''.#''(;(;E(BC"&111'5'F'F'H$ +77 ##It4##It4**95BBB**95BBB!||t'D'DU'R'R##I+F+##E*o=&&u-@@rp)rgrrrir)rrDrxrrOrDrgrrirrFrrfrK) rlpossible_fusion_pairsrctemplate_fusion_nodesrarrdrr fusion_resrYtemplate_pw_nodes ` ` ` rm_try_fusion_pairsScheduler._try_fusion_pairss G$ G$ G G G82LE $E 1''.E''.E#500Nd&@&@@}}.33EAA!33EA ))5%2$.$:$:##)00 &N*%77 %~T5O5OOOO2266~F+B5+P(+HFH1C-?FF~V1?.1?.!--##E+>W2rpcN[5nUR5HnUR5upVURnXs;d[ XV5(aM8UR U5 UR U5ULdeUR U5ULdeURXVXq5 M gr|)rrrrgrDrrrQ)rlrarcseen_pair_speedup_fnrYr`ra is_speedup_fns rm_finish_pending_fusions!Scheduler._finish_pending_fusions=s @J|.446N#1#B#B#D I*66M48J99 $ $] 3&&y1Y> >>&&y1Y> >>  } R7rpc [UVVs/sHup4[X45(dMUPM snn5n/nUH@up4[X45(aXE;aURX445 M.URX445 MB U$s snnfr|)rr?rBr)rlpossible_fusionsdeferred_prologue_fusionsn1n2epilogue_template_nodesnew_possible_fusionss rm_handle_template_overlap"Scheduler._handle_template_overlapUs#-. M.FB2DR2LR. M#  "&FB!"))b.K)00":$++RH5 ' $# Ns A= A= cURU5 [U5n[R[R 5(aB[R S5 UH'n[R SUR55 M) 0n0n/nURUU5n[R(d[R(aURX5nURUUUUU5 URX55 URXc5 UR!5 U(a&URUUUUU5 URXc5 [#USS9nUR%U5nU$)z Combine eligible nodes into FusedSchedulerNodes. This relies on two key functions to control the logic: - self.can_fuse(): checks if a fusion is legal - self.score_fusion(): assigns priority to a given fusion zfuse_nodes_once, candidates:z %scUR$r|rr\s rmr+Scheduler.fuse_nodes_once..s!++rpr)r2rrrrrrrget_possible_fusionsr(rrrxrjror\clearrrz) rlr>rrarrcrgrsrrs rmrScheduler.fuse_nodes_onceis^ !!%( '  " "7== 1 1   ; <#  )=)=)?@$   OQ  " 44     # #v':':#<<       !     $$[B //0ES##% $  " ")%    3 34I W{(=>..u5 rpcz[UR5nSn[UR5n[R SU5 [ [ RU55GH[upV[ RU5n[U5S:aM,UbX1:a GO)URU5(d[R SU5 MeUS- n[RS:n[ USRUSUS9n[RS [U5U5 UHn URU 5 M URU5 UR R#UR%5V s0sHoR'5U_M sn 5 UR(R+US5n U cGMMXR(U'GM^ [-US S 9UlUR/UR5Ul[RS UU[UR55 UR1UR5 gs sn f) z Groups parallel nodes rz2ComboKernels: Generating with num_ck_nodes = %s...rNz)ComboKernels: Not speeding up %d-th grouprTrz0ComboKernels: Combining %d nodes for %d-th groupcUR$r|rr\s rmr5Scheduler.create_combo_kernel_nodes..sq{{rprzDGenerated ComboKernel nodes: %d ComboKernels, totally %d -> %d nodes)rr>rrrr=rCrrxspeedup_by_combo_kernelr(rr+rr,rr1rrr3r~rrrzr2) rlrVrarmnum_nodes_orignumrr`r|rrLstreams rmr#Scheduler.create_combo_kernel_nodess!, TZZ F U' & D DT J NC3CCINI9~!'E,@// :: EsK QJE$;;a?O4! &&*. / K HHBI  """4(" OOK (  # # * *4?4I4I4KL4Kq{*4KL  ((,,Yq\:F!39##K0E FK-BC 33DJJ?   R    O  !!$**-Ms)H8 cLUHnURUR5 M gr|)r2r1)rlr>rs rmr2Scheduler.prune_redundant_depss D  % %d&=&= >rpc^^^ ^ /m [[[[4"5m SUU U U4Sjjn[R"[ 5nUHEnTR U5(aMUR5HnXFRU5 M MG UR5H nU"U5 M [R(ak[R"[ 5nUH,n[USS5n U (dMXRU5 M. UR5H nU"U5 M TRT 5m T RTRSS9 [ R#S[%T 55 T $)zN Helper to find all legal fusion opportunities, sorted by self.score_fusion() c>[U5HupUUS-US-[R-HnX#4nUT;aMTRU5 TR X#T5(aTR U5 MIUR 5(dUR5(dMuTR X2T5(dMTR X245 M M gr+)r=r()max_fusion_buffer_group_pairwise_attemptsrrrrrrx) r> node1_indexrrrrrrrbrls rmcheck_all_pairs7Scheduler.get_possible_fusions..check_all_pairss&/&6" "!Ok'FF'GE !.Cd{ HHSM}}U3CDD(//4++--1A1A1C1C&6JJ)//?!'7rprNT)rreversezfound %d possible fusionsr>rSrrv)rrr_rrrunfusable_noderrrr(aggressive_fusionr*get_possible_fusions_with_highest_priorityrscore_fusion_keyrrr) rlr>rrbuffer_names_groupingrr node_groupinggroup_groupingrrrrbs ` ` @@rmr}Scheduler.get_possible_fusionssV% 13D DEFH @ @(!, 7 7 =D""4((--/%*11$70 399;M M *<  # #(44T:Ngt45")006"0!6!6!8  ."9 JJ   $"7"7F4c:J6KLrpc^^^^^[["5mSUUUUU4SjjmUR5RR 5UR5RR 5-mUR RR 5UR RR 5-T- m[ UU4SjT55nU(a[X5"S5 U$)zf Finds whether there's a path from node1 to node2 (or vice-versa) caused indirectly by other fusions. c,>[U[5(a~UT;axTRU5 UR5R T5(ag[ TUR -5=(d# [UU4SjUR T- 55$g)NFc3N># UHnT"TRU5v M g7fr|rprrL found_pathrls rmrIScheduler.will_fusion_create_cycle..found_path..:s,H!DA#4#:#:1#=>>!D"%)rrrrissubsetr{rr)rcombined_ancestorscombined_namesrrlvisiteds rmr6Scheduler.will_fusion_create_cycle..found_path)s$ 233G8K D!++-667IJJ!  ?@CH!%2D!DHErpc3N># UHnT"TRU5v M g7fr|rprs rmr5Scheduler.will_fusion_create_cycle..Hs&WDVqJt66q9::DVrzwill create cycler )rrr_dictrrrr)rlrrcyclerrrrs` @@@@rmrO"Scheduler.will_fusion_create_cycles/02  2  % % ' - - 2 2 4'')//446 7  OO ! ! & & (5??+@+@+E+E+G G WDVWW  e #$7 8 rpc^^ SSKJm S U4SjjnU"U5nU"U5n[U 4SjU55n[U 4SjU55nURU5nSn UHn U [ U S5- n M TR X5n [RRRU S U -5(ag g![ a  gf=f) a Return true if fusing the two nodes can potentially increasing peak memory. The implementation is more like a heuristic since we don't really know if we are at peak or not when trying to fuse these two nodes. The order of nodes may change later which makes the peak memory estimation hard. Here is how we decide the LOWER BOUND of extra memory allocation if we fuse these 2 nodes: 1. find all buffers read by each node with a single user. These buffers are supposed to be reused if we don't fuses these 2 nodes 2. find the intersection of these buffers for the two node and sum the total buffer size. If we don't fuse these two nodes, we can at lease avoid this much memory allocation. Note that the extra memory allocation is not necessarily causing peak memory increase. This is just a heuristic. We return true only if the saving for fusion can not trade off the extra memory allocation. r)buffer_reuse_keycP>/nURRHnTRRUR5nU(dM1[ UR 5S:XdMLURR5(dMmURUR5 M U$r+) rrr[rrrr0rhas_tensor_outputr)rrrMrrls rm_find_single_user_inputsKScheduler.can_fusion_increase_peak_memory.._find_single_user_inputsdswF&&,,&&**277333syy>Q.3883M3M3O3OMM#((+-Mrpc34># UH nT"U5v M g7fr|rzrrrs rmr.r#S]c$4S$9$9]rTc34># UH nT"U5v M g7fr|rzrs rmrrsrrTrrFrT)rr_rzlist[ir.Buffer]) rrr intersectionr$rrrZrrstatically_known_gt) rlrrr lhs_dep_nodes rhs_dep_nodeslhs_reuse_keysrhs_reuse_keyscommon_reuse_keysmemory_overheadr bw_savingrs ` @rmcan_fusion_increase_peak_memory)Scheduler.can_fusion_increase_peak_memoryMs* 6 #   17 07 ##S]#SS##S]#SS*77G$C 3s1v;.%,,U:  77   / /i P P  s(C CCc[UR5Vs/sHoDR5PM snUR5Vs/sHoDR5PM sn-5n[SURR55n[SURR 55nXv-n[5n URRHAn UR U RU5(dM&U RU R5 MC [SURR 55[SURR 55-n [SURR55[SURR55-n X- n X- nX-n[U5U:$s snfs snf)Nc38# UHoRv M g7fr|rrs rmrEScheduler.fusion_prevent_too_many_reads_and_writes..s&T;SCxx;Src38# UHoRv M g7fr|rrs rmrr%R:Q3hh:Qrc38# UHoRv M g7fr|rrs rmrrs$ 7HH 7rc38# UHoRv M g7fr|rrs rmrrrrc38# UHoRv M g7fr|rrs rmrrs% 8HH 8rc38# UHoRv M g7fr|rrs rmrrsD+CCxx+Cr) rrr3rrr$can_buffer_be_removed_through_fusionrrr)rlrr thresholdrfused_node_namesnode1_write_namesnode2_read_namesreads_removed_through_fusionwrites_removed_through_fusionreall_read_namesall_write_names unique_reads unique_writesunique_io_bufferss rm(fusion_prevent_too_many_reads_and_writes2Scheduler.fusion_prevent_too_many_reads_and_writess&).): ;):]]_): ;+0??+<=+<4}}+<= > '&T5;L;L;S;S&TT%%R%:K:K:Q:Q%RR'7'K$:D%**11I88 0.11)..A 2$$ % 1 1 7 7$  C5+<+<+B+BC CD %% % 1 1 8 8%  D5+<+<+C+CD DE &D (G )8$% 11M <=s GG c[[URUR- 5[URUR- 55nUS:$)a This function prevents fusion for nodes that can increase memory footprint. This problem is more common in horizontal fusion, where nodes that are far apart in the original order get fused, lengthening the live intervals of tensors. This is very evident in models with activation checkpointing, where the recomputed nodes from different checkpointed regions get fused and significantly increase the memory footprint. The current attempt is a quick, possibly hacky, heuristic to prevent the fusion of nodes that are far away in the original order. A better but difficult to implement heuristic would be to use live intervals of the buffers, find region of peak pressure in the original program and prevent fusion that crosses that peak region. We might need special care or good approximation in this implementation, as fusion of node changes live intervals, and re-computing live intervals and peak memory after each fusion can introduce large compilation overhead. @)rrrr)rlrrproximity_scores rmare_long_distant_nodes Scheduler.are_long_distant_nodessE* %//1 2 %//1 2 ##rpc(0nURR5Vs0sHoURU_M nnURR5Vs0sHoURU_M nnUGHn[RR U5n Xhn Xxn [ U [5(a[ U [5(dS[U 5S[U 53XH'MsU R5U R5:wa(SU R5SU R53XH'M[U R5[U R5:waSXH'MU R5n U R5n X:wa SU SU 3XH'GM!U R5U R5:Xa SU SU 3XH'GMPSn[ U [R5(dSU R 3nS U SU S U3XH'GM [#U5$s snfs snf) ze Try to decide reasons why fusion fail due to no shared memory even though there are common buffers. znot MemoryDep: rzdifferent numel: broadcastzdifferent offset: zMismatch loop orders: rzLayout: zUnknown reason: z. )rrrrZrrrr4rrrYr get_offsetnormalize_with_stride_orderr+rr@r")rlrrcommon_buf_namesreasonsrnode1_name2depnode2_name2deprZrlhs_deprhs_deplhs_offrhs_off layout_strs rmdecide_fusion_fail_reason#Scheduler.decide_fusion_fail_reasons383D3D3U3U3WX3WC((C-3WX383D3D3U3U3WX3WC((C-3WX(H''$$X.C$.G$.Ggy11GY9W9W%d7m_F4=/J!  "g&7&7&99'(9(9(;'R@RCU55 M= [EU5nU"UUS 5nUcgUR&R(UUR&R(U'UUR&R(U'URGS S 5 URIX5n[U[J5(de[LROS U5 U$s sn f)a Attempts to enable fusion between two nodes by inverting indexing patterns. This optimization targets cases where node1 has a contiguous write and node2 has a contiguous write but discontiguous read. By inverting the indexing in node2's read and write operations, we can make them compatible with node1 for potential fusion. Args: node1: First scheduler node (source) node2: Second scheduler node (target for inversion) Returns: int: Fusion score if successful, 0 if optimization not applicable r c3@# UHoR5v M g7fr|rdrKs rmrAScheduler.shared_data_after_inverting_indexing..%s2>axxzz>r?c38# UHoRv M g7fr|rrs rmrr1s. 8HH 8rrrindex0index1c3$# UHov M g7fr|rz)rrs rmrrls%T7Std7Ssr)generate_inverse_formulaTFz!Shared memory after inversion: %d)(r(rrr buffer_namesrrrrrr<rdrr4rrrr var_namesrr subblocksget_read_exprs$torch._inductor.invert_expr_analysisrvarsrAdd make_argsrrZrrcombine_modular_indexing_pairsrrrr$rr)rlrrnode1_buffer_namesnode2_buffer_namescommon_buffer_namesnode2_unmet_dependencies node2_read node2_writer node1_writes node1_writenode2_read_exprs read_exprread_expr_indexwrite_expr_indexrrAsimplified_termstermsimplified_read_exprinverse_formulascores rm$shared_data_after_inverting_indexing.Scheduler.shared_data_after_inverting_indexings&44 2E>2 2 2#..;;="..;;=0E"$.. % 8 8. $   $ 8 ' (1 , u  && '! +s53D3D3K3K/Lq/P$u006678 4 1 1 8 89: *i00 9 9 161B1B1I1IJ1I## 1I J ??, ."??3 +y11 "++-    !2!2 2  K$4$44 ??k.. .#j6J6J2Kq2P u{{)) *a / ;;   22 2EKK666  7 &%Tu{{7Q7Q7S%TT  A %./0   228< <&O'  : :8 DD DD&O' Q[[%%a( z?a II'' 2D  # #  ??E 3 ##3423GTUW  " 7URAX55$S$s snfs snf)as Right now just greedily reorder the loop of node1 to be compatible with node2, but ideally we should have some heuristics to reorder the loop for node2 to be compatible with node1 if that's more efficient. Return the amount of shared data re-computed in this method. If no such recomputation happens, return -1 (not return 0 since 0 is a valid amount of shared data). c3@# UHoR5v M g7fr|rrKs rmr>Scheduler.shared_data_after_reordering_loop..s8 .1HHJJr?r rrrFz?Don't reorder loops since both nodes are reductions: %s v.s. %s)!r(rrrrrrrrrrrZrrrrrrr:r;rr4r#r dep_size_hintrrr&rr3rrr$r)rlrrrrr rrr candidatesrrr_numel reordereds rm!shared_data_after_reordering_loop+Scheduler.shared_data_after_reordering_loops 00C8 !&8 5 5      %"3"3"5"5"..;;="..;;=0E"383D3D3U3U3WX3WC((C-3WX383D3D3U3U3WX3WC((C-3WX .K$1G$1G3356689!!((::#--/!;  /" z?a $'zx7J7J17M#N '9--Z5S5S   w// /   "g&7&7&99))'22 !!##77II##%%77II  # #Q    KKT55eC D  kYXs 6K!*K&c[U[5(a6UR5(+=(a [UR5(+$[U[ 5(a~[UR[ R5(aURR5(+$UR5(+=(a [UR5(+$g)z. Is this node unfusable under any conditions. F) rrrrrWrrr+rrrs rmrScheduler.unfusable_nodes d2 3 3''))2U 3/  d5 6 6$))R%?%?@@9966888''))2U 3/ rpc:UR5[RR::agUR 5nUR 5nSnXEU-:a U"S5 g[ SUR555nU[RRRR4:Xa U"S5 gS SjnUR5n U R5(d5U"U R5(aUR!5(d U"S5 gg) zD Heuristics to avoid benchmarking predictably slow prologue fusions T皙?z@prologue fusion will not increase amount of bytes read in kernelFc3# UHTnURcMURR5H#nURS:XdMURv M% MV g7f)N call_function)rrr4r)rrLrs rmrEScheduler.check_prologue_fusion_heuristics_fusable..+sS .vv VV'')tt& AHH* .sA,AAz\prologue fusion will not increase attempt to fuse in padding bc it increases unaligned readscFURS:*=(a UR$)Nr)itemsizeis_floating_point)r$s rm low_prec_fpGScheduler.check_prologue_fusion_heuristics_fusable..low_prec_fp8s>>Q&B5+B+B BrpzVprologue fusion that must be upcast to fp32 not profitable for low precision templates)r$z torch.dtyperr{)rrZrinvoke_quant_opsrrrrrrkrlconstant_pad_ndr<r5rVr$rO) rl prologue_noderArW read_bytes write_bytesBYTES_THRESHOLD_MULTIPLIERrr$ template_bufs rm(check_prologue_fusion_heuristics_fusable2Scheduler.check_prologue_fusion_heuristics_fusable s   , , .!''2J2J J"88: #::< &)" 'AA B R S ",,.   uyy~~55==? ? n  C%??A 6688L..//!>>@@ h rpc^[U[5(a[U[5(dg[UR[R5(a)[UR[R5(dgUR 5(dUR 5(ag[ RS:XagURURpCUupVUupxUR5(d2UR5(dXh:wd[U5[U5:wag[URR5S:d#[URR5S:agTR[[URR555n TR[[URR555n [!X5[ R":agSU4Sjjn U "U5(d U "U5(ag/n [%['XW55H un upX:wdMU R)U 5 M" [U 5S:wagU SnUUUUnn[*R,R.R1UU5(aUUU4$[*R,R.R1UU5(aUUU4$g)a? Fusing two small pointwise nodes significantly reduces kernel overhead and launch overhead. However, slightly different sizes would prevent fusion. Here, we decide if expanding sizes of one node is profitible by allowing fusion, and returns the dimension to expand, node with smaller sizes, and new size after expand. Nrrc>URRHnURTR;aTRURnO%TRR UR5nU(dMe[ RRRX 5(dM[UR[5(aM g grN) rrrrZr[rrZrrUrrr-r)rrrrls rmhas_reusable_bufferIScheduler.get_expand_dim_for_pointwise_nodes..has_reusable_buffers((..99 ; ;; $ ; ;DII FI $ 0 0 4 4TYY ?II,,66yGG&y'<'<>TUU/rprr )rrrr+rrr(rrrrrrr r<rdrsmall_memory_access_thresholdr=rGrrZrrstatically_known_lt)rlrrn1_sizesn2_sizes n1_iter_sizesn1_reduce_sizes n2_iter_sizesn2_reduce_sizesnode1_write_memorynode2_write_memoryr1 mismatch_dimensionsidxn1_sizen2_size mismatch_dimmismatch_size1mismatch_size2s` rm"get_expand_dim_for_pointwise_nodes,Scheduler.get_expand_dim_for_pointwise_nodesHsl%//z%7W7W uzz2#4#4 5 55::r'8'899  ) ) + +u/M/M/O/O    )#\\5<<()1& )1&    !!##1=!S%77 u  '' (1 ,E4E4E4L4L0MPQ0Q"//T%:K:K:R:R5S0TU!//T%:K:K:R:R5S0TU " 722 3  u % %)TRR>:wa U"S5 gS%UUU4Sjjm[3U4SjUR@55(ag[T[[45(aTR5(d U"S5 gTRC5TRD-(a U"S5 gTR5(GaH[GT5(d U"S5 gTRI5(dTR5(a U"S5 gTRK5n U RM5n U (d U"S5 g[OSU RP55U - n TRS5U -(a U"S5 gTRU5(dTRU5(a U"S5 gTRW5mTSSHKnURY5nUH2n[[U4SjUR\55(aM)U"S5 g MM [T[^5(dT/O3TR`Vs/sHnUR5(dMUPM snn[-U5S :XdeUS n[-TSRb5S :XaU[-TSRbS R\5S :Xa,TSRbS R\S RULd U"S5 gUReTTU5(dgTR5(aTRU5(d%TRI5(d[gT5(d U"S5 gTRi5nUceURk5(a2[TR[R&5(d U"S 5 gTRS5[lRnRp-(d0TRS5[lRnRp-(a U"S!5 gTR5nTR5nUU:wa U"S"UU5 gAURsTTUS#9n[U[t5(deU(aCU[vRx:a/[vRz(aUR}TT5nUS :aUn[vR~(aZURTT5=n(aAUunnnURUU5 URsTT5n[U[t5(de[vR(a.U[vRx:aURTT5nUS :aUn[R[R5(a4[RS$TR5TR5U5 [lRRUTTU5(dgTRC5TRD-(acURTT5=(aJ [lRRUTTU5=(a! URU5RTT5$[lRRUTTU5=(a! URU5RTT5$s snf)&zR Determine if it is possible to combine node1 and node2 into a single fused node. FNTz/grouped node must not be fused with other nodesz node1 is nopz'node1 is extern but not a triton kernelz5node1's triton kernel doesn't support epilogue fusionz.node1 is extern but node2 is not SchedulerNodez3node1 is extern but node2.node is not SchedulerNodez4node1 is extern but node2.node.data is not Pointwiserrc3@># UHoRT:gv M g7fr|r)rrwritten_buffer_names rmr%Scheduler.can_fuse..sV>Us8822>Uz9epilogue reads from buffers other than the mutated outputc3*# UH oS:gv M g7f)rNrz)rusages rmrrI s;F5Fsz*node1 and node2 uses different buf layoutscV>UTL=(a UTL=(a TUR5;$r|)r)rrrrH s rm._is_other_node_that_references_mutation_bufferJScheduler.can_fuse.._is_other_node_that_references_mutation_buffers7 u,N#50N+z/K/K/MMrpc34># UH nT"U5v M g7fr|rz)rrrN s rmrrI s &D?tDD&rTznode2 is extern or nopznode1 must go before node2zprologue fusion turned offz2prologue fusion only supported for pointwise nodesz'template has no allowed prologue inputsc3@# UHoR5v M g7fr|r8)rrs rmrrI *sE_c<<>>_r?z;prologue fusion not implemented for kernel for these inputsz:template prologue can only fuse functional pointwise nodesr c3@># UHoRT;v M g7fr|r)rrGprologue_nodess rmrrI :sQytyyN:yrJ z7template prologue can only fuse nodes with a single usezEtemplate prologue can only fuse nodes with a single use into templateztemplate epilogue not satisfiedz6multi-output template epilogue requires ComputedBufferz#fusion for buffer explicit disabledzdevice mismatch (%s vs %s)rz%s and %s has %s shared data)rr_)Nrr~rrrrrrrrrcan_fuse_multi_outputs_templaterrrrr+rrrrrrrmutation_outputsrrrrinner_fn_free_symbolscollect_inner_fn_symbol_usagerr@r>rrr=rr5get_allowed_prologue_inpsrrbrErrrrr0rrrr- r8r/rVrZrno_fuse_buffer_namesrr$r(score_fusion_memory_thresholdrr $expand_dimension_for_pointwise_nodesrD rrr r&rrrrr3r8rcan_fuse_verticalcan_fuse_horizontal) rlrr can_reorderrrJstream2rWnode2_inner_fn_free_symbolssymbolusagesr4runsupported_prologue_argsr node_outsrrLtemplate_snodestemplate_snoder, rcdevice2shared_data_scorenew_shared_data_scoreexpand_analysis expand_dim smaller_node expand_sizerN rS rH s `` @@@rmrScheduler.can_fuses E>  ' ' ) )))--e4G))--e4G"w':w?Q e4 5 5&&u- - e4 5 5u%     4#3#3    $ ) )% 7$8 e1 2 2j '7 7  A B e3 4 4U=N=N=P=P   e6 7 7ejj"*D*DEE=>:://11KLe]33DEejj.99IJejjooy99JKuzz223q8 88"'**"="=a"@"E"E Ve>O>O>U>UVVVOP+0**//*O*O*Q '5FFvN;F;;; 6 uzz../14 44zz&&q)00EJJ4E4EE@A -   JJ u8:PQ R R%%'' ( )  $ $ & 8 , -     .u5501!!##u'8'8':':HI779H$,$F$F$H !(=>EX__EE'( & %%'*CCQR--//53Q3Q3S3SPQ"__.N&s+ ,,. $CQsyyQQQUV$%,"%);<<!&AAaA  '1, ,,,Q/NN2&../14r*2215;;<A"2&..q177:??>Q[@@sSS     ..00%%''259956 224L+ ++5577  B--AALM  " " $qww'C'C C  " " $qww'C'C C 5 6!!#""$ W  ,fg >  44 54M5 +S1111 !F$H$HH11$($J$J5RW$X !$)$9!  6 6#FFueT TO T6E 3Z{  < >ueL  9900eU$5M""6*>>ueL MCBs :iicUR5n[X5n[[5nURHtnUR R URUR5n[U[5(aURXaU5(aMaXWRU5 Mv URRGH n[U[5(d[U[5(dM0UR UR R URUR55n U (dMxU Hn [U[5(a)UR!X5(aU R#U 5 MA[U[5(dMXUR%XUR&5(dM{U R#U 5 M GM [)S[*R,R/UR15555n X-(a U"S5 gUR35n U HJnUR4UR75n XR8U R:-(dMBU"S5 g g)z Check if it is legal to fuse a consumer (node2) into a producer (node1). We can fuse them if all the reads of node2 either match corresponding writes in node1, or are written by nodes that can be scheduled before the fusion of node1 and node2. c3:# UHnURv M g7fr|rrs rmr.Scheduler.can_fuse_vertical..s$ U HHUr zmemory deps did not matchFz(intermediate nodes between node1 & node2T)rErrrrrrrrr6rrrrr4r5fusable_read_and_writer,.fusable_stardep_write_and_read_on_empty_tensorrrr r rHrrr[r5r1r)rlrrnode1_buf_namesrWremaining_deps_by_namerrcd remainingrMremaining_depsnode1_op_namesr#s rmr\ Scheduler.can_fuse_verticals 002%7B47H++C((,,SXXsxx@D#w''D,A,A#e,T,T " ( / / 4 , ##**Bb),,ZG5L5L.22%%))"''277;Iy#B!"i00T5P5P66"((,#GMM "((,$+&$$  445K5R5R5TU$    + + ,224"D&&t,==?G 7 7 @ J JJJ>? # rpcl^URUR5;agURRVs/sH!nURUR:XdMUPM# nn[ U5S:wagUSm[ T[5(ag[ T[5(de[TR[R5(ag[TR5TRR::dgUR URnU/n[ U["5(a UR$nSnUHen U RR&V s/sHn U RU:XdMU PM n n U (dMDUS- n[)U4SjU 55(aMe g US:*$s snfs sn f)NFrrc3$># UHn[U[5=(ai [UR[R 5(+=(a9 URTR:H=(a UR TR :Hv M g7fr|)rr4r#rr%TMPr)rrwrites rmr-Scheduler.fusable_weak_dep..sn +D 4+,+DJJAA,JJ%++-,II+,+sB B)rrErrr rrr5r4r#rr%r} rrr"rrCrrr) rlweak_deprrr~ mutating_writesrrelevant_reading_nodesnum_concurrent_reads reading_noderrelevant_readss ` rmrScheduler.fusable_weak_deps == 6 6 8 8 **11 1zzX222 1   1 $" eW % %%++++ u{{DHH 5 5 %//*ekk.F.FF++H,A,AB "' e7 8 8%*\\ " 2L)44:::D99 ):  " A %  + !3"$q((W 6sF,F,F11F1cz[U[5(GaURRURUR5nX2R:wdR[ UR [R5(d)[ UR [R5(ag[R(a:URUR:wa UR5nUR5nURUR5(agUR UR :H=(aa [UR 5[UR 5:=(a/ UR S[UR 5UR :H$[U["5(aURRURUR5nURRURUR5nURUR:XaURbX4:Xaggrb)rr4rrrr#rr%r} r(rr#rr[rjrrr5)rlrr~ read_name write_names rmrr Scheduler.fusable_read_and_writes dI & &--11$))TYYGIZZ'&tzz488<<&u{{DHH==00T]]enn5T~~')11%**== ekk)? Nc%**o5?II/EJJ0EJJ>  g & &--11$))TYYGI..225::uzzJJ UZZ'JJ*+rpcf[U[R5(dgUR5(dgURR UR UR 5nURR UR UR 5n[U[5(aXE:Xaggrb)rr+rrrrrr5)rlrr~ writing_noder r s rmrs 8Scheduler.fusable_stardep_write_and_read_on_empty_tensorFs,(B(BCC--//))--diiC **..uzz5::F eW % %)*Arpc@[RRX5$r|)rZrget_dep_size_hint)rlrrs rmr Scheduler.dep_size_hintSsww((::rpc ^^^U4SjnU(a9[RX5(a[RX5nU"USS5$[UR[ R 5(aURR5(d*UR5(dUR5(aURRURR-nURRURR-n S Sjn SnUHHn U H?n U "X5(dMU[TRU 5TRU 55- nMA MJ U"USS5$[URR5[URR5-n [URR5[URR5-n[X5S-[X5:aX:aX!p!URRURR-Vs/sH9nXRR;dXRR;dM7UPM; nnU"[!UU4SjU55SS5$URRURR-URRURR--n[!U4SjU55nSnUS:Xa'TR#X5(aTR%X5nU"UUS5$s snf) a The first term in our fusion score that estimates number of saved memory operations. This function scores fusion candidates based on shared memory access patterns. Higher scores indicate better fusion candidates. Scoring strategy: 1. If nodes share exact memory deps (same buffer + same indexing), return the sum of shared dep sizes (original behavior). 2. If no exact matches (score == 0), check for same-buffer reads with different indexing (e.g., split operations reading different slices). - Give bonus if nodes read from exactly the same set of buffers - Score based on overlap ratio: common_buffer_size / total_read_size - High overlap (>50%) suggests good cache locality benefit from fusion c">T(aXU4$X-$r|rz)r buffer_overlap_scoreis_mix_order_reductionreturn_is_mix_order_reductions rm_construct_return_value>Scheduler.score_fusion_memory.._construct_return_valueos-5KLL/ /rprTcX:Xag[U[[45(a4[U[[45(aURUR:H$grN)rr5r4r)dep1dep2s rm_match-Scheduler.score_fusion_memory.._matchsJ<dWi$899j7I.?? 99 11rpFrc3H># UHnTRUT5v M g7fr|r )rrrrls rmr0Scheduler.score_fusion_memory..s!IDSD&&sK88D"c3F># UHnTRU5v M g7fr|r rs rmrr s!J7ID&&s++7Ir)r r3r r3)rrrrrr+rrrrrrrrr rrr_can_use_buffer_overlap_scoring&_score_fusion_memory_by_buffer_overlap)rlrrrr rr r node1_deps node2_depsr node1_dep node2_dep node1_dep_len node2_dep_lenrrcommon_memory_depsr s` `` rmrScheduler.score_fusion_memoryVs2 0 %):)C)CE)Q)Q &66uDE*5!T: :5::r'A'ABBJJ0022  ""  ""**0053D3D3K3KKJ**0053D3D3K3KKJ E' !+Ii33 ..y94;M;Mi;X"",(+5!U; ;E--334s5;L;L;S;S7TT E--334s5;L;L;S;S7TT  } ,q 03}3T T,$u!,,22U5F5F5M5MMMC++111S&> > J7IJJ ! A:$>>uLL#'#N#N$ 'u.BEJJ3s 6M >M cpUR5(dUR5(agUR5(dUR5(ag[R(d[R(GaUR 5nUR 5nU(aU(dg[ SU55n[ SU55n[ 5nUGHnURHn [U R[5(dM$U RR5(dME[U R5(dMaU RR5n U bW[U [R5(a8U R5n X[-(aUR!U R5 MMUR!U R5 M GM U(aUHnURHn [U R[5(dM$U RR5(dMEU RU;dMWU RR5n U b=[U [R5(aU R5n Xk-(a gM g M X4Htn U R"R$HWn UR&R)U R*5nUcM-UR5(dMD[-U5(dMV g Mv g)a Check if buffer overlap scoring should be used for this node pair. Buffer overlap scoring handles split/cat patterns where nodes read from the same buffer at different indices. We skip it when: - Either node is a reduction (different memory access patterns) - Either node is a template - Both nodes are prologue/epilogue candidates for the same template, because horizontal fusion would prevent them from being absorbed into the template kernel. For example, in: q = a[:64, :]; k = a[64:, :] return mm(q + 2, k - 2) "q + 2" and "k - 2" both read from `a` and would get a high overlap score, but fusing them horizontally prevents prologue fusion into mm (resulting in 2 kernels instead of 1). We allow buffer overlap scoring when: - The node outputs are not actually in the template's allowed_prologue_inps, meaning they can't be prologue-fused anyway, so horizontal fusion doesn't prevent any optimization opportunity. FTc3@# UHoR5v M g7fr|r8rs rmr.+TmsLLNNmr?c3@# UHoR5v M g7fr|r8rs rmrr r r?)rrrr(rrrrr0rrr_r=r/r+r4rX rrrr1rrr8)rlrr node1_outputs node2_outputsnode1_output_namesnode2_output_names&node1_prologue_eligible_template_usersrrGrA allowed_inpsrrrFs rmr )Scheduler._can_use_buffer_overlap_scoringsx4     5#5#5#7#7     %"3"3"5"5   &":":":!--/M!--/M! !++Tm+T!T !++Tm+T!T    3%IID"499.?@@ II11337 BB )- (C(C(E (4)2+B+B::,9+R+R+TL1@ F J J499 U ACFFtyyQ%&%.6(C # &tyy2CDD $ 5 5 7 7 $ -S S-1II,G,G,IM,8Z -r/F/F>>0=/V/V/X #5#D+0$E(-!!*)&++11C#66::388DH ,$00227AA$2'rpc^^ ^ ^Sm S U U4Sjjm[SURR55n[SURR55nX4-m T (dg[U4SjURR55n[U4SjURR55n[ XV5nUS:Xag[U U4SjURR55n[U U4S jURR55n [ X5n X- n U [ R :aU $S$) a Score fusion based on buffer name overlap when exact dep matching fails. This handles the split/cat fusion case where nodes read from the same buffer but at different indices (e.g., different slices from a split operation). Scoring logic: - If nodes read from exactly the same buffers: high bonus (encourages fusion) - For common buffers: score based on overlap ratio - overlap_ratio = common_buffer_size / max(node1_total_reads, node2_total_reads) - If overlap_ratio > threshold (e.g., 0.5): give proportional score - If overlap_ratio < threshold: minimal/no score (not worth fusing) Note on dynamic shapes: - When deps have unbacked symbols (dynamic shapes), dep_size_hint returns 0 - In this case, we use count * 10 as a proxy for size - This ensures fusion still works for models with dynamic batch sizes Note on multiple deps from same buffer: - A node may have multiple MemoryDep entries for the same buffer name (e.g., 4 split reads from arg0_1 at different indices) - We sum ALL dep sizes for each buffer, not just take max - This ensures overlap ratio is calculated correctly when nodes read multiple slices from the same underlying buffer ryc:>TRU5nUS:aU$T$rr )rrFALLBACK_DEP_SIZErls rm get_dep_sizeFScheduler._score_fusion_memory_by_buffer_overlap..get_dep_sizeKs%%%c*D!84 :): :rpc38# UHoRv M g7fr|rrs rmrCScheduler._score_fusion_memory_by_buffer_overlap..Orrc38# UHoRv M g7fr|rrs rmrr Prrrc34># UH nT"U5v M g7fr|rzrrr s rmrr Y$ )@#L  )@rTc34># UH nT"U5v M g7fr|rzr s rmrr \r rTc3X># UHnURT;dMT"U5v M! g7fr|rrr common_namesr s rmrr g.% .xx<' L  .**c3X># UHnURT;dMT"U5v M! g7fr|rr s rmrr lr r )rr3rr$)rrrrrr(min_overlap_ratio)rlrrnode1_read_namesrnode1_total_read_sizenode2_total_read_sizemax_total_read_sizenode1_common_read_sizenode2_common_read_sizecommon_read_buffer_size overlap_ratior r r s` @@@rmr 0Scheduler._score_fusion_memory_by_buffer_overlap+sJ< ; ;&%R%:K:K:Q:Q%RR%%R%:K:K:Q:Q%RR(: !$$ ).):):)@)@$ ! !$$ ).):):)@)@$ ! ""7O ! # "%% ((..% "  "%% ((..% " #&&<"U0E (58P8P'P # VW rpc[U5S:XaU$0nUHup4UR5UR5:XdeUR5n[URU5R X455nXb;aX44/X&'MoX&R X445 M [ UR5[R"S5S9Sn[U5S:deU$)Nrrr) rrr$rget_fusion_pair_priorityrrrhr:r;)rlrr"possible_fusions_group_by_priorityrrrcfusion_pair_priority&possible_fusions_with_highest_prioritys rmr4Scheduler.get_possible_fusions_with_highest_prioritys  A %# #  +-LE##%)9)9);; ;;%%'F#&  (AA%O$ $MNL2H3HOON-25 . 4 4 6H>>55rpcD[RR"U/UQ76$)z Shim for list.sort(key=...) )rZr8 score_fusionrJs rmrScheduler.score_fusion_keys yy%%d3U33rpc[[RR55n[ UR 5H9nUR XR5 URUR5 M; g)zW Populate node.last_usage recursively (also for the nodes within a FusedSchedulerNode) N) rrZrrrr>rrrr)rlrrs rmrScheduler.compute_last_usagesV ))A)A)CDTZZ(D    35L5L M  & &t 7)rpc$[UR[RR- [RR R - 5GHnXR;a[URUnUR5(a5[RR RUR5 MlMnU[RR;dM[RRUn[U[R5(a+[RR RU5 M[U[R[R 45(aGM'UR"n[U[R$5(aUR'5(de[RR RUR"5 GM URR)5 g)z*Free any buffers that are no longer neededN)rrrZrrrUfreedr[re codegen_freerrrr+rrZr[rris_input_bufferr~)rlrrrstorages rm free_buffersScheduler.free_bufferss`  % %gg%% &gg""(( ) D '''&&t,<<>>GG((55chh?"---gg**40c2#5#566GG((55c:b&7&79M9M%NOO!hhG"7BMM::w?V?V?X?XXGG((55gllC) , !!'')rpcURR5HnUR5 M UR5 gr|)rkrflushr )rlrs rmr Scheduler.flushs.}}++-G MMO. rpc[U[[45(de[SS==S- ss'[R "[ SS95 UR5 UR5 SSS5 UR[RR5 UR5 g!,(df  NH=f)Nr extern_callsrF)increase_kernel_count) rrr%rrZset_kernel_handlerr0rrrrrUr )rlscheduler_nodes rmcodegen_extern_callScheduler.codegen_extern_calls  &(L M     ^,1,  ! !&u"E F  0 0 2  # # %G qww334  G Fs !B22 Cc|[UR5(aURc US35e[RR U5 [ UR5nUc[SUR35e[5(dURS:XaN[RRU5=nRS:a[U[R"55e[UR5(a.URS:Xd[![R"55eU"U5$)Nz( should have been normalized in loweringzUnsupported device type: rr|)rUrrrZradd_device_infor/rqr&rrget_device_propertiesmajorr7inspect currentframer8)rlrcdevice_schedulingrs rmcreate_backendScheduler.create_backends&++&&&,,*B h> ? B '5fkkB  $!:6;;-HI I|| v%%*ZZ%E%Ef%MM\TTWXX(w7K7K7MNN $$V[[E-A#G$8$8$:;; &&rpcUceXR;aURU5URU'URU$r|)rkr rbs rmrScheduler.get_backends@!!!  &$($7$7$?DMM& !}}V$$rpc^SU4SjjnUR5VVs0sH?nURcMURR5HnU"U5U4S_M MA nnn[UR 55nU(aJ[ U[ R"S5S9upg[RRRU5 ggs snnf)Nc>UTR;aMTRR[URR5VVs0sHupX_M snn5 TRW$s snnfr|)rrr=rr>)rLr:rls rm get_order*Scheduler.enter_context..get_orders^,,,$$++i >V,W>VdaQT>V,WX''* *-Xs A. rr)rLz torch.fx.Noderr$) rrrrrrr:r;rZrrU enter_context)rlrr rLrrrlasts` rmr Scheduler.enter_contexts +^^% %vv VV'')q\1 t #* %  w||~& 'x':':1'=>GA GG . .t 4  s C1Cc^URURn[U4SjU55=(a# XR;=(a XR ;$![a gf=f)NFc3n># UH*oR=(d UR5T;v M, g7fr|)rr3)rrGrs rmrAScheduler.can_buffer_be_removed_through_fusion..s)VPU C 3C CCPUr)r[r0KeyErrorrrr)rlrrr0s ` rmr.Scheduler.can_buffer_be_removed_through_fusionsj $$T*00E VPUV V 4111 4333   sA A('A(cURn[U[RRR 5(axUR =n(ae[U5upEU[R;dU[R;a0[U[RR5(deSU3$[RRRR(d[Rcg[U[ 5(a0UR"HnUR%U5nU(dMUs $ gURceUR'5(dUR)5S3$[UR[R*5(ag[UR[R,5(ag[/URSS5(ag[1UR5(ag UR3U5=n(aU$[RR4(a[7U5(ag g) zr Return the reason why we should partition the inductor graph on this node, or None if the node is cudagraphable. zcustom partition op: Nz6partition includes all ops when cudagraphs is disabledz opszDeviceCopy opszConditional opsunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opszdynamic shape ops)rrrrr+rrrPr(custom_should_partition_ops_ops OpOverloadrr^rHrrrshould_partitionrUr DeviceCopyrgrrT&_uses_cudagraph_unsafe_unbacked_symintcudagraph_skip_dynamic_graphsr-)rlrr0r4op_overload_packet_nameop_overload_namergrs rmr Scheduler.should_partitions)) gu11@@ A A%% %B %8DR8H 5 #'6+M+MM#v'I'II!"ejj&;&;<<<<./?.@AA &&--886>>FK d. / /..u56!M%yy$$${{}}oo'(- - dii / /# dii 0 0$ 49914 8 8) !$)) , ,0@@F F6 FM == 6 6-d33*rpc[5n[R(dU$URGHnURnUcM[ U[ RRR5(dMJURnUcM[[U5upVU[R;aU[R;aMUR5Hnn[RRR!U5n[#U[$R&[$R(45(dM]UR+U5 Mp GM U$)zS Collect output unbacked symints from ops in config.cudagraph_unsafe_unbacked_ops. )rr(cudagraph_unsafe_unbacked_opsr>rrrrr+rrrPr rZrrrr$r% UNBACKED_INTUNBACKED_FLOATr)rlunsafe_symintsrr0r4r r syms rm&_get_cudagraph_unsafe_unbacked_symints0Scheduler._get_cudagraph_unsafe_unbacked_symints]s 4><33! !JJDiiGgu'9'9'H'HII$$Bz8DR8H 5 #'v/S/SS$F,P,PP779gg&&//4!#(9(94;N;N'OPP"&&s+:'0rpcUR5nU(dg[U5nUHMn[RRR U5nUR HnXb;dM SU3s s $ MO g)Nz'uses cudagraph-unsafe unbacked symint: )r r-rZrrrr")rlrr node_symbolsr simplified_symfree_syms rmr 0Scheduler._uses_cudagraph_unsafe_unbacked_symintsnDDF5d; CWW--66s;N*77-DXJOO8 rpc0nUR[RR5 URH4nUR R 5Hup4URX'M M6 U$)zf Return a mapping from name strings to the corresponding graph inputs or base scheduler node outputs. )rrZrrr>rrhr)rlrJrrscheduler_buffers rmget_name_to_nodesScheduler.get_name_to_nodessdPR AGG001JJD*.*>*>*D*D*F&%5%:%: "+Grpc [[RR5VVs0sHup#X2_M nnn[[RR 55VVs0sHup#X2_M nnn/[Rl[U5HupgUR (aM/nURH#nURURU55 M% /n URH1n U RURU R555 M3 [RR R[UUU UR55 M gs snnfs snnf)zj computes a mapping from partition input/output indices to graph input/output indices for each partition. N)r=rZrrrpartition_mapsskip_cudagraph input_nodesrr output_nodesr3rQconstant_names) rl signaturesr> rname_to_graph_input_indexname_to_graph_output_index partition_id signature input_mappingoutput_mappingrs rmcompute_graph_partition_maps&Scheduler.compute_graph_partition_mapss;(11E1E'F% 'F)#DI'F "% (11I1I1K'L& 'L)#DI'L #& "$'0'< #L'' M!--$$%>%B%B4%HI. N!..%%&@&D&DT]]_&UV/ GG " " ) )! !",,  !(=% & s E'"E-c^SSjmSSjn[5R"SU56nUR"U4SjUR556 U"U5n[5nUHGn[R R RU5nURUR5 MI [[U[R"S5S95$) a9 Returns all symbol inputs which are required to be in scope to successfully perform codegen for this graph partition, including: - free symbols used in partition nodes - free symbols in partition input/node shapes, strides, and offsets. This is needed for recording cudagraphs for tensors with dynamic shapes. c[U[R5(a [5$[U[R5(a [ U5$[ S[U535e)z? Gets symbols used in input node shapes, strides, and offsets. zUnsupported input node type: )rr+rrrr(rrrs rmget_input_node_symbolsKScheduler.get_graph_partition_symbol_inputs..get_input_node_symbolssT $ 2 233!|#D")),,)$//*,I$t**VWWrpc&[SU55$)z Filters a set of symbols that are required for codegen. Skip symbols that are always internal to kernels, such as SymT.TMP, SymT.INDEX, and SymT.R0_INDEX. c3# UHVn[U[R[R[R[R 45(dMRUv MX g7fr|)r$r%SIZEFLOATr r rrs rmrVScheduler.get_graph_partition_symbol_inputs..filter_symbols..sI  A!  ))++  s AA  A r)symbolss rmfilter_symbolsCScheduler.get_graph_partition_symbol_inputs..filter_symbolss     rpc38# UHn[U5v M g7fr|r,r=s rmr>Scheduler.get_graph_partition_symbol_inputs..s Iyt,T22yrc38># UHupT"U5v M g7fr|rz)rrrr6 s rmrrB s N:Mwq$T**:Msrr)rz+ir.IRNode | sympy.Expr | ir.TorchBindObjectrOrderedSet[sympy.Symbol])r> rD rrD ) rrrrhrZrrrr"rr: attrgetter) rl partitionr( r? candidate_symbolsrr symplified_sr6 s @rm!get_graph_partition_symbol_inputs+Scheduler.get_graph_partition_symbol_inputss X= X % X  -  % ,7Al6H6H Iy I7    N+:K:K:M N ++<=(2 "A77++44Q7L JJ|00 1# &(*=*=f*EFGGrpc ^^/n[[RR55nTR 5nSUU4Sjjm[ [ U5[ U55GHjupg[5nUH,n URU RR55 M. URU5n [RRUV s/sHoRPM sn 5n [U RU R -V s/sH&n [#U [$5(aMU R&PM( sn 5U- n [U4SjU 55n [5nUHn URU R(5 M X- Vs/sH nX;dM UPM nnU RU5 U Vs0sHnX;dM XU_M nnU Vs0sHnX;dM XU;_M nnU Vs/sHnX;dM X;dMUPM nnU RU5 [U4SjU 55n U Vs/sHnT"U5(aMX_PM nnU Vs/sH$o[RR*;dM"UPM& nnTR-UU5n[/UUUUUU5nUR1U5 U R3XJ- 5nGMm USSS2$s sn fs sn fs snfs snfs snfs snfs snfs snf)z Gets signature for each graph partition, including input nodes, output nodes, and whether deallocating an input within graph partition. c>TRRUS5nUcg[URR[ 5(a,TR RUS5=n(aT"U5$gg)z Checks if buf_name resolves to a NoneLayout buffer (following mutation_real_name). Buffers with NoneLayout are not allocated so graph partition should not take them as inputs or outputs. NFT)r[rrrr@r?r)rZrris_unallocated_bufferrls rmrM FScheduler.get_graph_partition_signature..is_unallocated_buffersk ""&&x6C{#((//:66!% 7 7 ; ;Hd KK9K0;;rpc3Z># UH nTRRX5v M" g7fr|rrrrrls rmr:Scheduler.get_graph_partition_signature..N,/1D''++D771(+c3Z># UH nTRRX5v M" g7fr|rP rQ s rmrrR xrS rT Nr )rZr"rr{)rrZrrr# rGrrrrrr*rrrrrrr6rrrorI r<rr)rl partitionsskip_cudagraphsr+ unmet_output_namesrJrF r' output_namesrreturned_output_namesrrpartition_input_namesrrextra_input_namesr( input_deallocationextra_output_namesr) r* symbol_inputspartition_signaturerM s` @rmget_graph_partition_signature'Scheduler.get_graph_partition_signaturesZ '(@(@(BC--/   ,*- Z (?";* %I-7LL!##D$8$8$=$=$?@"%1$=$=>P$Q !'11<<.78id!!i8K"-!2!2[5G5G!G!GA)!W5!G "%//1/% ! 5?L !$++DOO<"2@!@D'@ ! " ( (): ;21D')4((1 2"1D'32221 "2"1D',0,L1 " " ( (); <$./1/% !21D,T2# "1 "7!6!''BSBS:S!6 !BB;M#:" #    1 2!6!K3K37K8 K8!K=?K=cURR5VVs0sH'up#U[RR;dM%X#_M) nnnUR R5VVs0sH'up%U[RR;dM%X%_M) nnnUR Vs/sH3nUR5[RR;dM1UPM5 nnURVs/sH%nU[RR;dM#UPM' n n[URUUUURU 5$s snnfs snnfs snfs snf)z Updates the partition signature by removing buffers specified in V.graph.removed_buffers. See [Note: Removed Graph Partition Arguments] ) r( rhrZrrr] r) maybe_get_namer* r<r_ r' ) rlr/ rrr( rr] rr) r* s rm.clean_removed_buffer_from_partition_signatures8Scheduler.clean_removed_buffer_from_partition_signaturessR!* 5 5 ; ; = = 177222 DL =  '99??A A 177222 DIA  ".. .""$AGG,C,CC .  "00 0177222 0  '  # #     $ $    )    s/$EE,$E E +0EE5"EEc^^^ ^ ^ ^ ^ SSKm [5m /m/m [U5VVs0sHup#X2_M snnm SUU U U U4Sjjm SU U 4SjjnUH8n[URR 5T U'T US:XdM0T "U5 M: /nSnU[U5:aT (dT(aT (a5T R T 5upsURU5 U"U5 T (aM5T(a5T R T5upsURU5 U"U5 T(aM5US- nU[U5:aT (aMT(aMU[U5:a [S5eU$s snnf)ad Reorder nodes to minimize the number of partitions via a bfs topological sort. This is the optimal reordering such that the number of partitions cannot be reduced further. This may be sub-optimal for other metrics such as peak memory. This does not change relative orders of two cudagraphable nodes, nor the relative order of two non_cudagraphable nodes. rNc>TUU4nTRU5(aTRTU5 gTRTU5 gr|)r heappush)rnode_with_indexcudagraphable_nodesheapq node_to_indexnon_cudagraphable_nodesrls rminsert_pending_nodesHScheduler.reorder_for_minimizing_partition..insert_pending_nodessA,T2D9O$$T**6H2ODrpc>URRH.nTUS:deTU==S-ss'TUS:XdM&T"U5 M0 gr)r succ_nodes)r succ_nodero node_to_indegrees rmupdate_indegreeCScheduler.reorder_for_minimizing_partition..update_indegreesO!]]55 ' 2Q666 +q0+#I.!3(3 6rprz Failed to schedule, while loop ran too long when reordering for minimizing the num of partitions rr_rrv) rl rFr=rr pred_nodesheappoprrq)rlr>r> rru r num_itersrrk rl ro rt rm rn s` @@@@@@rm reorder_for_minimizing_partition*Scheduler.reorder_for_minimizing_partitions_ 9=CEGI4=e4DE4Dys4DE  E E 4 4D%()A)A%B T "%*$T* -/ #e*$ #':)--(?@%%*) &--(;<%%&% NI#e*$ # #':': s5z ! ]FsE(c RSSKJnJn [[R R 55nU"UURUR[[R RR55U5upVURU5nU"XvU5upXS-:aU$U$)z` Reorder nodes to minimize the number of partitions if this only slightly increase peak memory. r)estimate_peak_memoryprepare_planning_infor ) rr~ r rrZrrr[r1rrr{ ) rlr>r~ r rdefault_peak_memoryrHreordered_nodesreorder_peak_memoryrs rmr0Scheduler.maybe_reorder_for_minimizing_partitions H"177#;#;#=> :O      # # qww++002 3  ; 7??F!5 "  s!: :" " rpc4/n/n/nSSjnUHnURU5SLnU(a,[UR5S:XaURU5 MIU(a U"U5(aURU5 MpURU5 M X#-U-$)z Reorder a node if it should be partitioned and has simple dependency: 1. move a partitioned node to the front if it has no dependency 2. move a partitioned node to the back if it is only used by OutputNode 3. otherwise do not reorder cUR5H8nURH%n[UR[5(aM$ g M: grb)rr0rrrc)rrrds rmonly_output_userPScheduler.reorder_for_partition_with_simple_dependency..only_output_user.s<'')99C%chh ;;$%*rpNrr )r rrr)rlr>frontmiddlebackr rr s rmr6Scheduler.reorder_for_partition_with_simple_dependency s*,*,(* D#44T:$F C(?(?$@A$E T"!&6t&<&< D! d#~$$rpc/nSn/n/nURHYnURU5SLnU(a)X&:wa$URU5 URU5 /nUnURU5 M[ U(a"URU5 URU5 [RR nUS:a^[ [X55HFunupU (aM[SU 55n X:dM*SXH'[RSUU U5 MH URXS9n URU 5 URX5 X4$)zz Given a list of BaseSchedulerNodes, split into a list of graph partitions and compute partition input/output signatures. TNrc3T# UHn[U[5(aMSv M g7frN)rrrKs rmr,Scheduler.graph_partition..`s$'!*A)!-CD!*s( (zFPartition %d has %d kernels, below minimum size %d, skipping cudagraph)rV rW )r>r rr(rcudagraph_min_partition_sizer=rGrcudagraphs_logrra r2 _log_graph_partitions) rlrV r' cur_partitionrW rnode_should_partitionmin_sizer:rF skip kernel_countr+ s rmrScheduler.graph_partition@sU+- ') JJD$($9$9$$?t$K !!H!!-0&&~6 " 2N   &    m ,  " "> 2==== a<(1#j2R(S$$It#&'!*'$L $.-1*&,,d($ )T"77!8  ))*5 ""::%%rpc [R[R5(dg[ S[ R R55nU(dg[SU55n[U5U- n[RS[U5UU5 [[X55Hunupx[RSU[U5UR(aSOS[UR5[UR55 UR(dMwUHn UR!U 5 M M g)Nc38# UHn[U5v M g7fr|)rU)rrcs rmr2Scheduler._log_graph_partitions..sO:NVF^^:Nrc3J# UHoR(aMSv M g7fr )r' r< s rmrr s!PZ?O?O!!Zs# #zCCreated %d graph partitions: %d cudagraphable, %d non-cudagraphablez3 Partition %d: %d nodes, %s, inputs=%d, outputs=%dznon-cudagraphable cudagraphable)r rrrrrZr device_typesrrrr=rGr' r( r) _log_non_cudagraphable_node) rlrV r+ has_gpu_devicecudagraphable_countnon_cudagraphable_countr:rF r/ rs rmr Scheduler._log_graph_partitionsws **7==99 O!'':N:NOO !!PZ!PP"%j/4G"G Q  O  #  *33z3N)O %A%  EI'0'?'?#_I))*I**+  '''%D44T:&*PrpcURU5nU(dgUR5nURbURR5OSnSU3/n[ UR5R nUR SU35 UbFURSSRSUR55S3nUR SU35 [RS USRU55 UbaURRS S5nU(a=UR5RS 5Hn [RS U 5 M ggg) z)Log details for a non-cudagraphable node.Nzreason=zir=rrc38# UHn[U5v M g7fr|)r")rr`s rmr8Scheduler._log_non_cudagraphable_node..s2Prks rmrScheduler.codegensO - .??))99((*]]4::. / . .s AA++ A9clSSKJn [RRn[ UR 5n[RR5 [RRSSU3UUS9 URU5 [[RRU5(deURU5nU[RRl [RRR5 [RRn[RRR[RR 5upxSSS5 [RRR#WW5 [RRR%XR5 [RRR&R)UR*V s/sHoR-5PM sn 5 g!,(df  N=fs sn f)z,Codegen a partition given its inputs/outputsr)SubgraphPythonWrapperCodegenT partition_) is_subgraph subgraph_nameparent_wrapper_codepartition_signaturesN)rr rZrrUr<rnset_current_wrapper_codeinit_wrapper_coder rre r write_prefixrgenerate is_inferencedefine_subgraph_launcher_fncodegen_partition_call allocatedrr) r3) rlrF r/ r r graph_partition_id graph_namepartition_coderrs rm_codegen_partition_wrapper$Scheduler._codegen_partition_wrappers Bgg22!$"?"?@ WW - - / GG % % *+=*>?$7%. &  MM) $agg224PQQ QQKKIVI8AAGG 5 GG - - /J ! 4 4 = =agg>R>R S N/02 88^T 334FR &&--)2)?)? @)?]]_)? @ 90 /: AsDH ?H1 H.cP^^^[RSUUU4Sjj5nU"5$)Nc3># TRTT5 TR(a[TRR5(a[TRRcS5e[ R RRTRR5 Sv TR(aL[TRR5(a([ R RR5 STlg!TR(aL[TRR5(a([ R RR5 STlf=f7f)Ndevice should have an index) %update_graph_partition_default_devicerurKrrrZrrUcodegen_device_guard_entercodegen_device_guard_exit)rV rlr+ srmctx1Scheduler.use_default_device_context..ctxs  6 6z: N**/@++00002288D1D$$??//55 3..3D//4444GG((BBD.2+ ..3D//4444GG((BBD.2+s B#E9'D+A%E9A&E66E9)rzIterator[None]) contextlibcontextmanager)rlrV r+ r s``` rmuse_default_device_context$Scheduler.use_default_device_contexts+  " " 3 3 # 3*u rpcN[U5S:XaUSR(dgSSjnSSjnSn[X5H upgUR(aMU"U5n O Ucg[X5H'upgUR(dMU"Xe5(aM' g XPlg)Nrrc6USR5nUceU$rr)rF partition_devices rmget_cudagraph_partition_deviceWScheduler.update_graph_partition_default_device..get_cudagraph_partition_devices'(|668 #/ //# #rpcDUHnUR5nX1:wdM g grbr)rF target_devicerrcs rmall_on_target_deviceMScheduler.update_graph_partition_default_device..all_on_target_devices("** "rp)rF r`rr)rF r`r rrr{)rr' rGru)rlrV r+ r r cudagraph_partition_devicerF r/ s rmr /Scheduler.update_graph_partition_default_devices z?a  1 (D(D  $  $ 5A   &*"$' $? I+++-KI-V*%@ & - $' $? I'''0D11 %@ 'A#rpcUR5up[U5S:a[SS==[U5- ss'URX5 [ X5H\up4[U5S:dS[U535eUR (aUR U5 MKURX45 M^ SSS5 [UR5n[RRRU5 US:as[RRceU[[RR5:Xd.SUS[[RR535egg!,(df  N=f) z Split nodes into partitions and codegen each partition into separate functions. This allows further applying different optimizations (e.g., cudagraph) to each function. rrcudagraph_partitionsz5Each partition must have at least one node but found NrzExpect z partition maps but got )rrrr rGr' r r r<rnrZrrUset_all_partition_namesr& )rlrV r+ rF r/ num_partitionss rmr Scheduler._codegen_partitions0sO "&!5!5!7 z?Q  Z !7 8C O K 8  , ,Z D(+J(C$ 9~*KCPYNK[\*++MM),33II)DEd;;< 44^D A 77))5 55!S)?)?%@@ .))A#aggF\F\B]A^_ @ E Ds A,E<< F c[R(aSSKn[R"5n[ 5n[ U5HnURS:Xa0URURRR:Xa OTURUR4nXd;d"SURSURS35eURU5 M URUlUR beUR(aG[R"R$(a([&R(R*R-5 [&R(R*R/5 UGHn[0R3[4R65(a4[0R9SUR;5UR=55 URAU5 [RB(aD[&R(R*RESURFRH55 URK5=n(GaXR:wd*URM5(dURO5(aURQ5 XR:wGa>UR(ai[SURRT5(aEURVbURY5 [&R(R*R[5 Xl[SURT5(aUR\cS 5eS n UR_5(a:[ UR`Rc55n U (a[eU 5S -OS n [&R(R*RgUR\U URh5 UR_5(aURbURkU5 [&R(R*RmS URFRH55 Xpl7URpRsURt5 URO5(aNURw[yUR{555upn UR}U5RXU 5 GOUURM5(aURU5 GO-UR5(a[R"[U5nUR}U5nS S KEJFn S S KGJHn S SKIJJn [UUUU45(aUnO[S[UU5<35eURU5 O[U[5(a!UR}U5RU5 Oc[U[[45(a!UR}U5RU5 O'[U[5(deUR5 [R"R(aUR}U5R5 URRsUR55 URRsUR55 [U[5(dWURK5nUbDURTS:wa4UR}U5R5(aURQ5 [SUR{555(a XplGMSUlGM URUR:wa[URce[SURRT5(a([&R(R*R[5 SUlURQ5 g![>a( [0R9SUR;55 GNgf=f)Nr_compile_innerzDuplicate stack frame :zs; did you add a decorator to one of the functions in this stack trace? If so, try using a context manager instead.z5Generating code for node %s with estimated runtime %fz6Generating code for node %s with estimated runtime 0.0c38# UHoRv M g7fr|rrs rmr%Scheduler._codegen..sD(>HH(>rr rc38# UHoRv M g7fr|rrs rmrr sC$:S$:r)CUDACombinedSchedulingr)XPUCombinedSchedulingz type(self)=rc3B# UHn[U[5v M g7fr|)rrrKs rmrr sJ9IA:a//9IrR)]r("check_stack_no_cycles_TESTING_ONLYtorch._dynamo.convert_frame traceback extract_stackrrrfilename_dynamo convert_frame__file__linenorrurrrrautotune_at_compile_timerZrrUwrite_get_raw_stream_headerregister_alignment_check_inputsrrrrrr3r rr size_assertscodegen_deferred_input_assertsrrrrurrr rKrcurrent_stream_idxgenerate_stream_ctx_exitr rrr~rrr rgenerate_stream_ctx_switching!codegen_deferred_alignment_copiesrsrrrrCrrrcodegen_templater rxrrrC codegen.cuda_combined_schedulingr rr#codegen.xpu.xpu_combined_schedulingr rrcodegen_combo_kernelrcodegen_mix_order_reductionrr codegen_noderrdebug_sync_kernel codegen_syncrrErrready_to_flushr)rlr>rstackrbframerrrc num_streamsunique_streamsr@rArBbackend_r rr rs rmr Scheduler._codegenPs8  4 4 .++-E7A|D!%JJ"22%--*E*E*N*NN~~u||4,U^^, <<>D .. IIO 224   t $""$$CCD(,(8(8(>(>D**v*111~~''''))JJL000**/@++0000 22> 99;,,FFH*0'(55%||7V9VV7&' 7799-78K8K8R8R8T-UN;IN 3a 7q(,,GG"LL' ;;++--$2E2E2Q2248 GG B BC$($4$4$:$:C !%   % % , ,T__ =!!484W4W)*51  (99!X!!((.""{{#=tD++F3T8V#%;=RS'G(KDJ=)9::,,T2D"9::  (DDTJD#5}"EFF  (55d;!$(>???? }}..  (557  ' ' . .t/D/D/F G  % % , ,T-E-E-G Hd$:;;*& v-((0??AAJJLJ9IJJJ%)"%)"EH   $"="= =&&2 22 !4!4!9!9::$$>>@! O!IIP s*3`.aacUSR5nU[RlX0lUceUR U5nUR X5$)rr)rrZrr+rrbenchmark_combo_kernel)rlrnode_benchmark_resultsrcrs rmr Scheduler.benchmark_combo_kernelsU1((* $!!!""6*--iPPrpc\^UnUSR5m[U4SjU55(dS5e[R(dgSSKJn S/pT0n[ U5HupxUR5n URU 5(a[RS5 URU 5upX4Xh'[R"U 5(a[RSU5 g XJ- nURU 5 M URX&5upnX- S :=(d US :n[R!["R$5(aSXM:dU(a$[RS['XM- S55 O#[RS[)XM- S55 X- U:=(d U$!Ua0n S [U 5;a[RS 5 S n A geS n A ff=f!Ua/n S [U 5;a[RS 5 S n A geS n A ff=f)rrc3H># UHoR5T:Hv M g7fr|r)rrrcs rmr4Scheduler.speedup_by_combo_kernel..sK?4??$.?r zsubkernel_nodesrr path1_listr r:rgrr"rrr ms2_clone _path2_list small_kernelrcs @rmr!Scheduler.speedup_by_combo_kernels   #..0K?KKK J K,,;rZ!#!/2HA)I##I..  R 55i@13 &-::b>>$$U! " IC   d #93< *.*E*E+ 'CK,9c  " "7== 1 1yL  E#)C2   I #0 $4 4Q$ *c!f4$$]     &#a&0  Y s=%A F= G6=G3$G.-G..G36H+<$H&%H&&H+crURUnURceURR5$r|)r[r get_layout)rlrZrs rmget_buffer_layoutScheduler.get_buffer_layout_s5x(xx###xx""$$rpc URHnUR5(dMURRHn[R R RUR5nU(dM?[U5S:XdMP[UR[[45(aMwUR5/:XdM[R RR!UR5 M M gr`)r>rUrrrZrrrrr;rr@r?r>rzero_dim_cpu_tensor_listr)rlrrrs rmrt$Scheduler.update_zero_dim_cpu_tensordsJJD{{}} ,,22DWW3377 BF+F3u< *"MMJ8I+J!!#OO-388<":. .rpc[U[5(aeURUn[RR R US9Ulg)z6Code-gen to enter the Stream context assigned to node.)rN)rrr~rZrrUcodegen_cuda_stream_enterr)rlr node_streams rmgenerate_stream_ctx_enter#Scheduler.generate_stream_ctx_entersLd$:;;;;))$/ #$77#7#7#Q#Q"$R$  rpcURce[RRR 5 SUlg)z1Code-gen to exit from the current Stream context.N)rrZrrUcodegen_cuda_stream_exitrks rmr "Scheduler.generate_stream_ctx_exits2''333 557#' rpc<XR;de[U[5(aSOURUnURU:XagURbUcgURcUbUR U5 gUR 5 UR U5 g)aMGenerate stream entering and exiting to properly run node in a multi-stream scenario. Stream context switching is only generated if ``node``'s assigned stream is different from the previous node's stream. NopKernelSchedulerNodes have stream=None and inherit the enclosing stream context (or do nothing if no context is active yet). N)r~rrr r( r )rlrrs rmr 'Scheduler.generate_stream_ctx_switchings*****$ 677 $$T*   " "f ,   $ $ 0V^   $ $ ,1C  * *4 0  ) ) +  * *4 0rp)rrnrrrkrrrrrsrurrrr[rZr1rJr~r>rjrrirrrrxr)r>zlist[ir.Operation]rrv)rz!dict[str, SchedulerDonatedBuffer]rurw)rZr"rr$)rZr"rr_rr{ry)rcrzrrv)rr"rrv)rrErr_)rj str | Nonerr{r)rgr_rrS)rrr>rrtuple[float, str]r|r>rrr{rrPrr")rrrcrrr1 )rir.MultiTemplateBufferrr{) rir.OperationBufferrr3 r:r$rrrrv)rrrr{)r>rrrPrz&tuple[LambdaFuture | None, ModuleType])rr_rr_rrd)rr_rr_)rr_rr_raOrderedSet[BaseSchedulerNode]rr_)rr_rr_rPr}rar5 )rT,dict[BaseSchedulerNode, list[PendingFusion]]rar5 rrv) rf1list[tuple[BaseSchedulerNode, BaseSchedulerNode]]rc&dict[BaseSchedulerNode, PendingFusion]rgr6 rar5 rr{)rar5 rcr8 )rrr7 rsr7 )r>rSrr{rrS)rVrPrrvr)r>rSrr{rr7 r!)rr_rr_rr$rr{)rr_rr_rz!tuple[str, ...] | OrderedSet[str]rr"r%r )r( r_rAr_rWrrr{)rr_rr_rz,tuple[int, SchedulerNode, sympy.Expr] | None)FT) rr_rr_r^ r{rr{rr{)r r6rr_rr_rr{)rr3r~ r4rr{)rr3r~ r5r rrr{rN)rr3rr{rr$)TFT) rr_rr_rr{r r{rr{rzint | tuple[int, int, bool])rrr7 rr7 )r>rrr)r r_rrv)rcrrBaseScheduling)rcrzrr9 rw )rr"rrrr{)rr_rr/ )rrD )r6dict[str, ir.IRNode | ir.TorchBindObject | sympy.Expr])r+ list[GraphPartitionSignature]rrv)rF r`r( r: rrD )rV list[PartitionType]rW z list[bool]rr; )r/ r<rr<)rz9tuple[list[PartitionType], list[GraphPartitionSignature]])rV r< r+ r; rrv)rF r`r/ r<rrv)rV r< r+ r; rz'contextlib.AbstractContextManager[None]rrrz%tuple[float, float, list[str | None]])r>rSrr{)rZr"rz ir.LayoutrO)rr/ )qr~rrrr&rrIrvrrrrpropertyrsetterrrrqrr]rr{r[rzrrtr}r|r}r rrrrrrrrrrrDrrKrQr\rjrorxrrr2r}rOrrrrr r rr- rD rr\ rrr rs r rr r rrrr r r r rr rr rIr r r# r2 rI ra re r{ rrrr r rr r r r r r rr rtr r# r( r r rrrs@rmr*r* s; o9b #6 p(S Q&&((7# ,"HMP^KZ+#Z , 6 S*4#&/<$6!F 808 8,%) *"  & >  >*6 >  >)0) )VVDp'8&'8+'8  '8  '8  '8R OS0AK / P&P/@P Pd > !3   0 !'  3 "D2$PD23D2  D2LO?PO?@O? L O? 3 O?  O?bS2S@S0$K$$U$(C&CC ! CJ3.j?6 &6 6  ; 6 p,&,/@, ,\7&7/@7 7r.2&.2/@.2MP.2 .2`$&$/@$ $6< <!<< < <|M&M/@M M^[ &[ /@[ [ z ;(;); ;  ;z`&`/@` 5`L"*. zM zM!zM zM $( zM  zMx;&;/@; ;z3)3)(93)BS3) 3)r%T   ' 7J   ;!.3*. mK mK!mK mK (, mK $( mK %mK^d d!d  dLR &R /@R R h6 Q6 :6@4@4 4 8*4 ) &'*% 5$    +:    =~!!F% " ? '1' 'RBH BHLBH " BHHK -K @JK &K Z" 0" " H?&? !?B& !>%,% %@5& B5&n";'";2";  ";H>0)  ) +)   ) V-;X 06-A--A;X-A -A^@ob Q4 Q . QN5`% H ( 11rpcr^\rSrSrSU4SjjrSSjrSSjrSSjrSSjrSSjr SSjr SS jr SS jr SSS jjr S S jrS!S jrSSjrS"SjrSSjrS#SjrS$SjrS%SjrS&SjrSS'SjjrSrU=r$)(r9 ic.>[TU]5 Xlgr|)rrr+)rlr+rs rmrBaseScheduling.__init__s "rpc\UR(aURR5 ggr|)r+r rks rmfree_buffers_in_scheduler(BaseScheduling.free_buffers_in_schedulers >> NN ' ' ) rpc[5$)z0Return a set of .codegen.common.BackendFeature()rrbs rmget_backend_features#BaseScheduling.get_backend_featuress |rpc[e)z? Check whether node1 and node2 can be vertically fused or not. rrs rmr\ BaseScheduling.can_fuse_vertical "!rpc[e)zA Check whether node1 and node2 can be horizontally fused or not. rrs rmr] "BaseScheduling.can_fuse_horizontalrK rpcUR5n[U[R5(dgUR 5(dg[UR [R 5(a[UR R5S:H=(ap [UR RS[R5=(a8 UR RSR5UR5:H$g)aF A Multi-Output Template (referenced in #144012) is a template node with MultiOutputLayout, and its output buffers are instances of MultiOutput. In this context, we verify whether node1 represents the Multi-Output Template and node2 corresponds to one of its outputs. If so, we further check if backend supports this fusion. Frr) r/rr+rrVrr=rrbrr3)rlrrr, s rmrT .BaseScheduling.can_fuse_multi_outputs_templates..0 ,(9(9::5577 ejj".. 1 1EJJ%%&!+Ouzz003RYY?OJJ%%a(113|7L7L7NN  rpcUR5(dUR5(a[RX5$[R X5(a [ X5$[ U[ 5(aURU5$[ U[5(aU[ U[5(a@[ UR[R5(de[RX5$[RX5$)z Fuse two nodes )rxrCrtrrrrr"rrrr+rr%r/rrs rmrtBaseScheduling.fuses     !1!1!3!3-225@ @  7 7 E E*58 8 6 7 7??5) ) 8 9 9j =? ? ejj"*D*DEE EE7EEeS S%**58 8rpc[e)zK Process the iteration sizes in case a transformation needs to be applied. r)rlrs rmrBaseScheduling.group_fnrK rpc[e)z Given a template node, generate a kernel. This function is only available for triton now. If the third-party backend behaves as a sub-class of TritonScheduling, it can override it or reuse it. r)rlrAepilogue_nodesrS s rmr BaseScheduling.codegen_templates "!rpc[ez4 Generate a kernel given a list of pre-fused nodes. r)rlr>rrs rmr.BaseScheduling.generate_kernel_code_from_nodess "!rpc[erX rrs rmr BaseScheduling.codegen_node "!rpc[er|rrs rmr *BaseScheduling.codegen_mix_order_reduction!rrpc[e)zd Generate synchronization code for the kernel. This method depends on the hardware characteristics. rrks rmr BaseScheduling.codegen_sync$r\ rpcg)z} Check whether the backend is requesting the scheduler to flush the generated kernel. If not supported, please return False. Frzrks rmr BaseScheduling.ready_to_flush*s rpc[e)zM Flush the generated kernel and python wrapper code to the source code file. rrks rmr BaseScheduling.flush1r\ rpc[e)rrrJs rmr$BaseScheduling.benchmark_fused_nodes7 "!rpc[e)zi Benchmark a compiled module and return the execution time in milliseconds on randomly generated inputs. r)rlrs rmr)BaseScheduling.benchmark_codegened_module@s "!rpcg)zt Return an unsigned integer which represents the priority of this fusion pair. The smaller is with higher priority. rrzrs rmr 'BaseScheduling.get_fusion_pair_priorityGsrpc[e)z Benchmark the list of nodes to combine and return the execution time and memory copy time in milliseconds on randomly generated inputs. r)rlrr s rmr %BaseScheduling.benchmark_combo_kernelPrg rpcU(a9SSKJn U"UU5n[RRR X$5 gg)Nr)'set_kernel_post_grad_provenance_tracing)rro rZrrUwrite_provenance_debug_handle)rl node_schedulerro debug_handles rmcodegen_commentBaseScheduling.codegen_commentYs<  UBL GG > >  rpr)r+zScheduler | Noneru)rcrrzOrderedSet[BackendFeature]r!r)rrrz"tuple[tuple[sympy.Expr, ...], ...])rAr_rU rrS rrr/ r|r2 )rz"FusedSchedulerNode | SchedulerNoderrv)rrrrvrwr0 )rrrr1 r%r= )rq rrr/ rrv)r~rrrrrD rG r\ r] rT rtrr rr r r r r rrr r rs rrrs@rmr9 r9 s#*"&"/@" ""&"/@" "&/@ 49&9/@9 9("3" +" "( "4 "4 "  "$%) "* " "" " "" "" " "0" ""&/@ "4" ."#'2   rpr9 )rz$torch._inductor.codecache.LocalCache)rgr_rr")rgr_rzCallable[[Any], Any] | None)rgr_rr)rrrr")rr_r1rKr[rrrv)r|)FusedSchedulerNode | GroupedSchedulerNoderrv)r|ru r+r*rrSrrv)rz)rzlist[list[int]]rrxrrvrz list[int])rr3 rbr4 rrvrQ) rr$rr$rr$rr$rrCrztuple[int, int])rrrrrr$rr$rr$rr$rrCrr{rw)rz ir.IRNoderrD )rr_rrD )rAr_rr{)rr_rr_) __future__rrr r{rr r rrr:rrrr rrrconcurrent.futuresrrrr r r r typing_extensionsr torch.utils._ordered_setrr+rrcollections.abcrrrtypesrtorch._inductor.codegen.wrapperrrrrrtorch._inductor.async_compiletorch.utils._pytreer~_pytreeretorch._dynamo.utilsrr torch._inductor.autotune_processrtorch._inductor.codecacherrtorch._inductor.irrtorch._inductor.metricsrr torch._inductor.stream_utilsr!%torch.fx.experimental.symbolic_shapesr"torch.utils._sympy.symbolr#r$r%torch.utils._tritonr&rr'r(r)r*r,analyze_preserves_zero_maskr-codegen.commonr.r/r0 comm_analysisr1r2r3r4r5r6excr7r8fx_utilsr9r:r;r<r=r>r?rr@rrArB runtime.hintsrCrDruntime.runtime_utilsrErFrrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrY virtualizedrZ getLoggerr~r_logginggetArtifactLoggerrr&rr rr`rrarb dataclassrdrrr)r~r_r rrrrrrrBrcr0rrrrrrrr%rCrrrrr rr-rmrlr"r(r-r8r=r?rBrDrFr*r9 rzrprmr s"   ,3BB'/)<< J5 $$$6E?7M8>OO*DDDMM;:2$ J:7&(! ^^ - -h A NN44XO >>;; $11(LI 34 y4 T]t_ DDD* (((YYx h8h8h8V 1_11H 1H 1V 22, ' #L T"  # *   *K *K4*K,*K *KZ"* 1"*J5.5S*%S*l : $: $ 8k**k*\[G0[G|:-+=:-zC:!3C:L b,bP#%+#+ + + +\0%01C0 08 1(((( ( # (  (6       # @    >%??,4$ &""P@nJ1nJ1bUxxrp