
    
9j6                     
   S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKrSSKJr  SSKJrJrJrJrJrJrJrJrJrJrJr  SSKJr  SS	KJrJrJr  SS
K J!r!  SSK"J#r#  SSK$J%r%J&r&  / 4S jr'S r(S\ RR                  S\*\+\+4   4S jr,S\ RR                  S\-\.\+   \*\+\+4   4   4S jr/S\ RR                  S\*\+\+4   4S jr0S\ RR                  S\*S\-\.\+   \*\+\+4   4   4S jr1S\ RR                  S\-\.\+   \*\+\+4   4   4S jr2 S!S\ RR                  S\*\+\+4   S\3S\-\ RR                  \*\+\+4   4   4S jjr4S\.\+   S\*\+\+4   4S jr5S\S\ RR                  S\-\ RR                  \*\+\
4   4   4S jr6S r7 " S S 5      r8g)"    N)literal_eval)which)Any   )SageMakerConfig)DynamoBackendPrecisionTypeis_fp8_availableis_hpu_availableis_mlu_availableis_musa_availableis_neuron_availableis_npu_availableis_sdaa_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)get_free_portis_port_in_usemerge_dicts)compare_versions   )parse_flag_from_env)DistributedTypeSageMakerDistributedTypec                     UR                  U5      u  p4[        U 5      R                  5        H0  u  pVU[        U5      R                  5       ;   d  M$  [	        X5U5        M2     U$ )z,
Filters out all `accelerate` specific args
)parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues          Q/home/wildlama/miniconda3/lib/python3.13/site-packages/accelerate/utils/launch.py_filter_argsr*   /   sU     )),7KH4j&&(
$x.%%''H5) ) O    c                      S V s/ s H  n [        U 5      (       d  M  U PM     nn [        U5      S:X  a  [        S5      eUS   n[        R                  " US/5      nSU;   a  USSSS	4$ US
SSS4$ s  sn f )z
Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
are: OpenMPI, Intel MPI, or MVAPICH.

Returns: Program name and arg names for hostfile, num processes, and processes per node
)mpirunmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )r   lenOSError
subprocesscheck_output)xmpi_appsmpi_appmpirun_versions       r)   _get_mpirun_argsr9   :   s     1=0aE!H0H=
8}tuu qkG,,g{-CDNn$dL+EE dFB.. >s
   A0A0r"   current_envc                 P   Sn[        U 5       H  nUR                  S5      (       d  M  [        X5      nUc  M+  US:X  a;  [        US   5      XS-   '   [        US   5      XS-   '   [        US   5      XS	-   '   Ml  [        [        X5      5      X UR	                  5        3'   M     U$ )
z&
Setup the FP8 environment variables.
ACCELERATE_fp8_fp8_override_linear_precisionr   FP8_OVERRIDE_FPROPr   FP8_OVERRIDE_DGRADr   FP8_OVERRIDE_WGRAD)r   
startswithgetattrstrupper)r"   r:   prefixargr(   s        r)   setup_fp8_envrH   R   s     FDz>>&!!D&E 99ADU1XK)= =>ADU1XK)= =>ADU1XK)= =><?@R<SK(399;- 89  r+   returnc                 @
   / nU R                   (       a  U R                  (       a  [        S5      e[        U SS5      nU R                  nU R
                  bo  [        5       u  pEpgn[        U SS5      n	U(       a  U(       a  [        X#-  5      OSn
UUUU R
                  UU
/-  nU(       a  X[        U5      /-  nU(       a  XU	/-  nU R                   (       dA  UR                  [        R                  5        U R                  (       a  UR                  S5        UR                  U R                  5        UR                  U R                  5        [        R                  R!                  5       n[        U R"                  =(       d    U R$                  5      US'   U R&                  (       a  S	US
'   U R(                  S:w  a  U R(                  b  [+        5       (       a  U R(                  US'   O[-        5       (       a  U R(                  US'   O[/        5       (       a  U R(                  US'   O[1        5       (       a  U R(                  US'   Ol[3        5       (       a  U R(                  US'   OM[5        5       (       a  U R(                  US'   O.[7        5       (       a  U R(                  US'   OU R(                  US'   US:  a(  U R8                  c   S5       eU R:                  c   S5       eUb  US:  d  US:  aE  U R8                  b  U R8                  OSUS'   U R:                  b  [        U R:                  5      OSUS'   [=        US   S5      (       a  SUS'   [        S5      US'    [?        U R@                  RC                  5       5      n[        U5      US"'   U R@                  RC                  5       S#:X  a%  [G        5       (       d  [I        S$5      e[K        X5      n [M        U RN                  RQ                  5       5      nURR                  US&'   U RT                  US''   [        U RV                  5      US('   [        U RX                  5      US)'   [        U RZ                  5      US*'   [        U R\                  5      US+'   U R^                  (       a  SUS,'   X4$ ! [         a>    [        SU R@                  RC                  5        S [>        RD                  " 5        S!35      ef = f! [         a>    [        S%U RN                  RQ                  5        S [L        RD                  " 5        S!35      ef = f)-zr
Prepares and returns the command list and an environment with the correct simple launcher environment variables.
0--module and --no_python cannot be used togethernum_processesNzbind-tosocket1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESSDAA_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESHABANA_VISIBLE_MODULESNEURON_RT_VISIBLE_CORESCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.z	127.0.0.1MASTER_ADDR29500MASTER_PORTFzgranularity=fine,compact,1,0KMP_AFFINITYKMP_BLOCKTIMEUnknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8rFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMIC*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONOMP_NUM_THREADSACCELERATE_CPU_AFFINITY)0	no_pythonmodule
ValueErrorrC   num_machinesmpirun_hostfiler9   rD   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   r   r   r   main_process_ipmain_process_portr   r	   mixed_precisionlowerlistr
   RuntimeErrorrH   r   dynamo_backendrE   r(   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicdynamo_use_regional_compilationnum_cpu_threads_per_processenable_cpu_affinity)r"   cmdrL   rq   mpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argbind_tonproc_per_noder:   r   r   s                 r)   prepare_simple_launcher_cmd_envr   d   s    C~~$++KLLD/48M$$L'ScSePL[$	84?LQ]]:;cf  
 	
 #m"455C))C>>

3>>";;JJtJJt##$JJt(()**//#K(+DHH,D(EK$%zz/5+,||u!9.2llK*+15K-.  26,,K./  26,,K./7;||K3448LLK01 ""59\\K1226,,K./a##/ 	
T	
/ %%1 	
V	
1 	!ma&7L1<L=A=Q=Q=]T%9%9cnM"DHDZDZDfS)?)?%@lsM";';<eDD&DN#'*1vO$
'(<(<(B(B(DE 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=%()I)I%JK!"14-.9  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   #R
 #S 
ASATc                    U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nUS:  a|  [        X-  5      U l        [        U5      U l        [        U R                  5      U l
        [        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [        U R                  5      S:H  nU(       aG  [        U5      (       a7  US::  a"  SU l        ["        R$                  " S	U S
35        O['        SU S35      eU R(                  (       a  U R*                  (       a  [-        S5      eU R(                  (       a  SU l        OU R*                  (       a  SU l        [.        R0                  R3                  5       nU R4                  (       a  SUS'   [        U SS5      nUS:w  a  U R6                  b  [9        5       (       a  XvS'   O|[;        5       (       a  XvS'   Oh[=        5       (       a  XvS'   OT[?        5       (       a  XvS'   O@[A        5       (       a  XvS'   O,[C        5       (       a  XvS'   O[E        5       (       a  XvS'   OXvS'   U RF                  RI                  5       n [K        U5      n[        U5      US'   U RF                  RI                  5       S:X  a%  [O        5       (       d  [Q        S5      e[S        X5      n [U        U RV                  RY                  5       5      n	U	RZ                  US!'   U R\                  US"'   [        U R^                  5      US#'   [        U R`                  5      US$'   [        U Rb                  5      US%'   U Rd                  (       GaQ  SUS&'   U Rf                  (       a  U Rh                  (       d  [-        S'5      e[k        U S(5      (       a  [        U Rl                  5      OS)US*'   [        U Rn                  5      US+'   [        U Rp                  5      RI                  5       US,'   [        U Rr                  5      RI                  5       US-'   [        U Rt                  5      US.'   U Rv                  b  [        U Rv                  5      US/'   U Rx                  b  [        U Rx                  5      US0'   U Rz                  b  [        U Rz                  5      US1'   U R|                  b  [        U R|                  5      US2'   [        U R~                  5      RI                  5       US3'   [        U R                  5      RI                  5       US4'   [        U Rf                  5      RI                  5       US5'   [        U Rh                  5      RI                  5       US6'   [        U R                  5      RI                  5       US7'   [        U S8S5      b  [        U R                  5      US9'   U R                  (       Ga  S:n
SUS;'   [        U R                  5      XjS<-   '   [        U R                  5      XjS=-   '   U R                  b  [        U R                  5      XjS>-   '   U R                  b  [        U R                  5      XjS?-   '   U R                  b  [        U R                  5      XjS@-   '   U R                  b  [        U R                  5      XjSA-   '   U R                  b  [        U R                  5      XjSB-   '   U R                  b  [        U R                  5      XjSC-   '   U R                  b  [        U R                  5      XjSD-   '   [        U R                  5      XjSE-   '   [        U R                  5      XjSF-   '   U R                  b  [        U R                  5      XjSG-   '   U R                  b  [        U R                  5      XjSH-   '   U R                  b  [        U R                  5      XjSI-   '   U R                  b  [        U R                  5      XjSJ-   '   U R                  b  [        U R                  5      XjSK-   '   U R                  b  [        U R                  5      XjSL-   '   U R                  b  [        U R                  5      XjSM-   '   U R                  b  [        U R                  5      XjSN-   '   U R                  b  [        U R                  5      XjSO-   '   U R                  b  [        U R                  5      XjSP-   '   U R                  b  [        U R                  5      XjSQ-   '   U R                  b  [        U R                  5      XjSR-   '   U R                  b  [        U R                  5      XjSS-   '   U R                  b  [        U R                  5      XjST-   '   U R                  b  [        U R                  5      XjSU-   '   U R                  b  [        U R                  5      XjSV-   '   [        U R                  5      USW'   U R                  (       a  S)USX'   U R                  (       a  [        X5      nU$ ! [,         a&    [-        SU S[J        RL                  " 5        S35      ef = f! [,         a>    [-        S U RV                  RY                  5        S[T        RL                  " 5        S35      ef = f)YzW
Prepares and returns an environment with the correct multi-GPU environment variables.
r   N<s  r   same_networkF:TPort `  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`fsdp_versionrN   FSDP_VERSIONFSDP_SHARDING_STRATEGYFSDP_RESHARD_AFTER_FORWARDFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGfsdp_ignored_modulesFSDP_IGNORED_MODULESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREEUSE_CUSTOM_FSDPNO_LOAD_OPTIMEOD_MASK_LOSSNO_SAVE_OPTIMOPTIMIZER_CPU_OFFLOADUSE_PRECISION_AWARE_OPTIMIZEROVERLAP_CPU_OPTIMIZER_D2H_H2D DECODER_LAST_PIPELINE_NUM_LAYERS	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERRECOMPUTE_GRANULARITYRECOMPUTE_METHODRECOMPUTE_NUM_LAYERSATTENTION_BACKENDEXPERT_MODEL_PARALLEL_SIZECONTEXT_PARALLEL_SIZEATTENTION_DROPOUTHIDDEN_DROPOUTATTENTION_SOFTMAX_IN_FP32EXPERT_TENSOR_PARALLEL_SIZECALCULATE_PER_TOKEN_LOSSUSE_ROTARY_POSITION_EMBEDDINGSrl   rm   )cr   r   rL   rq   r   rD   r   nnodesintmachine_rank	node_rankrC   master_addrmaster_portrdzv_endpointr   
standalonewarningswarnConnectionErrorro   rn   rp   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r   r   r	   r   r
   r   rH   r   r   rE   r(   r   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_stateshasattrr   fsdp_sharding_strategyfsdp_reshard_after_forwardfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointingr   use_megatron_lmmegatron_lm_tp_degreemegatron_lm_use_custom_fsdpmegatron_lm_no_load_optimmegatron_lm_eod_mask_lossmegatron_lm_no_save_optim!megatron_lm_optimizer_cpu_offload)megatron_lm_use_precision_aware_optimizer)megatron_lm_overlap_cpu_optimizer_d2h_h2d,megatron_lm_decoder_last_pipeline_num_layersmegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizer!megatron_lm_recompute_granularitymegatron_lm_recompute_method megatron_lm_recompute_num_layersmegatron_lm_attention_backend&megatron_lm_expert_model_parallel_size!megatron_lm_context_parallel_sizemegatron_lm_attention_dropoutmegatron_lm_hidden_dropout%megatron_lm_attention_softmax_in_fp32'megatron_lm_expert_tensor_parallel_size$megatron_lm_calculate_per_token_loss*megatron_lm_use_rotary_position_embeddingsr   r   use_parallelism_config%prepare_extend_env_parallelism_config)r"   rL   rq   r   r   need_port_checkr:   r   r   r   rF   s              r)   prepare_multi_gpu_envr     s 
   
 "!.				'!&&&M$$L**O..a!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;01 ""5<1229./**002Ou'8 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=}}}-3)*..t7S7Svww@Gn@]@]c$*;*;&<cfN# 14D4O4O0P,-478W8W4X4^4^4`01-01I1I-J-P-P-R)*-01I1I-J)*%%136t7Q7Q3RK/022>:=d>e>e:fK67&&2478S8S4TK01$$025d6O6O2PK.//243M3M/N/T/T/V+,.1$2K2K.L.R.R.T*+8;D<_<_8`8f8f8h4514T5Q5Q1R1X1X1Z-.7:4;];]7^7d7d7f344/6B25d6O6O2PK./4:01,/0J0J,K[()25d6V6V2W../))5478V8V4WK01))5478V8V4WK01))5478V8V4WK0111=<?@f@f<gK!88999EDGHvHvDwK!@@A99EDGHvHvDwK!@@A<<HGJAAHK!CCD -00J0J,K[()478Z8Z4[001--98;D<^<^8_K!44500<;>t?d?d;eK!77811=<?@f@f<gK!88955A@CDDnDn@oK!<<=11=<?@f@f<gK!889,,87:4;\;\7]K!33400<;>t?d?d;eK!778--98;D<^<^8_K!44566BADTEpEpAqK!==>11=<?@f@f<gK!889--98;D<^<^8_K!445**6589X9X5YK!11255A@CDDnDn@oK!<<=77CBEdFrFrBsK!>>?44@?B4ClCl?mK!;;<::FEH??FK!AAB &))I)I%JK!"14-."";DNm  u9/9JJ[\i\n\n\p[qqrsttu  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   i0 6#j# 00j #Ak+c                    SnSUS'   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R
                  5      XS-   '   [        U R                  5      XS	-   '   [        U R                  5      XS
-   '   U R                  S:  a  [        U R                  5      XS-   '   U R                  S:  aN  [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   U$ )zN
Extends `current_env` with context parallelism env vars if any have been set
PARALLELISM_CONFIG_rP   !ACCELERATE_USE_PARALLELISM_CONFIGDP_REPLICATE_SIZEDP_SHARD_SIZETP_SIZECP_SIZE
CP_BACKENDSP_SIZE
SP_BACKENDr   CP_COMM_STRATEGYSP_SEQ_LENGTHSP_SEQ_LENGTH_IS_VARIABLESP_ATTN_IMPLEMENTATION)rD   $parallelism_config_dp_replicate_size parallelism_config_dp_shard_sizeparallelism_config_tp_sizeparallelism_config_cp_sizeparallelism_config_cp_backendparallelism_config_sp_sizeparallelism_config_sp_backend#parallelism_config_cp_comm_strategy parallelism_config_sp_seq_length,parallelism_config_sp_seq_length_is_variable)parallelism_config_sp_attn_implementation)r"   r:   rF   s      r)   r  r    sB    #F7=K3403D4]4]0^K,,-,/0U0U,VK()&)$*I*I&JK"#&)$*I*I&JK"#),T-O-O)PK%&&)$*I*I&JK"#),T-O-O)PK%&&&*36t7_7_3`//0&&*03D4Y4Y0Z_,-<?@q@q<r8899<T=k=k9l556r+   c                 0   U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nSnU R
                  c  [        S   U l        US:  Ga@  U R
                  [        S   :w  Ga(  S/nUR                  S[        U R                  5      /5        U R
                  S:X  aE  [        SSS	5      (       a  [        S
5      eUR                  S[        U R                  5      S/5        O'UR                  SS[        U R
                  5      /5        U R                  b'  UR                  S[        U R                  5      /5        OgU R                  b'  UR                  S[        U R                  5      /5        O3UR                  S[        U R                  U R                  -  5      /5        U(       a  UR                  S[        U5      /5        UR                  S[        U5      /5        U R                  (       a  U R                   (       a  [        S5      eU R                  (       a  UR#                  S5        O"U R                   (       a  UR#                  S5        UR#                  U R$                  5        UR                  U R&                  5        OUS:  a  U R
                  [        S   :X  a|  [        X-  5      U l        [        U5      U l        [-        U R                  5      U l        [1        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [-        U R                  5      S:H  nU(       aG  [9        U5      (       a7  US::  a"  SU l        [<        R>                  " SU S35        O[A        SU S35      eU R                  (       a  U R                   (       a  [        S5      eU R                  (       a  SU l        OU R                   (       a  SU l        [B        RD                  RG                  5       nU RH                  (       a  SUS '   [1        U S!S"5      nUS":w  a  U RJ                  b  [M        5       (       a  XS#'   O|[O        5       (       a  XS$'   Oh[Q        5       (       a  XS%'   OT[S        5       (       a  XS&'   O@[U        5       (       a  XS''   O,[W        5       (       a  XS('   O[Y        5       (       a  XS)'   OXS*'    [[        U R\                  R_                  5       5      n	[c        S.[B        Rd                  Rg                  S-5      5      US.'   [        U	5      US/'   U R\                  R_                  5       S0:X  a%  [i        5       (       d  [k        S15      e[m        X5      n[        U Rn                  5      R_                  5       US2'   SUS3'   U Rp                  b  [        U Rp                  5      US4'   U Rr                  b  [        U Rr                  5      US5'   U Rt                  b&  [        U Rt                  5      R_                  5       US6'   U Rv                  b&  [        U Rv                  5      R_                  5       US7'   U Rx                  b&  [        U Rx                  5      R_                  5       US8'   U Rz                  b&  [        U Rz                  5      R_                  5       US9'   U R|                  b&  [        U R|                  5      R_                  5       US:'   U R~                  b  [        U R~                  5      US;'   U R                  (       a  S<US='   U R                  b  [        U R                  5      US>'   U R                  (       a  [        X5      nXW4$ ! [         a>    [        S+U R\                  R_                  5        S,[Z        R`                  " 5        S-35      ef = f)?zl
Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
r   Nr   r   	deepspeedr/   nossh<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr   Fr   Tr   r   r   r   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   
PYTHONPATHrc   rd   re   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErN   rm   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Dr   r   rL   rq   r   deepspeed_multinode_launcherr   rw   rD   deepspeed_hostfiler   rp   r   deepspeed_exclusion_filterdeepspeed_inclusion_filterro   rn   rs   rv   rx   r   r   r   r   rC   r   r   r   r   r   r   r   r   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r	   r   r   r   env_var_path_addpathabspathr
   r   rH   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_namesr  r  )
r"   rL   rq   r   r   r   r  r:   r   r   s
             r)   prepare_deepspeed_cmd_envrA    s   
 "!.				'!&&&M$$L**O..
C ((0,I!,L)aD==A^_`Aaam

L#d&=&=">?@,,7S(;; !NOOJJs4+<+<'=zJKJJ)<T=^=^9_`a**6JJ778 ,,8JJ778 JJc$*<*<@Q@Q*Q&RSTJJ_)=>?

OS):%;<=;;4>>OPP[[JJz"^^JJ}%

4''(

4,,-		d??C`abCcc!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;01 ""5<1229./
'(<(<(B(B(DE !1rwws?S TK03O0DK,-!!#u,!! E  $D614T5a5a1b1h1h1jK-..4K*+"9<T__9M56''3@CDDdDd@e<=)69$:P:P6Q6W6W6Y23$$0GJ4KhKhGiGoGoGqCD  ,CFtG`G`CaCgCgCi?@'9<T=Q=Q9R9X9X9Z56"".EHIdIdEeEkEkEmAB!!-:=d>X>X:Y6714-.))5BEdFhFhBi>?"";DNO  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

s   :#_ A`podc                     U R                   S:X  a*  [        SS9(       a  U R                  (       a  SUS'   OSUS'   U R                  (       a  SUS'   U(       a"  U R                  U l        U R                  U l        X4$ )	zQ
Prepares and returns an environment with the correct TPU environment variables.
bf16T)check_is_tpurN   XLA_DOWNCAST_BF16XLA_USE_BF16rP   rQ   )r   r   downcast_bf16r~   tpu_vmvmtpu_nametpu)r"   r:   rB  s      r)   prepare_tpurM  Q  sl     v%*@d*S/2K+,*-K'zz/5+,
++==r+   nargsc                 N   [        U 5      S:  a  0 $ S n[        R                  " 5       nUR                  U 5      u  p4[	        U5       H  u  pVUR                  S5      (       d  M  S nUS-   [        U5      :  a'  XES-      R                  S5      (       a  [        S5      eO[        S5      eUc  UR                  XaS9  Mw  UR                  XgS9  M     UR                  U 5      R                  R                  5        VV	s0 s H  u  pXS;   a  [        U	5      OU	_M     sn	n$ s  sn	nf )	Nr   c                 j     [        U 5      n U S-  U :X  a  [        U 5      $ U $ ! [         a    U s $ f = f)Nr   )floatr   rp   )ss    r)   _infer_type+_convert_nargs_to_dict.<locals>._infer_typej  s=    	aAAv{1vH 	H	s   # # 22)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)TrueFalse)r1   argparseArgumentParserr   	enumeraterB   rp   add_argument
parse_args__dict__r   r   )
rN  rS  r#   r&   unknownindexargumentrW  r'   r(   s
             r)   _convert_nargs_to_dictrc  e  s3   
5zA~	 $$&F((/JA$W-{++Fqy3w<'19%00==$ G  > ! C  ~##H#?##H#<# .* !++E2;;AACCJC 	.?%?l5!UJC  s   >D!sagemaker_configc                 "
   [        S5        U R                  [        R                  S'   U R                  b  U R                  [        R                  S'   O`UR
                  bH  UR                  b;  UR
                  [        R                  S'   UR                  [        R                  S'   O[        S5      e[        R                  R                  UR                  5      nU(       d  Sn[        R                  R                  UR                  5      nUR                  S5      (       d  [        S	U S
35      e[        S5        [        UR                  5      n [!        UR"                  R%                  5       5      n [)        UR*                  R-                  5       5      nS[/        U5      UR0                  UR2                  [/        UR4                  5      [/        UR6                  5      [/        UR8                  5      U R:                  R0                  S.nUR"                  R%                  5       S:X  a%  [=        5       (       d  [?        S5      e[A        X5      nS nU R:                  [B        RD                  :X  a  SSSS000nS n	U RF                  b  [        SU RF                   S35        0 n	[I        U RF                  5       n
[K        U
5       H6  u  pUS:X  a  M  URM                  S5      nUS   RO                  5       XS   '   M8     S S S 5        [        SU	 35        S nU RP                  b  [        SU RP                   S35        / n[I        U RP                  5       n
[K        U
5       HH  u  pUS:X  a  M  URM                  S5      nUS   US   RO                  5       S.nURS                  U5        MJ     S S S 5        [        SU 35        [        S 5        U RT                  UUU RV                  U RX                  U RZ                  U R\                  U R^                  U R`                  U Rb                  S!UUUUS".nU Rd                  b  [g        U Rd                  U5      nX4$ ! [         a>    [        SUR"                  R%                  5        S[         R&                  " 5        S35      ef = f! [         a>    [        SUR*                  R-                  5        S[(        R&                  " 5        S35      ef = f! , (       d  f       GN= f! , (       d  f       GNX= f)#Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerb   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr`   ra   rf   rP   )ACCELERATE_USE_SAGEMAKERrc   rg   rh   ri   rj   rk   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPErd   re   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)4printregionry   rz   profileaws_access_key_idaws_secret_access_keyr2   r5  dirnamerv   basenameendswithrp   rc  rx   r	   r   r   r   r   r   rE   rD   r(   r   r   r   r   distributed_typer
   r   rH   r   DATA_PARALLELsagemaker_inputs_fileopenr\  splitstripsagemaker_metrics_filers   rs  iam_role_namerw  rx  ry  rz  rq   ec2_instance_typeadditional_argsr   )rd  r"   ru  rt  r  r   r   r  r~  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts                   r)   prepare_sagemager_args_inputsr    s    

45'7'>'>BJJ#$ +$4$<$<

=!				+0J0J0V*.*@*@

&'.2.H.H

*+uvv !5!56J
''""4#7#78K&&ST_S``abcc	
34,T-F-FGO
'(<(<(B(B(DE
&t':':'@'@'BC %+&)/&:%3%9%9"&"2"2+.t/H/H+I),T-D-D)E69$:^:^6_1A1R1R1X1X	K !!#u,!! E  $D6L((,D,R,RR'.9d:K)LM --9./?/U/U.VV[\]"889T$T?6JJt$)*1 1&	 + : 	)*:);<= ..:/0@0W0W/XX]^_"99:d$T?6JJt$aDqTZZ\ "((5 + ; 	*+<*=>? 

%//"  .. 0 E E+;;&11)77*77);; %$*"/D$ ''3+;;TB!!m  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

@ :9 ;:s4   #Q 9#R" ;AS-AS?AR"AS*-
S<?
Tc                    [         R                  R                  U S5      R                  S5       Vs/ s H  n[	        U5      S:  d  M  UPM     nnUR                  [        U5      5        SR                  U5      $ s  snf )z
Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
caller to set it in os.environ.
r0   r   r   )ry   rz   getr  r1   rs   rD   join)env_var_namepath_to_addppathss       r)   r4  r4    sd    
 

|R8>>sCRC1s1vPQzQCER	LL[!"88E? Ss   A<	A<c                   (    \ rS rSrSrSS jrS rSrg)PrepareForLaunchi  aE  
Prepare a function that will launched in a distributed setup.

Args:
    launcher (`Callable`):
        The function to launch.
    distributed_type ([`~state.DistributedType`]):
        The distributed type to prepare for.
    debug (`bool`, *optional*, defaults to `False`):
        Whether or not this is a debug launch.
c                 <    Xl         [        U5      U l        X0l        g )N)launcherr   r  r~   )selfr  r  r~   s       r)   __init__PrepareForLaunch.__init__  s      /0@ A
r+   c                    U R                   (       a  [        [        R                  R	                  S5      5      n[        R                  R	                  S5      n[
        R                  R                  SU[
        R                  R                  XC5      US9  GOU R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                   [        R"                  4;   a  [%        U5      [        R                  S'   [        [        R                  R	                  SS5      5      n[        [        R                  R	                  SS	5      5      n[%        XV-  U-   5      [        R                  S
'   [%        S5      [        R                  S'   U R&                  " U6   g )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)r~   r   ry   rz   r  torchdistributedinit_process_group	FileStorer  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUMULTI_NEURONrD   r  )r  ra  r"   r  rdv_filenprocr   s          r)   __call__PrepareForLaunch.__call__!  s<   ::RZZ^^L9:Jzz~~&ABH00''11(G%	 1  ""%%%%&&%%%%%%(('
 
 (+5zBJJ|$

w23EBJJNN;:;I!$U%6%>!?BJJv&)!f

?#tr+   )r~   r  r  N)NOF)__name__
__module____qualname____firstlineno____doc__r  r  __static_attributes__ r+   r)   r  r    s    

r+   r  )F)9rZ  ry   r3   rt   r   astr   shutilr   typingr   r  commands.config.config_argsr   utilsr   r	   r
   r   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   r   utils.versionsr   r0   r   dataclassesr   r   r*   r9   	NamespacedictrD   rH   tupler   r   r  r  rA  boolrM  rc  r  r4  r  r  r+   r)   <module>r     s
    	  
      9    < D D - ! B -/ /0** c3h $b(*<*< btCyRVWZ\_W_R`G`Aa bJF 2 2 FtCH~ FR


+/
49d38n$%6aH$6$6 a5cDQTVYQYNAZ;[ aJ HM


+/S>@D
8tCH~-.('$s) 'S#X 'Ts"%s"-5-?-?s"
8tCH~-.s"l, ,r+   