
    
9j                        S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKr	S SK
r
S SKJrJr  S SKJr  S SKJrJr  S SKJr  S SKJrJrJr  S S	KJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&  \"" 5       (       a  S
r'S
r(OSr'Sr(S%S jr)S r*S r+S r,S r-S r.S r/S r0S r1S r2S r3S r4S r5S r6S%S jr7S%S jr8S r9S r:S r;S r<S  r=S! r>S" r?S# r@\AS$:X  a  \@" 5         gg)&    N)deepcopy)Path)
DataLoaderDataset)Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetRegressionModelare_the_same_tensors)DataLoaderConfigurationDistributedTypegathergather_objectis_bf16_availableis_cuda_availableis_datasets_availableis_fp16_availableis_hpu_availableis_mps_availableis_pytest_availableset_seedsynchronize_rng_statesgMbP?gư>c                 b    U(       a  [        UU [        U 5      S9n[        XUS9$ [        XSUS9$ )zBCreates a dataloader that can also use the `SeedableRandomSampler`)	generatordata_sourcenum_samples)
batch_sizesamplerT)r   shuffler   )r   lenr   )	train_setr   r   use_seedable_samplerr    s        c/home/wildlama/miniconda3/lib/python3.13/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloaderr&   9   s=     (!I

 )GLL)DT]^^    c                 4    [        SU R                   35        g )NzPrinting from the main process printprocess_indexstates    r%   
print_mainr.   H       	+E,?,?+@
ABr'   c                 4    [        SU R                   35        g )Nz%Printing from the local main process )r*   local_process_indexr,   s    r%   print_local_mainr2   L   s    	1%2K2K1L
MNr'   c                 4    [        SU R                   35        g )NPrinting from the last process r)   r,   s    r%   
print_lastr5   P   r/   r'   c                 :    [        SU SU R                   35        g )NPrinting from process : r)   )r-   process_idxs     r%   print_onr:   T   s    	";-r%2E2E1F
GHr'   c                     [        5       n U R                  n[        S5      nU R                  5          U R                  (       a=  [
        R                  " S5        [        US5       nUR                  S5        S S S 5        O&[        US5       nUR                  S5        S S S 5        S S S 5        U R                  5         U R                  (       a  [        U5       nSR                  UR                  5       5      nS S S 5         WR                  S5      (       d   S5       eUS:  a  UR                  S5      (       d   S5       eUR                  S5      U R                  S-
  :X  d)   S	UR                  S
5      S-    SU R                   35       e U R                  (       a%  UR#                  5       (       a  UR!                  5         U R                  5         [$        R&                  " 5       n[(        R*                  " U5         U R-                  [.        5      " U R0                  5        S S S 5        UR3                  5       R5                  5       nU R                  (       a  US:X  d
   U S35       eO,UR3                  5       R5                  5       S:X  d
   U S35       eUR7                  S5        UR9                  S5        [(        R*                  " U5         U R;                  [<        5      " U R0                  5        S S S 5        U R>                  (       a%  UR3                  5       R5                  5       S:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        [(        R*                  " U5         U RA                  [B        5      " U R0                  5        S S S 5        U RD                  (       a?  UR3                  5       R5                  5       SU R0                  R                  S-
   3:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        [G        U5       H  n[(        R*                  " U5         U RI                  [J        US9" U R0                  U5        S S S 5        U RL                  U:X  a5  UR3                  5       R5                  5       SU SU RL                   3:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        M     g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN{= f! [         a    UR!                  5         e f = f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN<= f)Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
 zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r4   )r+   r7   r8   )'r   num_processesr   main_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithendswithcountAssertionErrorunlinkexistsioStringIO
contextlibredirect_stdouton_main_processr.   r-   getvaluerstriptruncateseekon_local_main_processr2   is_local_main_processon_last_processr5   is_last_processrange
on_processr:   r+   )acceleratorr?   pathftextresultr9   s          r%   process_execution_checkrc   X   sr   -K--M./D		'	'	)&&JJsOdD!Q9: "! dD!Q23 " 
* !!#""$Z1771;;=)D 		??#DEEcGccEq }}%?@@^B^^@::89[=V=VYZ=ZZ %djj1I&JQ&N%O|\g\u\u[vwZ ""t{{}}!!#
A		#	#A	&##J/0A0AB 
'ZZ\  "F"";;lxGk=ll;zz|""$*=vhf,==*JJqMFF1I		#	#A	&))*:;K<M<MN 
'((zz|""$(PPPPzz|""$***JJqMFF1I		#	#A	&##J/0A0AB 
'""zz|""$*I+J[J[JiJilmJmIn(oooozz|""$***JJqMFF1I]+''*""8;"GHYHY[fg +$$3::<&&(.D[MQST_TmTmSn,oooo::<&&(B...	

1	q	 ,m "! "! 
*	) Z  	KKM	 
'	& 
'	& 
'	& +*s   4U&T98UUU U/BV &V &V2&W;&W9
U	U
U	U
U,/
U>V 
V/2
W
W
W%	c                  d    [        5       n U R                  S:X  a  [        S5        [        U 5        g )Nr   zTesting, testing. 1, 2, 3.)r
   r1   r*   r,   s    r%   init_state_checkre      s(    E  A%*+	%Lr'   c                     [        5       n [        S/5        [        [        R                  " 5       5      (       d   S5       eU R
                  [        R                  :X  a@  [        S/5        [        [        R                  R	                  5       5      (       d   S5       eO]U R
                  [        R                  :X  a?  [        S/5        [        [        R                  R	                  5       5      (       d   S5       e[        R                  " 5       n[        S/US9  [        UR                  5       5      (       d   S	5       eU R                  S
:X  a  [        S5        g g )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.r   )r   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r
   r   r   rg   get_rng_statedistributed_typer   	MULTI_GPUrh   	MULTI_XPUri   	Generator	get_stater1   r*   )r-   r   s     r%   rng_sync_checkrp      s   EG9% 3 3 566d8dd6!:!::x(#EJJ$<$<$>??mAmm?			?#<#<	<w'#EII$;$;$=>>l@ll>!IK=I>	 3 3 566j8jj6  A%-. &r'   c            	         [        5       n SU R                  -  n[        [        U5      SS9n[	        X R
                  U R                  U R                  SS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       e[        [        U5      SS9n[	        UU R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       eU R                  S:X  a  [        S	5        [        [        U5      SSS
9n[	        X R
                  U R                  U R                  SS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       e[        [        U5      SSS
9n[	        UU R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       eU R&                  S:X  a  [        S5        g g )N       r   T)put_on_devicer   %Wrong non-shuffled dataloader result.)ru   split_batchesz Non-shuffled dataloader passing.r   r!   !Wrong shuffled dataloader result.zShuffled dataloader passing.r
   r?   r   r\   r	   devicer+   appendr   rg   catequalcpuarangelongr*   tolistsortlistr1   r-   lengthdlrb   batchs        r%   dl_preparation_checkr      s   E%%%%F	E&Ma	0B	Ru/B/BEDWDWgk	lBFfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmD	E&Ma	0B	

B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmDa01	E&Ma	>B	Ru/B/BEDWDWgk	lBFfUm$ YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B FfUm$ YYv%%'F
KKMT%-((M*MM(  A%,- &r'   c            
         [        5       n SU R                  -  n[        [        U5      SS9n[	        X R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       e[        [        U5      SS9n[	        UU R
                  U R                  U R                  SSSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       eU R                  S:X  a  [        S	5        [        [        U5      SSS
9n[	        X R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       e[        [        U5      SSS
9n[	        UU R
                  U R                  U R                  SSSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       eU R&                  S:X  a  [        S5        g g )Nrr   rs   rt   T)ru   dispatch_batchesr   rv   )ru   rw   r   z(Non-shuffled central dataloader passing.rx   ry   z$Shuffled central dataloader passing.rz   r   s        r%   central_dl_preparation_checkr      s   E%%%%F	E&Ma	0B	
LL%--u/B/BRVim
B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmD	E&Ma	0B	

B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmDa89	E&Ma	>B	
LL%--u/B/BRVim
B FfUm$ YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B FfUm$ YYv%%'F
KKMT%-((M*MM(  A%45 &r'   c                     [        5       n  " S S[        5      n " S S5      nU" [        SU R                  -  5      5      nU" [	        U5      SS9n[        X4S9n[        XPR                  U R                  U R                  5      n[        UR                  S	5      (       a-  [        UR                  R                  U5      (       d   S
5       eg [        UR                  U5      (       d   S
5       eg )Nc                   &    \ rS rSrS rS rS rSrg)+custom_sampler_check.<locals>.CustomDataseti;  c                     Xl         g Ndata)selfr   s     r%   __init__4custom_sampler_check.<locals>.CustomDataset.__init__<  s    Ir'   c                 ,    [        U R                  5      $ r   )r"   r   r   s    r%   __len__3custom_sampler_check.<locals>.CustomDataset.__len__?  s    tyy>!r'   c                      U R                   U   $ r   r   )r   indexs     r%   __getitem__7custom_sampler_check.<locals>.CustomDataset.__getitem__B  s    99U##r'   r   N)__name__
__module____qualname____firstlineno__r   r   r   __static_attributes__ r'   r%   CustomDatasetr   ;  s    		"	$r'   r   c                   :    \ rS rSrS
S\S\S\4S jjrS rS rSr	g	)0custom_sampler_check.<locals>.CustomBatchSampleriE  dataset_lengthr   r!   c                 R    X l         [        R                  " U5      U l        X0l        g r   )r   npr   
data_indexr!   )r   r   r   r!   s       r%   r   9custom_sampler_check.<locals>.CustomBatchSampler.__init__F  s    (O ii7DO"Lr'   c              3      #    [        U 5      nU R                  (       a*  [        R                  R	                  U R
                  5      nOU R
                  n[        R                  " X!5      nU S h  vN   g  N7fr   )r"   r!   r   randompermutationr   array_split)r   num_batchesr   outputs       r%   __iter__9custom_sampler_check.<locals>.CustomBatchSampler.__iter__K  sM     d)K||		--doo>^^E7Fs   A.A80A61A8c                 n    [         R                  " [        U R                  5      U R                  -  5      $ r   )mathceilr"   r   r   r   s    r%   r   8custom_sampler_check.<locals>.CustomBatchSampler.__len__T  s#    99S1DOOCDDr'   )r   r   r!   N)T)
r   r   r   r   intboolr   r   r   r   r   r'   r%   CustomBatchSamplerr   E  s'    	#3 	#C 	#$ 	#
		Er'   r   rr   rs   rt   )batch_samplerr   z>Custom sampler was changed after calling `prepare_data_loader`)r
   r   r\   r?   r"   r   r	   r{   r+   hasattrr   
isinstance)r-   r   r   datasetr    r   s         r%   custom_sampler_checkr   8  s    E$ $E E$ E"u':':":;<G W!<G	G	3B	Ru/B/BEDWDW	XBr11"**88:LMM 	
L	
M "**,>?? 	
L	
?r'   c                  "   [        S5        [        SSS9n [        U SSS9n[        SS9n[	        US9nUR                  U5      n/ n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      n[        S5        UR                  S5        / n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      n[        R                  " XG5      (       d   S5       eg )N*   
   r   seed   Trx   r$   dataloader_config   xr   z;Did not obtain the same items with the same seed and epoch.)r   r   r   r   r   preparer\   r|   rg   r}   	set_epochallclose)r#   train_dlconfigr^   original_items_r   	new_itemss           r%   check_seedable_samplerr   f  s    RL!"5I)4@H$$?F7K""8,HN1XE!!%*-   YY~.N RLqI1XEU3Z(   		)$I>>.44s6ss4r'   c                  2   [        S5        [        SS9n [        U S9nUR                  S:  d   S5       e[	        [        [        S5      5      SSS9n[        USS	9nUR                  R                  R                  n[        U[        5      (       d   S
5       eg )Nr   Tr   r   r>   z)This test requires more than one process.r   rx   )
dataloaderr$   z:Sampler in BatchSamplerShard is not SeedableRandomSampler.)r   r   r   r?   r   r   r\   r	   r   r    r   r   )r   r^   r   prepared_data_loadertarget_samplers        r%   -check_seedable_sampler_in_batch_sampler_shardr     s    RL$$?F7K$$q(U*UU(DrO4HJ.!
 *77EEMMNn&;<< D<r'   c                  2   [        S5        Sn [        SSS9n[        USSS9n[        SU S9n[	        US9nUR                  U5      n/ n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      nSUl
        [	        US9nUR                  U5      n/ n	[        S	5       H   nU H  nU	R                  US
   5        M     M"     [        R                  " U	5      n	[        R                  " Xi5      (       a   S5       eg )Nr   r   r   r   Trx   )r$   	data_seedr   r   r   +   z1Obtained the same items with different data seed.)r   r   r   r   r   r   r\   r|   rg   r}   r   r   )
r   r#   r   r   r^   prepared_dlr   r   r   r   s
             r%   %check_seedable_sampler_with_data_seedr     s   RLI!"5I)4@H$$)TF7K%%h/KN1X E!!%*- !  YY~.N F7K%%h/KI1X EU3Z( !  		)$I~~n88m:mm88r'   c                    [        S5        UR                  S5        [        U SS9n[        XBX5      n[	        5       n[
        R                  R                  UR                  5       SS9n[        S5       Hs  nU Hj  n	UR                  5         U" U	S   5      n
[
        R                  R                  R                  XS   5      nUR                  5         UR                  5         Ml     Mu     XF4$ )Nr   r   r<   lrr   r   y)r   manual_seedr   r&   r   rg   optimSGD
parametersr\   	zero_gradnn
functionalmse_lossbackwardstep)r   r   r   r$   r#   r   model	optimizerepochr   r   losss               r%   mock_trainingr     s    RL"!b9I+I*cHE 0 0 2s;IqEOO5:&F88&&//c
CDMMONN   r'   c                    [        5       n[        R                  " 5       nSnUS-  UR                  -  n[	        XCUR                  -  X 5      u  pV[        UR                  5      (       d   S5       e[        UR                  5      (       d   S5       e[        5       n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S
 S9  [        R4                  R7                  UR                  U	R                  [8        [:        S S9  UR=                  S5        [?        SU S9n[        US9n[        XRX1R                  -  U 5      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S S9  [        R4                  R7                  UR                  U	R                  [8        [:        S S9  UR=                  S5        [A        5       (       d  [C        5       (       a  [=        S5        [         RD                  " 5         [        SS9n[        R&                  RG                  SS5      n	UR                  U	5      n	UR1                  U	SS9n[        RH                  " SS/5      RK                  [        RL                  URN                  S9nU" U5      n[Q        5       (       Ga  [=        S5        [         RD                  " 5         [?        U S9n[        SUS9n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S S9  [        R4                  R7                  UR                  U	R                  [8        [:        S  S9  [S        5       (       Ga  [U        5       (       Gd  [=        S!5        [         RD                  " 5         [?        U S9n[        SUS9n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S" S9  [        R4                  R7                  UR                  U	R                  [8        [:        S# S9  g g g )$Nrs      z0Did not obtain the same model on both processes.r<   r   r   r   r   r   c                     SU  3$ Nz>Did not obtain the same model on CPU or distributed training.
r   msgs    r%   <lambda> training_check.<locals>.<lambda>      YZ]Y^_r'   )atolrtolr   c                     SU  3$ r   r   r   s    r%   r   r     r   r'   zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)rw   r$   r   c                     SU  3$ r   r   r   s    r%   r   r     r   r'   c                     SU  3$ r   r   r   s    r%   r   r     r   r'   zSTraining yielded the same results on one CPU or distributed setup with batch split.zKeep fp32 wrapper check.fp16)mixed_precisionr   )keep_fp32_wrapperr>   )dtyper{   zBF16 training check.r   bf16)r  r   c                     SU  3$ r   r   r   s    r%   r   r   >      ]^a]bcr'   c                     SU  3$ r   r   r   s    r%   r   r   E  r  r'   zFP16 training check.c                     SU  3$ r   r   r   s    r%   r   r   d  r  r'   c                     SU  3$ r   r   r   s    r%   r   r   k  r  r'   )+r
   rg   rn   r?   r   r   abr   r&   r   r   r   r   r   r   r   r\   r   r   r   r   r   r   unwrap_modelr   testingassert_closeATOLRTOLr*   r   r   r   _reset_stateLinearTensortofloat16r{   r   r   r   )r$   r-   r   r   r   r#   	old_modelr^   r   r   r   r   r   r   r   r   model_with_fp32_wrapperinput_tensors                     r%   training_checkr    s   E!IJ!^e111F(e>Q>Q1QS\sI	,,`.``,	,,`.``,-K+I*cHE 0 0 2s;I!,!4!4Xi!PHYRL"1XEOO5:&F88&&//c
CD  &NN   $$U+//1E	MM_   
MM_   no/dYmn0ABK+j+>+>>@TH E 0 0 2s;I!,!4!4Xi!PHYRL"1XEOO5:&F88&&//c
CD  &NN   $$U+//1E	MM_   
MM_   kl .00()%%'!&91%##E*"-":":5TX":"Y ||QF+..U]];K]K].^(6 $%%%'3I]^!&L]^/	jg!KKOOE$4$4$63O?	%0%8%8)%T"b!qA!!uSz*xx**33F#JG$$T*  "  ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
 #3#5#5$%%%'3I]^!&L]^/	jg!KKOOE$4$4$63O?	%0%8%8)%T"b!qA!!uSz*xx**33F#JG$$T*  "  ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
; $6r'   c           	         [        5       nU R                  [        SUR                  -  5       Vs/ s H  n[	        US9PM     sn5      nUR                  USS9 n[        U5      S:X  d    SUR                   S[        U5       35       e S S S 5        U R                  [        SUR                  -  S-
  5       Vs/ s H  n[	        US9PM     sn5      nUR                  USS9 nUR                  (       a0  [        U5      S:X  d    SUR                   S[        U5       35       eO/[        U5      S:X  d    S	UR                   S[        U5       35       eS S S 5        UR                  5         U R                  [        SUR                  -  S-
  5       Vs/ s H  n[	        US9PM     sn5      nU R                  [        SUR                  -  5       Vs/ s H  n[	        US9PM     sn5      nXV4 GH  nUS
   nUR                  USS9 nUR                  S:X  a9  [        U5      [        U5      :X  d    SUR                   S[        U5       35       eO/[        U5      S:X  d    SUR                   S[        U5       35       e/ nU H  n	UR                  U	5        M     S S S 5        UR                  5         [        W5      n
U
S [        U5        Vs/ s H  oS
   PM	     nnX|:X  a  M   SU SU 35       e   g s  snf ! , (       d  f       GN{= fs  snf ! , (       d  f       GN= fs  snf s  snf ! , (       d  f       N= fs  snf )Nr   )kFapply_padding4Each process did not have two items. Process index: 
; Length: r>   z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: r  Tz9Single process did not receive all items. Process index: )Gathered results is incorrect. Expected: ; Got: )r
   	from_listr\   r?   dictsplit_between_processesr"   r+   r[   rF   r|   r   )datasets_Datasetr-   vr   resultsodd_data	even_dataexpected_outputresults_per_processrb   gathered_resultsrr   s                r%   $test_split_between_processes_datasetr.  o  sc   E%%%EDWDW@W:X&Y:XQtay:X&YZD		&	&t5	&	AW7|q  	
B5CVCVBWWabefmbnaop	
  
B
 %%%EDWDW@WZ[@[:\&]:\Qtay:\&]^D		&	&t5	&	AW  w<1$ MeNaNaMbblmpqxmylz{$ w<1$ ^_d_r_r^ss}  B  CJ  K  ~L  M$ 
B 
))eAH[H[D[^_D_>`*a>`4!9>`*abH **uQI\I\E\?]+^?]!D1I?]+^_I%s)**4t*D""a'7|s4y0 OPUPcPcOddnorszo{n|}0 7|q( J5K^K^J__ijmnujviwx( #%!#**62 " E 	!()<="2;SY"?@"?QC&"?@(v,UVeUffmntmu*vv(- &+ 'Z	A	A
 '^	A	A +b+^
 ED" AsC   K:0K?;L%A1LL(L-BL2M?
L
L%2
M 	c                     [        5       n [        [        SSU R                  -  5      5      nU R	                  U5       n[        U5      S:X  d    SU R                   S[        U5       35       e S S S 5        U R                  5         [        [        SSU R                  -  5      5      n[        [        SSU R                  -  S-
  5      5      nXC4 H  nUnU R	                  USS9 n[        R                  " [        U5      U R                  -  5      n[        U5      U:X  d-   SU R                   S	U R                   S[        U5       35       e/ nU H  nUR                  U5        M     S S S 5        U R                  5         [        W5      n	U	S [        U5       n
XZ:X  a  M   S
U SU
 35       e   g ! , (       d  f       GNK= f! , (       d  f       N`= f)Nr   r   r  r  r>   Tr  Process ; did not get the correct number of item(s). Process index: r   r!  )r
   r   r\   r?   r$  r"   r+   rF   r   r   r{   r|   r   )r-   r   r'  r)  r(  r*  num_samples_per_devicer+  rb   r,  r   s              r%   !test_split_between_processes_listr3    s   EaU00012D		&	&t	,7|q  	
B5CVCVBWWabefmbnaop	
  
- 
U1q5#6#6689IE!a%"5"55:;<H%**4t*D%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%!#**62 " E 	!()<=!+CI.(v,UVeUffmntmu*vv(' & 
-	, EDs   0F3&BG3
G
G	c                     [        5       n / SQn/ SQn[        R                  " / SQ5      nU R                  S;   Ga  XUS.n[	        U5      nU R                  U5       nU R                  S:X  a  US   US   S SU R                  -   :X  d   eOWU R                  S:X  a  US   US   S	S  :X  d   eO5U R                  S
:X  a%  US   US   SS  :X  d   SUS   S    SUS    35       eU R                  S:X  a  US   US   S SU R                  -   :X  d   eOCU R                  S:X  a  US   US   S	S  :X  d   eO!U R                  S
:X  a  US   US   SS  :X  d   eU R                  S:X  aV  [        R                  " US   US   S SU R                  -   5      (       d#   SUS   S SU R                  -    SUS    35       eOU R                  S:X  a<  [        R                  " US   US   S	S  5      (       d   SUS   S	S   SUS    35       eOKU R                  S
:X  a;  [        R                  " US   US   SS  5      (       d   SUS   SS   SUS    35       eS S S 5        U R                  5         g ! , (       d  f       N= f)N)r>   r   r   r            rs   )r
  r  cder`   gh)r>   r   r   )r
  r  r8  r   r
  rs   r   r   r   z
Expected: 
, Actual: r  r8  z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)	r
   rg   tensorr?   r   r$  r+   r   rF   )r-   r
  r  r8  r   	data_copyr'  s          r%   (test_split_between_processes_nested_dictrA    s
   E A0A-.Ai'Q'TN	**40G""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::ujSVXZI[H\\fgnorgsft<uu:""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::::""a'~~gclIcNC]Q%J]J]E]4^__ MdSViXrZ[_d_r_rZrNsMt  uB  CJ  KN  CO  BP  Q_ $$)~~gclIcN124FGG MdSViXYXZm_\ijqrujviwxG $$)~~gclIcN234GHH MdSViXZX[nM]]jkrsvkwjxyH/ 16 
7 10s   G6I,,
I:c                  .   [        5       n U R                  S:  a  [        R                  " / SQ/ SQ/5      R	                  U R
                  5      nU R                  U5       nU R                  S:X  a3  [        R                  " / SQ/5      R	                  U R
                  5      nO2[        R                  " / SQ/5      R	                  U R
                  5      n[        R                  R                  X#5        S S S 5        U R                  5         [        R                  " [        SU R                  -  5       Vs/ s H  oD/PM     sn5      R	                  U R
                  5      n[        R                  " [        SU R                  -  S-
  5       Vs/ s H  oD/PM     sn5      R	                  U R
                  5      nXV4 GH  nUR                  5        Vs/ s H  n[        R                  " U5      PM     nnU R                  USS9 n[        R                  " [        U5      U R                  -  5      n[        U5      U:X  d-   SU R
                   S	U R                   S
[        U5       35       e/ n	U H#  n
U	R!                  U
R	                  S5      5        M%     S S S 5        U R                  5         [#        W	5      nUS [        U5       nX|:X  a  GM   SU SU 35       e   g ! , (       d  f       GN= fs  snf s  snf s  snf ! , (       d  f       Np= f)Nr>   )r   r>   r   r   )r   r5  r6  r7  r   r   Tr  r0  r1  r  r   r   r!  )r
   r?   rg   r?  r  r{   r$  r+   r  r  rF   r\   r   r   r   r"   r|   r   )r-   r   r'  expectedir)  r(  r*  r2  r+  rb   r,  r   s                r%   #test_split_between_processes_tensorrE    s   EQ||\<89<<U\\J**40G""a' <<7::5<<H <<7::5<<HMM&&w9 1 	!5U5H5H1H+IJ+Iac+IJKNNu||\I||%E4G4G0G!0K*LM*LQS*LMNQQRWR^R^_H%48KKMBMq5<<?MB**4t*D%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%!#**699U+;< " E 	!()<=!+CI.(v,UVeUffmntmu*vv(# & 10 KMBDDs,   !BK%8K7K< LBL%
K4
L	c                     [        5       n U R                  S;   a  [        [        S5      5      n[	        U5      U R                  -  n[	        U5      U R                  -  nU R                  U5       nU R                  U:  a,  [	        U5      US-   :X  d   SUS-    S[	        U5       35       eO%[	        U5      U:X  d   SU S[	        U5       35       eS S S 5        U R                  5         g ! , (       d  f       N= f)N)r>   r   r   rs      r>   z2Each Process should have even elements. Expected: r>  )r
   r?   r   r\   r"   r$  r+   rF   )r-   r   num_samples_per_process
num_extrasr'  s        r%   #test_split_between_processes_evenlyrJ    s    El*E"I"%d)u/B/B"BY!4!44
**40G""Z/7|'>'BB HI`cdIdHeeopst{p|o}~B 7|'>> HI`Haaklopwlxkyz> 1 
 10s   0A"C++
C9c                      [        5       n U R                  5       SL d   eU R                  (       a  U R                  5         U R                  5       SL d   eU R                  5       SL d   eg )NFT)r   check_triggerrA   set_trigger)r^   s    r%   test_triggerrN    si    -K$$&%/// ""! $$&$... $$&%///r'   c                     SS K n [        R                  " 5         [        R                  R                  SS5      n[        5       n[        R                  " 5         U R                  [        5       nUR                  U5        S S S 5        S[        WR                  R                  S   5      ;   d   eS[        UR                  R                  S   5      ;   d   eg ! , (       d  f       N_= f)Nr   r>   z*`AcceleratorState` object has no attributez1This happens if `AcceleratorState._reset_state()`)pytestr
   r  rg   r   r  r   raisesAttributeErrorr   strvalueargs)rP  simple_modelr^   cms       r%   test_reinstantiated_staterX  +  s    !!#88??1a(L-K!!#	~	&"L) 
'73rxx}}Q?O;PPPP>#bhhmmTUFVBWWWW 
'	&s   .C
C'c                  
   [        5       n U R                  nUR                  S:X  a  [        S5        [	        5         UR                  5         UR                  [        R                  :X  a  [        R                  R                  5       nOUR                  nX!R                  :X  Ga   UR                  S:X  a  [        S5        [        5         UR                  S:X  a  [        S5        [        5         UR                  S:X  a  [        S5        [!        5         UR                  S:X  a  [        S5        [#        5         UR                  S:X  a  [        S5        [%        5         UR                  S:X  a  [        S5        ['        5       (       a  SS	KJn  [-        U5        O[        S
5        UR                  S:X  a  [        S5        [/        5         UR                  S:X  a  [        S5        [1        5         UR                  [        R2                  :w  a(  [5        5         [7        5         [9        5         [;        5         UR                  S:  a
  [=        5         UR                  [        R>                  :X  a  g UR                  S:X  a  [        S5        [A        SS9  [A        SS9  UR                  S:X  a  [        S5        [C        5         [E        5       (       a%  UR                  S:X  a  [        S5        [G        5         URI                  5         g )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z(
**Test split between processes evenly**z7
**Test split between processes as a datasets.Dataset**)r   z6Skipped because Hugging Face datasets is not availablez1
**Test random number generator synchronization**z 
**DataLoader integration test**r>   z
**Training integration test**Fr   Tz
**Breakpoint trigger test**z
**Test reinstantiated state**)%r   r-   r1   r*   re   rF   rk   r   rl   rg   rh   device_countr?   r+   rc   r3  rA  rE  rJ  r   datasetsr   r.  rp   r   XLAr   r   r   r   r   	DEEPSPEEDr  rN  r   rX  destroy_process_group)r^   r-   num_processes_per_noder%  s       r%   mainr`  ;  sL   -KE  A%"#	!:!::!&!8!8!:!&!4!4 !4!44!#01!!#@A)+!#@A02!#BC+-!#=>+-!#LM ""<01ABJK  A%BC  A%12!4!44$& -/Q57 !:!::  A%/0.-  A%-.N$$)34!#	!r'   __main__)F)BrQ   rO   r   rB   copyr   pathlibr   numpyr   rg   torch.utils.datar   r   
accelerater   accelerate.data_loaderr   r	   accelerate.stater
   accelerate.test_utilsr   r   r   accelerate.utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r&   r.   r2   r5   r:   rc   re   rp   r   r   r   r   r   r   r   r  r.  r3  rA  rE  rJ  rN  rX  r`  r   r   r'   r%   <module>rk     s  "  	       0 " M - Z Z   " DDDD_COCIFR/$9.x>6B+
\t4&n8$k
\-w`wB#LwD$0"X N"b zF r'   