
    
3j#                     R   S r SSKrSSKrSSKrSSKrSSKJr  SSKJrJrJ	r	J
r
JrJr  SSKrSSKJr  SSKJr  SSKJr  SS	KJr  \R.                  " \5      rS
r " S S5      rS\R8                  S\	S\
\   4S jr    SS\\	   S\\   S\\\\\
4      S\4S jjr  " S S\5      r!g)aR  A dataset reader that reads tarfile based datasets

This reader can extract image samples from:
* a single tar of image files
* a folder of multiple tarfiles containing imagefiles
* a tar of tars containing image files

Labels are based on the combined folder and/or tar name structure.

Hacked together by / Copyright 2020 Ross Wightman
    N)glob)ListTupleDictSetOptionalUnion)natural_key   )load_class_map)get_img_extensions)Readerz_tarinfos.picklec                   X    \ rS rSrSS\R
                  S\R                  4S jjrS rSr	g)	TarState   Ntftic                 *    Xl         X l        0 U l        g N)r   r   children)selfr   r   s      _/home/wildlama/miniconda3/lib/python3.13/site-packages/timm/data/readers/reader_image_in_tar.py__init__TarState.__init__!   s    #%#%-/    c                     S U l         g r   )r   r   s    r   resetTarState.reset&   s	    r   )r   r   r   )NN)
__name__
__module____qualname____firstlineno__tarfileTarFileTarInfor   r   __static_attributes__ r   r   r   r      s"    07?? 0w 0
r   r   r   parent_info
extensionsc                 
   Sn[        U 5       GH]  u  pEUR                  5       (       d  M  [        R                  R	                  UR                  5      u  pg[        R                  R                  U5      u  pU	R                  5       n	U	S:X  a  [        R                  " U R                  U5      SS9 n
[        UR                  [        R                  R                  US   U5      U/ / S9nU[        XUS9-  n[        R                  U SUR                   S	[!        US
   5       S35        US   R#                  U5        S S S 5        GM<  X;   d  GMD  US
   R#                  U5        US-  nGM`     U$ ! , (       d  f       GMt  = f)Nr   .tarr|)fileobjmodepathnamer0   r   r   samplesr*   z"/?. Extracted child tarinfos from . r3   z images.r   r   )	enumerateisfileosr0   splitsplitextlowerr$   openextractfiledictr2   join_extract_tarinfo_loggerdebuglenappend)r   r)   r*   sample_countir   dirnamebasenamer2   extctf
child_infos               r   r@   r@   *   sH   L2yy{{GGMM"''2GG$$X.	iik&=bnnR&8tD!rww||K4G'NSU`blnp
 0Z XX#EbggYbQTU_`iUjQkPlltuvJ'..z: ED 	"))"-AL    EDs   1BE22
F	class_name_to_idxcache_tarinfosortc                   ^^$^%^&^' U(       d	  [        SS9O
[        U5      nSn[        R                  R	                  U 5      (       a  [        R                  R                  U 5      S   R                  5       S:X  d   eU /n[        R                  R                  U 5      u  p[        R                  R                  U5      S   nSnO{U R                  [        R                  R                  5      R                  [        R                  R                  5      S   n[        [        R                  R                  U S5      SS9n[        U5      n[        U V	s/ s H"  n	[        R                  R                  U	5      PM$     sn	5      n
U(       d   S	U  S
35       e[        R!                  SU
S-  S S35        [#        / S9nSnUc
  U
S:  a  SOSnU(       a+  SU-   [$        -   n[        R                  R                  X5      n[        R                  R'                  U5      (       a_  [        R!                  SU S35        [)        US5       n[*        R,                  " U5      nS S S 5        [        US   5      U:X  d   S5       eGOB[/        U5       H  u  nnU(       a  SO>[        R                  R                  [        R                  R1                  U5      5      S   n[2        R(                  " USS9 n[#        [        R                  R5                  UU 5      US / / S9n[7        UUUS9n[        US   5      n[        R9                  U SU SU SU S U S!3
5        S S S 5        US   R;                  W5        M     U(       aD  [        R!                  S"U S35        [)        US#5       n[*        R<                  " X5        S S S 5        / m'/ m&Sm%Tc  Sm%/ nSS$.S% jm$U$U%UU&U'4S& jn[        R!                  S'5        US    H~  nU(       a  S OUS(   n[?        5       nSnUS    H2  nU" UUS)9nU(       a  [?        US*   S+9UR@                  US(   '   UU-  nM4     UU" UUS)9-  nU(       d  Mk  UR;                  UU45        M     AT%(       aB  [C        [E        [        T&5      [F        S,95      n[/        U5       VVs0 s H	  u  nnUU_M     snnm[        R!                  S-5        [I        T'T&5       V V!s/ s H  u  n n!U!T;   d  M  U TU!   4PM     n"n n!U(       a  [E        U"S. S,9n"[I        U"6 u  m'n#[J        RL                  " T'5      m'[J        RL                  " U#5      n#[        R!                  S/[        T'5       S0[        U5       S135        T'U#TU4$ s  sn	f ! , (       d  f       GN5= f! , (       d  f       GNM= f! , (       d  f       GN= fs  snnf s  sn!n f )2NT)as_setFr,   r   z*.tar)	recursivez'No .tar files found at specified path (z).z	Scanning i   z.2fzMB of tar files...)tartrees l       
 _z!Reading tar info from cache file .rbrS   z3Cached tartree len doesn't match number of tarfilesr-   )r/   r1   r4   r   /z. Extracted tarinfos from r5   z children, z	 samples.zWriting tar info to cache file wb)	leaf_onlyc                 D   [         R                  R                  " U6 R                  [         R                  R                  5      nU (       a,  UR                  [         R                  R                  5      S   $ UR                  [         R                  R                  S5      $ )NrQ   rU   )r8   r0   r?   stripsepr9   replace)rZ   r0   s     r   _label_from_paths+extract_tarinfos.<locals>._label_from_paths}   s^    ww||T"((5.7tzz"''++&r*[T\\"''++WZ=[[r   c                    > SnU S    Hq  nT" U S   [         R                  R                  UR                  5      5      nT(       d  UT;  a  ME  T	R                  X1U S   45        TR                  U5        US-  nMs     U$ )Nr   r3   r0   r   r   )r8   r0   rG   rD   )
infofnaddedslabelr_   build_class_maprL   labelsr3   s
        r   _add_samples&extract_tarinfos.<locals>._add_samples   sv    iA%d6lBGGOOAFF4KLE"u4E'ENNA4:./MM% QJE ! r   z+Collecting samples and building tar states.r2   )rc   r   )r   )keyz$Mapping targets and sorting samples.c                 8    [        U S   S   R                  5      $ Nr   )r
   r0   )ks    r   <lambda>"extract_tarinfos.<locals>.<lambda>   s    TUVWTXYZT[T`T`Har   zFinished processing z samples across z tar files.)'r   setr8   r0   r7   r:   r;   r9   r\   r]   r   r?   rC   sumgetsizerA   rb   r>   CACHE_FILENAME_SUFFIXexistsr<   pickleloadr6   rH   r$   relpathr@   rB   rD   dumpr   r   listsortedr
   zipnparray)(rootrL   rM   r*   rN   root_is_tartar_filenames	root_namenum_tarsf	tar_bytesrb   
cache_pathcache_filenamepfrF   rc   r0   r   r)   num_samplesnum_childrentarfilesri   tar_name	tar_stateparent_addedrK   child_addedsorted_labelsidxcre   lsamples_and_targetstargetsr_   rg   rh   r3   s(    `                                  @@@@r   extract_tarinfosr   ?   s    9C#40JJK	ww~~dww%b)//1V;;;''---GG$$Y/2	JJrww{{+11"''++>rB	RWW\\$8DI=!H?ARWW__Q'?@IG>tfBGG8LL9Yw.s33EFGDJ )J 6Ey+@@WW\\$7
	ww~~j!!8AFG*d#r;;r?D $4
#$0g2gg0}-EAr$2"''*:*:277;K;KB;O*PQR*SDbt,"D(AQU`blno.r;:V";z#:;c8*$>rd"\NR]^i]jjstv	 - ##K0 . LL::,aHIj$'2D% ( GFO  H+/ \	 	 LL>@J'&4K,?J	%j1J&zh?K9AZPTEU9V	"":f#56K'L	 2
 	[X>><OOXy12 ( 	VCK[AB2;M2JK2JQQV2JKLL79ADWfAUpAUAYZ^oYo4A034AUp$%8>ab/0GWhhwGhhwGLL'G~5Ec(m_T_`aG.88o @ $# -, ('b L qs=   )VV/A)V$V03W+W<W
V
V-	0
V?c                   F   ^  \ rS rSrSrSU 4S jjrS rS rS	S jrSr	U =r
$ )
ReaderImageInTar   zHMulti-tarfile dataset reader where there is one .tar file per class
    c                   > [         T	U ]  5         S nU(       a  [        X!5      nXl        [	        U R                  UUS9u  U l        U l        U l        nU R                  R                  5        VVs0 s H  u  pxX_M	     snnU l	        [        U5      S:X  a  US   S   c  SU l        US   S   U l        OSU l        [        U5      U l        X0l        g s  snnf )N)rL   rM   r   r   TF)superr   r   r   r   r3   r   rL   itemsclass_idx_to_namerC   r   r   r>   cache_tarfiles)
r   r   	class_mapr   rM   rL   r   rn   v	__class__s
            r   r   ReaderImageInTar.__init__   s      .y ?	GWII/'H
DdlD$:H
 483I3I3O3O3Q!R3Q41!$3Q!Rx=A(1+a."8#D%a[^DN$D!(^DN, "Ss   0Cc                 ,    [        U R                  5      $ r   )rC   r3   r   s    r   __len__ReaderImageInTar.__len__   s    4<<  r   c                     U R                   U   nU R                  U   nUu  pEnU(       a*  [        R                  R	                  U R
                  U5      OU R
                  nS nS n	U R                  (       a8  U R                  (       a  U R                  OU R                  U   n	U	R                  nUc-  [        R                  " U5      nU R                  (       a  Xl	        Ub  U R                  (       a#  U	R                  UR                     R                  OS n
U
cQ  [        R                  " UR                  U5      S9n
U R                  (       a  XR                  UR                     l	        U
nUR                  U5      U4$ )N)r.   )r3   r   r8   r0   r?   r   r   r   r   r   r$   r<   r   r2   r=   )r   indexsampletarget	sample_ti	parent_fnchild_ti
parent_absr   cache_staterJ   s              r   __getitem__ReaderImageInTar.__getitem__   s    e$e$)/&	h;DRWW\\$))Y7$))
,0,<,<$..$..QZB[KB:j)B""!#<@<O<O+&&x}}588UYC{ll2>>(+CD&&=@((7:B~~i(&00r   c                     U R                   U   S   R                  nU(       a  [        R                  R	                  U5      nU$ rm   )r3   r2   r8   r0   rH   )r   r   rH   absolutefilenames        r   	_filenameReaderImageInTar._filename   s7    <<&q)..ww''1Hr   )r   r   rL   r   r   r3   r   r   )rT   TN)FF)r    r!   r"   r#   __doc__r   r   r   r   r'   __classcell__)r   s   @r   r   r      s!    -*!12 r   r   )NNNT)"r   loggingr8   rv   r$   r   typingr   r   r   r   r   r	   numpyr}   timm.utils.miscr
   r   r   img_extensionsr   readerr   	getLoggerr    rA   rt   r   r%   strr@   boolr   r   r(   r   r   <module>r      s   
  	    : :  ' % . 


H
%*   t S . -1(,8<j9#D>j9  ~j9 U4#345	j9
 j9Z9v 9r   