
    +j*              	       &   S r SSKrSSKrSSKrSSKrSSKrSSKJr  SSKJ	r	  SSK
Jr  \" 5         \" SS15      rSr\ " S	 S
5      5       r\4S\S\S\\\	4   S-  4S jjrS\\\	4   S\SS4S jr  SS\S\R*                  S-  S\S-  S\4S jjrg)zMetadata extraction for asset scanning.

Tier 1: Filesystem metadata (zero parsing)
Tier 2: Safetensors header metadata (fast JSON read only)
    N)	dataclass)Any)init_mime_typesz.safetensorsz.sfti   c                      \ rS rSr% SrSr\\S'   Sr\\S'   Sr	\
\S'   Sr\S-  \S	'   Sr\\S
'   Sr\S-  \S'   Sr\\   S-  \S'   Sr\S-  \S'   Sr\\S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   Sr\S-  \S'   S\\\4   4S jrS\S\\   4S jrSr g)ExtractedMetadata   z/Metadata extracted from a file during scanning. filename	file_pathr   content_lengthNcontent_typeformat
base_modeltrained_wordsairFhas_preview_images
source_url
source_arnrepo_urlpreview_urlsource_hashrepo_idrevisionfilepathresolve_urlreturnc                    U R                   U R                  U R                  S.nU R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R
                  (       a  U R
                  US'   U R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R                  (       a  SUS'   U R                  (       a  U R                  US	'   U R                  (       a  U R                  US
'   U R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R                  (       a  U R                  US'   U R                   (       a  U R                   US'   U R"                  (       a  U R"                  US'   U$ )zJConvert to user_metadata dict for AssetReference.user_metadata JSON field.)r
   r   r   r   r   r   r   r   Tr   r   r   r   r   r   r   r   r   r   )r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )selfdatas     D/home/wildlama/comfy/ComfyUI/app/assets/services/metadata_extract.pyto_user_metadata"ExtractedMetadata.to_user_metadata9   s{    "11kk 

 >> $D#'#4#4D  ??!%D$($6$6D!88((DK"")-D%& ??!%D??!%D==#}}D"&"2"2D"&"2"2D <<"llDO==#}}D==#}}D"&"2"2D    reference_idc           	        ^^ / mSS[         S[         S-  S[        SS4UU4S jjjnS[         S[        [        -  S-  SS4UU4S jjnS[         S[        S-  SS4UU4S jjnU" S	U R                  5        U" S
U R
                  5        U" SU R                  5        U" SU R                  5        U" SU R                  5        U" SU R                  5        U R                  (       a  U R                  OSnU" SU5        U R                  (       a(  [        U R                  SS 5       H  u  pgU" SXvS9  M     U" SU R                  5        U" SU R                  5        U" SU R                  5        U" SU R                   5        U" SU R"                  5        U" SU R$                  5        U" SU R&                  5        U" SU R(                  5        U" SU R*                  5        T$ )z@Convert to asset_reference_meta rows for typed/indexed querying.keyvalNordinalr   c           
      p   > U(       a.  TR                  TU U[        U5      S:  a  US S OUS S S S.5        g g )Ni   asset_reference_idr&   r(   val_strval_numval_boolval_json)appendlen)r&   r'   r(   r$   rowss      r    add_str/ExtractedMetadata.to_meta_rows.<locals>.add_strk   sC    *6&-0X_s5Dz## $ $  r#   c           
      @   > Ub  TR                  TU SS US S S.5        g g Nr   r*   r0   r&   r'   r$   r2   s     r    add_num/ExtractedMetadata.to_meta_rows.<locals>.add_numw   s3    *6 #" $ $  r#   c           
      @   > Ub  TR                  TU SS S US S.5        g g r6   r7   r8   s     r    add_bool0ExtractedMetadata.to_meta_rows.<locals>.add_bool   s3    *6 ## # $  r#   r
   r   r   r   r   r   r   d   r   )r(   r   r   r   r   r   r   r   r   r   )r   )strintfloatboolr
   r   r   r   r   r   r   r   	enumerater   r   r   r   r   r   r   r   r   )	r   r$   r3   r9   r<   has_previewsiwordr2   s	    `      @r    to_meta_rowsExtractedMetadata.to_meta_rowsg   s   
	 
	3: 
	 
	D 
	 
	
	 
	3;#5 
	$ 
	 
	
	# 
	D4K 
	D 
	 
	 	
DMM* $"5"56 1 12$++& 	doo.txx 262I2It..t%|4 $T%7%7%=>9 ? 	doo.doo.
DMM*t//0t//0 		4<<(
DMM*
DMM*t//0r#    )!__name__
__module____qualname____firstlineno____doc__r
   r?   __annotations__r   r   r@   r   r   r   r   listr   r   rB   r   r   r   r   r   r   r   r   r   dictr   r!   rG   __static_attributes__rI   r#   r    r   r      s*   9 HcIsNC#L#*#FC "Jd
!&*M49t#*Ct$$ "Jd
!!Jd
!HcDj"Kt""Kt" GS4ZHcDjHcDj"Kt",$sCx. ,\F Fd Fr#   r   pathmax_sizer   c                     [        U S5       nUR                  S5      n[        U5      S:  a
   SSS5        g[        R                  " SU5      S   nXA:  a
   SSS5        gUR                  U5      n[        U5      U:  a
   SSS5        g[
        R                  " UR                  S5      5      sSSS5        $ ! , (       d  f       g= f! [        [
        R                  [        [        R                  4 a     gf = f)a7  Read only the JSON header from a safetensors file.

This is very fast - reads 8 bytes for header length, then the JSON header.
No tensor data is loaded.

Args:
    path: Absolute path to safetensors file
    max_size: Maximum header size to read (default 8MB)

Returns:
    Parsed header dict or None if failed
rb   Nz<Qr   zutf-8)openreadr1   structunpackjsonloadsdecodeOSErrorJSONDecodeErrorUnicodeDecodeErrorerror)rS   rT   fheader_byteslength_of_headerheader_datas         r    _read_safetensors_headerrg      s    $66!9L< 1$   &}}T<@C*  &&!12K;"22  ::k009:  T))+=v||L sP   C "B:C  B:C "!B:C $B:0	C :
CC C .C<;C<headermetac                    U R                  S0 5      n[        U[        5      (       d  gUR                  S5      =(       d)    UR                  S5      =(       d    UR                  S5      Ul        UR                  S5      nU(       a  [        U[        5      (       a   [
        R                  " U5      n[        5       nUR                  5        H9  n[        U[        5      (       d  M  UR                  UR                  5       5        M;     U(       a  [        U5      SS Ul        UR                  (       d  UR                  S5      n[        U[        5      (       a   [
        R                  " U5      n[        U[        5      (       a"  U V	s/ s H  n	[	        U	5      PM     sn	Ul        OUR                  S	5       V
s/ s H)  oR!                  5       (       d  M  U
R!                  5       PM+     sn
Ul        O6[        U[        5      (       a!  U V	s/ s H  n	[	        U	5      PM     sn	Ul        UR                  S
5      =(       d    UR                  S5      Ul        UR                  S5      nU(       a  SUl        UR                  S5      Ul        UR                  S5      Ul        UR                  S5      Ul        UR                  S5      Ul        UR                  S5      =(       d    UR                  S5      Ul        UR                  S5      =(       d    UR                  S5      Ul        UR                  S5      =(       d    UR                  S5      Ul        UR                  S5      =(       d    UR                  S5      Ul        UR                  S5      =(       d    UR                  S5      Ul        g! [
        R                   a     GNf = fs  sn	f s  sn
f ! [
        R                   aU    UR                  S	5       V
s/ s H)  oR!                  5       (       d  M  U
R!                  5       PM+     Os  sn
f sn
Ul         GNf = fs  sn	f )zXExtract metadata from safetensors header __metadata__ section.

Modifies meta in-place.
__metadata__Nss_base_model_versionzmodelspec.base_modelr   ss_tag_frequencyr>   r   ,r   zmodelspec.airssmd_cover_imagesTr   r   r   r   r   sshs_model_hashr   
hf_repo_idr   hf_revisionr   hf_filepathr   hf_url)get
isinstancerQ   r   r?   r\   r]   setvaluesupdatekeyssortedr   r`   rP   splitstripr   r   r   r   r   r   r   r   r   r   r   )rh   ri   st_metar   tag_freqall_tagsdataset_tagstwparsedxwcover_imagess               r    _extract_safetensors_metadatar      s     jj,Ggt$$ 	+, 	%;;-.	%;;|$ 	O KK 23MM377
	zz-0H!$H ( 1lD11OOL$5$5$78 !2 %+H%5ds%;"
 [[)b#UBfd++:@)A&Q#a&&)AD&=?XXc])X]ggi)!'')])XD& D!!24!5"Q#a&"!5D {{5!AW[[%ADH ;;23L"& kk,/DOkk,/DOKK
+DM{{=1D{{=1SW[[AR5SD ;;y)FW[[-FDLKK
+Iw{{=/IDMKK
+Iw{{=/IDM{{=1JW[[5JDK ## 		 *B)X'' U9;#%TA'')iaggi%T"U "6sm   AN7 '=N7 /O O O )O <OO,O Q7OO
O 'QP4P43QQabs_pathstat_resultrelative_filenamec                    [        5       nU=(       d    [        R                  R                  U 5      Ul        Xl        [        R                  R                  U 5      u  pEU(       a  UR                  S5      R                  5       OSUl	        [        R                  " U 5      u  pdXcl        Uc   [        R                  " U SS9nU(       a  UR                  Ul        UR                  5       ["        ;   a   [%        U 5      nU(       a   ['        Xs5        U$ U$ ! [         a     N^f = f! [(         a"  n[*        R,                  " SX5         SnAU$ SnAff = f)a  Extract metadata from a file using tier 1 and tier 2 methods.

Tier 1: Filesystem metadata from path and stat
Tier 2: Safetensors header parsing if applicable

Args:
    abs_path: Absolute path to the file
    stat_result: Optional pre-fetched stat result (saves a syscall)
    relative_filename: Optional relative filename to use instead of basename
        (e.g., "flux/123/model.safetensors" for model paths)

Returns:
    ExtractedMetadata with all available fields populated
.r	   NT)follow_symlinksz&Safetensors meta extract failed %s: %s)r   osrS   basenamer
   r   splitextlstriplowerr   	mimetypes
guess_typer   statr_   st_sizer   SAFETENSORS_EXTENSIONSrg   r   	Exceptionloggingdebug)	r   r   r   ri   _ext	mime_typerh   es	            r    extract_file_metadatar     s   & D &C)9)9()CDMNWWh'FA-0#**S/'')bDK''1LI! 	''(DAK )11 yy{,,)(3U-f; K4K  		  UFTTKUs*   .D D% 
D"!D"%
E/EE)NN)rN   r\   r   r   r   rZ   dataclassesr   typingr   utils.mime_typesr   	frozensetr   MAX_SAFETENSORS_HEADER_SIZEr   r?   r@   rQ   rg   r   r   r   rI   r#   r    <module>r      s       	  !  ,   #NF#;<  .  Q Q Qj  ;
	#s(^d>CKcNCK"3CK	CKP *.$(11$&1 Tz1 	1r#   