+j* 8dZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z e e ddhZ dZeGd d Zefd ed ed eee fdzfdZdeee fded dfdZ ddedejdzdedzd efdZdS)zMetadata extraction for asset scanning. Tier 1: Filesystem metadata (zero parsing) Tier 2: Safetensors header metadata (fast JSON read only) N) dataclass)Any)init_mime_typesz .safetensorsz.sfticeZdZUdZdZeed<dZeed<dZe ed<dZ edzed<dZ eed <dZ edzed <dZ eedzed <dZedzed <d Zeed<dZedzed<dZedzed<dZedzed<dZedzed<dZedzed<dZedzed<dZedzed<dZedzed<dZedzed<deeeffdZdedeefdZdS)ExtractedMetadataz/Metadata extracted from a file during scanning.filename file_pathrcontent_lengthN content_typeformat base_model trained_wordsairFhas_preview_images source_url source_arnrepo_url preview_url source_hashrepo_idrevisionfilepath resolve_urlreturnc$|j|j|jd}|jr |j|d<|jr |j|d<|jr |j|d<|jr |j|d<|jr |j|d<|jrd|d<|j r |j |d <|j r |j |d <|j r |j |d <|j r |j |d <|j r |j |d <|jr |j|d<|jr |j|d<|jr |j|d<|jr |j|d<|S)zJConvert to user_metadata dict for AssetReference.user_metadata JSON field.)r r r r r rrrTrrrrrrrrrr)r r r r r rrrrrrrrrrrrr)selfdatas D/home/wildlama/comfy/ComfyUI/app/assets/services/metadata_extract.pyto_user_metadataz"ExtractedMetadata.to_user_metadata9s "1k   > / $D    5#'#4D  ? 1!%D    7$($6D ! 8 #(DK  " .)-D% & ? 1!%D  ? 1!%D  = -#}D    3"&"2D    3"&"2D  < +"lDO = -#}D  = -#}D    3"&"2D  reference_idc ngddtdtdzdtddffd }dtdttzdzddffd }dtdtdzddffd }|d |j|d |j|d |j|d |j|d|j|d|j |j r|j nd}|d||j r0t|j ddD]\}}|d|||d|j |d|j|d|j|d|j|d|j|d|j|d|j|d|j|d|jS)z@Convert to asset_reference_meta rows for typed/indexed querying.rkeyvalNordinalrc |r<||t|dkr |ddn|dddddSdS)Niasset_reference_idr$r&val_strval_numval_boolval_json)appendlen)r$r%r&r"rowss radd_strz/ExtractedMetadata.to_meta_rows..add_strksn  *6&-0XX__s5D5zz## $ $  r!c J||dd|ddddSdSNrr(r.r$r%r"r0s radd_numz/ExtractedMetadata.to_meta_rows..add_numwsN *6 #" $ $r!c J||ddd|dddSdSr3r4r5s radd_boolz0ExtractedMetadata.to_meta_rows..add_boolsN *6 ## # $r!r r r r rrrdr)r&rrrrrrrrr)r)strintfloatboolr r r r rrrr enumeraterrrrrrrrr) rr"r1r6r8 has_previewsiwordr0s ` @r to_meta_rowszExtractedMetadata.to_meta_rowsgs   3:  D         3;#5 $        # D4K D         DM*** $"5666 1222$+&&&  do...tx   262ISt..t %|444   :$T%7%=>> : :4q99999  do... do... DM*** t/000 t/000  4<((( DM*** DM*** t/000 r!)__name__ __module__ __qualname____doc__r r:__annotations__r r r;r r rrlistrrr=rrrrrrrrrdictrr rBr!rrrs99HcIsNC#L#*###FC"Jd !!!&*M49t#***Ct$$$$"Jd !!!!Jd !!!HcDj"Kt""""Kt"""GS4ZHcDjHcDj"Kt""",$sCx.,,,,\FFdFFFFFFr!rpathmax_sizerc8 t|d5}|d}t|dkr ddddStjd|d}||kr ddddS||}t||kr ddddSt j|dcdddS#1swxYwYdS#tt j ttj f$rYdSwxYw)aWRead only the JSON header from a safetensors file. This is very fast - reads 8 bytes for header length, then the JSON header. No tensor data is loaded. Args: path: Absolute path to safetensors file max_size: Maximum header size to read (default 8MB) Returns: Parsed header dict or None if failed rbNz}t|tr'| | ?|rt|dd|_ n#t j $rYnwxYw|j s|d}t|tr t j|}t|trd |D|_ n$d |d D|_ n_#t j $r'd |d D|_ Yn*wxYwt|trd |D|_ |dp|d|_|d} | rd|_|d|_|d|_|d|_|d|_|dp|d|_|dp|d|_|dp|d|_|dp|d|_|dp|d|_dS) z`Extract metadata from safetensors header __metadata__ section. Modifies meta in-place. __metadata__Nss_base_model_versionzmodelspec.base_modelrss_tag_frequencyr9rc,g|]}t|SrJr:.0xs r z1_extract_safetensors_metadata..s)A)A)AQ#a&&)A)A)Ar!c^g|]*}||+SrJstripriws rrkz1_extract_safetensors_metadata..s-)X)X)Xaggii)X!'')))X)X)Xr!,c^g|]*}||+SrJrmros rrkz1_extract_safetensors_metadata..s-%T%T%TA!''))%Taggii%T%T%Tr!c,g|]}t|SrJrgrhs rrkz1_extract_safetensors_metadata..s!5!5!5Q#a&&!5!5!5r!rz modelspec.airssmd_cover_imagesTrrrrrsshs_model_hashr hf_repo_idr hf_revisionr hf_filepathrhf_url)get isinstancerIrr:rTrUsetvaluesupdatekeyssortedrrXrHsplitrrrrrrrrrrr) r`rast_metartag_freqall_tags dataset_tagstwparsed cover_imagess r_extract_safetensors_metadatarsjj,,G gt $ $  +,, % ;;- . . % ;;| $ $ OKK 233M M377  z-00H!$H ( 1 1 9 9 lD119OOL$5$5$7$7888 <%+H%5%5dsd%;"#    D    6 [[ ) ) b#   6 UBfd++Y)A)A&)A)A)AD&&)X)XRXXc]])X)X)XD&' U U U%T%T#%T%T%T""" U D ! ! 6!5!5"!5!5!5D {{5!!AW[[%A%ADH;;233L'"&kk,//DOkk,//DOKK ++DM{{=11D{{=11SW[[AR5S5SD;;y))FW[[-F-FDLKK ++Iw{{=/I/IDMKK ++Iw{{=/I/IDM{{=11JW[[5J5JDs&BD33EE:AG3HHabs_path stat_resultrelative_filenamect}|ptj||_||_tj|\}}|r'|dnd|_ tj |\}}||_ |( tj |d}n#t$rYnwxYw|r |j|_|t"vrPt%|}|r? t'||n-#t($r }t+jd||Yd}~nd}~wwxYw|S)aExtract metadata from a file using tier 1 and tier 2 methods. Tier 1: Filesystem metadata from path and stat Tier 2: Safetensors header parsing if applicable Args: abs_path: Absolute path to the file stat_result: Optional pre-fetched stat result (saves a syscall) relative_filename: Optional relative filename to use instead of basename (e.g., "flux/123/model.safetensors" for model paths) Returns: ExtractedMetadata with all available fields populated .rNT)follow_symlinksz&Safetensors meta extract failed %s: %s)rosrKbasenamer r splitextlstriplowerr mimetypes guess_typer statrWst_sizer SAFETENSORS_EXTENSIONSr_r Exceptionloggingdebug) rrrra_ext mime_typer`es rextract_file_metadatars|&   D&C)9)9()C)CDMDN W  h ' 'FAs-08#**S//'')))bDK'11LIq!D '(DAAAKK    D 2)1 yy{{,,,)(33  U U-fd;;;; U U U FRSTTTTTTTT U Ks*/C CCD"" E ,EE )NN)rFrTrrrrR dataclassesrtypingrutils.mime_typesr frozensetrMAX_SAFETENSORS_HEADER_SIZErr:r;rIr_rrrrJr!rrs  !!!!!!,,,,,,#NF#;<<. QQQQQQQ Qj ;  #s(^d>CK cNCK"3CK CKCKCKCKP*.$(111$&1Tz1 111111r!