+jAPddlZddlZddlmZddlmZmZmZddlZddl m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZddlmZmZddlmZmZm Z m!Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z*m+Z+m,Z,dd l-m.Z.Gd d eZ/GddeZ0edZ1de1de2e3fdZ4de2e3fdZ5de2e3fdZ6 d9de1de7de7de8e3dzfdZ9de1de8e3fdZ:de2e3de;fdZ d:d!e2e3d"e8e3d#e7d$e7de=e2ee8e3e;ff d%Z?d&e2ed'e8e3de;fd(Z@dZAd)ZBd*ZCeAd+fde=e1dfd,e;d-e;de2fd.ZD d;d/e3d0e3d1e3d2e7d3e7d4ege7fdzd5eEe3e#fdzde;fd6ZF d;d7e2d2e7d3e7d4ege7fdzd5eEe3e#fdzde=e;e2e3ff d8ZGdS)<N)Path)CallableLiteral TypedDict)add_missing_tag_for_asset_idbulk_update_enrichment_levelbulk_update_is_missingbulk_update_needs_verifydelete_orphaned_seed_assetdelete_references_by_idsensure_tags_existget_asset_by_hashget_reference_by_idget_references_for_prefixesget_unenriched_references(mark_references_missing_outside_prefixesreassign_asset_referencesremove_missing_tag_for_asset_idset_reference_system_metadataupdate_asset_hash_and_mime) SeedAssetSpecbatch_insert_seed_assets) get_mtime_ns is_visiblelist_files_recursivelyverify_file_unchanged)HashCheckpointcompute_blake3_hash)extract_image_dimensions)extract_file_metadata)compute_relative_filenameget_comfy_models_folders!get_name_and_tags_from_asset_path)create_sessioncBeZdZUeed<eed<eed<eed<eed<dS)_RefInforef_id file_pathexistsstat_unchanged needs_verifyN)__name__ __module__ __qualname__str__annotations__bool2/home/wildlama/comfy/ComfyUI/app/assets/scanner.pyr&r&.sE KKKNNN LLLr3r&c@eZdZUedzed<eed<eeed<dS)_AssetAccumulatorNhashsize_dbrefs)r,r-r.r/r0intlistr&r2r3r4r6r66s; * LLL x.r3r6modelsinputoutputrootreturnc\|dkr7g}tD]\}}||d|DS|dkr1tjt jgS|dkr1tjt jgSgS)Nr=cLg|]!}tj|"Sr2)ospathabspath).0ps r4 z)get_prefixes_for_root..Ds&222q""222r3r>r?)r"extendrDrErF folder_pathsget_input_directoryget_output_directory)r@bases_bucketpathss r4get_prefixes_for_rootrQ?s x688  NGU LL    22E2222 w @ B BCCDD x A C CDDEE Ir3cd}d|DS)z3Get all known asset prefixes across all root types.r<c6g|]}t|D]}|Sr2)rQ)rGr@rHs r4rIz*get_all_known_prefixes..Os. I I I$-B4-H-H I IA I I I Ir3r2) all_rootss r4get_all_known_prefixesrULs&CI I I) I I IIr3cg}tD]\}}tj|pg}|D]}tdt |jDs.tj||}|sFtj |}d}t |}|D]8}| tj |rd}n9|r| |Ɍ|S)Nc34K|]}t|VdS)N)r)rGparts r4 z'collect_models_files..Ws*IIDz$''IIIIIIr3FT) r"rKget_filename_listallrparts get_full_pathrDrErFis_relative_toappend) out folder_namerN rel_filesrel_pathabs_pathallowedabs_pbs r4collect_models_filesrhRsC688%% U 2;??E2 ! % %HIIDNN4HIIIII #1+xHHH wx00HGNNE  ''(:(:;;"GE % 8$$$ % Jr3Fcollect_existing_pathsupdate_missing_tagsc Pt|}|s|rtndSt|||}i}|D] }||j}||j|jgd}|||j<d} d} t|j|dtj |j d} nh#t$rd} YnZt$rd} tjd |j Yn3t $r'} d} tjd |j | Yd} ~ nd} ~ wwxYw|d |j|j | | |jd g} g} g}g}g}t}|D]5\}}|d }|d }t+d|D}t-d|D}|D]}|ds||d&|dr>||d|dr| |d|ds#|ds| |d|\|r|rt/||nE|D]B}|dr8|tj|dC@|rk|D]%}|ds||d&|r@ t7||nq#t8$r } tjd|| Yd} ~ nLd} ~ wwxYwnC|rA t=||dn-#t8$r } tjd|| Yd} ~ nd} ~ wwxYw|D]B}|dr8|tj|dC7t?||t|fd|D}tA||dtA||dtC|| dtC|| d|r|ndS)aReconcile asset references with filesystem for a root. - Toggle needs_verify per reference using mtime/size stat check - For hashed assets with at least one stat-unchanged ref: delete stale missing refs - For seed assets with all refs missing: delete Asset and its references - Optionally add/remove 'missing' tags based on stat check in this root - Optionally return surviving absolute paths Args: session: Database session root: Root type to scan collect_existing_paths: If True, return set of surviving file paths update_missing_tags: If True, update 'missing' tags based on file status Returns: Set of surviving absolute paths if collect_existing_paths=True, else None N)include_missing)r7r8r9FTr8follow_symlinks)mtime_dbr8 stat_resultzPermission denied accessing %szOSError checking %s: %sr9)r'r(r)r*r+r7c3&K|] }|dV dS)r*Nr2rGrs r4rYz2sync_references_with_filesystem..s(>>AA./>>>>>>r3c3(K|] }|d VdS)r)Nr2rrs r4rYz2sync_references_with_filesystem..s(88aak/888888r3r)r'r*r+r()asset_idz-Failed to remove missing tag for asset %s: %s automatic)ruoriginz*Failed to add missing tag for asset %s: %scg|]}|v| Sr2r2)rGr' stale_sets r4rIz3sync_references_with_filesystem..s#WWW&vY?V?Vv?V?V?Vr3)value)"rQsetrgetru asset_hash size_bytesrmtime_nsrDstatr(FileNotFoundErrorPermissionErrorloggingdebugOSErrorr_ reference_idr+itemsanyr[r addrErFr Exceptionwarningrr r r )sessionr@rirjprefixesrowsby_assetrowaccr*r)e to_set_verifyto_clear_verify stale_ref_idsto_mark_missingto_clear_missing survivorsaida_hashr9 any_unchanged all_missingrsrys @r4sync_references_with_filesystemrhs{.%T**H 9.8suuuD8 &+>   D.0H  ll3<(( ;>cnbQQC%(HS\ " GF2IGCM4HHHNN !   FFF K K KF M:CM J J J J J G G GF M3S]A F F F F F F F F G F * ] "0 # 0        "M!#O!M!#O"$%%INN$$-?-?SV6{>>>>>>> 88488888  2 2AX; &&q{333!" 8 ''( 444^$8#**1X;777%& 2q/@ 2$$Qx[111 > G  G*7C8888GGA{G! bgooa n&E&EFFF   V 6 6{6!((8555" 3GcJJJJJ OGa ! V V,Ws;WWWWW V V V LcSTUUUUUUUU V ? ?A{ ? bgooa n==>>> ?Wm444M""IWWWWOWWWO7O4@@@@7$4EBBBBWm4@@@@WoUCCCC. 899D8sT78B00 D>%D% D.DD#K55 L?LL&L99 M#MM#c8 t5}t||dd}||p tcdddS#1swxYwYdS#t$r.}t jd||tcYd}~Sd}~wwxYw)zzSync a single root's references with the filesystem. Returns survivors (existing paths) or empty set on failure. T)rirjNzfast DB scan failed for %s: %s)r$rcommitr{rr exception)r@sessrrs r4sync_root_safelyrs     &7'+$( I KKMMM% & & & & & & & & & & & & & & & & & & :D!DDDuu s@A!7A A!AA!AA!! B+#BBBrc t5}t||}||cdddS#1swxYwYdS#t$r }t jd|Yd}~dSd}~wwxYw)zMark references as missing when outside the given prefixes. This is a non-destructive soft-delete. Returns count marked or 0 on failure. Nz!marking missing assets failed: %sr)r$rrrrr)rrcountrs r4$mark_missing_outside_prefixes_safelyrs     r?)rJrhrrKrLrM)rrPs r4collect_paths_for_rootsr sE5 )++,,,% +L,L,N,NOOPPP5 +L,M,O,OPPQQQ Lr3TrPexisting_pathsenable_metadata_extractioncompute_hashescg}t}d}|D]/}tj|}||vr|dz }, tj|d} n#t $rYPwxYw| js\t|\} } t|} d} |rt|| | } d}|rF t|\}}d|z}n-#t$r }tj d||Yd}~nd}~wwxYw| r| jnd}||| jt!| | | | | ||dd || 1|||fS) aeBuild asset specs from paths, returning (specs, tag_pool, skipped_count). Args: paths: List of file paths to process existing_paths: Set of paths that already exist in the database enable_metadata_extraction: If True, extract tier 1 & 2 metadata compute_hashes: If True, compute blake3 hashes (slow for large files) rTrmNrprelative_filenameblake3:Failed to hash %s: %s) rdr~r info_nametagsfnamemetadatar7 mime_typejob_id)r{rDrErFrrst_sizer#r!r rrrr content_typer_rupdate)rPrrrspecstag_poolskippedrHrfstat_pnamer rel_fnamerr}digest_rrs r4build_asset_specsrs"$EHG //"" N " " qLG  WUD999FF    H ~  6u== d-e44  % ,""+H"&  C C/66 &/  C C C 7BBBBBBBB C.6?H))4  !$n(00!"$"&     (G ##s*A A&%A&-C C/C**C/rrc|sdSt5}|rt||t||d}||jcdddS#1swxYwYdS)zBInsert asset specs into database, returning count of created refs.r)rowner_idN)r$r rr inserted_refs)rrrresults r4insert_asset_specsr`s q   $T  . dH - - -)$ebIII # $$$$$$$$$$$$$$$$$$s?AA#&A#ri max_levellimitcg}|D]$}|t|%|sgSt5}t||||cdddS#1swxYwYdS)a Get assets that need enrichment for the given roots. Args: roots: Tuple of root types to scan max_level: Maximum enrichment level to include limit: Maximum number of rows to return Returns: List of UnenrichedReferenceRow )rrN)rJrQr$r)rrrrr@rs r4get_unenriched_assets_for_rootsrrsH55-d334444     T( (iu                     sAA #A r(rruextract_metadata compute_hashinterrupt_checkhash_checkpointsct} tj|d} n#t$r|cYSwxYwt | } t |} d} d} |r"t || | } | r| j} t}d}|rd t | }| j }d}|}| |}|ftj|d}|j t |ks|j |j krd}| |dnt |}t|||\}}|||||_ ||_ |||<|S|| |dtj|d}t |}||krtjd|nd|}| p| du}|rt"}n-#t$$r }tjd||Yd}~nd}~wwxYwt'||}| |j | kr0|tjd |tS|rf| rd| }| r=| d r(t1|| }|r||t5||||rqt7||}|rL|j|krAt;|||j|t=||| rt?||j| n't?|||| n| rt?||| tA||g||!|S) aEnrich a single asset with metadata and/or hash. Args: session: Database session (caller manages lifecycle) file_path: Absolute path to the file reference_id: ID of the reference to update asset_id: ID of the asset to update (for mime_type and hash) extract_metadata: If True, extract safetensors header and mime type compute_hash: If True, compute blake3 hash interrupt_check: Optional non-blocking callable that returns True if the operation should be interrupted (e.g. paused or cancelled) hash_checkpoints: Optional dict for saving/restoring hash progress across interruptions, keyed by file path Returns: New enrichment level achieved TrmNr)r checkpointz1File modified during hashing, discarding hash: %srrz?Ref %s mtime changed during enrichment, discarding stale resultzimage/)r)"ENRICHMENT_STUBrDrrrr!r rENRICHMENT_METADATArr|r file_sizepoprrrENRICHMENT_HASHEDrrrollbackinfoto_user_metadata startswithrrrridrr rrr)rr(rrurrrr new_levelrinitial_mtime_nsrrr full_hash mtime_before size_beforercur_statrnew_checkpoint stat_after mtime_after metadata_okrrefsystem_metadatadimsexistings r4 enrich_assetrs56 ID999 $F++))44I IH,( '     , -I+I I.C- C'//L .KJ+-11)<< )!wy$GGGH"+|H/E/EEE)3x7GGG%) (,,Y===='3H'='= %8 /%&&& "FN ~#/N4N.:N+/:N,2@$Y/   + $$Y555DAAAJ&z22K{** SU^____.f.. "22Jhd6J 2 1I C C C O3Y B B B B B B B B C g| 4 4C {cl&666 M    NHN"3355  ---h77 -+IKKKD -&&t,,,%g|_MMM K$Wi88  P x// %gxl S S S &w 9 9 9 V*7HK9UUUU &w)Y O O O O K"7H JJJJ <.)DDD NN s, //>CF4A.F44 G>GGrc d}g}t5}|D]}| |rn t||j|j|j||||} | |jkr|dz }n||jc#t$rS} tj d|j| | ||jYd} ~ d} ~ wwxYwdddn #1swxYwY||fS)aEnrich a batch of assets. Uses a single DB session for the entire batch, committing after each individual asset to avoid long-held transactions while eliminating per-asset session creation overhead. Args: rows: List of UnenrichedReferenceRow from get_unenriched_assets_for_roots extract_metadata: If True, extract metadata for each asset compute_hash: If True, compute hash for each asset interrupt_check: Optional non-blocking callable that returns True if the operation should be interrupted (e.g. paused or cancelled) hash_checkpoints: Optional dict for saving/restoring hash progress across interruptions, keyed by file path Returns: Tuple of (enriched_count, failed_reference_ids) rN)r(rrurrrrrzFailed to enrich %s: %s) r$rr(rruenrichment_levelr_rrrr) rrrrrenriched failed_idsrrrrs r4enrich_assets_batchrst2HJ   4T 4 4C*/@/@* 4(!m!$!1 \%5!-$3%5    s333MHH%%c&6777 4 4 4 93=!LLL !!#"233333333 4)4444444444444442 Z s<C%AA98C%9 CA C C%CC%%C),C))FF)TF)TFNN)HrrDpathlibrtypingrrrrKapp.assets.database.queriesrrr r r r r rrrrrrrrrapp.assets.services.bulk_ingestrrapp.assets.services.file_utilsrrrrapp.assets.services.hashingrr$app.assets.services.image_dimensionsr$app.assets.services.metadata_extractr app.assets.services.path_utilsr!r"r#app.database.dbr$r&r6RootTyper;r/rQrUrhr1r{rrr:rtuplerrrrrrrdictrrr2r3r4rs //////////$ LKKKKKKKIIIIIIFFFFFF +*****y  . /  T#Y    JS JJJJ d3i2$) % 99 9!9 9  X_ 9999D8C( 49      53#7 DI    (, C$C$ 9C$HC$!%C$ C$  4 C# -. C$C$C$C$N $d=1 $SX $# $ $ $ $ %  3          B"159=EEEE E  E  Eb$h'$.E3./$6E EEEET"159= 5 5 5 5 5 b$h'$. 5 3./$6 5  3S > 5 5 5 5 5 5 r3