+jrdZddlZddlZddlZddlZddlmZmZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZddlmZGdd eZGd d e ZGd d e ZeGddZeGddZ e egdfZ!GddZ"e"Z#dS)zHBackground asset seeder with thread management and cancellation support.N) dataclassfield)Enum)Callable) ENRICHMENT_METADATAENRICHMENT_STUBRootTypebuild_asset_specscollect_paths_for_rootsenrich_assets_batchget_all_known_prefixesget_prefixes_for_rootget_unenriched_assets_for_rootsinsert_asset_specs$mark_missing_outside_prefixes_safelysync_root_safely)dependencies_availableceZdZdZdS)ScanInProgressErrorzBRaised when an operation cannot proceed because a scan is running.N)__name__ __module__ __qualname____doc__1/home/wildlama/comfy/ComfyUI/app/assets/seeder.pyrrsLLLLrrc"eZdZdZdZdZdZdZdS)StatezSeeder state machine states.IDLERUNNINGPAUSED CANCELLINGN)rrrrrr r!r"rrrrr s(&& DG FJJJrrceZdZdZdZdZdZdS) ScanPhasezScan phase options.fastenrichfullN)rrrrFASTENRICHFULLrrrr$r$)s# D F DDDrr$cLeZdZUdZdZeed<dZeed<dZeed<dZ eed<dS)Progressz*Progress information for a scan operation.rscannedtotalcreatedskippedN) rrrrr-int__annotations__r.r/r0rrrr,r,1sU44GSE3NNNGSGSrr,c\eZdZUdZeed<edzed<eeZ ee ed<dS) ScanStatusz#Current status of the asset seeder.stateNprogress)default_factoryerrors) rrrrrr2r,rlistr8strrrrr4r4;sR-- LLLod333FDI33333rr4ceZdZdZd1dZd1dZdefdZdej dddfd e e d fd ed e dzd ededef dZ d2d e e d fd e dzd edefdZ d2d e e d fd e dzdedefdZ d3d e e d fdedefdZdefdZdefdZdefdZdefdZ d4d e e d fdzd edzd e dzd edzdedzdedefdZd5dedzdefdZdefdZd6deddfdZdefdZd1dZdefdZdefd Zdefd!Z d"e!d#e"ddfd$Z# d7d%edzd&edzd'edzd(edzddf d)Z$d*Z%d+e!ddfd,Z&d e e d fddfd-Z'd1d.Z(d e e d fde eeeffd/Z)d e e d fde eeffd0Z*dS)8 _AssetSeederzBackground asset scanning manager. Spawns ephemeral daemon threads for scanning. Each scan creates a new thread that exits when complete. Use the module-level ``asset_seeder`` instance. returnNctj|_tj|_d|_d|_g|_d|_ tj |_ tj |_ |j d|_tj|_d|_d|_d|_d|_d|_dS)NrF) threadingRLock_lockrr_state _progress_last_progress_errors_threadEvent _cancel_event _run_gateset_rootsr$r*_phase_compute_hashes _prune_first_progress_callback _disabled_pending_enrichselfs r__init__z_AssetSeeder.__init__Os_&& j *./3"$ 04 &_.."** ,. !* %*"';?$,0rc<d|_tjddS)z=Disable the asset seeder, preventing any scans from starting.TzAsset seeder disabledN)rPlogginginforRs rdisablez_AssetSeeder.disablecs  ,-----rc|jS)z&Check if the asset seeder is disabled.)rPrRs r is_disabledz_AssetSeeder.is_disabledhs ~rmodelsinputoutputFroots.phaseprogress_callback prune_firstcompute_hashesc|jrtjddStjd||j|j5|jtjkr"tjd ddddStj |_t|_ g|_ ||_ ||_||_||_||_|j|jt-j|jdd|_|j ddddS#1swxYwYdS) aStart a background scan for the given roots. Args: roots: Tuple of root types to scan (models, input, output) phase: Scan phase to run (FAST, ENRICH, or FULL for both) progress_callback: Optional callback called with progress updates prune_first: If True, prune orphaned assets before scanning compute_hashes: If True, compute blake3 hashes (slow) Returns: True if scan was started, False if already running z(Asset seeder is disabled, skipping startFz!Seeder start (roots=%s, phase=%s)z,Asset seeder already running, skipping startNr<T)targetnamedaemon)rPrVdebugrWvaluerArBrrr r,rCrErKrLrNrMrOrHclearrIrJr?Thread _run_scanrFstart)rSr_r`rarbrcs rrmz_AssetSeeder.startls( >  MD E E E5 8%MMM Z  {ej(( KLLL         -DK%ZZDNDLDKDK +D #1D &7D #   $ $ & & & N   $+~#DL L   )                  s+E8B;EEEcJ||tj||dS)a^Start a fast scan (phase 1 only) - creates stub records. Args: roots: Tuple of root types to scan progress_callback: Optional callback for progress updates prune_first: If True, prune orphaned assets before scanning Returns: True if scan was started, False if already running Fr_r`rarbrc)rmr$r()rSr_rarbs r start_fastz_AssetSeeder.start_fasts0 zz./#    rcJ||tj|d|S)a`Start an enrichment scan (phase 2 only) - extracts metadata and hashes. Args: roots: Tuple of root types to scan progress_callback: Optional callback for progress updates compute_hashes: If True, compute blake3 hashes Returns: True if scan was started, False if already running Fro)rmr$r))rSr_rarcs r start_enrichz_AssetSeeder.start_enrichs1 zz"/)    rc|j5|||r ddddS|j^t|jd}||t ||jd<|jdp||jd<n ||d|_t jd|jddddn #1swxYwYdS)aStart an enrichment scan now, or queue it for after the current scan. If the seeder is idle, starts immediately. Otherwise, the enrich request is stored and will run automatically when the current scan finishes. Args: roots: Tuple of root types to scan compute_hashes: If True, compute blake3 hashes Returns: True if started immediately, False if queued for later r_rcNTr_rczEnrich scan queued (roots=%s)F)rArrrQrJupdatetuplerVrW)rSr_rcexisting_rootss renqueue_enrichz_AssetSeeder.enqueue_enrichsf$Z Y Y  u^ LL  Y Y Y Y Y Y Y Y#/!$T%9'%B!C!C%%e,,,05n0E0E$W-()9:Ln$%566 #&4(($ L8$:Nw:W X X X Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y usC BC  C C cf|j5|jtjtjfvr ddddSt jd|jjtj|_|j |j  ddddS#1swxYwYdS)zRequest cancellation of the current scan. Returns: True if cancellation was requested, False if not running or paused NFz Asset seeder cancelling (was %s)T) rArBrr r!rVrWrir"rHrJrIrRs rcancelz_AssetSeeder.cancels Z  {5=%,"???         L;T[=N O O O*DK   " " $ $ $ N                     s!B&A#B&&B*-B*c*|S)zStop the current scan (alias for cancel). Returns: True if stop was requested, False if not running )rzrRs rstopz_AssetSeeder.stops {{}}rc |j5|jtjkr ddddSt jdtj|_|j ddddS#1swxYwYdS)zPause the current scan. The scan will complete its current batch before pausing. Returns: True if pause was requested, False if not running NFzAsset seeder pausingT) rArBrr rVrWr!rIrjrRs rpausez_AssetSeeder.pausesZ  {em++         L/ 0 0 0,DK N " " "                   sA8?A88A<?A<c2|j5|jtjkr ddddSt jdtj|_|jdddn #1swxYwY| didS)zResume a paused scan. This is a noop if the scan is not in the PAUSED state Returns: True if resumed, False if not paused NFzAsset seeder resumingzassets.seed.resumedT) rArBrr!rVrWr rIrJ _emit_eventrRs rresumez_AssetSeeder.resumesZ ! !{el** ! ! ! ! ! ! ! ! L0 1 1 1-DK N    ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! .333tsA6>A66A:=A:@timeoutcftjd|j5|j}|j}|j} |j} |j} dddn #1swxYwY|| |sdS||n| } | ||n|||n|| ||n| ||n| S)a(Cancel any running scan and start a new one. Args: roots: Roots to scan (defaults to previous roots) phase: Scan phase (defaults to previous phase) progress_callback: Progress callback (defaults to previous) prune_first: Prune before scan (defaults to previous) compute_hashes: Compute hashes (defaults to previous) timeout: Max seconds to wait for current scan to stop Returns: True if new scan was started, False if failed to stop previous zAsset seeder restart requestedNrFro) rVrWrArKrLrOrNrMrzwaitrm) rSr_r`rarbrcr prev_roots prev_phase prev_callback prev_prune prev_hashescbs rrestartz_AssetSeeder.restart&s ,  5666 Z / /JJ 3M*J.K  / / / / / / / / / / / / / / / yyy)) 5"3"?  ]zz ,%%* ,%%* '2'> J"0"<+    s$A  AAc|j5|j}dddn #1swxYwY|dS||| S)zWait for the current scan to complete. Args: timeout: Maximum seconds to wait, or None for no timeout Returns: True if scan completed, False if timeout expired or no scan running NTr)rArFjoinis_alive)rSrthreads rrz_AssetSeeder.waitSsZ " "\F " " " " " " " " " " " " " " " >4 G $$$??$$$$s   c |j5|jp|j}t|j|r't |j|j|j|j ndt|j cdddS#1swxYwYdS)z2Get the current status and progress of the seeder.r-r.r/r0N)r5r6r8) rArCrDr4rBr,r-r.r/r0r9rE)rSsrcs r get_statusz_AssetSeeder.get_statuscs Z  .7D$7Ck K)KK DL))                     sA A55A9<A9c||||j5d|_ddddS#1swxYwYdS)zGracefully shutdown: cancel any running scan and wait for thread. Args: timeout: Maximum seconds to wait for thread to exit rN)rzrrArF)rSrs rshutdownz_AssetSeeder.shutdownts  ' """ Z  DL                  sAA A c|j5|jtjkrt dtj|_dddn #1swxYwY t sKtjd |j5| ddddS#1swxYwYdSt}t|}|dkrtj d|||j5| dddS#1swxYwYS#|j5| dddw#1swxYwYwxYw)aMark references as missing when outside all known root prefixes. This is a non-destructive soft-delete operation. Assets and their metadata are preserved, but references are flagged as missing. They can be restored if the file reappears in a future scan. This operation is decoupled from scanning to prevent partial scans from accidentally marking assets belonging to other roots. Should be called explicitly when cleanup is desired, typically after a full scan of all roots or during maintenance. Returns: Number of references marked as missing Raises: ScanInProgressError: If a scan is currently running z0Cannot mark missing assets while scan is runningNz:Database dependencies not available, skipping mark missingrzMarked %d references as missing) rArBrrrr rrVwarning_reset_to_idler rrW)rS all_prefixesmarkeds rmark_missing_outside_prefixesz*_AssetSeeder.mark_missing_outside_prefixessc&Z ( ({ej(()F -DK  ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( &)++ P & &##%%% & & & & & & & & & & & & & & & & & & 233L9,GGFzz >GGG & &##%%% & & & & & & & & & & & & & & & & & &##%%% & & & & & & & & & & & & & & & &sk6A  AA"DB""B&)B&/9D/DD D E$E9 EE E E EcN|j|_tj|_d|_dS)zFReset state to IDLE, preserving last progress. Caller must hold _lock.N)rCrDrrrBrRs rrz_AssetSeeder._reset_to_idles "nj rc4|jS)z)Check if cancellation has been requested.)rHis_setrRs r _is_cancelledz_AssetSeeder._is_cancelleds!((***rch|j p|jS)a>Non-blocking check: True if paused or cancelled. Use as interrupt_check for I/O-bound work (e.g. hashing) so that file handles are released immediately on pause rather than held open while blocked. The caller is responsible for blocking on _check_pause_and_cancel() afterward. )rIrrHrRs r_is_paused_or_cancelledz$_AssetSeeder._is_paused_or_cancelleds/>((***Id.@.G.G.I.IIrc|js|di|j|S)a=Block while paused, then check if cancelled. Call this at checkpoint locations in scan loops. It will: 1. Block indefinitely while paused (until resume or cancel) 2. Return True if cancelled, False to continue Returns: True if scan should stop, False to continue zassets.seed.paused)rIrrrrrRs r_check_pause_and_cancelz$_AssetSeeder._check_pause_and_cancelsW~$$&& 7   12 6 6 6 !!###r event_typedatac ddlm}t|dr$|jr|j||dSdSdS#t $rYdSwxYw)z.Emit a WebSocket event if server is available.r) PromptServerinstanceN)serverrhasattrr send_sync Exception)rSrrrs rrz_AssetSeeder._emit_events  + + + + + +|Z00 B\5J B%// DAAAAA B B B B    DD s8A A Ar-r.r/r0cd}d}|j5|j ddddS| ||j_| ||j_| ||j_| ||j_|jrB|j}t|jj|jj|jj|jj}dddn #1swxYwY|r!|r! ||dS#t$rYdSwxYwdSdS)z'Update progress counters (thread-safe).Nr) rArCr-r.r/r0rOr,r)rSr-r.r/r0callbackr6s r_update_progressz_AssetSeeder._update_progresss-1$( Z  ~%        ")0& ',$")0&")0&& 2# N2.. N2 N2                (    """""        s) B/BB//B36B3? C CCmessagec|j5t|j|jkr|j|ddddS#1swxYwYdS)z:Add an error message (thread-safe), capped at _MAX_ERRORS.N)rAlenrE _MAX_ERRORSappend)rSrs r _add_errorz_AssetSeeder._add_errors Z - -4<  4#333 ##G,,, - - - - - - - - - - - - - - - - - -s8A  AAcddl}|D]g}|dkr8tjdtj|j@t|}|rtjd||hdS)z)Log the directories that will be scanned.rNr\z!Asset scan [models] directory: %szAsset scan [%s] directories: %s) folder_pathsrVrWospathabspath models_dirr)rSr_rrootprefixess r_log_scan_configz_AssetSeeder._log_scan_configs T TDx 7GOOL$;<< 166TL!BD(SSS T Trc tj}|j}|j}d}d}d}d}d} t s|d|dddi |r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS|jr8t!} t#| } | dkrtjd | |rtjdd} |r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS|||t*jt*jfvr||\} } }| | |}}}|rd} |r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS|dt3||||d|t*jt*jfvr|rd} |r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS||\}}|rd} |r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS|dt3||dtj|z }tjd||j|||||d|j||||t;|ddnh#t<$r[}|d|tjd|ddtA|iYd}~nd}~wwxYw|r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d ddddS#1swxYwYdS#|r-|d|jr |jjnd||d|j 5| |j } | Ed|_ | | d | d  stjd | d dddw#1swxYwYwxYw)z,Main scan loop running in background thread.Frz#Database dependencies not availablezassets.seed.errorrzassets.seed.cancelled)r-r.r/Nr_rcrtz.Pending enrich scan could not start (roots=%s)z%Marked %d refs as missing before scanz(Asset scan cancelled after pruning phaseTzassets.seed.fast_complete)r_r/r0r.zassets.seed.enrich_complete)r_enrichedz:Scan(%s, %s) done %.3fs: created=%d enriched=%d skipped=%dzassets.seed.completed)r`r.r/rr0elapsedz Scan failed: zAsset scan failed)!time perf_counterrKrLrrrrCr-rArrQrrrVrrNr rrWrrr$r(r*_run_fast_phaser9r)_run_enrich_phaseriroundr exceptionr:)rSt_startr_r` cancelled total_createdtotal_enrichedskipped_existing total_pathspendingrrr/r0pathsenrich_cancelledres rrlz_AssetSeeder._run_scans #%%     n )++  EFFF  ' EFd   +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                  s  R577 =lKKA::L!H&QQQ++--  GHHH N   +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                  ]  ! !% ( ( (888*.*>*>u*E*E'%?FQV/ //11 $Ix   +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                  G  /!%e#0#3!, )9>::://11 $IX   +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                  g483I3I%3P3P0 .# $IL   +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                  [  1!%e$2'))G3G LL        '"[(, ./$Wa00      G G G OO/A// 0 0 0  1 2 2 2   09c!ff2E F F F F F F F F G    +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                     +=A^#R4>#9#9QR!,#0  ##%%%.&+/D(,,%g.'./?'@- L#G,                s;V;A#DDDA)V;Fast scan: build_asset_specs took %.3fs (%d specs, %d skipped))r0i?z2Fast scan cancelled after %d/%d files (created=%d)c(h|]}|dD]}|S)tagsr).0spects r z/_AssetSeeder._run_fast_phase..s)DDDtF|DD!!DDDDrzBatch insert failed at offset z: z Batch insert failed at offset %dN)r-r/assets.seed.progress)r`r-r.r/zHFast scan complete: %.3fs total (created=%d, skipped=%d, total_paths=%d))rrrJrrurrVrhrr rrr9r rangerWrrrr)rSr_ t_fast_startrrexisting_pathst_syncr t_collectrrt_specsspecstag_pool batch_sizelast_progress_timeprogress_intervalibatch batch_tagsr/rr-nows rrz_AssetSeeder._run_fast_phasesQ (**  #&55"$$ 7 7A++-- :$&69999  ! !"21"5"5 6 6 6 6 G    & (       ' ' ) ) 6 "2A5 5%'' '.. B    ) + JJ   %jj  K000  !5kkK& I I   #%%,=  ', - - - )x)  L    ' ) JJ     &6777  ' ' ) ) @ "2K? ? !.00q#e**j11! )! )A++-- D HJJ!  %&6 CCCC!a*n,-EDDDDDJ I,UJ??(  I I I I I Ia I IJJJ!"DaHHHHHHHH I#e**nG#%%C  ! !'= ! I I I''+<<<  *!'#*!$U#0 &)" c%jj-HHH V    , .       . ;;sJ(( K,20K''K,cd}d}tj}d}|jst}nt}|dt |ddtd}d}i} |rtj d |d|fSt||| } rfd | D} | snt| d|j|j | \} } || z }| | dkr0|d z }||kr$tjd|t!n;nd}tj} | |z |kr|dd|d| }d|fS)zRun phase 2: enrich existing records with metadata and hashes. Returns: Tuple of (cancelled, total_enriched) rdrrr&)r_r`rTz%Enrich scan cancelled after %d assets) max_levellimitc&g|] }|jv |Sr) reference_id)rrskip_idss r z2_AssetSeeder._run_enrich_phase..%s%VVVAq~X7U7Ua7U7U7Ur)extract_metadata compute_hashinterrupt_checkhash_checkpointszKEnrich phase stopping: %d consecutive batches with no progress (%d skipped)r)r`rF)rrrMrrrr9rJrrVrWrr rrurr)rSr_rrrrtarget_max_levelconsecutive_emptymax_consecutive_emptyr unenrichedr failed_idsrrs @rrz_AssetSeeder._run_enrich_phases  !.00# 3.  2   !5kkH 5 5   !UU !/12 )++-- , DnUUU^++9* J WVVVVVVV  #6!%!1 $ %&!#%%C''+<<<  *!)$2&)"e2 )hn$$r)r=N)r[NF)r[F)NNNNNr)N)r)NNNN)+rrrrrTrXboolrZr$r*rvr ProgressCallbackrmrprrrxrzr|r~rfloatrrr4rrr1rrrrrr:dictrrrrrrlrrrrrr<r<Gs?1111(.... T 'D$>59!$ ,,Xs]#,,,d2 ,  ,  , ,,,,`'D59!   Xs]# ,d2        4'D59$   Xs]# ,d2        4'D$""Xs]#""  """"H     dt &.2"&59#'&*+ + Xs]#d*+ 4+ ,d2 + D[ + t + +  + + + + Z%%EDL%D%%%% J"          (&s(&(&(&(&T +t++++JJJJJ $ $ $ $ $c$# "" ##t#Tz#t # t #  ####JK-#-$---- TeHcM&: Tt T T T TyyyyvjRj<j<j<jr sRNN  ((((((((                            322222MMMMM)MMMD   4444444 4XJ,-D %D %D %D %D %D %D %D %N|~~ r