
    +jQ                     D   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
c mc mZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d d	l*m+Z+m,Z,m-Z-m.Z. d d
l/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 d dl6m7Z7 	 	 	 	 	 	 	 	 d=de8de8de9de9de8dz  de8dz  de8de8dz  de3dee8         de8de:de0fdZ;	 	 d>dee8         de3de8dz  de9fd Z<	 	 	 	 d?de8de3d!ee8         de8de8dz  de:fd"Z=	 	 	 	 	 	 d@de8d#e8de3de>e8         dz  de8de8de8dz  de8dz  de1fd$Z?d%ed&e8d'e8dz  d(e@dz  de@e8ef         ddfd)ZAd*ZBd%ed&e8d'e8de8dz  d+e@dz  ddfd,ZCd%ed-e8d.e8d+e@dz  ddf
d/ZDd#e8dz  d0e8de8fd1ZE G d2 d3eF          ZG G d4 d5eF          ZH	 	 	 	 	 	 	 	 dAd6e8d#e8dz  de>e8         dz  de@dz  d7e8dz  de8d8e8dz  de8dz  de8dz  de2fd9ZI	 	 dBde8d#e8de>e8         de8de8dz  de2fd:ZJ	 	 	 	 	 dCd;e8d#e8de>e8         dz  de@dz  de8de8dz  de8dz  de2dz  fd<ZKdS )D    N)AnySequence)Session)add_tags_to_referencecount_active_siblingscreate_stub_assetensure_tags_existfetch_reference_and_assetget_asset_by_hashget_reference_by_file_pathget_reference_tagsget_or_create_referencelist_references_by_asset_idreference_existsremove_missing_tag_for_asset_idset_reference_metadataset_reference_system_metadataset_reference_tagsupdate_asset_hash_and_mimeupsert_assetupsert_referencevalidate_tags_exist)get_utc_nownormalize_tags)batch_insert_seed_assets)get_size_and_mtime_ns)extract_image_dimensions)compute_relative_filename!get_name_and_tags_from_asset_pathresolve_destination_from_tagsvalidate_path_within_base)IngestResultRegisterAssetResultUploadResultUserMetadataextract_asset_dataextract_reference_data)create_session  manualFabs_path
asset_hash
size_bytesmtime_ns	mime_type	info_nameowner_id
preview_iduser_metadatatags
tag_originrequire_existing_tagsreturnc           	         t           j                            |           }|pi }d}d}d}d}d }t                      5 }|rt	          ||          sd }t          ||||          \  }}}t          ||j        ||pt           j                            |          ||          \  }}t          ||          }|r|j        }|r|j
        |k    r||_
        t          t          |	                    }|r'|rt          ||           t          ||||
|            t          |||j        |j        |           t%          |||||j                   	 t)          ||j                   n*# t*          $ r t-          j        d|j                   Y nw xY w|                                 d d d            n# 1 swxY w Y   t3          |||||	          S )
NF)r-   r.   r0   )asset_id	file_pathnamer/   r2   )reference_idr5   origincreate_if_missing)r=   r;   current_metadatar4   )r=   r;   r0   current_system_metadata)r:   z*Failed to clear 'missing' tag for asset %s)asset_createdasset_updatedref_createdref_updatedr=   )ospathabspathr(   r   r   r   idbasenamer   r3   r   listr   r   _update_metadata_with_filenamer;   r4   _maybe_store_image_dimensionssystem_metadatar   	Exceptionlogging	exceptioncommitr"   )r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   locatorrB   rC   rD   rE   r=   sessionassetrefnorms                         :/home/wildlama/comfy/ComfyUI/app/assets/services/ingest.py_ingest_file_from_pathrY   4   s    gooh''G!'RMMMKK#L			 >W 	"#GZ88 "!
.:!!	/
 /
 /
+}m $4X7bg..w77$
 $
 $
 [ )'::  	6L ,cn
::!+!$t**--D 	( 7'666%!-%*?&?    +)-!$!2+    *)!#(+(;   	V+GehGGGGG 	V 	V 	VJEHUUUUU	V 	}> > > > > > > > > > > > > > >@ ##!   s6   DF/E%$F/%$F	F/FF//F36F3
file_pathsjob_idc                     d}| D ]_}t           j                            |          s"	 t          |||          r|dz  };# t          $ r t          j        d|           Y \w xY w|S )zoRegister a batch of output file paths as assets.

    Returns the number of files successfully registered.
    r   )r4   r[      zFailed to register output: %s)rF   rG   isfileingest_existing_filerO   rP   rQ   )rZ   r4   r[   
registeredr,   s        rX   register_output_filesra      s     J 	I 	Iw~~h'' 		I#f     a
 	I 	I 	I=xHHHHH	Is   A  A"!A"
extra_tagsc                     t           j                            |           }t          |           \  }}t	          j        | d          d         }t          |           \  }	}
t          t          	                    |
t          |          z                       }t                      5 }t          ||          }|t                      }||_        ||_        d|_        d|_        ||_        d|_        |j        }|r^t)          ||j        |j                  }|dk    r&t-          |||p|j                  }|j        |_        nd|_        ||_        |r||_        |                                 	 ddd           dS | |||	|t           j                            |           dd||d
}|rt;          ||           t=          ||g|          }|                                 |j        dk    cddd           S # 1 swxY w Y   dS )	am  Register an existing on-disk file as an asset stub.

    If a reference already exists for this path, updates mtime_ns, job_id,
    size_bytes, and resets enrichment so the enricher will re-hash it.

    For brand-new paths, inserts a stub record (hash=NULL) for immediate
    UX visibility.

    Returns True if a row was inserted or updated, False otherwise.
    Fstrictr   N)r.   r0   T)
r,   r.   r/   r1   r5   fnamemetadatahashr0   r[   )r2   ) rF   rG   rH   r   	mimetypes
guess_typer   rK   dictfromkeysr(   r   r   r/   r[   
is_missing
deleted_at
updated_atenrichment_levelrU   r   rI   r   r0   r:   rh   r.   rR   rJ   r	   r   	won_paths)r,   r4   rb   r2   r[   rS   r.   r/   r0   r<   	path_tagsr5   rT   existing_refnowrU   siblings	new_assetspecresults                       rX   r_   r_      sz   " gooh''G0::J$Xe<<<Q?I7AAOD)i$z*:*::;;<<D			 /$W1'7CC#--C$,L!"(L&+L#&*L#&)L#,-L) &E 4 1%(LOTTa<< 1#-"+">u! ! !I
 -6LL))!%EJ'1E$  4*3NN;/$ /$ /$ /$ /$ /$ /$ /$@ !$ W%%h//"
 
  	-gt,,,)'D6HMMM!#_/$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$ /$s   %CG39A-G33G7:G7r<   c                 &   |pi }t                      5 }t          ||           }	|	st          d|            |r|	j        st	          ||	j        |           |rt          ||          sd }t          ||	j        |||          \  }
}|sy|r|
j        |k    r||
_        t          ||
j                  }t          t          |
          t          |	          |d          }|                                 |cd d d            S t          |          }|
j        rt!          |
j                  nd }|r||d<   |rt#          ||
j        |	           t%          ||	j        |
j        |
j        
           |t)          ||
j        ||           t          ||
j                  }|                    |
           t          t          |
          t          |	          |d          }|                                 |cd d d            S # 1 swxY w Y   d S )Nr-   zNo asset with hash )r:   r0   )r:   r2   r<   r3   r=   F)rV   rU   r5   createdfilenamer=   r4   )r:   new_reference_idrA   )r=   r5   r>   T)r(   r   
ValueErrorr0   r   rI   r   r   r3   r   r#   r'   r&   rR   rk   r;   r   r   (_backfill_image_dimensions_from_siblingsrN   r   refresh)r-   r<   r4   r5   r6   r2   r0   r3   rT   rU   rV   rD   	tag_namesrx   new_metacomputed_filenames                   rX   _register_existing_assetr      s    "'RM			 GW!'jAAA 	A?:??@@@ 	XU_ 	X&wYWWWW 	"#GZ88 "!
2X!
 
 
[  	 ,cn
::!+*7HHHI(*3//(//	  F NNAG G G G G G G GD &&HK_5cmDDD[_ 	5#4HZ  	" V&    	1X V$'$7		
 	
 	
 	
  V!	    'wSVDDD	$&s++$U++	
 
 
 	OG G G G G G G G G G G G G G G G G Gs   C$HC5HH
H
rT   r=   r;   r@   c                     |rt          |          nd }|pi }t          |          }|                                D ]
\  }}	|	||<   |r||d<   ||k    rt          | ||           d S d S )Nr}   r~   )r   rk   itemsr   )
rT   r=   r;   r@   r4   r   current_metar   kvs
             rX   rL   rL   K  s     AJS1)<<<t#)rLL!!H##%%  1 10<%"	
 	
 	
 	
 	
 	
      )kindwidthheightrA   c                     |r|                     d          sdS t          ||          }|sdS |pi }t          |          }|                    |           ||k    rt	          | ||           dS dS )zPopulate ``kind``/``width``/``height`` on system_metadata for image refs.

    Non-image MIME types are a no-op. Pre-existing keys (e.g. enricher-written
    safetensors metadata, download provenance) are preserved by merge.
    zimage/N)r0   r=   rN   )
startswithr   rk   updater   )rT   r=   r;   r0   rA   dimscurrentmergeds           rX   rM   rM   f  s      I00:: #ICCCD %+G']]F
MM$%%"	
 	
 	
 	
 	
 	
 r   r:   r   c                    |pi }|                     d          dk    r
d|v rd|v rdS t          | |          D ]}|j        |k    r|j        pi }|                     d          dk    r1|                     d          }|                     d          }t	          |          t
          us"t	          |          t
          us|dk    s|dk    rt          |          }	d|	d<   ||	d<   ||	d<   |	|k    rt          | ||	            dS dS )a  Copy image dimension keys from any sibling reference of the same asset.

    The from-hash path doesn't read the file bytes, so dimensions can't be
    extracted there directly. When another reference of the same asset already
    carries image dimensions, copy them onto the new reference so consumers
    see consistent metadata regardless of how the asset was registered.

    Best-effort: missing siblings, non-image siblings, or absent dimension
    keys leave the target reference unchanged.
    r   imager   r   Nr   r   )getr   rI   rN   typeintrk   r   )
rT   r:   r   rA   r   siblingmetar   r   r   s
             rX   r   r     sM     &+G{{6g%%'W*<*<WATAT.wAA  :)))&,"88Fw&&!!(##KKs""F||3&&zz{{g vw!xW)- &   
 	3 r   fallbackc                 x    t           j                            | pd                                p|          }|r|n|S )Nr)   )rF   rG   rJ   strip)r<   r   ns      rX   _sanitize_filenamer     s;    
$*"++--9::A11xr   c                       e Zd ZdS )HashMismatchErrorN)__name__
__module____qualname__r*   r   rX   r   r     s        Dr   r   c                   $     e Zd Zdef fdZ xZS )DependencyMissingErrormessagec                 X    || _         t                                          |           d S )N)r   super__init__)selfr   	__class__s     rX   r   zDependencyMissingError.__init__  s(    !!!!!r   )r   r   r   strr   __classcell__)r   s   @rX   r   r     sD        " " " " " " " " " " "r   r   	temp_pathclient_filenameexpected_hashc	                    	 t          j        |           \  }	}
nM# t          $ r!}t          t	          |                    d }~wt
          $ r}t          d|           d }~ww xY wd|	z   }|r9||                                                                k    rt          d          t                      5 }t          ||          }d d d            n# 1 swxY w Y   |t          j        t
                    5  | r3t          j                            |           rt          j        |            d d d            n# 1 swxY w Y   t%          |p||	          }t'          |||pi |pg d|||          }t)          |j        |j        |j        d	          S |st1          d
          t3          |          \  }}|rt          j        j        |g|R  n|}t          j        |d           |p|pd                                }|rBt          j                            t          j                            |                    d         nd}dt=          |          cxk     rdk    rn n|nd}|	 | }t          j                            t          j                            ||                    }tA          ||           |pVtC          j"        t          j                            |          d          d         ptC          j"        |d          d         pd}	 t          j#        | |           n$# t
          $ r}t          d|           d }~ww xY w	 tI          |          \  }}n$# tJ          $ r}t          d|           d }~ww xY wtM          |||||t%          |p||	          |||pi |dd          }|j'        }|st          d          t                      5 }tQ          |||          }|st          d          |\  }} tS          ||j*                  }!d d d            n# 1 swxY w Y   t)          tW          |          tY          |           |!|j-        	          S )Nzfailed to hash uploaded file: blake3:z0Uploaded file hash does not match provided hash.rz   r   r+   r-   r<   r4   r5   r6   r2   r0   r3   FrV   rU   r5   created_newz'tags are required for new asset uploadsT)exist_okr)   r]   r      rd   application/octet-streamz)failed to move uploaded file into place: z!failed to stat destination file: )r-   r,   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7    failed to create asset referencer=   r2   "inconsistent DB state after ingestr{   ).hashingcompute_blake3_hashImportErrorr   r   rO   RuntimeErrorr   lowerr   r(   r   
contextlibsuppressrF   rG   existsremover   r   r$   rV   rU   r5   r   r    joinmakedirssplitextrJ   lenrH   r!   ri   rj   replacer   OSErrorrY   r=   r
   r   rI   r'   r&   rB   )"r   r<   r5   r4   r   r2   r   r0   r3   digest_er-   rT   existingdisplay_namerx   base_dirsubdirsdest_dirsrc_for_ext_extexthashed_basenamedest_abscontent_typer.   r/   ingest_resultr=   pairrV   rU   r   s"                                     rX   upload_from_temp_pathr     s   A/	::	 - - -$SVV,,, A A A?A??@@@AV#J T}':':'<'<'B'B'D'DDD RSSS			 EW$WDDDE E E E E E E E E E E E E E E  ++ 	% 	% %RW^^I66 %	)$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% *$*A/FSSS)!'-2!	
 	
 	
 
,	
 
 
 	
  DBCCC5d;;Hg3:Hrw|H/w////HK4(((("0d0b7799KALT27BG,,[99::1==RTDc$ii%%%%2%%%%%$$2C&&&Owrw||HoFFGGHh111 RW--k::5III!L 	&>>>qA	&% L

9h'''' L L LJqJJKKKLD4X>>
HH D D DBqBBCCCD +$T%<_vNNN#)r#  M !-L ?=>>>			 EW(,
 
 
  	ECDDD
U&wSVDDD	E E E E E E E E E E E E E E E "3'' ''!/	   s    
A$A  A$AA$5CCC96D;;D?D?L, ,
M6MMM$ $
N.N  N?P''P+.P+c                 Z   	 t          |           \  }}n# t          $ r g }Y nw xY wt          g ||          }	 t          j        |           \  }}nM# t
          $ r!}	t          t          |	                    d}	~	wt          $ r}	t          d|	           d}	~	ww xY wd|z   }
t          |           \  }}|pt          j        | d          d         pd}t          | |
|||t          ||          ||d	d

  
        }|j        }|st          d          t!                      5 }t#          |||          }|st          d          |\  }}t%          ||j                  }ddd           n# 1 swxY w Y   t)          t+          |          t-          |          ||j                  S )aA  Register an already-saved file in the asset database without moving it.

    Tags are derived from the filesystem path (root category + subfolder names),
    merged with any caller-provided tags, matching the behavior of the scanner.
    If the path is not under a known root, only the caller-provided tags are used.
    Nzfailed to hash file: r   Frd   r   r   r   upload)
r,   r-   r.   r/   r0   r1   r2   r5   r6   r7   r   r   r   r{   r   )r   r   r   r   r   r   r   r   rO   r   r   ri   rj   rY   r   r=   r(   r
   r   rI   r$   r'   r&   rB   )r,   r<   r5   r2   r0   r   rr   merged_tagsr   r   r-   r.   r/   r   r   r=   rT   r   rV   rU   r   s                        rX   register_file_in_placer   3  st   8BB99   			 !49!4t!455K8/99	 - - -$SVV,,, 8 8 861667778V#J0::J Xe444Q7 	&% 
 +$TF;;;#  M !-L ?=>>>			 EW(,
 
 
  	ECDDD
U&wSVDDD	E E E E E E E E E E E E E E E "3'' ''!/	   sA    $$A 
BA99BBB%?E00E47E4hash_strc           
      v   |                                                                  }	 t          |t          |d|v r|                    dd          d         n|          |pi |pg d|||          }n&# t
          $ r t          j        d|           Y d S w xY wt          |j	        |j
        |j        d          S )	N:r]   r   r+   r   z,create_from_hash: no asset found for hash %sFr   )r   r   r   r   splitr   rP   warningr$   rV   rU   r5   )	r   r<   r5   r4   r2   r0   r3   	canonicalrx   s	            rX   create_from_hashr   u  s       &&((I) #SI=M=MysA66q99S\   (-2!
 
 
    F	RRRtt Jl[	   s   A
A3 3BB)NNr)   NNr*   r+   F)NN)Nr*   r)   N)NNr+   r)   NN)NNNNr)   NNN)r)   N)NNr)   NN)Lr   rP   ri   rF   typingr   r   sqlalchemy.ormr   app.assets.services.hashingassetsservicesr   app.assets.database.queriesr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   app.assets.helpersr   r   app.assets.services.bulk_ingestr   app.assets.services.file_utilsr   $app.assets.services.image_dimensionsr   app.assets.services.path_utilsr   r   r    r!   app.assets.services.schemasr"   r#   r$   r%   r&   r'   app.database.dbr(   r   r   boolrY   ra   r_   rK   r   rk   rL   _IMAGE_DIMENSION_KEYSrM   r   r   rO   r   r   r   r   r   r*   r   rX   <module>r      s            				                 " " " " " " - - - - - - - - - - - -                                         * ; : : : : : : : D D D D D D @ @ @ @ @ @ I I I I I I                           + * * * * * ! !"&"'] ]]] ] 	]
 Tz] Tz] ] d
]  ] 3-] ]  ] ] ] ] ]D #'  $J 		   2 #' "F$ F$F$F$ F$ 	F$
 $JF$ 
F$ F$ F$ F$X #'! !S SS
S  S s)d
	S
 S S TzS d
S S S S Sn


 Tz
 Tk	

 S>
 

 
 
 
0 4 


 
 Tz	

 "D[
 

 
 
 
<--- - "D[	-
 
- - - -` S4Z  3  3        
	 	 	 	 		 	 	 	" " " " "Y " " " !!%"& $ !m mm
*m s)d
m $;	m
 4Zm m :m Tzm d
m m m m mh  ? ??
? s)? 	?
 Tz? ? ? ? ?J "!% !! !!
! s)d
! $;	!
 ! Tz! d
! D! ! ! ! ! !r   