
    +j                        S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
r
S SKrS SKJrJr  S r " S S\R                   5      r " S S	\R                   5      rSIS
 jr " S S\R                   5      r " S S\R                   5      rS rS r " S S\R                   5      r " S S\R                   5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S \5      r " S! S"\5      r  " S# S$\5      r! " S% S&\R                   5      r" " S' S(\5      r# " S) S*\5      r$ " S+ S,\5      r% " S- S.\5      r& " S/ S0\5      r' " S1 S2\5      r( " S3 S4\5      r) " S5 S6\5      r* " S7 S8\5      r+ " S9 S:\5      r, " S; S<\5      r- " S= S>\R                   5      r. " S? S@\R                   5      r/ " SA SB\R                   5      r0 " SC SD\R                   5      r1 " SE SF\5      r2SG\24SH jr3g)J    N)Image)override)ComfyExtensionioc                    U (       d  [        S5      e/ nU  H  n[        R                  R                  X5      n[        R
                  " [        R                  U5      nUR                  S:X  a  UR                  S 5      nUR                  S5      n[        R                  " U5      R                  [        R                  5      S-  n[        R                   " U5      S   nUR#                  U5        M     U$ )a/  Utility function to load and process a list of images.

Args:
    image_files: List of image filenames
    input_dir: Base directory containing the images
    resize_method: How to handle images of different sizes ("None", "Stretch", "Crop", "Pad")

Returns:
    torch.Tensor: Batch of processed images
zNo valid images found in inputIc                     U S-  $ )Ngp? )is    :/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_dataset.py<lambda>)load_and_process_images.<locals>.<lambda>$   s    a7m    RGB     o@N)
ValueErrorospathjoinnode_helperspillowr   openmodepointconvertnparrayastypefloat32torch
from_numpyappend)image_files	input_diroutput_imagesfile
image_pathimg	img_array
img_tensors           r   load_and_process_imagesr,      s     9::MWW\\)2
!!%**j988s?))34Ckk% HHSM((4u<	%%i07
Z(  r   c                   4    \ rS rSr\S 5       r\S 5       rSrg)LoadImageDataSetFromFolderNode-   c                     [         R                  " S/ SQSSSS[         R                  R                  S[        R
                  " 5       SS	9/[         R                  R                  S
SSS9/S9$ )NLoadImageDataSetFromFolderzload folderzload from folderzload datasetzload imagesimport datasetzLoad Image (from Folder)imagezvLoad a dataset of images from a specified folder and return a list of images. Supported formats: PNG, JPG, JPEG, WEBP.TfolderzThe folder to load images from.optionstooltipimagesList of loaded imagesdisplay_nameis_output_listr8   node_idsearch_aliasesr<   categorydescriptionis_experimentalinputsoutputs)r   SchemaComboInputfolder_pathsget_input_subfoldersr   Outputclss    r   define_schema,LoadImageDataSetFromFolderNode.define_schema.   sy    yy0o3 Q (==?=   !)#'3   
 	
r   c                 D  ^ [         R                  R                  [        R                  " 5       U5      n/ SQn[         R
                  " U5       V^s/ s H!  m[        U4S jU 5       5      (       d  M  TPM#     nn[        XR5      n[        R                  " U5      $ s  snf )N.pngz.jpgz.jpegz.webpc              3   b   >#    U  H$  nTR                  5       R                  U5      v   M&     g 7fr   lowerendswith.0extfs     r   	<genexpr>9LoadImageDataSetFromFolderNode.execute.<locals>.<genexpr>N   s'     G6Fs1779%%c**6F   ,/)
r   r   r   rI   get_input_directorylistdiranyr,   r   
NodeOutput)rM   r5   sub_input_dirvalid_extensionsrZ   r$   output_tensors       `  r   execute&LoadImageDataSetFromFolderNode.executeG   s    \%E%E%GP= ZZ.
.G6FGG . 	 

 0K}}]++
s   B4Br
   N__name__
__module____qualname____firstlineno__classmethodrN   re   __static_attributes__r
   r   r   r.   r.   -   s(    
 
0 	, 	,r   r.   c                   4    \ rS rSr\S 5       r\S 5       rSrg)"LoadImageTextDataSetFromFolderNodeT   c                    [         R                  " S/ SQSSSS[         R                  R                  S[        R
                  " 5       SS	9/[         R                  R                  S
SSS9[         R                  R                  SSSS9/S9$ )NLoadImageTextDataSetFromFolderr2   zLoad Image-Text (from Folder)r4   zLoad a dataset of pairs of images and text captions from a specified folder and return them as a list. Supported formats: PNG, JPG, JPEG, WEBP.Tr5   z1The folder to load images and text captions from.r6   r9   r:   r;   textszList of text captionsr>   )	r   rF   rG   rH   rI   rJ   r   rK   StringrL   s    r   rN   0LoadImageTextDataSetFromFolderNode.define_schemaU   s    yy4o8 j (==?O   !)#'3   
 		  !(#'3 ! 
 	
r   c           
        ^^ [         R                  " SU 35        [        R                  R	                  [
        R                  " 5       U5      n/ SQn/ n[        R                  " U5       GH1  m[        R                  R	                  UT5      n[        U4S jU 5       5      (       a  UR                  U5        MQ  [        R                  R                  U5      (       d  Mw  SnTR                  S5      S   R                  5       (       a  [        TR                  S5      S   5      nUR                  [        R                  " U5       V^s/ s H?  m[        U4S jU 5       5      (       d  M  [        R                  R	                  UT5      PMA     snU-  5        GM4     U Vs/ s H4  owR                  [        R                  R!                  U5      S   S5      PM6     nn/ n	U H  n
[        R                  R	                  X*5      n[        R                  R#                  U5      (       aE  [%        US	S
S9 mTR'                  5       R)                  5       nU	R                  U5        S S S 5        M  U	R                  S5        M     [+        XB5      n[         R                  " S[-        U5       SU S35        [.        R0                  " X5      $ s  snf s  snf ! , (       d  f       GM  = f)NzLoading images from folder: rQ   c              3   b   >#    U  H$  nTR                  5       R                  U5      v   M&     g 7fr   rT   )rX   rY   items     r   r[   =LoadImageTextDataSetFromFolderNode.execute.<locals>.<genexpr>}   s'     J9I#4::<((--9Ir]      _r   c              3   b   >#    U  H$  nTR                  5       R                  U5      v   M&     g 7fr   rT   rW   s     r   r[   ry      s'     SBR3qwwy11#66BRr]   .txtrutf-8encoding Loaded z images from .)logginginfor   r   r   rI   r^   r_   r`   r#   isdirsplitisdigitintextendreplacesplitextexistsr   readstripr,   lenr   ra   )rM   r5   rb   rc   r$   r   repeatrZ   caption_file_pathcaptionscaption_filecaption_pathcaptionrd   rx   s          `      @r   re   *LoadImageTextDataSetFromFolderNode.executes   s3   3F8<=\%E%E%GP=JJ}-D77<<t4DJ9IJJJ""4(t$$::c?1%--// C!34F"" "$D!1!1ASBRSS .T1-!1
  .& @K
?J!IIbgg&&q)!,f5{ 	 
 -L77<<DLww~~l++,g>!ffhnn.GOOG, ?> # . 0Kws=12-aPQ}}]551
 ?>s   K
.$K
%;K70K
K'	r
   Nrg   r
   r   r   ro   ro   T   s(    
 
: )6 )6r   ro   c                    [         R                  " USS9  / n[        U 5       GH  u  pV[        U[        R
                  5      (       Ga  UR                  5       S:X  a$  UR                  S   S:X  a  UR                  S5      nUR                  5       S:X  a_  UR                  S   S;   aL  UR                  S   S::  a9  UR                  S   S:  a&  UR                  S   S:  a  UR                  SSS5      nUR                  5       R                  5       n[        R                  " US	-  SS
5      R                  [        R                  5      n[         R"                  " U5      nO[%        S['        U5       35      eU(       a
  U SUS S3n	O'[(        R*                  " X!5      u    ppU SUS SUS S3n	[         R,                  R/                  X5      nUR1                  U5        UR3                  U	5        GM     U$ )a  Utility function to save a list of image tensors to disk.

Args:
    image_list: List of image tensors (each [1, H, W, C] or [H, W, C] or [C, H, W])
    output_dir: Directory to save images to
    prefix: Filename prefix

Returns:
    List of saved filenames
Texist_ok   r   rz      )rz   r   r      r      zExpected torch.Tensor, got r{   05drR   05)r   makedirs	enumerate
isinstancer!   Tensordimshapesqueezepermutecpunumpyr   clipr   uint8r   	fromarrayr   typerI   get_save_image_pathr   r   saver#   )
image_list
output_dirprefix	overwritesaved_filesidxr+   r*   r)   filenamer{   counterresolved_prefixfilepaths                 r   save_images_to_folderr      s    KK
T*K$Z0j%,,//~~1$)9)9!)<)A'//2
 ~~1$)9)9!)<	)I$$Q'1,"((+a/"((+a/!+!3!3Aq!!<J #(..0I	E 11c:AA"((KI //),C:4
;K:LMNN  3s)40H0<0P0PQW0d-Aq1)*!GB<qS	FH77<<
58$C 1F r   c                   4    \ rS rSr\S 5       r\S 5       rSrg)SaveImageDataSetToFolderNode   c                 8   [         R                  " S/ SQSSSSSS[         R                  R                  SSS	9[         R                  R                  S
SSS9[         R                  R                  SSSSS9[         R
                  R                  SSSS/SS9// SS9$ )NSaveImageDataSetToFolder)save foldersave to foldersave datasetsave imagesexport datasetz#Save Image (to Folder) (DEPRECATED)r4   zGSave a dataset of images to a specified folder. Supported formats: PNG.Tr9   List of images to save.r8   folder_namedataset?Name of the folder to save images to (inside output directory).defaultr8   filename_prefix!Prefix for saved image filenames.r   r8   advancedr   r   	incrementPWhether to overwrite existing files or increment filenames to avoid overwriting.r   r7   r8   )r?   r@   r<   rA   rB   rC   is_output_nodeis_input_listrD   rE   is_deprecatedr   rF   r   rH   rt   rG   rL   s    r   rN   *SaveImageDataSetToFolderNode.define_schema   s    yy.m>a x1JK		!%]   
 		%#?!	    '(+6n	  ( =
 	
r   c                    US   nUS   nUS   n[         R                  R                  [        R                  " 5       U5      n[        XX4S:H  5      n[        R                  " S[        U5       SU S35        [        R                  " 5       $ )Nr   r   Saved z images to r   )r   r   r   rI   get_output_directoryr   r   r   r   r   ra   )rM   r9   r   r   r   r   r   s          r   re   $SaveImageDataSetToFolderNode.execute   s|     "!n)!,AwWW\\,"C"C"E{S
+FWbQbcvc+./{:,aHI}}r   r
   Nrg   r
   r   r   r   r      s)     
  
D 
 
r   r   c                   8    \ rS rSr\S 5       r\SS j5       rSrg) SaveImageTextDataSetToFolderNodei  c                 t   [         R                  " S/ SQSSSSSS[         R                  R                  SSS	9[         R                  R                  S
SSSS9[         R                  R                  SSSS9[         R                  R                  SSSSS9[         R
                  R                  SSSS/SS9// S9
$ )NSaveImageTextDataSetToFolder)r   r   r   r   z	save textr   zSave Image-Text (to Folder)r4   zSave a dataset of pairs of images and text captions to a specified folder. Images are saved as PNG files and captions are saved as TXT files with the same filename_prefix.Tr9   r   r   rs   zList of text captions to save.)optionalforce_inputr8   r   r   r   r   r   r   r   r   r   r   r   r   
r?   r@   r<   rA   rB   rC   r   r   rD   rE   r   rL   s    r   rN   .SaveImageTextDataSetToFolderNode.define_schema  s    yy2z6 F x1JK		! $<   
 		!%]   
 		%#?!	    '(+6n	  %2 E#
 #	
r   Nc                 8   US   nUS   nUS   n[         R                  R                  [        R                  " 5       U5      n[        XX4S:H  5      nU(       av  [        [        Xu5      5       H^  u  nu  pU	R                  SS5      n[         R                  R                  Xk5      n[        USSS9 nUR                  U
5        S S S 5        M`     [        R                  " S[        U5       S	U S
35        [        R                  " 5       $ ! , (       d  f       M  = f)Nr   r   rR   r}   wr   r   r   z images and captions to r   )r   r   r   rI   r   r   r   zipr   r   writer   r   r   r   ra   )rM   r9   r   r   r   rs   r   r   r   r   r   caption_filenamer   rZ   s                 r   re   (SaveImageTextDataSetToFolderNode.execute.  s     "!n)!,AwWW\\,"C"C"E{S
+FWbQbc ,5c+6M,N((h#+#3#3FF#C !ww||JI,g>!GGG$ ?> -O 	vc+.//G
|STUV}}	 ?>s   1D


D	r
   r   rg   r
   r   r   r   r     s)    $
 $
L  r   r   c                 :   U R                  5       S:X  a$  U R                  S   S:X  a  U R                  S5      n U R                  5       R	                  5       S-  R                  SS5      R                  [        R                  5      n[        R                  " U5      $ )zConvert tensor to PIL Image.r   r   rz   r   )r   r   r   r   r   r   r   r   r   r   r   )r+   r*   s     r   tensor_to_pilr   G  sx    ~~1!1!1!!4!9''*
!'')C/55a=DDRXXNI??9%%r   c                     [         R                  " U 5      R                  [         R                  5      S-  n[        R
                  " U5      S   $ )zConvert PIL Image to tensor.r   r   )r   r   r   r    r!   r"   )r)   r*   s     r   pil_to_tensorr   O  s9    $$RZZ058II&u--r   c                       \ rS rSrSrSr/ rSrSr/ r	Sr
SrSr\S 5       r\S 5       r\S 5       r\S 5       r\S	 5       r\S
 5       rSrg)ImageProcessingNodeiX  a<  Base class for image processing nodes that operate on images.

Child classes should set:
    node_id: Unique node identifier (required)
    search_aliases: List of search aliases (optional)
    display_name: Display name (optional, defaults to node_id)
    description: Node description (optional)
    extra_inputs: List of additional io.Input objects beyond "images" (optional)
    is_group_process: None (auto-detect), True (group), or False (individual) (optional)
    is_output_list: True (list output) or False (single output) (optional, default True)
    is_deprecated: True if the node is deprecated (optional, default False)

Child classes must implement ONE of:
    _process(cls, image, **kwargs) -> tensor  (for single-item processing)
    _group_process(cls, images, **kwargs) -> list[tensor]  (for group processing)
NFc                    U R                   b  U R                   $ [        nSnU R                   H  nSUR                  ;   d  M  Un  O   SnU R                   H  nSUR                  ;   d  M  Un  O   USL=(       a    X!LnUSL=(       a    XALnU(       a  U(       a  [	        U R
                   S35      eU(       d  U(       d  [	        U R
                   S35      eU$ zDetect whether this node uses group or individual processing.

Returns:
    bool: True if group processing, False if individual processing
N_process_group_processzj: Cannot override both _process and _group_process. Override only one, or set is_group_process explicitly.z1: Must override either _process or _group_process)is_group_processr   __mro____dict__r   rh   rM   
base_classprocess_definerklassgroup_definerhas_process	has_groups          r   _detect_processing_mode+ImageProcessingNode._detect_processing_moder  s     +''' )
 [[EU^^+"' ! [[E5>>1 % ! &T1Wo6W!-Q-2Q	9<<. !I I  9<<. QR  r   c                 ^   [        U[        R                  5      (       a^  UR                  S:w  a!  [	        S[        UR                  5       35      e[        UR                  S   5       Vs/ s H	  o!X"S-    PM     sn$ / nU H  n[        U[        R                  5      (       a  UR                  S:w  a/  [	        S[        U5      R                   S[        USS5       35      eUR                  [        UR                  S   5       Vs/ s H	  o$X"S-    PM     sn5        M     U$ s  snf s  snf )	z1Normalize to a flat list of [1, H, W, C] tensors.r   z$Expected 4D image tensor, got shape r   rz   zExpected 4D image tensor, got z shape r   N)r   r!   r   ndimr   tupler   ranger   rh   getattrr   )rM   r9   r   flatrx   s        r   _ensure_image_list&ImageProcessingNode._ensure_image_list  s    fell++{{a #GfllH[G\!]^^+0a+AB+Aa1qSM+ABBDdELL11TYY!^ #A$t*BUBUAVV]^efjlsuy^z]{!|}}KKeDJJqM.BC.Ba!.BCD   C Ds   +D%D*
c                 &   U R                   c  [        U R                   S35      eU R                  5       nU R                  b  U R                  OUn[
        R                  R                  SU(       a  SOSS9/nUR                  U R                  5        [
        R                  " U R                   U R                  U R                  =(       d    U R                   U R                  U R                  SUU[
        R                  R                  SUSS9/S	9	$ )
N  must set node_id class variabler9   zList of images to process.zImage to process.r   TzProcessed imagesr;   	r?   r@   r<   rA   rB   rC   r   rD   rE   )r?   NotImplementedErrorrh   r   r=   r   r   rH   r   extra_inputsrF   r@   r<   rA   rB   rK   )rM   is_groupoutput_is_listrD   s       r   rN   !ImageProcessingNode.define_schema  s    ;;%6V&WXX..0 #&"4"4"@Ch 	
 HHNN4<0BU  
 	c&&'yyKK--))8S[[\\ "!)#1.   
 	
r   c                 r   U R                  5       nU(       a  U R                  U5      n0 nUR                  5        H6  u  pV[        U[        5      (       a  [        U5      S:X  a	  US   XE'   M2  XdU'   M8     U(       a  U R                  " U40 UD6nOU R                  " U40 UD6n[        R                  " U5      $ zEExecute the node. Routes to _process or _group_process based on mode.rz   r   )
r   r  itemsr   listr   r   r   r   ra   )rM   r9   kwargsr  paramskvresults           r   re   ImageProcessingNode.execute  s     ..0++F3F LLNDA!T""s1v{aD	q		 # ''9&9F \\&3F3F}}V$$r   c                 2    [        U R                   S35      e)zOverride this method for single-item processing.

Args:
    image: tensor - Single image tensor
    **kwargs: Additional parameters (already extracted from lists)

Returns:
    tensor - Processed image
 must implement _process methodr  rh   )rM   r4   r  s      r   r   ImageProcessingNode._process       "S\\N2Q"RSSr   c                 2    [        U R                   S35      e)zOverride this method for group processing.

Args:
    images: list[tensor] - List of image tensors
    **kwargs: Additional parameters (already extracted from lists)

Returns:
    list[tensor] - Processed images
% must implement _group_process methodr  )rM   r9   r  s      r   r   "ImageProcessingNode._group_process        "||nAB
 	
r   r
   )rh   ri   rj   rk   __doc__r?   r@   r<   rB   r  r   r=   r   rl   r   r  rN   re   r   r   rm   r
   r   r   r   r   X  s    " GNLKLNM) )V   '
 '
R % %0 
T 
T 
 
r   r   c                       \ rS rSrSrSr/ rSrSr/ r	Sr
SrSr\S 5       r\S 5       r\S 5       r\S 5       r\S	 5       rS
rg)TextProcessingNodei  a1  Base class for text processing nodes that operate on texts.

Child classes should set:
    node_id: Unique node identifier (required)
    search_aliases: List of search aliases (optional)
    display_name: Display name (optional, defaults to node_id)
    description: Node description (optional)
    extra_inputs: List of additional io.Input objects beyond "texts" (optional)
    is_group_process: None (auto-detect), True (group), or False (individual) (optional)
    is_output_list: True (list output) or False (single output) (optional, default True)
    is_deprecated: True if the node is deprecated (optional, default False)

Child classes must implement ONE of:
    _process(cls, text, **kwargs) -> str  (for single-item processing)
    _group_process(cls, texts, **kwargs) -> list[str]  (for group processing)
NFc                    U R                   b  U R                   $ [        nSnU R                   H  nSUR                  ;   d  M  Un  O   SnU R                   H  nSUR                  ;   d  M  Un  O   USL=(       a    X!LnUSL=(       a    XALnU(       a  U(       a  [	        U R
                   S35      eU(       d  U(       d  [	        U R
                   S35      eU$ r   )r   r'  r   r   r   rh   r   s          r   r   *TextProcessingNode._detect_processing_mode'  s     +''' (
 [[EU^^+"' ! [[E5>>1 % ! &T1Wo6W!-Q-2Q	9<<. !I I  9<<. QR  r   c                    U R                   c  [        U R                   S35      eU R                  5       n[        R
                  R                  SU(       a  SOSS9/nUR                  U R                  5        [        R                  " U R                   U R                  =(       d    U R                   SSUU[        R
                  R                  SU R                  SS	9/S
9$ )Nr  rs   zList of texts to process.zText to process.r   textTzProcessed textsr;   )r?   r<   rA   rC   r   rD   rE   )r?   r  rh   r   r   rt   rH   r   r  rF   r<   rK   r=   )rM   r  rD   s      r   rN    TextProcessingNode.define_schemaS  s    ;;%6V&WXX..0 IIOO7?3EW  
 	c&&'yyKK))8S[[ "		  !(#&#5#5- ! 
 	
r   c                    U R                  5       n0 nUR                  5        H6  u  pV[        U[        5      (       a  [	        U5      S:X  a	  US   XE'   M2  XdU'   M8     U(       a  U R
                  " U40 UD6nOU R                  " U40 UD6nU R                  (       a$  [        R                  " U(       a  U5      $ U/5      $ [        R                  " U/5      $ r  )
r   r  r   r  r   r   r   r=   r   ra   )rM   rs   r  r  r  r  r  r  s           r   re   TextProcessingNode.executer  s     ..0 LLNDA!T""s1v{aD	q		 # ''88F \\%262F ==8BB&BB ==&**r   c                 2    [        U R                   S35      e)zOverride this method for single-item processing.

Args:
    text: str - Single text string
    **kwargs: Additional parameters (already extracted from lists)

Returns:
    str - Processed text
r  r  )rM   r+  r  s      r   r   TextProcessingNode._process  r   r   c                 2    [        U R                   S35      e)zOverride this method for group processing.

Args:
    texts: list[str] - List of text strings
    **kwargs: Additional parameters (already extracted from lists)

Returns:
    list[str] - Processed texts
r"  r  )rM   rs   r  s      r   r   !TextProcessingNode._group_process  r$  r   r
   )rh   ri   rj   rk   r%  r?   r@   r<   rB   r  r   r=   r   rl   r   rN   re   r   r   rm   r
   r   r   r'  r'    s    " GNLKLNM) )V 
 
< + +6 
T 
T 
 
r   r'  c                   t    \ rS rSrSrSrSrSrSr\	R                  R                  SSS	S
SS9/r\S 5       rSrg)ResizeImagesByShorterEdgeNodei  ResizeImagesByShorterEdgez*Resize Images by Shorter Edge (DEPRECATED)image/transformzeResize images so that the shorter edge matches the specified dimension while preserving aspect ratio.Tshorter_edge   rz       z&Target dimension for the shorter edge.r   minmaxr8   c                     [        U5      nUR                  u  pEXE:  a  Un[        XRU-  -  5      nOUn[        XBU-  -  5      nUR                  Xg4[        R
                  R                  5      n[        U5      $ r   )r   sizer   resizer   
ResamplingLANCZOSr   )rM   r4   r7  r)   r   hnew_wnew_hs           r   r   &ResizeImagesByShorterEdgeNode._process  ss    E"xx5 EA-./E EA-./Ejj%)9)9)A)ABS!!r   r
   Nrh   ri   rj   rk   r?   r<   rA   rB   r   r   IntrH   r  rl   r   rm   r
   r   r   r4  r4    sZ    )G?L HyKM
< 	 	
L 
" 
"r   r4  c                   t    \ rS rSrSrSrSrSrSr\	R                  R                  SSS	S
SS9/r\S 5       rSrg)ResizeImagesByLongerEdgeNodei  ResizeImagesByLongerEdgez)Resize Images by Longer Edge (DEPRECATED)r6  zdResize images so that the longer edge matches the specified dimension while preserving aspect ratio.Tlonger_edgei   rz   r9  z%Target dimension for the longer edge.r:  c                 R   / nU H  n[        U5      nUR                  u  pgXg:  a  Un[        XrU-  -  5      n	OUn	[        XbU-  -  5      nUR                  X4[        R
                  R                  5      nUR                  [        U5      5        M     [        R                  " USS9$ )Nr   r   )r   r>  r   r?  r   r@  rA  r#   r   r!   cat)
rM   r4   rK  resized_imagesimage_ir)   r   rB  rC  rD  s
             r   r   %ResizeImagesByLongerEdgeNode._process  s    G(C88DAu#Aq12#Aq12**e^U-=-=-E-EFC!!-"45  yyQ//r   r
   NrF  r
   r   r   rI  rI    sZ    (G>L HxKM
; 	 	
L 0 0r   rI  c            	           \ rS rSrSr/ SQrSrSrSr\	R                  R                  SSS	S
SS9\	R                  R                  SSS	S
SS9/r\S 5       rSrg)CenterCropImagesNodei  CenterCropImagescropcuttrimzCrop Image (Center)r6  z1Center crop an image to the specified dimensions.widthr8  rz   r9  Crop width.r:  heightCrop height.c                 $   [        U5      n[        SUR                  U-
  S-  5      n[        SUR                  U-
  S-  5      n[	        UR                  XR-   5      n[	        UR                  Xc-   5      nUR                  XVXx45      n[        U5      $ )Nr   r   )r   r<  rY  r[  r;  rV  r   )	rM   r4   rY  r[  r)   lefttoprightbottoms	            r   r   CenterCropImagesNode._process  s    E"1syy5(Q./!cjj6)a/0CIIt|,SZZ.hh512S!!r   r
   Nrh   ri   rj   rk   r?   r@   r<   rA   rB   r   rG  rH   r  rl   r   rm   r
   r   r   rS  rS    sj     G*N&LHEK
WcqdMR
Xst^TL
 " "r   rS  c            
           \ rS rSrSr/ SQrSrSrSr\	R                  R                  SSS	S
SS9\	R                  R                  SSS	S
SS9\	R                  R                  SSSSSS9/r\S 5       rSrg)RandomCropImagesNodei  RandomCropImagesrU  zCrop Image (Random)r6  z3Randomly crop an image to the specified dimensions.rY  r8  rz   r9  rZ  r:  r[  r\  seedr       Random seed.c                    [         R                  R                  US-  5        [        U5      n[	        SUR
                  U-
  5      n[	        SUR                  U-
  5      nUS:  a#  [         R                  R                  SUS-   5      OSnUS:  a#  [         R                  R                  SUS-   5      OSn	[        UR
                  X-   5      n
[        UR                  X-   5      nUR                  XX45      n[        U5      $ )N    r   rz   )r   randomrg  r   r<  rY  r[  randintr;  rV  r   )rM   r4   rY  r[  rg  r)   max_leftmax_topr^  r_  r`  ra  s               r   r   RandomCropImagesNode._process  s    
		ty)*E"q#))e+,af,-5=\ryy  HqL1q3:Q;bii7Q;/ACIIt|,SZZ.hh512S!!r   r
   Nrc  r
   r   r   re  re    s     G*N(LHGK 	WcqdMR
Xst^T
A1*<n 	 	
L 
" 
"r   re  c            
           \ rS rSrSrSS/rSrSrSr\	R                  R                  SS	S
SSSS9\	R                  R                  SS	SSSSS9/r\S 5       rSrg)NormalizeImagesNodei   NormalizeImages	normalizeznormalize colorszNormalize Image Colorszimage/colorz3Normalize images using mean and standard deviation.mean      ?              ?zMean value for normalization.Tr   r;  r<  r8   r   stdgMbP?z%Standard deviation for normalization.c                     X-
  U-  $ r   r
   )rM   r4   ru  rz  s       r   r   NormalizeImagesNode._process9  s    ##r   r
   Nrh   ri   rj   rk   r?   r@   r<   rA   rB   r   FloatrH   r  rl   r   rm   r
   r   r   rr  rr     s    G!34N+LHGK
3 	 	
 	; 	 	
L& $ $r   rr  c                   v    \ rS rSrSrS/rSrSrSr\	R                  R                  SSS	S
SS9/r\S 5       rSrg)AdjustBrightnessNodei>  AdjustBrightness
brightnesszAdjust Brightnessimage/adjustmentsz"Adjust the brightness of an image.factorrx  rw         @zCBrightness factor. 1.0 = no change, <1.0 = darker, >1.0 = brighter.r:  c                 *    X-  R                  SS5      $ )Nrw  rx  clamprM   r4   r  s      r   r   AdjustBrightnessNode._processN  s    %%c3//r   r
   Nr}  r
   r   r   r  r  >  s\     G >N&L H6K
Y 	 	
L 0 0r   r  c                   v    \ rS rSrSrS/rSrSrSr\	R                  R                  SSS	S
SS9/r\S 5       rSrg)AdjustContrastNodeiS  AdjustContrastcontrastzAdjust Contrastr  z Adjust the contrast of an image.r  rx  rw  r  zMContrast factor. 1.0 = no change, <1.0 = less contrast, >1.0 = more contrast.r:  c                 8    US-
  U-  S-   R                  SS5      $ )Nrv  rw  rx  r  r  s      r   r   AdjustContrastNode._processc  s#    &,33C==r   r
   Nr}  r
   r   r   r  r  S  s\    G<N$L H4K
c 	 	
L > >r   r  c                   |    \ rS rSrSr/ SQrSrSrSrSr	\
R                  R                  SS	S	S
SS9/r\S 5       rSrg)ShuffleDatasetNodeih  ShuffleDatasetshuffle	randomizemixzShuffle Images Listimage/batchz/Randomly shuffle the order of images in a list.Trg  r   rh  ri  r:  c                     [         R                  R                  US-  5        [         R                  R                  [	        U5      5      nU Vs/ s H  oAU   PM	     sn$ s  snf )Nrk  )r   rl  rg  permutationr   )rM   r9   rg  indicesr   s        r   r   !ShuffleDatasetNode._group_processu  sI    
		ty)*))''F4#*+7aq	7+++s   A r
   N)rh   ri   rj   rk   r?   r@   r<   rA   rB   r   r   rG  rH   r  rl   r   rm   r
   r   r   r  r  h  s]    G2N(LHCK
A1*<n 	 	
L , ,r   r  c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)ShuffleImageTextDatasetNodei|  z:Special node that shuffles both images and texts together.c                 l   [         R                  " S/ SQSSSSS[         R                  R                  SSS	9[         R                  R                  S
SSS9[         R
                  R                  SSSSSS9/[         R                  R                  SSSS9[         R                  R                  S
SSS9/S9	$ )NShuffleImageTextDatasetr  zShuffle Pairs of Image-Textr  z<Randomly shuffle the order of pairs of image-text in a list.Tr9   zList of images to shuffle.r   rs   zList of texts to shuffle.)r8   r   rg  r   rh  ri  r:  zShuffled imagesr;   zShuffled textsr  )r   rF   r   rH   rt   rG  rK   rL   s    r   rN   )ShuffleImageTextDatasetNode.define_schema  s    yy-:8$X x1MN		1LZ^_**  
 !)#'-   
 		  !(GW ! 	'
 	
r   c                 0   US   n[         R                  R                  US-  5        [         R                  R                  [	        U5      5      nU Vs/ s H  oQU   PM	     nnU Vs/ s H  oRU   PM	     nn[
        R                  " Xg5      $ s  snf s  snf )Nr   rk  )r   rl  rg  r  r   r   ra   )rM   r9   rs   rg  r  r   shuffled_imagesshuffled_textss           r   re   #ShuffleImageTextDatasetNode.execute  s{    Aw
		ty)*))''F4.56g!9g6,34Gq(G4}}_== 74s   B(Br
   N	rh   ri   rj   rk   r%  rl   rN   re   rm   r
   r   r   r  r  |  s,    D
 
@ > >r   r  c                   >    \ rS rSrSrS/rSrSrSrSr	\
S 5       rS	rg
)TextToLowercaseNodei  TextToLowercase	lowercasez&Convert Text to Lowercase (DEPRECATED)r+  zConvert text to lowercase.Tc                 "    UR                  5       $ r   )rU   rM   r+  s     r   r   TextToLowercaseNode._process      zz|r   r
   Nrh   ri   rj   rk   r?   r@   r<   rA   rB   r   rl   r   rm   r
   r   r   r  r    4    G=N;LH.KM r   r  c                   >    \ rS rSrSrS/rSrSrSrSr	\
S 5       rS	rg
)TextToUppercaseNodei  TextToUppercase	uppercasez&Convert Text to Uppercase (DEPRECATED)r+  zConvert text to uppercase.Tc                 "    UR                  5       $ r   )upperr  s     r   r   TextToUppercaseNode._process  r  r   r
   Nr  r
   r   r   r  r    r  r   r  c                   x    \ rS rSrSr/ SQrSrSrSr\	R                  R                  SSS	S
SS9/r\S 5       rSrg)TruncateTextNodei  TruncateText)truncaterW  shortenzTruncate Textr+  z"Truncate text to a maximum length.
max_lengthM   rz   i'  zMaximum text length.r:  c                     US U $ r   r
   )rM   r+  r  s      r   r   TruncateTextNode._process  s    KZ  r   r
   Nrc  r
   r   r   r  r    sW    G1N"LH6K
"!@V 	 	
L ! !r   r  c                   p    \ rS rSrSrSrSrSrSr\	R                  R                  SSS	S
9/r\S 5       rSrg)AddTextPrefixNodei  AddTextPrefixzAdd Text Prefix (DEPRECATED)r+  zAdd a prefix to all texts.Tr   r   zPrefix to add.r   c                 
    X!-   $ r   r
   )rM   r+  r   s      r   r   AddTextPrefixNode._process  s
    }r   r
   Nrh   ri   rj   rk   r?   r<   rA   rB   r   r   rt   rH   r  rl   r   rm   r
   r   r   r  r    M    G1LH.KM
		"6FGL  r   r  c                   p    \ rS rSrSrSrSrSrSr\	R                  R                  SSS	S
9/r\S 5       rSrg)AddTextSuffixNodei  AddTextSuffixzAdd Text Suffix (DEPRECATED)r+  zAdd a suffix to all texts.Tsuffixr   zSuffix to add.r   c                 
    X-   $ r   r
   )rM   r+  r  s      r   r   AddTextSuffixNode._process  s
    }r   r
   Nr  r
   r   r   r  r    r  r   r  c                       \ rS rSrSrSrSrSrSr\	R                  R                  SSS	S
9\	R                  R                  SSSS
9/r\S 5       rSrg)ReplaceTextNodei  ReplaceTextzReplace Text (DEPRECATED)r+  zReplace text in all texts.Tfindr   zText to find.r   r   zText to replace with.c                 $    UR                  X#5      $ r   )r   )rM   r+  r  r   s       r   r   ReplaceTextNode._process  s    ||D**r   r
   Nr  r
   r   r   r  r    sc    G.LH.KM
		OD
			27NOL
 + +r   r  c                   8    \ rS rSrSrSrSrSrSr\	S 5       r
Srg	)
StripWhitespaceNodei  StripWhitespacezStrip Whitespace (DEPRECATED)r+  z5Strip leading and trailing whitespace from all texts.Tc                 "    UR                  5       $ r   )r   r  s     r   r   StripWhitespaceNode._process  r  r   r
   N)rh   ri   rj   rk   r?   r<   rA   rB   r   rl   r   rm   r
   r   r   r  r    s-    G2LHIKM r   r  c            	           \ rS rSrSrSr/ SQrSrSrSr	Sr
\R                  R                  S	S
SSSSS9/r\S 5       rSrg)ImageDeduplicationNodei  zMRemove duplicate or very similar images from a list using perceptual hashing.ImageDeduplication)deduplicatezremove duplicateszsimilarity filterzDeduplicate Imagesr  z4Remove duplicate or very similar images from a list.Tsimilarity_thresholdgffffff?rw  rx  zmSimilarity threshold (0-1). Higher means more similar. Images above this threshold are considered duplicates.ry  c                    [        U5      S:X  a  / $ S nS nU Vs/ s H
  oS" U5      PM     nn/ n[        [        U5      5       Hg  nSn	U HB  n
U" Xh   Xj   5      nSUS-  -
  nX:  d  M  Sn	[        R                  " SU S	U
 S
US S35          O   U	(       a  MV  UR	                  U5        Mi     U Vs/ s H  oU   PM	     nn[        R                  " S[        U5       S[        U5       S35        U$ s  snf s  snf )z1Remove duplicate images using perceptual hashing.r   c                 &  ^ [        U 5      nUR                  S[        R                  R                  5      R                  S5      n[        UR                  5       5      n[        U5      [        U5      -  mSR                  U4S jU 5       5      nU$ )zMCompute a simple perceptual hash by resizing to 8x8 and comparing to average.)   r  Lr   c              3   6   >#    U  H  oT:  a  S OSv   M     g7f)10Nr
   )rX   pavgs     r   r[   NImageDeduplicationNode._group_process.<locals>.compute_hash.<locals>.<genexpr>:  s     HA3wC 7s   )r   r?  r   r@  rA  r   r  getdatasumr   r   )r+   r)   	img_smallpixels	hash_bitsr  s        @r   compute_hash;ImageDeduplicationNode._group_process.<locals>.compute_hash0  ss    
+C

65+;+;+C+CDLLSQI)++-.Ff+F+CHHHIr   c                 8    [        S [        X5       5       5      $ )z2Compute Hamming distance between two hash strings.c              3   .   #    U  H  u  pX:g  v   M     g 7fr   r
   )rX   c1c2s      r   r[   RImageDeduplicationNode._group_process.<locals>.hamming_distance.<locals>.<genexpr>?  s     @.?FBrx.?s   )r  r   )hash1hash2s     r   hamming_distance?ImageDeduplicationNode._group_process.<locals>.hamming_distance=  s    @c%.?@@@r   Frx  g      P@TzImage z is similar to image z (similarity: z.3fz), skippingzDeduplication: kept z out of  images)r   r  r   r   r#   )rM   r9   r  r  r  r)   hasheskeep_indicesr   is_duplicatejdistance
similarityunique_imagess                 r   r   %ImageDeduplicationNode._group_process)  s#    v;!I		A
 066v,s#v6 s6{#A L!+FIvyA HtO4
5#'LLL #8>*UXIYYde  "  <##A& $" -99LqL9"3}#5"6hs6{m7S	
 3 7* :s   C54C:r
   N)rh   ri   rj   rk   r%  r?   r@   r<   rA   rB   r   r   r~  rH   r  rl   r   rm   r
   r   r   r  r    si    W"GLN'LHHK
" D 	 	
	L 1 1r   r  c                   2   \ rS rSrSrSr/ SQrSrSrSr	Sr
S	r\R                  R                  S
SSSSS9\R                  R                  SSSSSSS9\R                  R                  SSSSSSS9\R                  R                  SSSSSSS9/r\S 5       rSrg)ImageGridNodei^  z3Combine multiple images into a single grid/collage.	ImageGrid)gridcollagecombinezMake Image Gridr  z+Arrange multiple images into a grid layout.TFcolumnsr   rz      zNumber of columns in the grid.r:  
cell_width       i   zWidth of each cell in the grid.ry  cell_heightz Height of each cell in the grid.paddingr   2   zPadding between images.c                    [        U5      S:X  a  [        S5      e[        U5      nXb-   S-
  U-  nX#-  US-
  U-  -   nXt-  US-
  U-  -   n	[        R                  " SX4S5      n
[	        U5       He  u  pX-  nX-  n[        U5      nUR                  X44[        R                  R                  5      nXU-   -  nXU-   -  nU
R                  UUU45        Mg     [        R                  " SU SU SU S	U SU	 S
35        [        U
5      $ )zArrange images into a grid.r   z(Cannot create grid from empty image listrz   r   )r   r   r   Created xz grid with z	 images ())r   r   r   newr   r   r?  r@  rA  paster   r   r   )rM   r9   r  r	  r  r  
num_imagesrows
grid_widthgrid_heightr  r   r+   rowcolr)   r  ys                     r   r   ImageGridNode._group_process  s=    v;!GHH [
$q(W4 )Wq[G,CC
(D1H+?? yy 99E  )0OC.C-C  
+C**j68H8H8P8PQC G+,AW,-A JJsQF#  1 	wiqk*YzlRST_S``ab	
 T""r   r
   N)rh   ri   rj   rk   r%  r?   r@   r<   rA   rB   r   r=   r   rG  rH   r  rl   r   rm   r
   r   r   r  r  ^  s    =G1N$LH?KN
4 	 	
 	5 	 	
 	6 	 	
 	qaR9R]a 	 	
1L: ## ##r   r  c                   H    \ rS rSrSrSr/ SQrSrSrSr	Sr
Sr\S	 5       rS
rg)MergeImageListsNodei  z.Merge multiple image lists into a single list.MergeImageLists)r  z
merge listz	make listzMerge Image Lists (DEPRECATED)r  z*Concatenate multiple image lists into one.Tc                 L    [         R                  " S[        U5       S35        U$ )zASimply return the images list (already merged by input handling).zMerged image list contains r  r   r   r   )rM   r9   s     r   r   "MergeImageListsNode._group_process  s#    
 	23v;-wGHr   r
   N)rh   ri   rj   rk   r%  r?   r@   r<   rA   rB   r   r   rl   r   rm   r
   r   r   r  r    s;    8G6N3LH>KM r   r  c                   @    \ rS rSrSrSrSrSrSrSr	Sr
\S 5       rS	rg
)MergeTextListsNodei  z-Merge multiple text lists into a single list.MergeTextListszMerge Text Lists (DEPRECATED)r+  z)Concatenate multiple text lists into one.Tc                 L    [         R                  " S[        U5       S35        U$ )z@Simply return the texts list (already merged by input handling).zMerged text list contains z textsr!  )rM   rs   s     r   r   !MergeTextListsNode._group_process  s#    
 	1#e*VDEr   r
   N)rh   ri   rj   rk   r%  r?   r<   rA   rB   r   r   rl   r   rm   r
   r   r   r$  r$    s6    7G2LH=KM r   r$  c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)ResolutionBucketi  zIBucket latents and conditions by resolution for efficient batch training.c                 *   [         R                  " S/ SQSSSSS[         R                  R                  SSS	9[         R                  R                  S
SS	9/[         R                  R                  SSSS9[         R                  R                  S
SSS9/S9	$ )Nr)  )zbucket by resolutionzgroup by resolutionzbatch by resolutionzResolution Bucketmodel/trainingz,Group latents and conditionings into bucketsTlatentsz-List of latent dicts to bucket by resolution.r   conditioningz7List of conditioning lists (must match latents length).z8List of batched latent dicts, one per resolution bucket.r;   z3List of condition lists, one per resolution bucket.r  )r   rF   LatentrH   ConditioningrK   rL   s    r   rN   ResolutionBucket.define_schema  s    yy&a,%F 		K    %%"U & 	 		  !*#'V ! 
 &&!/#'Q ' %
 	
r   c                    [        U5      [        U5      :w  a$  [        S[        U5       S[        U5       S35      e/ n/ n[        X5       HQ  u  pVUS   nUR                  S   n[	        U5       H)  n	UR                  Xy   5        UR                  Xi   5        M+     MS     0 n
[        X45       H]  u  pUR                  S   UR                  S   pX4nX;  a  / / S.X'   X   S	   R                  U5        X   S
   R                  U5        M_     / n/ nU
R                  5        Hs  u  u  pn[        R                  " US	   SS9nUR                  SU05        UR                  US
   5        [        R                  " SU SU S[        US	   5       S35        Mu     [        R                  " S[        U
5       S[        U5       S35        [        R                  " UU5      $ )NNumber of latents (') does not match number of conditions (z).samplesr   )r,  
conditionsr,  r7  rM  zResolution bucket (r  z):  samplesr  z resolution buckets from )r   r   r   r   r  r#   r  r!   stackr   r   r   ra   )rM   r,  r-  flat_latentsflat_conditionslatent_dictcondr4  
batch_sizer   bucketslatentrB  r   keyoutput_latentsoutput_conditionsbucket_datastacked_latentss                      r   re   ResolutionBucket.execute  s    w<3|,,%c'l^3Z[^_k[lZmmop 
 !$W!;K!),G q)J :&##GJ/&&tw/ ' "< >LF<<#V\\"%5q&C!+-R@L#**62L&--d3 ? #*==?FQK#kk+i*@aHO!!9o">? $$[%>?LL%aS!CK	4J0K/LHU $3 	xG~-Fs<GXFYYabc}}^->??r   r
   Nr  r
   r   r   r)  r)    s.    S
 
B 6@ 6@r   r)  c                   <    \ rS rSrSr\S 5       r\SS j5       rSrg)MakeTrainingDataseti5  zHEncode images with VAE and texts with CLIP to create a training dataset.c                    [         R                  " SS/SSSSS[         R                  R                  SSS	9[         R                  R                  S
SS	9[         R
                  R                  SSS	9[         R                  R                  SSSSS9/[         R                  R                  SSSS9[         R                  R                  SSSS9/S9	$ )NrH  zencode datasetzMake Training Datasetr+  zeEncode images with VAE and texts with CLIP to create a training dataset of latents and conditionings.Tr9   zList of images to encode.r   vaez)VAE model for encoding images to latents.r   z-CLIP model for encoding text to conditioning.rs   zoList of text captions. Can be length n (matching images), 1 (repeated for all), or omitted (uses empty string).)r   r8   r   r,  List of latent dictsr;   r-  List of conditioning listsr  )
r   rF   r   rH   VaeCliprt   r.  rK   r/  rL   s    r   rN   !MakeTrainingDataset.define_schema7  s    yy),-0% x1LM#N   $S   		! N $	     		  !*#'2 ! 
 &&!/#'8 ' /#
 #	
r   Nc           	      .   US   nUS   n[        U5      nUb  [        U5      S:X  a  S/n[        U5      S:X  a  US:  a  XE-  nO-[        U5      U:w  a  [        S[        U5       SU SU S35      e[        R                  " SU S	35        / nU H6  nUR	                  US S 2S S 2S S 2S S
24   5      nUR                  SU05        M8     [        R                  " S[        U5       S35        / n	U H]  n
U
S:X  a!  UR                  UR                  S5      5      nO"UR                  U
5      nUR                  U5      nU	R                  U5        M_     [        R                  " S[        U5       S[        U	5       S35        [        R                  " Xi5      $ )Nr   r   rz   zNumber of texts (z#) does not match number of images (z ). Text list should have length z
, 1, or 0.z	Encoding z images with VAE...r   r4  z texts with CLIP...zCreated dataset with z latents and z conditioning.)
r   r   r   r   encoder#   encode_from_tokens_scheduledtokenizer   ra   )rM   r9   rJ  r   rs   r  latents_listr+   latent_tensorconditioning_listr+  r=  tokenss                r   re   MakeTrainingDataset.execute^  s    !fAw [
=CJ!ODEu:?zA~&EZ:%#CJ</RS]R^ _00:|:G  	y,?@A JJJz!Q2A2+'>?MM :; ! 	yU,?@ADrz88r9JKt,88@$$T*  	#C$5#6mCHYDZC[[ij	
 }}\==r   r
   r   r  r
   r   r   rH  rH  5  s,    R$
 $
L +> +>r   rH  c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)SaveTrainingDataseti  z?Save encoded training dataset (latents + conditioning) to disk.c                 2   [         R                  " SSS/SSSSSS[         R                  R                  SS	S
9[         R                  R                  SSS
9[         R
                  R                  SSSS9[         R                  R                  SSSSSSS9// S9
$ )NrZ  r   r   zSave Training Datasetr+  zeSave encoded training dataset (latents + conditioning) to disk for efficient loading during training.Tr,  z.List of latent dicts from MakeTrainingDataset.r   r-  z4List of conditioning lists from MakeTrainingDataset.r   training_datasetz9Name of folder to save dataset (inside output directory).r   
shard_sizei  rz   i z!Number of samples per shard file.ry  r   )r   rF   r.  rH   r/  rt   rG  rL   s    r   rN   !SaveTrainingDataset.define_schema  s    yy),n=0% 		L    %%"R &  		!.W   
   ?!  . A!
 !	
r   c                    US   nUS   n[        U5      [        U5      :w  a$  [        S[        U5       S[        U5       S35      e[        R                  R	                  [
        R                  " 5       U5      n[        R                  " USS9  [        U5      nXd-   S-
  U-  n[        R                  " SU S	U S
U S35        [        U5       H  nX-  n	[        X-   U5      n
XU
 X)U
 S.nSUS S3n[        R                  R	                  X\5      n[        US5       n[        R                  " X5        S S S 5        [        R                  " SUS-    SU SU SX-
   S3	5        M     UUUS.n[        R                  R	                  US5      n[        US5       n[        R                   " XSS9  S S S 5        [        R                  " SU S	U S35        ["        R$                  " 5       $ ! , (       d  f       N= f! , (       d  f       NQ= f)Nr   r2  r3  z/). Something went wrong in dataset preparation.Tr   rz   zSaving z samples to z shards in ...)r,  r-  shard_04d.pklwbzSaved shard /: z (z	 samples))num_samples
num_shardsr]  zmetadata.jsonr   r   )indentzSuccessfully saved r   )r   r   r   r   r   rI   r   r   r   r   r  r;  r   r!   r   jsondumpr   ra   )rM   r,  r-  r   r]  r   rg  rh  	shard_idx	start_idxend_idx
shard_datashard_filename
shard_pathrZ   metadatametadata_paths                    r   re   SaveTrainingDataset.execute  s    "!n]
 w<3|,,%c'l^3Z[^_k[lZm n? @  WW\\,"C"C"E{S

J. 'l!.2zA
k],zl+j\QTU	

 z*I!.I)0+>G #W5 ,w ?J  &i_D9NjAJj$'1

:) ( LLy1}oQzl"^<LBwObNcclm# +. '$$

 ZA-%IIh!, & 	*;-|J<qQR}}% (' &%s   G,G=,
G:	=
Hr
   Nr  r
   r   r   rZ  rZ    s,    I"
 "
H < <r   rZ  c                   8    \ rS rSrSr\S 5       r\S 5       rSrg)LoadTrainingDataseti  z(Load encoded training dataset from disk.c                     [         R                  " SSS/SSSS[         R                  R                  SS	S
S9/[         R                  R                  SSSS9[         R                  R                  SSSS9/S9$ )Nrv  r3   ztraining datazLoad Training Datasetr+  zULoad encoded training dataset (latents + conditioning) from disk for use in training.Tr   r\  zFName of folder containing the saved dataset (inside output directory).r   r,  rK  r;   r-  rL  r>   )r   rF   rt   rH   r.  rK   r/  rL   s    r   rN   !LoadTrainingDataset.define_schema  s    yy),o>0%o 		!.d    		  !*#'2 ! 
 &&!/#'8 ' 
 	
r   c           
         [         R                  R                  [        R                  " 5       U5      n[         R                  R                  U5      (       d  [        SU 35      e[        [         R                  " U5       Vs/ s H5  nUR                  S5      (       d  M  UR                  S5      (       d  M3  UPM7     sn5      nU(       d  [        SU 35      e[        R                  " S[        U5       SU S35        / n/ nU H  n[         R                  R                  X'5      n[        US5       n[        R                   " U5      n	S S S 5        UR#                  W	S	   5        UR#                  U	S
   5        [        R                  " SU S[        U	S	   5       S35        M     [        R                  " S[        U5       SU S35        [$        R&                  " XV5      $ s  snf ! , (       d  f       N= f)NzDataset directory not found: ra  rc  zNo shard files found in zLoading z shards from r`  rbr,  r-  r   rf  r8  zSuccessfully loaded z samples from r   )r   r   r   rI   r   r   r   sortedr_   
startswithrV   r   r   r   r   r!   loadr   r   ra   )
rM   r   dataset_dirrZ   shard_filesall_latentsall_conditioning
shard_filerq  ro  s
             r   re   LoadTrainingDataset.execute  s    ggll<#D#D#FTww~~k**<[MJKK  K00A<<) ./jj.@ 0
 7}EFFxK 01{m3OP %Jk>Jj$'1"ZZ]
 ( z)45##J~$>?LL7:,bZ	5J1K0LHUV & 	"3{#3"4N;-qQ	
 }}[;;=& ('s   G%!G%9G%5G**
G8	r
   Nr  r
   r   r   rv  rv    s+    2
 
: '< '<r   rv  c                   L    \ rS rSr\S\\\R                        4S j5       r	Sr
g)DatasetExtensioniB  returnc                 &  #    [         [        [        [        [        [
        [        [        [        [        [        [        [        [        [        [        [         ["        [$        [&        [(        [*        [,        [.        [0        [2        [4        [6        /$ 7fr   )r.   ro   r   r   r4  rI  rS  re  rr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  rH  rZ  rv  r)  )selfs    r   get_node_listDatasetExtension.get_node_listC  sf      +.(,)(   '"C"
 "	
s   BBr
   N)rh   ri   rj   rk   r   r  r   r   	ComfyNoder  rm   r
   r   r   r  r  B  s)    #
T$r||*<%= #
 #
r   r  r  c                     #    [        5       $ 7fr   )r  r
   r   r   comfy_entrypointr  j  s     s   )r4   T)4r   r   rj  r   r   r!   PILr   typing_extensionsr   rI   r   comfy_api.latestr   r   r,   r  r.   ro   r   r   r   r   r   r   r'  r4  rI  rS  re  rr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r)  rH  rZ  rv  r  r  r
   r   r   <module>r     s    	     &   /<$,R\\ $,NI6 I6X1h/2<< /d;r|| ;B&.r
",, r
j[
 [
B"$7 "<0#6 0B". ",". ":$- $<0. 0*>, >*,, ,(+>",, +>b
, 

, 
!) !"* * +( + 	, 	F0 FRK#' K#\- (+ ,\@r|| \@~U>",, U>pd",, dNH<",, H<\%
~ %
P 0 r   