
    +j                     ^   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZd dlmZmZ d Z G d dej                  Z G d d	ej                  ZdKdZ G d dej                  Z G d dej                  Zd Zd Z G d dej                  Z G d dej                  Z G d de          Z G d de          Z G d de          Z G d de          Z G d d e          Z G d! d"e          Z G d# d$e          Z  G d% d&e          Z! G d' d(ej                  Z" G d) d*e          Z# G d+ d,e          Z$ G d- d.e          Z% G d/ d0e          Z& G d1 d2e          Z' G d3 d4e          Z( G d5 d6e          Z) G d7 d8e          Z* G d9 d:e          Z+ G d; d<e          Z, G d= d>e          Z- G d? d@ej                  Z. G dA dBej                  Z/ G dC dDej                  Z0 G dE dFej                  Z1 G dG dHe          Z2dIe2fdJZ3dS )L    N)Image)override)ComfyExtensionioc                    | st          d          g }| D ]}t          j                            ||          }t	          j        t          j        |          }|j        dk    r|	                    d           }|
                    d          }t          j        |                              t          j                  dz  }t          j        |          d         }|                    |           |S )aK  Utility function to load and process a list of images.

    Args:
        image_files: List of image filenames
        input_dir: Base directory containing the images
        resize_method: How to handle images of different sizes ("None", "Stretch", "Crop", "Pad")

    Returns:
        torch.Tensor: Batch of processed images
    zNo valid images found in inputIc                     | dz  S )Ngp? )is    :/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_dataset.py<lambda>z)load_and_process_images.<locals>.<lambda>$   s
    a7m     RGB     o@N)
ValueErrorospathjoinnode_helperspillowr   openmodepointconvertnparrayastypefloat32torch
from_numpyappend)image_files	input_diroutput_imagesfile
image_pathimg	img_array
img_tensors           r   load_and_process_imagesr+      s      ;9:::M 	) 	)W\\)T22
!%*j998s??))3344Ckk%  HSMM((44u<	%i007
Z((((r   c                   :    e Zd Zed             Zed             ZdS )LoadImageDataSetFromFolderNodec                     t          j        dg dddddt           j                            dt	          j                    d	          gt           j                            d
dd          g          S )NLoadImageDataSetFromFolderzload folderzload from folderzload datasetzload imagesimport datasetzLoad Image (from Folder)imagezvLoad a dataset of images from a specified folder and return a list of images. Supported formats: PNG, JPG, JPEG, WEBP.TfolderzThe folder to load images from.optionstooltipimagesList of loaded imagesdisplay_nameis_output_listr6   node_idsearch_aliasesr:   categorydescriptionis_experimentalinputsoutputs)r   SchemaComboInputfolder_pathsget_input_subfoldersr   Outputclss    r   define_schemaz,LoadImageDataSetFromFolderNode.define_schema.   s    y0ooo3 Q (=??=    !)#'3    
 
 
 	
r   c                     t           j                            t          j                    |          }g dfdt          j        |          D             }t          ||          }t          j        |          S )N.pngz.jpgz.jpegz.webpc                 L    g | ]t          fd D                        S )c              3   f   K   | ]+}                                                     |          V  ,d S r   lowerendswith.0extfs     r   	<genexpr>zDLoadImageDataSetFromFolderNode.execute.<locals>.<listcomp>.<genexpr>N   s9      GGs17799%%c**GGGGGGr   )any)rV   rX   valid_extensionss    @r   
<listcomp>z:LoadImageDataSetFromFolderNode.execute.<locals>.<listcomp>K   sN     
 
 
GGGG6FGGGGG

 
 
r   )	r   r   r   rG   get_input_directorylistdirr+   r   
NodeOutput)rK   r3   sub_input_dirr#   output_tensorr[   s        @r   executez&LoadImageDataSetFromFolderNode.executeG   s    \%E%G%GPP===
 
 
 
Z..
 
 

 0]KK}]+++r   N__name__
__module____qualname__classmethodrL   rb   r
   r   r   r-   r-   -   sH        
 
 [
0 	, 	, [	, 	, 	,r   r-   c                   :    e Zd Zed             Zed             ZdS )"LoadImageTextDataSetFromFolderNodec                 &   t          j        dg dddddt           j                            dt	          j                    d	          gt           j                            d
dd          t           j                            ddd          g          S )NLoadImageTextDataSetFromFolderr0   zLoad Image-Text (from Folder)r2   zLoad a dataset of pairs of images and text captions from a specified folder and return them as a list. Supported formats: PNG, JPG, JPEG, WEBP.Tr3   z1The folder to load images and text captions from.r4   r7   r8   r9   textszList of text captionsr<   )	r   rD   rE   rF   rG   rH   r   rI   StringrJ   s    r   rL   z0LoadImageTextDataSetFromFolderNode.define_schemaU   s    y4ooo8 j (=??O    !)#'3    
 	  !(#'3 !  
 
 
 	
r   c                    t          j        d|            t          j                            t          j                    |          }g dg }t          j        |          D ]t          j                            |          t          fdD                       r|	                               Tt          j        
                              rd}                    d          d                                         r(t                              d          d                   }|                    fdt          j                  D             |z             d |D             }g }|D ]}t          j                            ||          }t          j                            |          rft!          |d	d
          5 }	|	                                                                }
|	                    |
           d d d            n# 1 swxY w Y   |	                    d           t'          ||          }t          j        dt)          |           d| d           t+          j        ||          S )NzLoading images from folder: rN   c              3   f   K   | ]+}                                                     |          V  ,d S r   rR   )rV   rW   items     r   rY   z=LoadImageTextDataSetFromFolderNode.execute.<locals>.<genexpr>}   s9      JJ#4::<<((--JJJJJJr      _r   c                     g | ]=t          fd D                       t          j                                      >S )c              3   f   K   | ]+}                                                     |          V  ,d S r   rR   rU   s     r   rY   zHLoadImageTextDataSetFromFolderNode.execute.<locals>.<listcomp>.<genexpr>   s9      SS3qwwyy11#66SSSSSSr   )rZ   r   r   r   )rV   rX   r   r[   s    @r   r\   z>LoadImageTextDataSetFromFolderNode.execute.<locals>.<listcomp>   s^       SSSSBRSSSSST1--  r   c                     g | ];}|                     t          j                            |          d          d          <S )rq   .txt)replacer   r   splitextrV   rX   s     r   r\   z>LoadImageTextDataSetFromFolderNode.execute.<locals>.<listcomp>   sH     
 
 
:;AIIbg&&q))!,f55
 
 
r   rutf-8encoding Loaded z images from .)logginginfor   r   r   rG   r]   r^   rZ   r"   isdirsplitisdigitintextendexistsr   readstripr+   lenr   r_   )rK   r3   r`   r#   repeatcaption_file_pathcaptionscaption_filecaption_pathrX   captionra   rp   r   r[   s               @@@r   rb   z*LoadImageTextDataSetFromFolderNode.executes   s   <F<<===\%E%G%GPP===J}-- 	 	D7<<t44DJJJJ9IJJJJJ ""4((((t$$ ::c??1%--// 5 C!344F""    !#D!1!1  
   
 
?J
 
 
 - 	$ 	$L7<<|DDLw~~l++ $,g>>> -!ffhhnn..GOOG,,,- - - - - - - - - - - - - - - ####/]KKPs=11PPPPPQQQ}]H555s   <HH	H	Nrc   r
   r   r   ri   ri   T   sH        
 
 [
: )6 )6 [)6 )6 )6r   ri   r2   Tc                    t          j        |d           g }t          |           D ]\  }}t          |t          j                  r |                                dk    r&|j        d         dk    r|                    d          }|                                dk    rY|j        d         dv rJ|j        d         dk    r9|j        d         dk    r(|j        d         dk    r|	                    ddd          }|
                                                                }t          j        |d	z  dd
                              t          j                  }t!          j        |          }nt%          dt'          |                     |r
| d|dd}	n(t)          j        ||          \  }
}
}}
}| d|dd|dd}	t           j                            ||	          }|                    |           |                    |	           |S )a  Utility function to save a list of image tensors to disk.

    Args:
        image_list: List of image tensors (each [1, H, W, C] or [H, W, C] or [C, H, W])
        output_dir: Directory to save images to
        prefix: Filename prefix

    Returns:
        List of saved filenames
    Texist_ok   r   rq      )rq   r   r      r      zExpected torch.Tensor, got rr   05drO   05)r   makedirs	enumerate
isinstancer    Tensordimshapesqueezepermutecpunumpyr   clipr   uint8r   	fromarrayr   typerG   get_save_image_pathr   r   saver"   )
image_list
output_dirprefix	overwritesaved_filesidxr*   r)   r(   filenamerr   counterresolved_prefixfilepaths                 r   save_images_to_folderr      s%    K
T****K$Z00 !% !%Zj%,// 	O~~1$$)9!)<)A)A'//22
 ~~1$$)9!)<	)I)I$Q'1,,"(+a//"(+a//!+!3!3Aq!!<!<J #((..00I	E 11c::AA"(KKI /),,CCM4
;K;KMMNNN  	G 0030000HH0<0PQWYc0d0d-Aq'1o)FFGFFFFFFFH7<<
H558$$$$r   c                   :    e Zd Zed             Zed             ZdS )SaveImageDataSetToFolderNodec                 R   t          j        dg dddddddt           j                            dd	          t           j                            d
dd          t           j                            dddd          t           j                            ddddgd          gg d          S )NSaveImageDataSetToFolder)save foldersave to foldersave datasetsave imagesexport datasetz#Save Image (to Folder) (DEPRECATED)r2   zGSave a dataset of images to a specified folder. Supported formats: PNG.Tr7   List of images to save.r6   folder_namedataset?Name of the folder to save images to (inside output directory).defaultr6   filename_prefix!Prefix for saved image filenames.r   r6   advancedr   r   	incrementPWhether to overwrite existing files or increment filenames to avoid overwriting.r   r5   r6   )r=   r>   r:   r?   r@   rA   is_output_nodeis_input_listrB   rC   is_deprecatedr   rD   r   rF   rm   rE   rJ   s    r   rL   z*SaveImageDataSetToFolderNode.define_schema   s    y.mmm>a x1JKK	!%]    
 	%#?!	     '(+6n	   ( =
 
 
 	
r   c                 6   |d         }|d         }|d         }t           j                            t          j                    |          }t          ||||dk              }t          j        dt          |           d| d           t          j
                    S )Nr   r   Saved z images to r   )r   r   r   rG   get_output_directoryr   r   r   r   r   r_   )rK   r7   r   r   r   r   r   s          r   rb   z$SaveImageDataSetToFolderNode.execute   s     "!n)!,AwW\\,"C"E"E{SS
+FJQUWbQbccHc+..HH:HHHIII}r   Nrc   r
   r   r   r   r      sI         
  
 [ 
D 
 
 [
 
 
r   r   c                   <    e Zd Zed             Zedd            ZdS ) SaveImageTextDataSetToFolderNodec                    t          j        dg dddddddt           j                            dd	          t           j                            d
ddd          t           j                            ddd          t           j                            dddd          t           j                            ddddgd          gg 
  
        S )NSaveImageTextDataSetToFolder)r   r   r   r   z	save textr   zSave Image-Text (to Folder)r2   zSave a dataset of pairs of images and text captions to a specified folder. Images are saved as PNG files and captions are saved as TXT files with the same filename_prefix.Tr7   r   r   rl   zList of text captions to save.)optionalforce_inputr6   r   r   r   r   r   r   r   r   r   r   r   r   
r=   r>   r:   r?   r@   rA   r   r   rB   rC   r   rJ   s    r   rL   z.SaveImageTextDataSetToFolderNode.define_schema  s    y2zzz6 F x1JKK	! $<    
 	!%]    
 	%#?!	     '(+6n	   %2 E#
 #
 #
 #	
r   Nc                 p   |d         }|d         }|d         }t           j                            t          j                    |          }t          ||||dk              }|rt          t          ||                    D ]}\  }\  }	}
|	                    dd          }t           j                            ||          }t          |dd          5 }|
                    |
           d d d            n# 1 swxY w Y   ~t          j        dt          |           d	| d
           t          j                    S )Nr   r   rO   rv   wr{   r|   r   z images and captions to r   )r   r   r   rG   r   r   r   ziprw   r   writer   r   r   r   r_   )rK   r7   r   r   r   rl   r   r   r   r   r   caption_filenamer   rX   s                 r   rb   z(SaveImageTextDataSetToFolderNode.execute.  sm    "!n)!,AwW\\,"C"E"E{SS
+FJQUWbQbcc  	%,5c+u6M6M,N,N % %((h#+#3#3FF#C#C !w||J8HII,g>>> %!GGG$$$% % % % % % % % % % % % % % % 	Uc+..UU
UUUVVV}s   C11C5	8C5	r   rc   r
   r   r   r   r     sN        $
 $
 [$
L    [  r   r   c                 Z   |                                  dk    r&| j        d         dk    r|                     d          } |                                                                 dz                      dd                              t          j                  }t          j
        |          S )zConvert tensor to PIL Image.r   r   rq   r   )r   r   r   r   r   r   r   r   r   r   r   )r*   r)   s     r   tensor_to_pilr   G  s    ~~1!1!!4!9!9''**
!!''))C/55a==DDRXNNI?9%%%r   c                     t          j        |                               t           j                  dz  }t	          j        |          d         S )zConvert PIL Image to tensor.r   r   )r   r   r   r   r    r!   )r(   r)   s     r   pil_to_tensorr   O  s:    $$RZ0058II&&u--r   c                       e Zd ZdZdZg ZdZdZg ZdZ	dZ
dZed             Zed             Zed             Zed             Zed             Zed	             ZdS )
ImageProcessingNodeap  Base class for image processing nodes that operate on images.

    Child classes should set:
        node_id: Unique node identifier (required)
        search_aliases: List of search aliases (optional)
        display_name: Display name (optional, defaults to node_id)
        description: Node description (optional)
        extra_inputs: List of additional io.Input objects beyond "images" (optional)
        is_group_process: None (auto-detect), True (group), or False (individual) (optional)
        is_output_list: True (list output) or False (single output) (optional, default True)
        is_deprecated: True if the node is deprecated (optional, default False)

    Child classes must implement ONE of:
        _process(cls, image, **kwargs) -> tensor  (for single-item processing)
        _group_process(cls, images, **kwargs) -> list[tensor]  (for group processing)
    NFc                     | j         | j         S t          }d}| j        D ]}d|j        v r|} nd}| j        D ]}d|j        v r|} n|duo||u}|duo||u}|r|rt	          | j         d          |s|st	          | j         d          |S zDetect whether this node uses group or individual processing.

        Returns:
            bool: True if group processing, False if individual processing
        N_process_group_processzj: Cannot override both _process and _group_process. Override only one, or set is_group_process explicitly.z1: Must override either _process or _group_process)is_group_processr   __mro____dict__r   rd   rK   
base_classprocess_definerklassgroup_definerhas_process	has_groups          r   _detect_processing_modez+ImageProcessingNode._detect_processing_moder  s     +'' )
 [ 	 	EU^++"' ,
 [ 	 	E5>11 % 2
 &T1WoZ6W!-Q-z2Q	 	9 	< I I I    	9 	<RRR   r   c                    t          t          j                  rUj        dk    r$t	          dt          j                             fdt          j        d                   D             S g }D ]t          t          j                  rj        dk    r6t	          dt                    j	         dt          dd                     |                    fd	t          j        d                   D                        |S )
z1Normalize to a flat list of [1, H, W, C] tensors.r   z$Expected 4D image tensor, got shape c                 *    g | ]}||d z            S rq   r
   rV   r   r7   s     r   r\   z:ImageProcessingNode._ensure_image_list.<locals>.<listcomp>  s%    BBBaF1QqS5MBBBr   r   zExpected 4D image tensor, got z shape r   Nc                 *    g | ]}||d z            S r   r
   )rV   r   rp   s     r   r\   z:ImageProcessingNode._ensure_image_list.<locals>.<listcomp>  s%    CCCa!eCCCr   )r   r    r   ndimr   tupler   ranger   rd   getattrr   )rK   r7   flatrp   s    ` @r   _ensure_image_listz&ImageProcessingNode._ensure_image_list  s    fel++ 	C{a !]flH[H[!]!]^^^BBBB5a+A+ABBBB 	E 	EDdEL11 ~TY!^^ !|$t**BU!|!|^efjlsuy^z^z!|!|}}}KKCCCCeDJqM.B.BCCCDDDDr   c                    | j         t          | j         d          |                                 }| j        | j        n|}t
          j                            d|rdnd          g}|                    | j	                   t          j
        | j         | j        | j        p| j         | j        | j        d||t
          j                            d|d          g		  	        S )
N  must set node_id class variabler7   zList of images to process.zImage to process.r   TzProcessed imagesr9   	r=   r>   r:   r?   r@   rA   r   rB   rC   )r=   NotImplementedErrorrd   r   r;   r   r   rF   r   extra_inputsrD   r>   r:   r?   r@   rI   )rK   is_groupoutput_is_listrB   s       r   rL   z!ImageProcessingNode.define_schema  s   ;%&W&W&WXXX..00 #&"4"@Ch 	
 HNN4<U00BU   
 	c&'''yK-)8S[\ "!)#1.    
 
 
 	
r   c                 h   |                                  }|r|                     |          }i }|                                D ]>\  }}t          |t                    rt          |          dk    r|d         ||<   9|||<   ?|r | j        |fi |}n | j        |fi |}t          j	        |          S zEExecute the node. Routes to _process or _group_process based on mode.rq   r   )
r   r   itemsr   listr   r   r   r   r_   )rK   r7   kwargsr  paramskvresults           r   rb   zImageProcessingNode.execute  s     ..00 	4++F33F LLNN 	 	DAq!T"" s1vv{{aDq		q		 	4'S'99&99FF "S\&33F33F}V$$$r   c                 0    t          | j         d          )zOverride this method for single-item processing.

        Args:
            image: tensor - Single image tensor
            **kwargs: Additional parameters (already extracted from lists)

        Returns:
            tensor - Processed image
         must implement _process methodr  rd   )rK   r2   r  s      r   r   zImageProcessingNode._process       "S\"R"R"RSSSr   c                 0    t          | j         d          )a  Override this method for group processing.

        Args:
            images: list[tensor] - List of image tensors
            **kwargs: Additional parameters (already extracted from lists)

        Returns:
            list[tensor] - Processed images
        % must implement _group_process methodr  )rK   r7   r  s      r   r   z"ImageProcessingNode._group_process  $     "|BBB
 
 	
r   )rd   re   rf   __doc__r=   r>   r:   r@   r  r   r;   r   rg   r   r   rL   rb   r   r   r
   r   r   r   r   X  s         " GNLKLNM) ) [)V   [ '
 '
 ['
R % % [%0 
T 
T [
T 
 
 [
 
 
r   r   c                       e Zd ZdZdZg ZdZdZg ZdZ	dZ
dZed             Zed             Zed             Zed             Zed             ZdS )	TextProcessingNodeae  Base class for text processing nodes that operate on texts.

    Child classes should set:
        node_id: Unique node identifier (required)
        search_aliases: List of search aliases (optional)
        display_name: Display name (optional, defaults to node_id)
        description: Node description (optional)
        extra_inputs: List of additional io.Input objects beyond "texts" (optional)
        is_group_process: None (auto-detect), True (group), or False (individual) (optional)
        is_output_list: True (list output) or False (single output) (optional, default True)
        is_deprecated: True if the node is deprecated (optional, default False)

    Child classes must implement ONE of:
        _process(cls, text, **kwargs) -> str  (for single-item processing)
        _group_process(cls, texts, **kwargs) -> list[str]  (for group processing)
    NFc                     | j         | j         S t          }d}| j        D ]}d|j        v r|} nd}| j        D ]}d|j        v r|} n|duo||u}|duo||u}|r|rt	          | j         d          |s|st	          | j         d          |S r   )r   r  r   r   r   rd   r   s          r   r   z*TextProcessingNode._detect_processing_mode'  s     +'' (
 [ 	 	EU^++"' ,
 [ 	 	E5>11 % 2
 &T1WoZ6W!-Q-z2Q	 	9 	< I I I    	9 	<RRR   r   c                    | j         t          | j         d          |                                 }t          j                            d|rdnd          g}|                    | j                   t	          j	        | j         | j
        p| j         dd||t          j                            d| j        d	          g
          S )Nr  rl   zList of texts to process.zText to process.r   textTzProcessed textsr9   )r=   r:   r?   rA   r   rB   rC   )r=   r  rd   r   r   rm   rF   r   r  rD   r:   rI   r;   )rK   r  rB   s      r   rL   z TextProcessingNode.define_schemaS  s    ;%&W&W&WXXX..00 IOO7?W33EW   
 	c&'''yK)8S[ "	  !(#&#5- !  
 
 
 	
r   c                 |   |                                  }i }|                                D ]>\  }}t          |t                    rt	          |          dk    r|d         ||<   9|||<   ?|r | j        |fi |}n | j        |fi |}| j        rt          j	        |r|n|g          S t          j	        |g          S r  )
r   r	  r   r
  r   r   r   r;   r   r_   )rK   rl   r  r  r  r  r  r  s           r   rb   zTextProcessingNode.executer  s     ..00 LLNN 	 	DAq!T"" s1vv{{aDq		q		 	3'S'8888FF "S\%22622F  	+=8!A&BBB =&***r   c                 0    t          | j         d          )zOverride this method for single-item processing.

        Args:
            text: str - Single text string
            **kwargs: Additional parameters (already extracted from lists)

        Returns:
            str - Processed text
        r  r  )rK   r  r  s      r   r   zTextProcessingNode._process  r  r   c                 0    t          | j         d          )zOverride this method for group processing.

        Args:
            texts: list[str] - List of text strings
            **kwargs: Additional parameters (already extracted from lists)

        Returns:
            list[str] - Processed texts
        r  r  )rK   rl   r  s      r   r   z!TextProcessingNode._group_process  r  r   )rd   re   rf   r  r=   r>   r:   r@   r  r   r;   r   rg   r   rL   rb   r   r   r
   r   r   r  r    s         " GNLKLNM) ) [)V 
 
 [
< + + [+6 
T 
T [
T 
 
 [
 
 
r   r  c                   x    e Zd ZdZdZdZdZdZej	        
                    dddd	d
          gZed             ZdS )ResizeImagesByShorterEdgeNodeResizeImagesByShorterEdgez*Resize Images by Shorter Edge (DEPRECATED)image/transformzeResize images so that the shorter edge matches the specified dimension while preserving aspect ratio.Tshorter_edge   rq       z&Target dimension for the shorter edge.r   minmaxr6   c                 
   t          |          }|j        \  }}||k     r|}t          |||z  z            }n|}t          |||z  z            }|                    ||ft          j        j                  }t          |          S r   )r   sizer   resizer   
ResamplingLANCZOSr   )rK   r2   r$  r(   r   hnew_wnew_hs           r   r   z&ResizeImagesByShorterEdgeNode._process  s    E""x1q55 E\A-.//EE E\A-.//Ejj%)9)ABBS!!!r   Nrd   re   rf   r=   r:   r?   r@   r   r   IntrF   r  rg   r   r
   r   r   r!  r!    sv        )G?L HyKM
< 	 	
 	
L 
" 
" [
" 
" 
"r   r!  c                   x    e Zd ZdZdZdZdZdZej	        
                    dddd	d
          gZed             ZdS )ResizeImagesByLongerEdgeNodeResizeImagesByLongerEdgez)Resize Images by Longer Edge (DEPRECATED)r#  zdResize images so that the longer edge matches the specified dimension while preserving aspect ratio.Tlonger_edgei   rq   r&  z%Target dimension for the longer edge.r'  c                 j   g }|D ]}t          |          }|j        \  }}||k    r|}t          |||z  z            }	n|}	t          |||z  z            }|                    ||	ft          j        j                  }|                    t          |                     t          j
        |d          S )Nr   r   )r   r+  r   r,  r   r-  r.  r"   r   r    cat)
rK   r2   r7  resized_imagesimage_ir(   r   r/  r0  r1  s
             r   r   z%ResizeImagesByLongerEdgeNode._process  s     
	6 
	6G((C8DAq1uu#Aq122#Aq122**eU^U-=-EFFC!!-"4"45555yQ////r   Nr2  r
   r   r   r5  r5    sv        (G>L HxKM
; 	 	
 	
L 0 0 [0 0 0r   r5  c                       e Zd ZdZg dZdZdZdZej	        
                    dddd	d
          ej	        
                    dddd	d          gZed             ZdS )CenterCropImagesNodeCenterCropImagescropcuttrimzCrop Image (Center)r#  z1Center crop an image to the specified dimensions.widthr%  rq   r&  Crop width.r'  heightCrop height.c                 <   t          |          }t          d|j        |z
  dz            }t          d|j        |z
  dz            }t	          |j        ||z             }t	          |j        ||z             }|                    ||||f          }t          |          S )Nr   r   )r   r)  rD  rF  r(  rA  r   )	rK   r2   rD  rF  r(   lefttoprightbottoms	            r   r   zCenterCropImagesNode._process  s    E""1sy5(Q.//!cj6)a/00CIte|,,SZv..hhc5&122S!!!r   Nrd   re   rf   r=   r>   r:   r?   r@   r   r3  rF   r  rg   r   r
   r   r   r>  r>    s         G***N&LHEK
WcqdMRR
Xst^TTL
 " " [" " "r   r>  c            	           e Zd ZdZg dZdZdZdZej	        
                    dddd	d
          ej	        
                    dddd	d          ej	        
                    ddddd          gZed             ZdS )RandomCropImagesNodeRandomCropImagesr@  zCrop Image (Random)r#  z3Randomly crop an image to the specified dimensions.rD  r%  rq   r&  rE  r'  rF  rG  seedr       Random seed.c                     t           j                            |dz             t          |          }t	          d|j        |z
            }t	          d|j        |z
            }|dk    r#t           j                            d|dz             nd}|dk    r#t           j                            d|dz             nd}	t          |j        ||z             }
t          |j        |	|z             }|	                    ||	|
|f          }t          |          S )N    r   rq   )r   randomrQ  r   r)  rD  rF  randintr(  rA  r   )rK   r2   rD  rF  rQ  r(   max_leftmax_toprI  rJ  rK  rL  s               r   r   zRandomCropImagesNode._process  s    
	ty)***E""q#)e+,,af,--5=\\ry  HqL111q3:Q;;bi7Q;///ACIte|,,SZv..hhc5&122S!!!r   NrM  r
   r   r   rO  rO    s         G***N(LHGK 	WcqdMRR
Xst^TT
A1*<n 	 	
 	
L 
" 
" [
" 
" 
"r   rO  c            	           e Zd ZdZddgZdZdZdZej	        
                    ddd	d
dd          ej	        
                    dddd
dd          gZed             ZdS )NormalizeImagesNodeNormalizeImages	normalizeznormalize colorszNormalize Image Colorszimage/colorz3Normalize images using mean and standard deviation.mean      ?              ?zMean value for normalization.Tr   r(  r)  r6   r   stdgMbP?z%Standard deviation for normalization.c                     ||z
  |z  S r   r
   )rK   r2   r^  rc  s       r   r   zNormalizeImagesNode._process9  s    ##r   Nrd   re   rf   r=   r>   r:   r?   r@   r   FloatrF   r  rg   r   r
   r   r   r[  r[     s        G!34N+LHGK
3 	 	
 	
 	; 	 	
 	
L& $ $ [$ $ $r   r[  c                   z    e Zd ZdZdgZdZdZdZej	        
                    dddd	d
          gZed             ZdS )AdjustBrightnessNodeAdjustBrightness
brightnesszAdjust Brightnessimage/adjustmentsz"Adjust the brightness of an image.factorra  r`         @zCBrightness factor. 1.0 = no change, <1.0 = darker, >1.0 = brighter.r'  c                 4    ||z                       dd          S )Nr`  ra  clamprK   r2   rl  s      r   r   zAdjustBrightnessNode._processN  s    %%c3///r   Nre  r
   r   r   rh  rh  >  sx         G >N&L H6K
Y 	 	
 	
L 0 0 [0 0 0r   rh  c                   z    e Zd ZdZdgZdZdZdZej	        
                    dddd	d
          gZed             ZdS )AdjustContrastNodeAdjustContrastcontrastzAdjust Contrastrk  z Adjust the contrast of an image.rl  ra  r`  rm  zMContrast factor. 1.0 = no change, <1.0 = less contrast, >1.0 = more contrast.r'  c                 @    |dz
  |z  dz                        dd          S )Nr_  r`  ra  ro  rq  s      r   r   zAdjustContrastNode._processc  s&    &,33C===r   Nre  r
   r   r   rs  rs  S  sx        G<N$L H4K
c 	 	
 	
L > > [> > >r   rs  c                       e Zd ZdZg dZdZdZdZdZe	j
                            dddd	d
          gZed             ZdS )ShuffleDatasetNodeShuffleDatasetshuffle	randomizemixzShuffle Images Listimage/batchz/Randomly shuffle the order of images in a list.TrQ  r   rR  rS  r'  c                     t           j                            |dz             t           j                            t	                              }fd|D             S )NrU  c                      g | ]
}|         S r
   r
   r   s     r   r\   z5ShuffleDatasetNode._group_process.<locals>.<listcomp>y  s    +++aq	+++r   )r   rV  rQ  permutationr   )rK   r7   rQ  indicess    `  r   r   z!ShuffleDatasetNode._group_processu  sQ    
	ty)***)''F44++++7++++r   N)rd   re   rf   r=   r>   r:   r?   r@   r   r   r3  rF   r  rg   r   r
   r   r   rx  rx  h  s        G222N(LHCK
A1*<n 	 	
 	
L , , [, , ,r   rx  c                   >    e Zd ZdZed             Zed             ZdS )ShuffleImageTextDatasetNodez:Special node that shuffles both images and texts together.c                    t          j        dg ddddddt           j                            dd	          t           j                            d
dd          t           j                            ddddd          gt           j                            ddd          t           j                            d
dd          g	  	        S )NShuffleImageTextDatasetrz  zShuffle Pairs of Image-Textr~  z<Randomly shuffle the order of pairs of image-text in a list.Tr7   zList of images to shuffle.r   rl   zList of texts to shuffle.)r6   r   rQ  r   rR  rS  r'  zShuffled imagesr9   zShuffled textsr  )r   rD   r   rF   rm   r3  rI   rJ   s    r   rL   z)ShuffleImageTextDatasetNode.define_schema  s    y-:::8$X x1MNN	1LZ^__**   
 !)#'-    
 	  !(GW !  	'
 
 
 	
r   c                    |d         }t           j                            |dz             t           j                            t	                              }fd|D             }fd|D             }t          j        ||          S )Nr   rU  c                      g | ]
}|         S r
   r
   r   s     r   r\   z7ShuffleImageTextDatasetNode.execute.<locals>.<listcomp>  s    6666!9666r   c                      g | ]
}|         S r
   r
   )rV   r   rl   s     r   r\   z7ShuffleImageTextDatasetNode.execute.<locals>.<listcomp>  s    444q%(444r   )r   rV  rQ  r  r   r   r_   )rK   r7   rl   rQ  r  shuffled_imagesshuffled_textss    ``    r   rb   z#ShuffleImageTextDatasetNode.execute  s    Aw
	ty)***)''F446666g6664444G444}_n===r   Nrd   re   rf   r  rg   rL   rb   r
   r   r   r  r  |  sO        DD
 
 [
@ > > [> > >r   r  c                   >    e Zd ZdZdgZdZdZdZdZe	d             Z
dS )	TextToLowercaseNodeTextToLowercase	lowercasez&Convert Text to Lowercase (DEPRECATED)r  zConvert text to lowercase.Tc                 *    |                                 S r   )rS   rK   r  s     r   r   zTextToLowercaseNode._process      zz||r   Nrd   re   rf   r=   r>   r:   r?   r@   r   rg   r   r
   r   r   r  r    M        G=N;LH.KM  [  r   r  c                   >    e Zd ZdZdgZdZdZdZdZe	d             Z
dS )	TextToUppercaseNodeTextToUppercase	uppercasez&Convert Text to Uppercase (DEPRECATED)r  zConvert text to uppercase.Tc                 *    |                                 S r   )upperr  s     r   r   zTextToUppercaseNode._process  r  r   Nr  r
   r   r   r  r    r  r   r  c                   |    e Zd ZdZg dZdZdZdZej	        
                    dddd	d
          gZed             ZdS )TruncateTextNodeTruncateText)truncaterB  shortenzTruncate Textr  z"Truncate text to a maximum length.
max_lengthM   rq   i'  zMaximum text length.r'  c                     |d |         S r   r
   )rK   r  r  s      r   r   zTruncateTextNode._process  s    KZK  r   NrM  r
   r   r   r  r    sy        G111N"LH6K
"!@V 	 	
 	
L ! ! [! ! !r   r  c                   t    e Zd ZdZdZdZdZdZej	        
                    ddd	          gZed
             ZdS )AddTextPrefixNodeAddTextPrefixzAdd Text Prefix (DEPRECATED)r  zAdd a prefix to all texts.Tr   r~   zPrefix to add.r   c                     ||z   S r   r
   )rK   r  r   s      r   r   zAddTextPrefixNode._process  s    }r   Nrd   re   rf   r=   r:   r?   r@   r   r   rm   rF   r  rg   r   r
   r   r   r  r    g        G1LH.KM
	"6FGGL   [  r   r  c                   t    e Zd ZdZdZdZdZdZej	        
                    ddd	          gZed
             ZdS )AddTextSuffixNodeAddTextSuffixzAdd Text Suffix (DEPRECATED)r  zAdd a suffix to all texts.Tsuffixr~   zSuffix to add.r   c                     ||z   S r   r
   )rK   r  r  s      r   r   zAddTextSuffixNode._process  s    f}r   Nr  r
   r   r   r  r    r  r   r  c                       e Zd ZdZdZdZdZdZej	        
                    ddd	          ej	        
                    d
dd	          gZed             ZdS )ReplaceTextNodeReplaceTextzReplace Text (DEPRECATED)r  zReplace text in all texts.Tfindr~   zText to find.r   rw   zText to replace with.c                 .    |                     ||          S r   )rw   )rK   r  r  rw   s       r   r   zReplaceTextNode._process  s    ||D'***r   Nr  r
   r   r   r  r    s~        G.LH.KM
	ODD
		27NOOL
 + + [+ + +r   r  c                   8    e Zd ZdZdZdZdZdZed             Z	dS )StripWhitespaceNodeStripWhitespacezStrip Whitespace (DEPRECATED)r  z5Strip leading and trailing whitespace from all texts.Tc                 *    |                                 S r   )r   r  s     r   r   zStripWhitespaceNode._process  r  r   N)
rd   re   rf   r=   r:   r?   r@   r   rg   r   r
   r   r   r  r    sF        G2LHIKM  [  r   r  c                       e Zd ZdZdZg dZdZdZdZdZ	e
j                            dd	d
ddd          gZed             ZdS )ImageDeduplicationNodezMRemove duplicate or very similar images from a list using perceptual hashing.ImageDeduplication)deduplicatezremove duplicateszsimilarity filterzDeduplicate Imagesr~  z4Remove duplicate or very similar images from a list.Tsimilarity_thresholdgffffff?r`  ra  zmSimilarity threshold (0-1). Higher means more similar. Images above this threshold are considered duplicates.rb  c                    t                    dk    rg S d d }fdD             }g }t          t                              D ]i}d}|D ]K} |||         ||                   }	d|	dz  z
  }
|
|k    r#d}t          j        d	| d
| d|
dd            nL|s|                    |           jfd|D             }t          j        dt          |           dt                     d           |S )z1Remove duplicate images using perceptual hashing.r   c                 X   t          |           }|                    dt          j        j                                      d          }t          |                                          }t          |          t          |          z  d
                    fd|D                       }|S )zMCompute a simple perceptual hash by resizing to 8x8 and comparing to average.)   r  Lr~   c              3   ,   K   | ]}|k    rd ndV  dS )10Nr
   )rV   pavgs     r   rY   zNImageDeduplicationNode._group_process.<locals>.compute_hash.<locals>.<genexpr>:  s/      HHAq3wwCHHHHHHr   )r   r,  r   r-  r.  r   r
  getdatasumr   r   )r*   r(   	img_smallpixels	hash_bitsr  s        @r   compute_hashz;ImageDeduplicationNode._group_process.<locals>.compute_hash0  s    
++C

65+;+CDDLLSQQI)++--..Ff++F+CHHHHHHHHHIr   c                 P    t          d t          | |          D                       S )z2Compute Hamming distance between two hash strings.c              3   (   K   | ]\  }}||k    V  d S r   r
   )rV   c1c2s      r   rY   zRImageDeduplicationNode._group_process.<locals>.hamming_distance.<locals>.<genexpr>?  s*      @@FBrRx@@@@@@r   )r  r   )hash1hash2s     r   hamming_distancez?ImageDeduplicationNode._group_process.<locals>.hamming_distance=  s)    @@c%.?.?@@@@@@r   c                 &    g | ]} |          S r
   r
   )rV   r(   r  s     r   r\   z9ImageDeduplicationNode._group_process.<locals>.<listcomp>B  s#    666,,s##666r   Fra  g      P@TzImage z is similar to image z (similarity: z.3fz), skippingc                      g | ]
}|         S r
   r
   r   s     r   r\   z9ImageDeduplicationNode._group_process.<locals>.<listcomp>W  s    999q999r   zDeduplication: kept z out of  images)r   r   r   r   r"   )rK   r7   r  r  hasheskeep_indicesr   is_duplicatejdistance
similarityunique_imagesr  s    `          @r   r   z%ImageDeduplicationNode._group_process)  s    v;;!I	 	 		A 	A 	A
 7666v666 s6{{## 	' 	'A L! 	 	++F1IvayAA HtO4
!555#'LLeeeee*eeee   E 6   '##A&&& :999L999S3}#5#5SSs6{{SSS	
 	
 	
 r   N)rd   re   rf   r  r=   r>   r:   r?   r@   r   r   rf  rF   r  rg   r   r
   r   r   r  r    s        WW"GLLLN'LHHK
" D 	 	
 	
	L 1 1 [1 1 1r   r  c                   B   e Zd ZdZdZg dZdZdZdZdZ	dZ
ej                            d	d
ddd          ej                            dddddd          ej                            dddddd          ej                            dd
dddd          gZed             ZdS )ImageGridNodez3Combine multiple images into a single grid/collage.	ImageGrid)gridcollagecombinezMake Image Gridr~  z+Arrange multiple images into a grid layout.TFcolumnsr   rq      zNumber of columns in the grid.r'  
cell_width       i   zWidth of each cell in the grid.rb  cell_heightz Height of each cell in the grid.paddingr   2   zPadding between images.c                 B   t          |          dk    rt          d          t          |          }||z   dz
  |z  }||z  |dz
  |z  z   }||z  |dz
  |z  z   }	t          j        d||	fd          }
t	          |          D ]m\  }}||z  }||z  }t          |          }|                    ||ft          j        j                  }|||z   z  }|||z   z  }|
	                    |||f           nt          j        d| d| d| d	| d|	 d
           t          |
          S )zArrange images into a grid.r   z(Cannot create grid from empty image listrq   r   )r   r   r   Created xz grid with z	 images ())r   r   r   newr   r   r,  r-  r.  paster   r   r   )rK   r7   r  r  r  r  
num_imagesrows
grid_widthgrid_heightr  r   r*   rowcolr(   r  ys                     r   r   zImageGridNode._group_process  s}    v;;!GHHH [[
W$q(W4 z)Wq[G,CC
[(D1H+?? y[ 99EE  )00 	$ 	$OC.C-C  
++C**j+68H8PQQC zG+,A{W,-A JJsQF####bwbbbb*bbzbbT_bbb	
 	
 	
 T"""r   N)rd   re   rf   r  r=   r>   r:   r?   r@   r   r;   r   r3  rF   r  rg   r   r
   r   r   r  r  ^  s       ==G111N$LH?KN
4 	 	
 	
 	5 	 	
 	
 	6 	 	
 	
 	qaR9R]a 	 	
 	
1L: ## ## [## ## ##r   r  c                   H    e Zd ZdZdZg dZdZdZdZdZ	dZ
ed             Zd	S )
MergeImageListsNodez.Merge multiple image lists into a single list.MergeImageLists)r
  z
merge listz	make listzMerge Image Lists (DEPRECATED)r~  z*Concatenate multiple image lists into one.Tc                 P    t          j        dt          |           d           |S )zASimply return the images list (already merged by input handling).zMerged image list contains r  r   r   r   )rK   r7   s     r   r   z"MergeImageListsNode._group_process  s+    
 	G3v;;GGGHHHr   N)rd   re   rf   r  r=   r>   r:   r?   r@   r   r   rg   r   r
   r   r   r  r    s]        88G666N3LH>KM  [  r   r  c                   @    e Zd ZdZdZdZdZdZdZdZ	e
d             ZdS )	MergeTextListsNodez-Merge multiple text lists into a single list.MergeTextListszMerge Text Lists (DEPRECATED)r  z)Concatenate multiple text lists into one.Tc                 P    t          j        dt          |           d           |S )z@Simply return the texts list (already merged by input handling).zMerged text list contains z textsr  )rK   rl   s     r   r   z!MergeTextListsNode._group_process  s+    
 	D#e**DDDEEEr   N)rd   re   rf   r  r=   r:   r?   r@   r   r   rg   r   r
   r   r   r  r    sR        77G2LH=KM  [  r   r  c                   >    e Zd ZdZed             Zed             ZdS )ResolutionBucketzIBucket latents and conditions by resolution for efficient batch training.c                 D   t          j        dg ddddddt           j                            dd	          t           j                            d
d	          gt           j                            ddd          t           j                            d
dd          g	  	        S )Nr  )zbucket by resolutionzgroup by resolutionzbatch by resolutionzResolution Bucketmodel/trainingz,Group latents and conditionings into bucketsTlatentsz-List of latent dicts to bucket by resolution.r   conditioningz7List of conditioning lists (must match latents length).z8List of batched latent dicts, one per resolution bucket.r9   z3List of condition lists, one per resolution bucket.r  )r   rD   LatentrF   ConditioningrI   rJ   s    r   rL   zResolutionBucket.define_schema  s    y&aaa,%F 	K     %%"U &  	 	  !*#'V !  
 &&!/#'Q '  %
 
 
 	
r   c                 l   t          |          t          |          k    r0t          dt          |           dt          |           d          g }g }t          ||          D ]b\  }}|d         }|j        d         }t	          |          D ]8}	|                    ||	                    |                    ||	                    9ci }
t          ||          D ]q\  }}|j        d         |j        d         }}||f}||
vrg g d|
|<   |
|         d	                             |           |
|         d
                             |           rg }g }|
                                D ]\  \  }}}t          j        |d	         d          }|                    d|i           |                    |d
                    t          j
        d| d| dt          |d	                    d           t          j
        dt          |
           dt          |           d           t          j        ||          S )NNumber of latents (') does not match number of conditions (z).samplesr   )r  
conditionsr  r  r9  zResolution bucket (r  z):  samplesr  z resolution buckets from )r   r   r   r   r   r"   r	  r    stackr   r   r   r_   )rK   r  r  flat_latentsflat_conditionslatent_dictcondr  
batch_sizer   bucketslatentr/  r   keyoutput_latentsoutput_conditionsbucket_datastacked_latentss                      r   rb   zResolutionBucket.execute  s    w<<3|,,,,pc'llpp[^_k[l[lppp  
 !$Wl!;!; 	0 	0K!),G q)J :&& 0 0##GAJ///&&tAw////0
 o>> 		4 		4LFD<#V\"%5qAa&C'!!+-R@@CL#**6222CL&--d3333 #*==?? 
	 
	FQK#k+i*@aHHHO!!9o">??? $$[%>???LUaUU!UUK	4J0K0KUUU    	bGbbs<GXGXbbbccc}^->???r   Nr  r
   r   r   r  r    sT        SS
 
 [
B 6@ 6@ [6@ 6@ 6@r   r  c                   @    e Zd ZdZed             Zedd            ZdS )MakeTrainingDatasetzHEncode images with VAE and texts with CLIP to create a training dataset.c                    t          j        ddgdddddt           j                            dd	          t           j                            d
d	          t           j                            dd	          t           j                            dddd          gt           j                            ddd          t           j	                            ddd          g	  	        S )Nr  zencode datasetzMake Training Datasetr  zeEncode images with VAE and texts with CLIP to create a training dataset of latents and conditionings.Tr7   zList of images to encode.r   vaez)VAE model for encoding images to latents.r   z-CLIP model for encoding text to conditioning.rl   zoList of text captions. Can be length n (matching images), 1 (repeated for all), or omitted (uses empty string).)r   r6   r   r  List of latent dictsr9   r  List of conditioning listsr  )
r   rD   r   rF   VaeCliprm   r  rI   r  rJ   s    r   rL   z!MakeTrainingDataset.define_schema7  s   y),-0% x1LMM#N    $S    	! N $	      	  !*#'2 !  
 &&!/#'8 '  /#
 #
 #
 #	
r   Nc           	         |d         }|d         }t          |          }|t          |          dk    rdg}t          |          dk    r|dk    r||z  }n9t          |          |k    r&t          dt          |           d| d| d          t          j        d| d	           g }|D ]@}|                    |d d d d d d d d
f                   }|                    d|i           At          j        dt          |           d           g }	|D ]p}
|
dk    r)|                    |                    d                    }n*|                    |
          }|                    |          }|	                    |           qt          j        dt          |           dt          |	           d           t          j	        ||	          S )Nr   r~   rq   zNumber of texts (z#) does not match number of images (z ). Text list should have length z
, 1, or 0.z	Encoding z images with VAE...r   r  z texts with CLIP...zCreated dataset with z latents and z conditioning.)
r   r   r   r   encoder"   encode_from_tokens_scheduledtokenizer   r_   )rK   r7   r!  r   rl   r  latents_listr*   latent_tensorconditioning_listr  r  tokenss                r   rb   zMakeTrainingDataset.execute^  s+    !fAw [[
=CJJ!OODEu::??zA~~J&EEZZ:%%GCJJ G GS] G G0:G G G   	@@@@AAA  	< 	<JJJz!!!QQQ2A2+'>??MM :;;;; 	@U@@@AAA 	+ 	+Drzz88r9J9JKKt,,88@@$$T****jC$5$5jjCHYDZDZjjj	
 	
 	
 }\+<===r   r   r  r
   r   r   r  r  5  sT        RR$
 $
 [$
L +> +> +> [+> +> +>r   r  c                   >    e Zd ZdZed             Zed             ZdS )SaveTrainingDatasetz?Save encoded training dataset (latents + conditioning) to disk.c                 L   t          j        dddgddddddt           j                            dd	
          t           j                            dd
          t           j                            ddd          t           j                            dddddd          gg 
  
        S )Nr/  r   r   zSave Training Datasetr  zeSave encoded training dataset (latents + conditioning) to disk for efficient loading during training.Tr  z.List of latent dicts from MakeTrainingDataset.r   r  z4List of conditioning lists from MakeTrainingDataset.r   training_datasetz9Name of folder to save dataset (inside output directory).r   
shard_sizei  rq   i z!Number of samples per shard file.rb  r   )r   rD   r  rF   r  rm   r3  rJ   s    r   rL   z!SaveTrainingDataset.define_schema  s    y),n=0% 	L     %%"R &   	!.W    
   ?!   . A!
 !
 !
 !	
r   c                 z   |d         }|d         }t          |          t          |          k    r0t          dt          |           dt          |           d          t          j                            t          j                    |          }t          j        |d           t          |          }||z   dz
  |z  }t          j	        d| d	| d
| d           t          |          D ]}||z  }	t          |	|z   |          }
||	|
         ||	|
         d}d|dd}t          j                            ||          }t          |d          5 }t          j        ||           d d d            n# 1 swxY w Y   t          j	        d|dz    d| d| d|
|	z
   d	           |||d}t          j                            |d          }t          |d          5 }t          j        ||d           d d d            n# 1 swxY w Y   t          j	        d| d	| d           t#          j                    S )Nr   r
  r  z/). Something went wrong in dataset preparation.Tr   rq   zSaving z samples to z shards in ...)r  r  shard_04d.pklwbzSaved shard /: z (z	 samples))num_samples
num_shardsr2  zmetadata.jsonr   r   )indentzSuccessfully saved r   )r   r   r   r   r   rG   r   r   r   r   r   r(  r   r    r   jsondumpr   r_   )rK   r  r  r   r2  r   r;  r<  	shard_idx	start_idxend_idx
shard_datashard_filename
shard_pathrX   metadatametadata_paths                    r   rb   zSaveTrainingDataset.execute  s$    "!n]
 w<<3|,,,,@c'll @ @[^_k[l[l @ @ @   W\\,"C"E"E{SS

J.... 'll!J.2zA
UkUUzUUjUUU	
 	
 	

 z** 	 	I!J.I)j0+>>G #9W#45 ,Yw-> ? J :i9999Nj.AAJj$'' *1
:q)))* * * * * * * * * * * * * * * Lmy1}mmzmm^mmwYbObmmm    '$$
 

 ZAA-%% 	-Ih!,,,,	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	Q;QQJQQQRRR}s$   E77E;	>E;	 HHHNr  r
   r   r   r/  r/    sO        II"
 "
 ["
H < < [< < <r   r/  c                   >    e Zd ZdZed             Zed             ZdS )LoadTrainingDatasetz(Load encoded training dataset from disk.c                    t          j        dddgddddt           j                            dd	d
          gt           j                            ddd          t           j                            ddd          g          S )NrI  r1   ztraining datazLoad Training Datasetr  zULoad encoded training dataset (latents + conditioning) from disk for use in training.Tr   r1  zFName of folder containing the saved dataset (inside output directory).r   r  r"  r9   r  r#  r<   )r   rD   rm   rF   r  rI   r  rJ   s    r   rL   z!LoadTrainingDataset.define_schema  s    y),o>0%o 	!.d     	  !*#'2 !  
 &&!/#'8 '  
 
 
 	
r   c           
         t           j                            t          j                    |          }t           j                            |          st          d|           t          d t          j        |          D                       }|st          d|           t          j
        dt          |           d| d           g }g }|D ]}t           j                            ||          }t          |d          5 }t          j        |          }	d d d            n# 1 swxY w Y   |                    |	d                    |                    |	d	                    t          j
        d
| dt          |	d                    d           t          j
        dt          |           d| d           t!          j        ||          S )NzDataset directory not found: c                 f    g | ].}|                     d           |                    d          ,|/S )r5  r7  )
startswithrT   ry   s     r   r\   z/LoadTrainingDataset.execute.<locals>.<listcomp>  sP       <<)) /0jj.@.@  r   zNo shard files found in zLoading z shards from r4  rbr  r  r   r:  r  zSuccessfully loaded z samples from r   )r   r   r   rG   r   r   r   sortedr^   r   r   r   r   r    loadr   r   r_   )
rK   r   dataset_dirshard_filesall_latentsall_conditioning
shard_filerE  rX   rC  s
             r   rb   zLoadTrainingDataset.execute  s&    gll<#D#F#FTTw~~k** 	LJ[JJKKK  K00  
 
  	GEEEFFFOK 0 0OO{OOOPPP % 		W 		WJk:>>Jj$'' +1"Z]]
+ + + + + + + + + + + + + + + z)4555##J~$>???LU:UUZ	5J1K1KUUUVVVVQ3{#3#3QQ;QQQ	
 	
 	
 }[*:;;;s   D##D'	*D'	Nr  r
   r   r   rI  rI    sN        22
 
 [
: '< '< ['< '< '<r   rI  c                   L    e Zd Zedeeej                          fd            ZdS )DatasetExtensionreturnc                 Z  K   t           t          t          t          t          t
          t          t          t          t          t          t          t          t          t          t          t           t"          t$          t&          t(          t*          t,          t.          t0          t2          t4          t6          gS r   )r-   ri   r   r   r!  r5  r>  rO  r[  rh  rs  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r/  rI  r  )selfs    r   get_node_listzDatasetExtension.get_node_listC  sg       +.(,)(   '"C"
 "	
r   N)	rd   re   rf   r   r
  r   r   	ComfyNoder[  r
   r   r   rW  rW  B  sE        #
T$r|*<%= #
 #
 #
 X#
 #
 #
r   rW  rX  c                  "   K   t                      S r   )rW  r
   r   r   comfy_entrypointr^  j  s      r   )r2   T)4r   r   r>  r   r   r    PILr   typing_extensionsr   rG   r   comfy_api.latestr   r   r+   r\  r-   ri   r   r   r   r   r   r   r  r!  r5  r>  rO  r[  rh  rs  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r/  rI  rW  r^  r
   r   r   <module>rb     s    				             & & & & & &         / / / / / / / /  <$, $, $, $, $,R\ $, $, $,NI6 I6 I6 I6 I6 I6 I6 I6X1 1 1 1h/ / / / /2< / / /d; ; ; ; ;r| ; ; ;B& & &. . .r
 r
 r
 r
 r
", r
 r
 r
j[
 [
 [
 [
 [
 [
 [
 [
B" " " " "$7 " " "<0 0 0 0 0#6 0 0 0B" " " " ". " " "," " " " ". " " ":$ $ $ $ $- $ $ $<0 0 0 0 0. 0 0 0*> > > > >, > > >*, , , , ,, , , ,(+> +> +> +> +>", +> +> +>b
 
 
 
 
, 
 
 

 
 
 
 
, 
 
 
! ! ! ! !) ! ! !"    *       *   + + + + +( + + + 	 	 	 	 	, 	 	 	F F F F F0 F F FRK# K# K# K# K#' K# K# K#\    -   (    +   ,\@ \@ \@ \@ \@r| \@ \@ \@~U> U> U> U> U>", U> U> U>pd d d d d", d d dNH< H< H< H< H<", H< H< H<\%
 %
 %
 %
 %
~ %
 %
 %
P 0      r   