
    +jJ                     &   d Z ddlZddlmZ ddlZddlmc mZ ddl	m
Z
 ddlZd$dZd%dZd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Z G d d          Z G d d          Zdddddddej        dedededed ed!eej                 d"ej        fd#ZdS )&u)  
Optical-flow-warped noise for VOID Pass 2 refinement.

Adapted from RyannDaGreat/CommonSource (MIT License, Ryan Burgert):
  https://github.com/RyannDaGreat/CommonSource
  - noise_warp.py  (NoiseWarper / warp_xyωc / regaussianize / get_noise_from_video)
  - raft.py        (RaftOpticalFlow)

Only the code paths that ``comfy_extras/nodes_void.py::VOIDWarpedNoise`` actually
uses (torch THWC uint8 input, no background removal, no visualization, no disk
I/O, default warp/noise params) have been inlined.  External ``rp`` utilities
have been replaced with equivalents from torch.nn.functional / einops.  The
RAFT optical-flow model itself is loaded offline via ``OpticalFlowLoader`` in
``nodes_void.py`` and passed into ``get_noise_from_video`` by the caller; this
module never downloads weights at runtime.
    N)Optional)	rearrangeTc                    | j         dk    r$t          dt          | j                             | j        \  }}}t	          |t
          t          f          rVt	          |t                    sAt          dt          ||z                      }t          dt          ||z                      }n|\  }}||f||fk    r|r| 	                                n| S i }	|dv rd|	d<   t          j        | d         f||f|d|	d	         }
|
S )
a3  Resize a CHW tensor.

    ``size`` is either a scalar factor or a (h, w) tuple.  ``interp`` is one
    of ``"bilinear"``, ``"nearest"``, ``"area"``.  When ``copy`` is False and
    the requested size matches the input, returns the input tensor as is
    (faster but callers must not mutate the result).
       z5_torch_resize_chw expects a 3D CHW tensor, got shape    )bilinearbicubicFalign_cornersNsizemoder   )ndim
ValueErrortupleshape
isinstanceintfloatboolmaxcloneFinterpolate)imager   interpcopy_in_hin_wnew_hnew_wkwargsouts              </home/wildlama/comfy/ComfyUI/comfy_extras/void_noise_warp.py_torch_resize_chwr%       s#    zQXE%+DVDVXX
 
 	
 KMAtT$e%% jt.D.D As4$;''((As4$;''((uu~$%% $/u{{}}}%/F((("'
-d
P5%.v
P
P
P
PQR
SCJ    r   c                    | j         dk    r$t          dt          | j                             |j        |j        k    r9t          dt          |j                   dt          |j                             | j        \  }}}|t	          j        ||j        |j                  z   }|t	          j        ||j        |j                  dddf         z   }||dz
  z  dz  dz
  }	||dz
  z  dz  dz
  }
t	          j        |	|
gd	
          d         	                    | j                  }t          j        | d         ||dd          d         }|S )zRelative remap of a CHW image via ``F.grid_sample``.

    Equivalent to ``rp.torch_remap_image(image, dx, dy, relative=True, interp=interp)``
    for ``interp`` in {"bilinear", "nearest"}.  Out-of-bounds samples are 0.
    r   z9_torch_remap_relative expects a 3D CHW tensor, got shape z1_torch_remap_relative: dx and dy must match, got  vs devicedtypeNr      dimTzeros)r   r
   padding_moder   )r   r   r   r   torcharanger*   r+   stacktor   grid_sample)r   dxdyr   r   hwx_absy_absx_normy_normgridr#   s                r$   _torch_remap_relativer@   =   sk    zQ\ekHZHZ\\
 
 	
 
x28fbhffUZ[][cUdUdff
 
 	
 kGAq!a	BBBBEa	BBB111d7KKEq1uo"Q&Fq1uo"Q&F;'R000699%+FFD
-dTd  	C Jr&   c                    | j         dk    r$t          dt          | j                             | j        \  }}}|j        ||fk    s|j        ||fk    r?t          d| d| dt          |j                   dt          |j                             |                                t          j        ||j        t
          j                  z   }|                                t          j        ||j        t
          j                  dddf         z   }|d	k    ||k     z  |d	k    z  ||k     z                      d
          }||z  |z                       d
          |         }	t          | d          |         }
t          j
        ||z  |f| j        | j                  }|                    d	|	|
           t          |d||          S )zScatter-add a CHW image using relative floor-rounded (dx, dy) offsets.

    Equivalent to ``rp.torch_scatter_add_image(image, dx, dy, relative=True,
    interp='floor')``.  Out-of-bounds targets are dropped.
    r   z?_torch_scatter_add_relative expects a 3D CHW tensor, got shape z,_torch_scatter_add_relative: dx/dy must be (, z
), got dx=z dy=r)   Nr   r-   c h w -> (h w) cr+   r*   (h w) c -> c h wr9   r:   )r   r   r   r   longr2   r3   r*   reshaper   r0   r+   
index_add_)r   r7   r8   in_cr   r   xyvalidindices
flat_imager#   s               r$   _torch_scatter_add_relativerP   Z   s    zQbeTYT_N`N`bb
 
 	
 {D$	xD$<28d|#;#;=4 = =4 = =BHoo= =+0??= =
 
 	

 			ELbiuzJJJJA
		ELbiuzJJJ111d7SSA1fT"a1f-T:CCBGGE4x!|$$R((/G5"455e<J
+td{D)U\
R
R
RCNN1gz***S,====r&   c                     | j         \  }}}t          | d          }t          j        |dddd          \  }}}t          |d||          }|||fS )zFind unique pixel values in a CHW tensor.

    Returns ``(unique_colors [U, C], counts [U], index_matrix [H, W])`` where
    ``index_matrix[i, j]`` is the index of the unique color at that pixel.
    rC   r   TF)r/   return_inversereturn_countssortedz(h w) -> h wrF   )r   r   r2   unique)	r   r   r9   r:   flatunique_colorsinverse_indicescountsindex_matrixs	            r$   unique_pixelsr[   {   sn     kGAq!U.//D-2\!DU. . .*M?F _nQGGGL&,..r&   c                 X   | j         \  }}}t          |                                                                          dz   }t	          | d          }t          j        ||j         d         f|j        |j                  }|	                    d|
                    d          |           |S )z>For each unique index, sum the CHW image values at its pixels.r   rC   rD   r   r-   )r   r   r   itemr   r2   r0   r+   r*   rI   view)r   rZ   r   r9   r:   urV   r#   s           r$   sum_indexed_valuesr`      s    kGAq!L##%%&&*AU.//D
+q$*Q-(
4;
O
O
OCNN1l''++T222Jr&   c                 r    | j         \  }}||                     d                   }t          |d||          S )z@Build a CHW image from an index matrix and a (U, C) color table.r-   rE   rF   )r   r^   r   )rZ   rW   r9   r:   rV   s        r$   indexed_to_imagerb      s>    DAq**2../DT-a8888r&   c                 <   | j         \  }}}t          | dd                   \  }}}t          j        |           }t	          ||          }t          ||t          |d          z            }||z
  }	t          |t          |d                    }
| |
dz  z  |	z   }||
fS )aK  Variance-preserving re-sampling of a CHW noise tensor.

    Wherever the noise contains groups of identical pixel values (e.g. after
    a nearest-neighbor warp that duplicated source pixels), adds zero-mean
    foreign noise within each group and scales by ``1/sqrt(count)`` so the
    output is unit-variance gaussian again.
    Nr   zu -> u 1      ?)r   r[   r2   
randn_liker`   rb   r   )noiser   hswsrY   rZ   foreign_noisesummedmeanedzeroed_foreigncounts_imageoutputs               r$   regaussianizero      s     IAr2+E"1"I66Av|$U++M|<<FlFYvz5R5R,RSSF"V+N#L)FJ2O2OPPL\S((>9F<r&   c                     | j         \  }}}t          j        t          j        || j        | j                  t          j        || j        | j                  d          \  }}t          j        ||g          S )zIReturn a (2, H, W) tensor of (x, y) pixel coordinates matching ``image``.r)   ij)indexing)r   r2   meshgridr3   r*   r+   r4   )r   r   r9   r:   rL   rK   s         r$   xy_meshgrid_like_imagert      sr    kGAq!>Qu|5;???Qu|5;???  DAq
 ;1vr&   c                     t          j        | dd                   }t          j        | dd                   }t          j        |||| g          S )uT   Pack a (C, H, W) noise tensor into a state tensor (3+C, H, W) = [dx, dy, ω, noise].Nr   )r2   
zeros_like	ones_likecat)rf   r0   oness      r$   noise_to_staterz      sJ    U2A2Y''E?5!9%%D9eUD%0111r&   c                     | dd         S )z.Unpack the noise channels from a state tensor.r   N )states    r$   state_to_noiser~      s    9r&   c           
         |j         | j         k    rt          d|j          d| j                    | j        dk    r$t          dt          | j                             | j        \  }}}|j        d||fk    r*t          d| d| dt          |j                             | j         }d	\  }}d}d}	d}
||	z
  }||z
  }|d
k    rt          d| d          | |
         d
k                                    st          d          t          |           }t          j        |           }d
|d|<   d||
<   d
|| d<   t          j        |           }t          | d|         |d
          |d          d          |d|<   t          | | d         |d
          |d          d          || d<   d||
         ||
         d
k    <   | 
                                }|d|xx         |z  cc<   ||d|         z                                   }||         d
k    ||         |k     z  ||         d
k    z  ||         |k     z  }t          j        |d          ||          }|d|                                         |d|xx         z  cc<   d
|d|<   fd} |t          j        d||| j        |                    d
k    }t          j        |||          }t          j        ||gd          }t!          || d                   \  || d<   }||
         |d
         z  ||
<   ||
                                         ||
<   t          j        |dd          \  }}t          j        |          } |||
         d                   d
         ||
<    ||d|         ||
         d         z            ||
         d         z  |d|<    ||| d         ||
         d         z             |||
         d         dz                                            z  || d<   t          j        |||          }||
         ||
                                         z  ||
<   ||
xx         dz  cc<   ||
xx         dz  cc<   |S )u   Warp a noise-warper state tensor along the given optical flow.

    ``state`` has shape ``(3+c, h, w)`` (= dx, dy, ω, c noise channels).
    ``flow`` has shape ``(2, h, w)`` (= dx, dy).
    z@warp_state: flow and state must be on the same device, got flow=z state=r   z4warp_state: state must be 3D (3+C, H, W), got shape r,   z%warp_state: flow must have shape (2, rB   z), got )r   r   r   zDwarp_state: state has no noise channels (expected 3+C with C>0, got z
 channels)z/warp_state: all weights in state[2] must be > 0Nr   nearestc                 >    t          | d         d                   S )Nr   r   )rP   )tensorscat_xys    r$   scatzwarp_state.<locals>.scat  s    *671:wqzJJJr&   rD   r.   )chunksr/   gh㈵>gH.?)r*   r   r   r   r   allrt   r2   
empty_liker@   r   roundwherery   r+   rx   ro   
nan_to_numchunksqrtmean)r}   flowxyocr9   r:   r*   x_chy_chxyxyww_chcocr?   init
pre_expand
pre_shrinkpos	in_boundsr   shrink_maskconcatrm   expandshrinkrn   r   s                             @r$   
warp_stater      s,    {el"";; ;,1L; ;
 
 	
 zQW5CUCUWW
 
 	
 JD!QzaAYVAVVVV5CTCTVV
 
 	
 \FJD$	
B
CDs
A	BAvvcSWccc
 
 	
 $K!O  "" LJKKK!%((DE""DD"IDJD!I !%((J+E#2#Ja47(IVVJssO,UB344[47(T!WHiXXJstt./JtZ%*+ JssOOOtOOO*SbS/!
(
(
*
*CTaCIM2c$i1nETUVWIio-tZ@@J"o##%%GssOOOwOOOJssOK K K K K
 $uz!QVLLLMMPQQK [$
;;J Y
J/Q777F -faRSSk : :FA233K$<,q/1F4L$<**,,F4LVA1===Jj))F4
4(.//2F4L$z#2#D)9$)??@@6$<PTCUUF3B3K$z1"##D)9$)??@@444!#D D
dffFA233K [ff55F$<&,"3"3"5"55F4L
4LLLDLLL
4LLLVLLLMr&   c                   B    e Zd ZdZej        fdZed             Zd Z	dS )NoiseWarpera/  Maintain a warpable noise state and emit gaussian noise per frame.

    Simplified from RyannDaGreat/CommonSource/noise_warp.py::NoiseWarper:
    ``scale_factor``, ``post_noise_alpha``, ``progressive_noise_alpha``, and
    ``warp_kwargs`` are all dropped since VOIDWarpedNoise always uses defaults.
    c                     |dk    s|dk    s|dk    rt          d| d| d|           || _        || _        || _        || _        || _        t          j        |||||          }t          |          | _	        d S )Nr   z/NoiseWarper: c/h/w must all be positive, got c=z h=z w=rD   )
r   r   r9   r:   r*   r+   r2   randnrz   _state)selfr   r9   r:   r*   r+   rf   s          r$   __init__zNoiseWarper.__init__,  s    66Q!VVqAvvQ!QQQQaQQ   
Aq!5@@@$U++r&   c                     t          | j                  }| j        dd         }||z  |dz                                  z  S )Nr,   r   )r~   r   r   )r   nweightss      r$   rf   zNoiseWarper.noise:  sB    
 4;''+ac"7{gl002222r&   c                    |j         |j         k    r9t          dt          |j                    dt          |j                              t          j        ||g                              | j        | j                  }|j         \  }}}t          || j	        | j
        fdd          }|j         dd          \  }}|dxx         ||z  z  cc<   |dxx         ||z  z  cc<   t          | j        |          | _        | S )	Nz'NoiseWarper: dx and dy must match, got r(   r   Tr   r   r   )r   r   r   r2   r4   r5   r*   r+   r%   r9   r:   r   r   )	r   r7   r8   r   r   oflowhoflowwflowhflowws	            r$   __call__zNoiseWarper.__call__C  s    8rx`%//``uUWU]``   {B8$$''TZ@@ J66 '7$OOOz"##u
 	Q56>!Q56>! d33r&   N)
__name__
__module____qualname____doc__r2   float32r   propertyrf   r   r|   r&   r$   r   r   $  sc          /4m , , , , 3 3 X3    r&   r   c                   &    e Zd ZdZddZd Zd ZdS )RaftOpticalFlowai  RAFT-large wrapper around a pre-loaded torchvision model.

    ``model`` must be the ``torchvision.models.optical_flow.raft_large`` module
    with its weights already populated; this class is load-agnostic so the
    caller owns downloading/offload concerns (see ``OpticalFlowLoader`` in
    ``nodes_void.py``).  ``__call__`` returns a ``(2, H, W)`` flow.
    Nc                    |t           j                                        }t          |t          j                  st	          j        |          n|}|                    |          }|                                 || _        || _        d S N)	comfymodel_managementget_torch_devicer   r2   r*   r5   evalmodel)r   r   r*   s      r$   r   zRaftOpticalFlow.__init__e  sq    >+<<>>F-7-M-MYf%%%SY  




r&   c                     |                     | j        t          j                  }|j        \  }}}|dz  dz  }|dz  dz  }t          |||fdd          }|dz  dz
  }|d          S )N   r   Fr   r,   r   )r5   r*   r2   r   r   r%   )r   	image_chwr   r   r9   r:   r    r!   s           r$   _preprocesszRaftOpticalFlow._preprocesso  sq    T[%-88+1aa1a1!%%%PPP	AT{r&   c                    |j         |j         k    r9t          dt          |j                    dt          |j                              |j         \  }}}t          j                    5  |                     |          }|                     |          }|                     ||          }|d         d         }	|	j         dd         ||fk    rt          |	||fdd	          }	ddd           n# 1 swxY w Y   |	S )
z:``from_image``, ``to_image``: CHW float tensors in [0, 1].z9RaftOpticalFlow: from_image and to_image must match, got r(   r-   r   r   Nr   Fr   )r   r   r   r2   no_gradr   r   r%   )
r   
from_imageto_imager   r9   r:   img1img2list_of_flowsr   s
             r$   r   zRaftOpticalFlow.__call__x  sa   x~--LZ-..L L49(.4I4IL L   "1a]__ 	O 	O##J//D##H--D JJtT22M $Q'Dz"##1a&(((1vzNNN	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O s   (A9C--C14C1r   )r   r   r   r   r   r   r   r|   r&   r$   r   r   \  sP                  r&   r      rd   r       )noise_channelsresize_framesresize_flowdownscale_factorr*   video_framesraftr   r   r   r   r*   returnc                r   t          |t                    r|dk     rt          d|          | j        dk    s| j        d         dk    r$t          dt          | j                             | j        t          j        k    rt          d| j                   |t          j                                        }t          |t          j                  st          j        |          n|}|j        d	k    rt          j        d
           | j        d         }|                     |                              dddd                              t          j                  dz  }|dk    rot)          dt          |j        d         |z                      }	t)          dt          |j        d         |z                      }
t+          j        ||	|
fd          }|j        \  }}}}||z  }||z  }|z  s|z  rt          j        d||           t          j                    5  t1          ||||          }|j        z  }|j        z  }t          j        ||||ft          j        |          }fd} ||j                                      ddd          |d<   |d         }t;          d|          D ]k}||         } |||                              |          } ||d         |d                     ||j                                      ddd          ||<   |}l	 ddd           n# 1 swxY w Y   |S )aT  Produce optical-flow-warped gaussian noise from a video.

    Args:
        video_frames: ``(T, H, W, 3)`` uint8 torch tensor.
        raft: Pre-loaded RAFT optical-flow wrapper (see ``RaftOpticalFlow``).
        noise_channels: Channels in the output noise.
        resize_frames: Pre-RAFT frame scale factor.
        resize_flow: Post-flow up-scale factor applied to the optical flow;
            the internal noise state is allocated at
            ``(resize_flow * resize_frames * H, resize_flow * resize_frames * W)``.
        downscale_factor: Area-pool factor applied to the noise before return;
            should evenly divide the internal noise resolution.
        device: Target device.  Defaults to ``comfy.model_management.get_torch_device()``.

    Returns:
        ``(T, H', W', noise_channels)`` float32 noise tensor on ``device``.
    r   z>get_noise_from_video: resize_flow must be a positive int, got    r-   r   zEget_noise_from_video: video_frames must have shape (T, H, W, 3), got zHget_noise_from_video: video_frames must be uint8 in [0, 255], got dtype NcpuzVOIDWarpedNoise: running get_noise_from_video on CPU; this will be slow (minutes for ~45 frames).  Use CUDA for interactive use.r   r,   g     o@      ?arear   ztVOIDWarpedNoise: internal noise size %dx%d is not divisible by downscale_factor %d; output noise may have artifacts.)r   r9   r:   r*   rD   c                 :    t          | dz  dd          }|z  S )Nr   r   Fr   )r%   )	noise_chwdownr   s     r$   	downscalez'get_noise_from_video.<locals>.downscale  s.     %Y6F0FUZ[[[D***r&   )r   r   r   r   r   r   r+   r2   uint8	TypeErrorr   r   r   r*   typeloggingwarningr5   permuter   r   r   r   r   r   r9   r:   emptyrf   range)r   r   r   r   r   r   r*   Tframesr    r!   r   HW
internal_h
internal_wwarperdown_hdown_wrn   r   previcurrr   s        `                   r$   get_noise_from_videor     s   6 k3'' 
;??\[\\
 
 	
 A!3B!71!<!</+,,/ /
 
 	
 U[((.%+. .
 
 	

 ~'88::)3FEL)I)IUU\&!!!vF{eL	
 	
 	

 	1A__V$$,,Q1a88;;EMJJURFAs6<?]:;;<<As6<?]:;;<<vUENHHHJAq!QqJqJ$$ 

5E(E 
D
$4	
 	
 	
 
  
j
 
 
 ----/u}V
 
 
	+ 	+ 	+ 	+ 	+ Ifl++33Aq!<<q	ayq! 	 	A!9D4d##&&v..DF47DG$$$!	&,//771a@@F1IDD	)              6 Ms   !C>L,,L03L0)T)r   )r   r   typingr   r2   torch.nn.functionalnn
functionalr   einopsr   comfy.model_managementr   r%   r@   rP   r[   r`   rb   ro   rt   rz   r~   r   r   r   Tensorr   r   r*   r   r|   r&   r$   <module>r      s   "                              :   :> > >B/ / /  9 9 9     ,  2 2 2  
Y Y Yx1 1 1 1 1 1 1 1p+ + + + + + + +l %)` ` `,`
` 	`
 ` ` ` U\"` \` ` ` ` ` `r&   