
    +j0                        S r SSKrSSKJr  SSKJs  Jr  SSKr\R                  R                  r	 " S S\R                  5      rS rS rS rS rS	 rS
 r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      rS r " S S\R                  5      r " S S\R                  5      rg)z7FILM: Frame Interpolation for Large Motion (ECCV 2022).    Nc                   >   ^  \ rS rSrSrSSS\4U 4S jjrS rSrU =r	$ )
FilmConv2d   z6Conv2d with optional LeakyReLU and FILM-style padding.TNc           	         > [         TU ]  5         US-  (       + U l        UR                  XX3S-  (       a  US-  OSXVS9U l        U(       a  [
        R                  " S5      U l        g S U l        g )N   r   )kernel_sizepaddingdevicedtypeg?)super__init__even_padConv2dconvnn	LeakyReLU
activation)	selfin_channelsout_channelssizer   r
   r   
operations	__class__s	           P/home/wildlama/comfy/ComfyUI/comfy_extras/frame_interpolation_models/film_net.pyr   FilmConv2d.__init__   sf     1H%%kTophp[_cd[dvw  AG%  U	/9",,s+t    c                     U R                   (       a  [        R                  " US5      nU R                  U5      nU R                  b  U R	                  U5      nU$ )N)r      r   r   )r   Fpadr   r   )r   xs     r   forwardFilmConv2d.forward   sC    ==a&AIIaL??&"Ar   )r   r   r   )
__name__
__module____qualname____firstlineno____doc__opsr   r"   __static_attributes____classcell__r   s   @r   r   r      s"    @CGPT\`mp D r   r   c                    U R                   nUR                  S   UR                  S   peUS S 2S4   R                  5       US-  -  nUS S 2S4   R                  5       US-  -  n[        R                  " US S S S 24   U-   US S S 2S 4   U-   /SS9n	[
        R                  " U R                  5       U	SSS	S
9R                  U5      $ )Nr      r         ?r   dimbilinearborderF)modepadding_modealign_corners)r   shapefloattorchstackr   grid_sampleto)
imageflowgrid_xgrid_yr   HWdxdygrids
             r   
_warp_corerF      s    KKE::a=$**Q-q	ad			q3w	'B	ad			q3w	'B;;tT1}-2F4D=4IB4NOUVWD==:Hdijmmnsttr   c                     U /n[        SU5       H,  n[        R                  " U SS5      n UR                  U 5        M.     U$ Nr   r   )ranger   
avg_pool2dappend)r=   pyramid_levelspyramid_s       r   build_image_pyramidrO   '   s?    gG1n%UAq)u & Nr   c                     U S   nU/nU S S S S S2    HU  n[         R                  " XR                  SS SS S9R                  S5      R	                  U5      nUR                  U5        MW     UR                  5         U$ )Nr      r2   )r   r4   scale_factor)r   interpolater7   mul_add_rK   reverse)residual_pyramidr>   flow_pyramidresidual_flows       r   flow_pyramid_synthesisr[   /   s    BD6L)#2.tt4}}T(;(;Aa(@z`dejjklmrr  tA  BD! 5 r   c           	      H    U  Vs/ s H  o"US S 2S S S 4   -  PM     sn$ s  snf N )rM   scalarr=   s      r   multiply_pyramidr`   9   s+    =DEWEF1dD$.//WEEEs   c                 Z    [        X5       VVs/ s H  u  p4U" X45      PM     snn$ s  snnf r]   )zip)feature_pyramidrY   warp_fnfeaturesr>   s        r   pyramid_warprf   =   s(    :=o:\]:\GH#:\]]]s   'c           	      v    [        X5       VVs/ s H  u  p#[        R                  " X#/SS9PM     snn$ s  snnf Nr   r0   )rb   r9   cat)pyramid1pyramid2f1f2s       r   concatenate_pyramidsrn   A   s0    585LM5L62EIIrhA&5LMMMs   "5c                   >   ^  \ rS rSrSSSSS\4U 4S jjrS rSrU =r$ )	SubTreeExtractorE   r.   @   rR   Nc                    > [         T
U ]  5         / n[        U5       HC  nX(-  n	UR                  [        R
                  " [        XSXEUS9[        XSXEUS95      5        U	nME     [        R                  " U5      U l        g Nr.   r
   r   r   )	r   r   rI   rK   r   
Sequentialr   
ModuleListconvs)r   r   channelsn_layersr
   r   r   rx   iout_chr   s             r   r   SubTreeExtractor.__init__F   sx    xA]FLL;&Zde61VU_`b c !K ! ]]5)
r   c                     Un/ n[        U R                  5       H@  u  pVU" U5      nUR                  U5        XRS-
  :  d  M(  [        R                  " USS5      nMB     U$ rH   )	enumeraterx   rK   r   rJ   )r   r=   nheadrM   r{   layers          r   r"   SubTreeExtractor.forwardQ   sX    !$**-HA;DNN4 q5y||D!Q/	 .
 r   )rx   	r$   r%   r&   r'   r)   r   r"   r*   r+   r,   s   @r   rp   rp   E   s!    #$rAdRVcf 	* r   rp   c                   >   ^  \ rS rSrSSSSS\4U 4S jjrS rSrU =r$ )	FeatureExtractor\   r.   rr   rR   Nc           	      N   > [         TU ]  5         [        XX4XVS9U l        X0l        g )Nru   )r   r   rp   extract_sublevels
sub_levels)r   r   ry   r   r
   r   r   r   s          r   r   FeatureExtractor.__init__]   s(    !1+jo  "H$r   c                    [        [        U5      5       Vs/ s H7  nU R                  X   [        [        U5      U-
  U R                  5      5      PM9     nn/ n[        [        U5      5       H  nX2   S   n[        SU R                  5       H(  nXb::  d  M
  [
        R                  " XSX&-
     U   /SS9nM*     UR                  U5        X R                  S-
  :  d  Mq  S X2U R                  -
  S-   '   M     U$ s  snf )Nr   r   r0   )rI   lenr   minr   r9   ri   rK   )r   image_pyramidr{   sub_pyramidsrc   re   js          r   r"   FeatureExtractor.forwardb   s    !&s='9!:<!:A ..}/?SEWZ[E[]a]l]lAmn!: 	 <s=)*A#q)H1doo.6$yy(4G4J)KQRSH / ""8,OOa''8<0145 + <s   >C8)r   r   r   r,   s   @r   r   r   \   s!    #$raTXeh %
 r   r   c                   8   ^  \ rS rSrSS\4U 4S jjrS rSrU =r$ )FlowEstimatorr   Nc                 l  > [         TU ]  5         [        R                  " 5       U l        [        U5       H*  nU R                  R                  [        XSXEUS95        UnM,     U R                  R                  [        XS-  SXEUS95        U R                  R                  [        US-  SSSXEUS95        g )Nr.   ru   r   r   Fr   r
   r   r   )r   r   r   rw   _convsrI   rK   r   )	r   r   	num_convsnum_filtersr
   r   r   rN   r   s	           r   r   FlowEstimator.__init__s   s    mmoy!AKKz+Afnxyz%K " 	:k!3CQvoyz{:kQ&61W]  xB  C  	Dr   c                 h    [         R                  " X/SS9nU R                   H  nU" U5      nM     U$ rh   )r9   ri   r   )r   
features_a
features_bnetr   s        r   r"   FlowEstimator.forward|   s1    ii0a8KKDs)C  
r   )r   r   r,   s   @r   r   r   r   s    CGt`c D r   r   c                   >   ^  \ rS rSrSSSSS\4U 4S jjrS rSrU =r$ )	PyramidFlowEstimator   rr   r.   r.   r.   r.       rr         Nc                   > [         T
U ]  5         US-  n/ n[        [        U5      5       H-  n	UR	                  [        XrU	   X9   XEUS95        XqU	S-   -  -  nM/     US   U l        [        R                  " US S S S S2   5      U l	        g )Nr   ru   r   rQ   )
r   r   rI   r   rK   r   
_predictorr   rw   _predictors)r   filters
flow_convsflow_filtersr
   r   r   r   
predictorsr{   r   s             r   r   PyramidFlowEstimator.__init__   s    l
s:'AmKA`f  AK  L  Mq1u--K ( %R.==CR2)>?r   c                    [        U5      nU R                  US   US   5      nU/n[        US-
  [        U R                  5      S-
  S5       Vs/ s H  owU R                  4PM     nnU[	        U R                  5       V	V
s/ s H"  u  p[        U R                  5      S-
  U	-
  U
4PM$     sn
n	-  nU Ho  u  p{[
        R                  " XQU   R                  SS SS9R                  S5      nU" X   U" X'   U5      5      nUR                  U5        UR                  U5      nMq     UR                  5         U$ s  snf s  sn
n	f )NrQ   r   r   rR   r2   r   r4   )r   r   rI   r   r   r   rT   r7   rU   rK   rV   rW   )r   feature_pyramid_afeature_pyramid_brd   levelsv	residualsr{   stepskp	predictor
v_residuals                r   r"   PyramidFlowEstimator.forward   s=   &'OO-b13DR3HIC	/4VaZTEUEUAVYZAZ\^/_`/_!T__%/_`4K[K[A\]A\3t''(1,q0!4A\]]!LAa&:&@&@1&EJW\\]^_A"#4#7ARAUWX9YZJZ(z"A	 "
 	 a]s   D7)D<)r   r   r   r,   s   @r   r   r      s(    !lI[dhpt  BE @ r   r   c                 R   ^ [        U4S j[        U 5       5       5      S-   S-   S-  $ )Nc              3   .   >#    U  H
  nTU-  v   M     g 7fr]   r^   ).0r{   r   s     r   	<genexpr>'_get_fusion_channels.<locals>.<genexpr>   s     3l1ls   r.   r   )sumrI   )levelr   s    `r   _get_fusion_channelsr      s'    3eEl33a7!;q@@r   c                   >   ^  \ rS rSrSSSSS\4U 4S jjrS rSrU =r$ )	Fusion   rR   r.   rr   Nc                   > [         TU ]  5         UR                  USSXES9U l        [        R
                  " 5       U l        [        X5      nSn[        U5      S S S2    H  n	X:  a  X9-  OX2-  n
U R                  R                  [        R
                  " [        XzSSXEUS9[        Xx=(       d    U
-   U
SXEUS	9[        XSXEUS	9/5      5        U
n[        X5      U
S-  -
  nM     g )
Nr.   r   )r   r
   r   r   rQ   r   Fr   ru   )r   r   r   output_convr   rw   rx   r   rI   rK   r   )r   rz   specialized_layersr   r
   r   r   r   increaser{   r   r   s              r   r   Fusion.__init__   s    %,,WaQv,c]]_
*8=x2&A,-,B7<IfKJJbmm;Q5QWq{|;*AkBKQR[a  |F  G;Qv_ij-l m n &K+A7+:JJH 'r   c           
      R   US   n[        U R                  5       Hy  u  p4[        U R                  5      S-
  U-
  nUS   " [        R                  " X!U   R
                  SS SS95      nUS   " US   " [        R                  " X   U/SS95      5      nM{     U R                  U5      $ )	NrQ   r   r   r   rR   nearestr   r0   )	r   rx   r   r   rT   r7   r9   ri   r   )r   rM   r   r   layersr{   s         r   r"   Fusion.forward   s    bk"4::.IADJJ!#a'A)AMM#AJ4D4DQq4IPYZ[C)F1IeiiS0Aq&IJKC / $$r   )rx   r   r   r,   s   @r   r   r      s"     !aDX\il K% %r   r   c            
       r   ^  \ rS rSrSSSSSSSS	S	\4
U 4S
 jjrS rS rS rS r	S r
SS jrSS jrSrU =r$ )FILMNet         r.   rR   rr   r   r   Nc           	         > [         TU ]  5         Xl        X l        [	        SXTXU
S9U l        [        XVXxXS9U l        [        XCXXXS9U l	        0 U l
        g rt   )r   r   rL   fusion_pyramid_levelsr   extractr   predict_flowr   fuse_warp_grids)r   rL   r   specialized_levelsr   r   r   r   r
   r   r   r   s              r   r   FILMNet.__init__   s[    ,%:"'7vgqr0lin  G:7Y^v	r   c                     U R                   R                  R                  S   S   R                  R                  R
                  $ )Nr   )r   r   rx   r   weightr   )r   s    r   	get_dtypeFILMNet.get_dtype   s4    ||--33A6q9>>EEKKKr   c                 8    SUS   -  US   -  UR                   -  $ )Ni  r   r   )itemsize)r   r7   r   s      r   memory_used_forwardFILMNet.memory_used_forward   s"    eAhq)ENN::r   c           
      h   X4U R                   ;   a  g0 U l         [        U R                  5       H  n[        R                  " SSU-  -
  * SSU-  -
  U[        R
                  US9[        R                  " SSU-  -
  * SSU-  -
  U[        R
                  US94U R                   X4'   US-  US-  p!M     g)z.Pre-compute warp grids for all pyramid levels.Nr   )r   r
   r   )r   rI   rL   r9   linspacefloat32)r   rA   rB   r
   rN   s        r   _build_warp_gridsFILMNet._build_warp_grids   s    6T%%%t**+AQU|QQYW]^QU|QQYW]^(DaV$ 616q ,r   c                 t    U R                   UR                  S   UR                  S   4   u  p4[        XX45      $ )Nr   r.   )r   r7   rF   )r   r=   r>   r?   r@   s        r   warpFILMNet.warp   s6    ))4::a=$**Q-*HI%v66r   c                 T    [        XR                  5      nU R                  U5      nX#4$ )zRExtract image and feature pyramids for a single frame. Can be cached across pairs.)rO   rL   r   )r   imgr   rc   s       r   extract_featuresFILMNet.extract_features   s)    +C1D1DE,,}5--r   c                     [        U[        R                  5      (       a  UR                  SS9R	                  5       OUnU R                  XU/US9$ )N)r   r   r.   r0   )cache)
isinstancer9   Tensormeanitemforward_multi_timestep)r   img0img1timestepr   ts         r   r"   FILMNet.forward   sH    3=h3U3UHMMiM(--/[c**4s%*HHr   c                 p   U R                  UR                  S   UR                  S   UR                  5        U(       a  SU;   a  US   OU R                  U5      u  pVU(       a  SU;   a  US   OU R                  U5      u  px[	        U R                  XhU R                  5      5      SU R                   n	[	        U R                  XU R                  5      5      SU R                   n
U R                  n[        USU USU 5      [        USU USU 5      /nAAAA/ n[        R                  " X1R                  UR                  S9n[        [        U5      5       H  nXUS-    n[        U
U5      n[        U	SU-
  5      n[        US   UU R                  5      n[        US   UU R                  5      n[!        UUUU5       VVVVs/ s H!  u  nnnn[        R"                  " UUUU/SS	9PM#     nnnnnAAAAUR%                  U R'                  U5      5        AM     [        R"                  " USS	9$ s  snnnnf )
zLCompute flow once, synthesize at multiple timesteps. Expects batch=1 inputs.r   r.   r   r   N)r
   r   r   r   r0   )r   r7   r
   r   r[   r   r   r   rn   r9   tensorr   rI   r   r`   rf   rb   ri   rK   r   )r   r   r   	timestepsr   
image_pyr0	feat_pyr0
image_pyr1	feat_pyr1fwd_flowbwd_flowfplp2wresults
dt_tensorsidxbatch_dt
bwd_scaled
fwd_scaled
fwd_warped
bwd_warpedfwbwbfffaligneds                             r   r   FILMNet.forward_multi_timestep   s   tzz!}djjmT[[I166U?fPTPePefjPk
166U?fPTPePefjPk
)$*;*;IRVR[R[*\]^y_c_y_yz)$*;*;IRVR[R[*\]^y_c_y_yz ((#Jt$4ioF#Jt$4ioFH
Iy\\)KKtzzR
Y(C!cAg.H)(H=J)(ALAJ%c!fj$))DJ%c!fj$))DJ-0ZU_-`b-`>2r2r yy"b"b!1q9-`  bJ
JNN499W-. ) yya((bs   (H0)r   r   r   r   r   rL   )r/   Nr]   )r$   r%   r&   r'   r)   r   r   r   r   r   r   r"   r   r*   r+   r,   s   @r   r   r      sN    &'qUVcdCU^bjn{~L;
"7.I
) )r   r   )r(   r9   torch.nnr   torch.nn.functional
functionalr   	comfy.opscomfyr)   disable_weight_initModuler   rF   rO   r[   r`   rf   rn   rp   r   r   r   r   r   r   r^   r   r   <module>r     s    =     ii## $uF^Nryy .ryy ,BII "299 6A
%RYY %2I)bii I)r   