
    +j0                        d Z ddlZddlmZ ddlmc mZ ddlZej	        j
        Z	 G d dej                  Zd Zd Zd Zd Zd	 Zd
 Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Zd Z G d dej                  Z G d dej                  ZdS )z7FILM: Frame Interpolation for Large Motion (ECCV 2022).    Nc                   2     e Zd ZdZdddef fd	Zd Z xZS )
FilmConv2dz6Conv2d with optional LeakyReLU and FILM-style padding.TNc                     t                                                       |dz   | _        |                    ||||dz  r|dz  nd||          | _        |rt          j        d          nd | _        d S )N   r   )kernel_sizepaddingdevicedtypeg?)super__init__even_padConv2dconvnn	LeakyReLU
activation)	selfin_channelsout_channelssizer   r	   r
   
operations	__class__s	           P/home/wildlama/comfy/ComfyUI/comfy_extras/frame_interpolation_models/film_net.pyr   zFilmConv2d.__init__   s     1H%%k<Thlophp[w[_cd[d[dvw  AG  OT%  U  U	/9C",s+++t    c                     | j         rt          j        |d          }|                     |          }| j        |                     |          }|S )N)r      r   r   )r   Fpadr   r   )r   xs     r   forwardzFilmConv2d.forward   sJ    = 	'a&&AIIaLL?&""Ar   )__name__
__module____qualname____doc__opsr   r    __classcell__r   s   @r   r   r      sd        @@CGPT\`mp D D D D D D      r   r   c                    | j         }|j        d         |j        d         }}|d d df                                         |dz  z  }|d d df                                         |dz  z  }t          j        |d d d d f         |z   |d d d d f         |z   gd          }	t          j        |                                 |	ddd	
                              |          S )Nr      r         ?r   dimbilinearborderF)modepadding_modealign_corners)r
   shapefloattorchstackr   grid_sampleto)
imageflowgrid_xgrid_yr
   HWdxdygrids
             r   
_warp_corerA      s    KE:a=$*Q-qA	aaad				q3w	'B	aaad				q3w	'B;tT111}-2F4D=4IB4NOUVWWWD=:Hdijjjmmnstttr   c                     | g}t          d|          D ]-}t          j        | dd          } |                    |            .|S Nr   r   )ranger   
avg_pool2dappend)r8   pyramid_levelspyramid_s       r   build_image_pyramidrJ   '   sO    gG1n%%  UAq))uNr   c                 0   | d         }|g}| d d         d d d         D ]b}t          j        ||j        dd         dd                               d                              |          }|                    |           c|                                 |S )Nr      r-   )r   r/   scale_factor)r   interpolater2   mul_add_rF   reverse)residual_pyramidr9   flow_pyramidresidual_flows       r   flow_pyramid_synthesisrV   /   s    BD6L)#2#.ttt4 " "}T(;AaC(@z`deeejjklmmrr  tA  B  BD!!!!r   c                      fd| D             S )Nc                 2    g | ]}|d d d d d f         z  S N ).0r8   scalars     r   
<listcomp>z$multiply_pyramid.<locals>.<listcomp>:   s0    EEEEEF111dD$.//EEEr   rZ   )rH   r\   s    `r   multiply_pyramidr^   9   s    EEEEWEEEEr   c                 <    fdt          | |          D             S )Nc                 .    g | ]\  }} ||          S rZ   rZ   )r[   featuresr9   warp_fns      r   r]   z pyramid_warp.<locals>.<listcomp>>   s)    ]]]$GGHd##]]]r   zip)feature_pyramidrT   rb   s     `r   pyramid_warprf   =   s'    ]]]]#o|:\:\]]]]r   c                 6    d t          | |          D             S )Nc                 D    g | ]\  }}t          j        ||gd           S r   r+   r4   cat)r[   f1f2s      r   r]   z(concatenate_pyramids.<locals>.<listcomp>B   s/    MMM62rEIr2hA&&&MMMr   rc   )pyramid1pyramid2s     r   concatenate_pyramidsrp   A   s     MMS85L5LMMMMr   c                   2     e Zd Zdddddef fd	Zd Z xZS )SubTreeExtractorr)   @   rM   Nc                 L   t                                                       g }t          |          D ]W}||z  }	|                    t	          j        t          ||	d|||          t          |	|	d|||                               |	}Xt	          j        |          | _        d S Nr)   r	   r
   r   )	r   r   rD   rF   r   
Sequentialr   
ModuleListconvs)r   r   channelsn_layersr	   r
   r   ry   iout_chr   s             r   r   zSubTreeExtractor.__init__F   s    x 	! 	!A]FLL;&Zdeee661V5U_```b b c c c !KK]5))


r   c                     |}g }t          | j                  D ]D\  }} ||          }|                    |           ||dz
  k     rt          j        |dd          }E|S rC   )	enumeratery   rF   r   rE   )r   r8   nheadrH   r|   layers          r   r    zSubTreeExtractor.forwardQ   sn    !$*-- 	0 	0HAu5;;DNN4   1q5yy|D!Q//r   r!   r"   r#   r%   r   r    r&   r'   s   @r   rr   rr   E   s[        #$rAdRVcf 	* 	* 	* 	* 	* 	*      r   rr   c                   2     e Zd Zdddddef fd	Zd Z xZS )FeatureExtractorr)   rs   rM   Nc                     t                                                       t          ||||||          | _        || _        d S )Nrv   )r   r   rr   extract_sublevels
sub_levels)r   r   rz   r   r	   r
   r   r   s          r   r   zFeatureExtractor.__init__]   sQ    !1+x\bjo  }G  "H  "H  "H$r   c                      fdt          t                              D             }g }t          t                              D ]}||         d         }t          d j                  D ]/}||k    r't          j        ||||z
           |         gd          }0|                    |           | j        dz
  k    rd || j        z
  dz   <   |S )Nc           
          g | ]A}                     |         t          t                    |z
  j                            BS rZ   )r   minlenr   )r[   r|   image_pyramidr   s     r   r]   z,FeatureExtractor.forward.<locals>.<listcomp>c   sZ     < < < ..}Q/?SEWEWZ[E[]a]lAmAmnn < < <r   r   r   r+   )rD   r   r   r4   rk   rF   )r   r   sub_pyramidsre   r|   ra   js   ``     r   r    zFeatureExtractor.forwardb   s   < < < < <!&s='9'9!:!:< < <s=))** 	= 	=A#Aq)H1do.. T T66$y(LQ4G4J)KQRSSSH""8,,,DOa'''8<Q0145r   r   r'   s   @r   r   r   \   s[        #$raTXeh % % % % % %
      r   r   c                   ,     e Zd Zddef fd	Zd Z xZS )FlowEstimatorNc                    t                                                       t          j                    | _        t          |          D ]1}| j                            t          ||d|||                     |}2| j                            t          ||dz  d|||                     | j                            t          |dz  ddd|||                     d S )Nr)   rv   r   r   Fr   r	   r
   r   )r   r   r   rx   _convsrD   rF   r   )	r   r   	num_convsnum_filtersr	   r
   r   rI   r   s	           r   r   zFlowEstimator.__init__s   s    mooy!! 	& 	&AKz+{Af\anxyyyzzz%KK:k;!3CQv]boyzzz{{{:kQ&61W]ej  xB  C  C  C  	D  	D  	D  	D  	Dr   c                 `    t          j        ||gd          }| j        D ]} ||          }|S )Nr   r+   )r4   rk   r   )r   
features_a
features_bnetr   s        r   r    zFlowEstimator.forward|   s@    iZ0a888K 	 	D$s))CC
r   r   r'   s   @r   r   r   r   sZ        CGt`c D D D D D D      r   r   c                   2     e Zd Zdddddef fd	Zd Z xZS )PyramidFlowEstimatorrs   r)   r)   r)   r)       rs         Nc                    t                                                       |dz  }g }t          t          |                    D ]A}	|                    t          |||	         ||	         |||                     |||	dz   z  z  }B|d         | _        t          j        |d d         d d d                   | _	        d S )Nr   rv   r   rL   )
r   r   rD   r   rF   r   
_predictorr   rx   _predictors)r   filters
flow_convsflow_filtersr	   r
   r   r   
predictorsr|   r   s             r   r   zPyramidFlowEstimator.__init__   s    l
s:'' 	. 	.AmKAUV`fns  AK  L  L  L  M  M  M7q1u--KK$R.=CRC2)>??r   c                 j    t          |          }                     |d         |d                   }|g} fdt          |dz
  t           j                  dz
  d          D             }| fdt	           j                  D             z  }|D ]\  }}	t          j        |||         j        dd         d                              d          } |	||          |||         |                    }
|	                    |
           |
                    |
          }|                                 |S )	NrL   c                 "    g | ]}|j         fS rZ   )r   )r[   r|   r   s     r   r]   z0PyramidFlowEstimator.forward.<locals>.<listcomp>   s     ```!!T_%```r   r   r   c                 N    g | ]!\  }}t          j                  d z
  |z
  |f"S )r   )r   r   )r[   kpr   s      r   r]   z0PyramidFlowEstimator.forward.<locals>.<listcomp>   s7    ]]]A3t'((1,q0!4]]]r   rM   r-   r   r/   )r   r   rD   r   r   r   rO   r2   rP   rF   rQ   rR   )r   feature_pyramid_afeature_pyramid_brb   levelsv	residualsstepsr|   	predictor
v_residuals   `          r   r    zPyramidFlowEstimator.forward   sP   &''OO-b13DR3HIIC	````uVaZTEUAVAVYZAZ\^/_/_```]]]]4K[A\A\]]]]! 	# 	#LAya&7&:&@1&EJWWW\\]^__A"#4Q#7ARSTAUWX9Y9YZZJZ(((z""AAr   r   r'   s   @r   r   r      sg        !lI[dhpt  BE @ @ @ @ @ @      r   r   c                 f    t          fdt          |           D                       dz   dz   dz  S )Nc              3   "   K   | ]	}|z  V  
d S rY   rZ   )r[   r|   r   s     r   	<genexpr>z'_get_fusion_channels.<locals>.<genexpr>   s'      331333333r   r)   r   )sumrD   )levelr   s    `r   _get_fusion_channelsr      s;    3333eEll33333a7!;q@@r   c                   2     e Zd Zdddddef fd	Zd Z xZS )FusionrM   r)   rs   Nc                 <   t                                                       |                    |dd||          | _        t	          j                    | _        t          ||          }d}t          |          d d d         D ]}	|	|k     r||	z  n||z  }
| j        	                    t	          j        t          ||
dd|||          t          ||p|
z   |
d|||	          t          |
|
d|||	          g                     |
}t          |	|          |
dz  z
  }d S )
Nr)   r   )r   r	   r
   r   rL   r   Fr   rv   )r   r   r   output_convr   rx   ry   r   rD   rF   r   )r   r{   specialized_layersr   r	   r
   r   r   increaser|   r   r   s              r   r   zFusion.__init__   sf   %,,WaQv]b,cc]__
*8W==x2& 	K 	KA,-0B,B,B7a<<TfIfKJbm;Q5QW_dq{|||;(*AkBKQR[ain  |F  G  G  G;QvU_ijjj-l m m n n n &K+Aw77+:JJHH	K 	Kr   c           	         |d         }t          | j                  D ]\  }}t          | j                  dz
  |z
  } |d         t          j        |||         j        dd         d                    } |d          |d         t          j        ||         |gd                              }|                     |          S )	NrL   r   r   r   rM   nearestr   r+   )	r   ry   r   r   rO   r2   r4   rk   r   )r   rH   r   r   layersr|   s         r   r    zFusion.forward   s    bk"4:.. 	L 	LIAvDJ!#a'A&)AM#GAJ4DQqS4IPYZZZ[[C&)IF1IeiS0Aq&I&I&IJJKKCC$$$r   r   r'   s   @r   r   r      sa         !aDX\il K K K K K K% % % % % % %r   r   c            
       b     e Zd Zdddddddddef
 fd		Zd
 Zd Zd Zd Zd Z	ddZ
ddZ xZS )FILMNet      r)   rM   rs   r   r   Nc                    t                                                       || _        || _        t	          d||||	|
          | _        t          |||||	|
          | _        t          |||||	|
          | _	        i | _
        d S ru   )r   r   rG   fusion_pyramid_levelsr   extractr   predict_flowr   fuse_warp_grids)r   rG   r   specialized_levelsr   r   r   r   r	   r
   r   r   s              r   r   zFILMNet.__init__   s    ,%:"'7JvUZgqrrr0*l[ain  |F  G  G  G:'976Y^kuvvv	r   c                 Z    | j         j        j        d         d         j        j        j        S )Nr   )r   r   ry   r   weightr
   )r   s    r   	get_dtypezFILMNet.get_dtype   s#    |-3A6q9>EKKr   c                 :    d|d         z  |d         z  |j         z  S )Ni  r   r   )itemsize)r   r2   r
   s      r   memory_used_forwardzFILMNet.memory_used_forward   s     eAhq)EN::r   c           	      D   ||f| j         v rdS i | _         t          | j                  D ]v}t          j        dd|z  z
   dd|z  z
  |t          j        |          t          j        dd|z  z
   dd|z  z
  |t          j        |          f| j         ||f<   |dz  |dz  }}wdS )z.Pre-compute warp grids for all pyramid levels.Nr   )r
   r	   r   )r   rD   rG   r4   linspacefloat32)r   r<   r=   r	   rI   s        r   _build_warp_gridszFILMNet._build_warp_grids   s    q6T%%%Ft*++ 	" 	"AQU|QQYW]^^^QU|QQYW]^^^(DaV$ 616qAA	" 	"r   c                 v    | j         |j        d         |j        d         f         \  }}t          ||||          S )Nr   r)   )r   r2   rA   )r   r8   r9   r:   r;   s        r   warpzFILMNet.warp   s8    )4:a=$*Q-*HI%vv666r   c                 ^    t          || j                  }|                     |          }||fS )zRExtract image and feature pyramids for a single frame. Can be cached across pairs.)rJ   rG   r   )r   imgr   re   s       r   extract_featureszFILMNet.extract_features   s0    +C1DEE,,}55o--r   r*   c                     t          |t          j                  r(|                    d                                          n|}|                     |||g|          S )N)r   r   r)   r+   )cache)
isinstancer4   Tensormeanitemforward_multi_timestep)r   img0img1timestepr   ts         r   r    zFILMNet.forward   sW    3=h3U3UcHMMiM((--///[c**4s%*HHHr   c                 ~   |                      |j        d         |j        d         |j                   |rd|v r|d         n|                     |          \  }}|rd|v r|d         n|                     |          \  }}t	          |                     ||| j                            d| j                 }	t	          |                     ||| j                            d| j                 }
| j        }t          |d|         |d|                   t          |d|         |d|                   g}~~~~g }t          j
        ||j        |j                  }t          t          |                    D ]}|||dz            }t          |
|          }t          |	d|z
            }t          |d         || j                  }t          |d         || j                  }d	 t!          ||||          D             }~~~~|                    |                     |                     ~t          j        |d
          S )zLCompute flow once, synthesize at multiple timesteps. Expects batch=1 inputs.r   r)   r   r   N)r	   r
   r   r   c                 L    g | ]!\  }}}}t          j        ||||gd           "S ri   rj   )r[   fwbwbfffs        r   r]   z2FILMNet.forward_multi_timestep.<locals>.<listcomp>   sN     b b b)2r2r y"b"b!1q999 b b br   r+   )r   r2   r	   r   rV   r   r   r   rp   r4   tensorr
   rD   r   r^   rf   rd   rF   r   rk   )r   r   r   	timestepsr   
image_pyr0	feat_pyr0
image_pyr1	feat_pyr1fwd_flowbwd_flowfplp2wresults
dt_tensorsidxbatch_dt
bwd_scaled
fwd_scaled
fwd_warped
bwd_warpedaligneds                         r   r   zFILMNet.forward_multi_timestep   sR   tz!}djmT[III16 k6U??fPTPePefjPkPk
I16 k6U??fPTPePefjPkPk
I)$*;*;IyRVR[*\*\]]^y_c_y^yz)$*;*;IyRVR[*\*\]]^y_c_y^yz (#Jtt$4ioFF#Jtt$4ioFFH
Iy\)DKtzRRR
Y(( 
	 
	C!#cAg+.H)(H==J)(ALAAJ%c!fj$)DDJ%c!fj$)DDJb b-0ZU_-`-`b b bGJ
JNN499W--...ya((((r   )r*   NrY   )r!   r"   r#   r%   r   r   r   r   r   r   r    r   r&   r'   s   @r   r   r      s        &'qUVcdCU^bjn{~     L L L; ; ;
" 
" 
"7 7 7. . .I I I I
) ) ) ) ) ) ) )r   r   )r$   r4   torch.nnr   torch.nn.functional
functionalr   	comfy.opscomfyr%   disable_weight_initModuler   rA   rJ   rV   r^   rf   rp   rr   r   r   r   r   r   r   rZ   r   r   <module>r     s   = =                    i#       $u u u    F F F^ ^ ^N N N    ry   .    ry   ,    BI   "    29   6A A A
% % % % %RY % % %2I) I) I) I) I)bi I) I) I) I) I)r   