
    3j                     B   S SK Jr  S SKJr  S SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  SSKJr  SS	KJr  SS
KJr  SSKJrJr  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJ r   SSK!J"r"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  SSK*J+r+J,r,  SSK-J.r.  SSK/J0r0J1r1  SSK2J3r3  \Rh                  " \55      r6\" SS9\ " S S\5      5       5       r7\" SS9\ " S S \5      5       5       r8 " S! S"\15      r9 " S# S$\05      r:\ " S%S&9 " S' S(\.5      5       r; " S) S*\	Rx                  5      r=\ " S+ S,\+5      5       r> " S- S.\'5      r? " S/ S0\	Rx                  5      r@S1\R                  S2\R                  S3\R                  S4\R                  4S5 jrB " S6 S7\	Rx                  5      rC " S8 S9\	Rx                  5      rD\ " S: S;\5      5       rES<\R                  S=\FS4\G\R                  \R                  4   4S> jrHS?\R                  S@\ISA\IS4\R                  4SB jrJ\" SCS9 " SD SE\E5      5       rK/ SFQrLg)G    )Callable)	dataclassN)strict)nnpad_sequence   )PreTrainedConfig)create_bidirectional_mask)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutput
TensorTypeauto_docstringcan_return_tuplelogging)no_inherit_decorator)requires   )CONFIG_MAPPING
AutoConfig)AutoModelForKeypointDetection)CLIPMLP)apply_rotary_pos_emb)LlamaAttentioneager_attention_forward)SuperGlueImageProcessorPil)SuperGlueImageProcessorSuperGlueImageProcessorKwargs)SuperPointConfigzETH-CVG/lightglue_superpoint)
checkpointc                      ^  \ rS rSr% SrSrS\0rSr\	\
-  S-  \S'   Sr\\S'   Sr\\S	'   S
r\\S'   Sr\S-  \S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\-  \S'   Sr\\S'   U 4S jrS rSrU =r$ )LightGlueConfig,   a  
keypoint_detector_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `SuperPointConfig`):
    The config object or dictionary of the keypoint detector.
descriptor_dim (`int`, *optional*, defaults to 256):
    The dimension of the descriptors.
depth_confidence (`float`, *optional*, defaults to 0.95):
    The confidence threshold used to perform early stopping
width_confidence (`float`, *optional*, defaults to 0.99):
    The confidence threshold used to prune points
filter_threshold (`float`, *optional*, defaults to 0.1):
    The confidence threshold used to filter matches

Examples:
    ```python
    >>> from transformers import LightGlueConfig, LightGlueForKeypointMatching

    >>> # Initializing a LightGlue style configuration
    >>> configuration = LightGlueConfig()

    >>> # Initializing a model from the LightGlue style configuration
    >>> model = LightGlueForKeypointMatching(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
	lightgluekeypoint_detector_configN   descriptor_dim	   num_hidden_layers   num_attention_headsnum_key_value_headsgffffff?depth_confidencegGz?width_confidence皙?filter_thresholdg{Gz?initializer_rangegelu
hidden_act        attention_dropoutTattention_biasc                   > U R                   c  U R                  U l         [        U R                  [        5      (       aY  U R                  R                  SS5      U R                  S'   [        U R                  S      " S0 U R                  DSS0D6U l        OU R                  c  [        S   " SS9U l        U R                  S-  U l        U R                  U l	        [        TU ],  " S0 UD6  g )N
model_type
superpointattn_implementationeager)r=   r    )r/   r.   
isinstancer(   dictgetr   r*   intermediate_sizehidden_sizesuper__post_init__)selfkwargs	__class__s     i/home/wildlama/miniconda3/lib/python3.13/site-packages/transformers/models/lightglue/modular_lightglue.pyrF   LightGlueConfig.__post_init__Z   s    ##+'+'?'?D$ d33T:::>:W:W:[:[\hjv:wD)),7,:4;X;XYe;f,g -//-EL-D) **2,:<,H]d,eD)!%!4!4q!8..''    c                 T    U R                   U R                  -  S:w  a  [        S5      eg)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z1descriptor_dim % num_heads is different from zeroN)r*   r.   
ValueError)rG   s    rJ   validate_architecture%LightGlueConfig.validate_architecturel   s,    !9!99Q>PQQ ?rL   )rD   rC   r(   r/   )__name__
__module____qualname____firstlineno____doc__r;   r   sub_configsr(   rA   r"   __annotations__r*   intr,   r.   r/   r0   floatr1   r3   r4   r6   strr8   r9   boolrF   rO   __static_attributes____classcell__rI   s   @rJ   r%   r%   ,   s    6 J-z:K?Cd%55<CNCs  &*t*"e""e"!e!#u#J%(us{(ND($R RrL   r%   a  
    Base class for outputs of LightGlue keypoint matching models. Due to the nature of keypoint detection and matching,
    the number of keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the
    batch of images, the maximum number of matches is set as the dimension of the matches and matching scores. The mask
    tensor is used to indicate which values in the keypoints, matches, matching_scores and prune tensors are keypoint
    matching information.
    )custom_introc                   f   \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Srg)LightGlueKeypointMatchingOutputr   aW  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Loss computed during training.
matches (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Index of keypoint matched in the other image.
matching_scores (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Scores of predicted matches.
keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`):
    Absolute (x, y) coordinates of predicted keypoints in a given image.
prune (`torch.IntTensor` of shape `(batch_size, num_keypoints)`):
    Pruning mask indicating which keypoints are removed and at which layer.
mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`):
    Mask indicating which values in matches, matching_scores, keypoints and prune are keypoint matching
    information.
hidden_states (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, 2, num_channels,
    num_keypoints)` returned when `output_hidden_states=True` is passed or when
    `config.output_hidden_states=True`
attentions (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, 2, num_heads, num_keypoints,
    num_keypoints)` returned when `output_attentions=True` is passed or when
    `config.output_attentions=True`
Nlossmatchesmatching_scores	keypointsprunemaskhidden_states
attentionsr?   )rQ   rR   rS   rT   rU   rc   torchFloatTensorrW   rd   re   rf   rg   	IntTensorrh   ri   tuplerj   r\   r?   rL   rJ   ra   ra   r   s    0 &*D%

d
")(,GU%,04OU&&-4*.Iu  4'.$(E5??T!(%)D%

d
")59M5**+d2926Je''(4/6rL   ra   c                       \ rS rSrSrg)LightGlueImageProcessorKwargs   r?   N)rQ   rR   rS   rT   r\   r?   rL   rJ   rp   rp      s    rL   rp   c                   r   ^  \ rS rSr S	SSS\\\   -  S\S\\\	\
R                  4      4U 4S jjjrSrU =r$ )
LightGlueImageProcessor   outputsra   target_sizes	thresholdreturnc                 $   > [         TU ]  XU5      $ NrE   post_process_keypoint_matchingrG   ru   rv   rw   rI   s       rJ   r|   6LightGlueImageProcessor.post_process_keypoint_matching   s     w5gYWWrL   r?   r7   )rQ   rR   rS   rT   r   listrn   rY   rA   rZ   rk   Tensorr|   r\   r]   r^   s   @rJ   rs   rs      sZ    
 	X2X !4;.X 	X
 
d3$%	&X XrL   rs   rk   backendsc                   p   ^  \ rS rSr\" SS9 SSSS\\\   -  S\S\\	\
S	4      4U 4S
 jjj5       rSrU =r$ )LightGlueImageProcessorPil   r   r   ru   ra   rv   rw   rx   ztorch.Tensorc                 $   > [         TU ]  XU5      $ rz   r{   r}   s       rJ   r|   9LightGlueImageProcessorPil.post_process_keypoint_matching   s     w5gYWWrL   r?   r   )rQ   rR   rS   rT   r   r   r   rn   rY   rA   rZ   r|   r\   r]   r^   s   @rJ   r   r      sc    z"
 	X2X !4;.X 	X
 
d3&'	(X #XrL   r   c            
          ^  \ rS rSrS\4U 4S jjr S
S\R                  S\S-  S\	\R                     \	\R                  \R                  4   -  4S jjr
S	rU =r$ )LightGluePositionalEncoder   configc                    > [         TU ]  5         [        R                  " SUR                  UR
                  -  S-  SS9U l        g )Nr   Fbias)rE   __init__r   Linearr*   r.   	projectorrG   r   rI   s     rJ   r   #LightGluePositionalEncoder.__init__   s:    1f&;&;v?Y?Y&Y]^&^ejkrL   rf   output_hidden_statesNrx   c                     U R                  U5      nUR                  SSS9n[        R                  " U5      n[        R                  " U5      nXV4nU(       a  XC4nU$ U4nU$ )Nr   dim)r   repeat_interleaverk   cossin)rG   rf   r   projected_keypoints
embeddingscosinessinesoutputs           rJ   forward"LightGluePositionalEncoder.forward   sl     #nnY7(::1":E
))J'		*%%
6J*2 R\P]rL   )r   F)rQ   rR   rS   rT   r%   r   rk   r   r[   rn   r   r\   r]   r^   s   @rJ   r   r      sb    l l
 LQ		=AD[		u||	uU\\5<<%?@	@	 	rL   r   c                   >  ^  \ rS rSrS\S\4U 4S jjr    SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\
\   S\	\R                  \R                  S-  4   4S jjrSrU =r$ )LightGlueAttention   r   	layer_idxc                 &   > [         TU ]  5         U ?g rz   )rE   r   
rotary_embrG   r   r   rI   s      rJ   r   LightGlueAttention.__init__   s    OrL   Nri   position_embeddingsattention_maskencoder_hidden_statesencoder_attention_maskrH   rx   c                    UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	US Ln
U
(       a  UOUnU
(       a  UOUnU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUb  Uu  nn[        XUU5      u  p[        R                  " U R                  R                  [        5      nU" U U	UUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                   " / UQSP76 R#                  5       nU R%                  U5      nUU4$ )Nr      r   r7   )dropoutscaling)shapehead_dimq_projview	transposek_projv_projr   r   get_interfacer   _attn_implementationr   trainingr8   r   reshape
contiguouso_proj)rG   ri   r   r   r   r   rH   input_shapehidden_shapequery_statesis_cross_attentioncurrent_statescurrent_attention_mask
key_statesvalue_statesr   r   attention_interfaceattn_outputattn_weightss                       rJ   r   LightGlueAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST2$>2D.-;M!7Sa[[055lCMMaQRS
{{>277EOOPQSTU**HC';LVY[^'_$L(?(M(MKK,,.E)
 %8"	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((rL   r?   )NNNN)rQ   rR   rS   rT   r%   rX   r   rk   r   rn   r   r   r   r\   r]   r^   s   @rJ   r   r      s     3  IM.2596:*)||*) #5<<#=>E*) t+	*)
  %||d2*) !&t 3*) -.*) 
u||U\\D00	1*) *)rL   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueMLP   r   c                    > [         TU ]  U5        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  SS9U l        g )NT)elementwise_affine)rE   r   r   r   rC   fc1	LayerNorm
layer_normr   s     rJ   r   LightGlueMLP.__init__   sG     99V55v7O7OP,,v'?'?TXYrL   ri   rx   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rz   )r   r   activation_fnfc2)rG   ri   s     rJ   r   LightGlueMLP.forward  sB    /6**=9/rL   )r   r   rQ   rR   rS   rT   r%   r   rk   r   r   r\   r]   r^   s   @rJ   r   r      s1    Z Z
U\\ ell  rL   r   c                     ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\R                  S\R                  S\	S	-  S
\	S	-  S\
\R                  \
\R                     S	-  \
\R                     S	-  4   4S jjrSrU =r$ )LightGlueTransformerLayeri  r   r   c                    > [         TU ]  5         [        X5      U l        [	        U5      U l        [        X5      U l        [	        U5      U l        g rz   )rE   r   r   self_attentionr   self_mlpcross_attention	cross_mlpr   s      rJ   r   "LightGlueTransformerLayer.__init__  s@    0C$V,1&D%f-rL   descriptorsrf   r   r   Noutput_attentionsrx   c                    U(       a  SOS nU(       a  SOS nU(       a  Xa4-   nUR                   u  pn
U R                  UUUUS9u  p[        R                  " X/SS9nU R	                  U5      nX-   nU(       a  X4nUR                  SSX5      R                  S5      R                  XU
5      nUb6  UR                  SSSSU	5      R                  S5      R                  USSU	5      OS nU R                  UUUUS9u  nn[        R                  " UU/SS9nU R                  U5      nUU-   nU(       a4  UU4nUUR                  XU
5      4-   W-   UR                  XU
5      4-   U-   nU(       a	  X|4-   U4-   nXU4$ )Nr?   )r   r   r   r   r   r   r   )r   r   r   )	r   r   rk   catr   r   flipr   r   )rG   r   rf   r   r   r   all_hidden_statesall_attentions
batch_sizenum_keypointsr*   attention_outputself_attentionsintermediate_statesoutput_statesself_attention_descriptorsself_attention_hidden_statesr   r   cross_attention_outputcross_attentionscross_intermediate_statescross_output_statescross_attention_hidden_statess                           rJ   r   !LightGlueTransformerLayer.forward  s    #7BD0d 1N B4?4E4E1
> -1,?,? ))/	 -@ -
) $ii(GRP&9:%0%@",?+O( '..r1mTT!WWZ? 	 ) ""2q!Q>CCAFNNz[\^_ano 	 483G3G&"7#9/	 4H 4
0 0 %*II/IKa.bhj$k!"nn-FG03FF-FH[,\)!-55jQ_`bc./ &&z.QST 0	0  +.@@DTCVVN~==rL   )r   r   r   r   )FF)rQ   rR   rS   rT   r%   rX   r   rk   r   r[   rn   r   r\   r]   r^   s   @rJ   r   r     s    . .3 . -2).H>\\H> <<H> 	H>
 #TkH>  $;H> 
u||U5<<047u||9Lt9SS	TH> H>rL   r   
similaritymatchability0matchability1rx   c                    U R                   u  p4n[        R                  R                  U5      [        R                  R                  U5      R	                  SS5      -   n[        R                  R                  U S5      n[        R                  R                  U R	                  SS5      R                  5       S5      R	                  SS5      nU R                  X4S-   US-   4S5      n	Xx-   U-   U	SS2SU2SU24'   [        R                  R                  UR                  S5      * 5      U	SS2SS2S4'   [        R                  R                  UR                  S5      * 5      U	SS2SSS24'   U	$ )z;create the log assignment matrix from logits and similarityr   r   r   r   N)	r   r   
functional
logsigmoidr   log_softmaxr   new_fullsqueeze)
r   r   r   r   num_keypoints_0num_keypoints_1certaintiesscores0scores1scoress
             rJ   sigmoid_log_double_softmaxr
  ^  sM    4>3C3C0J--**=9BMM<T<TUb<c<m<mnoqr<ssKmm''
A6Gmm''
(<(<R(D(O(O(QSTU__`bdfgG  *.A?UVCV!WYZ[F4;4E4SF1 0 00111=3H3H3L2LMF1crc2:11=3H3H3L2LMF1b#2#:MrL   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jr	S	r
U =r$ )
LightGlueMatchAssignmentLayerim  r   c                    > [         TU ]  5         UR                  U l        [        R                  " U R                  U R                  SS9U l        [        R                  " U R                  SSS9U l        g )NTr   r   )rE   r   r*   r   r   final_projectionmatchabilityr   s     rJ   r   &LightGlueMatchAssignmentLayer.__init__n  sY    $33 "		$*=*=t?R?RY] ^IId&9&914HrL   r   rh   rx   c                    UR                   u  p4nU R                  U5      nU[        R                  " U R                  UR
                  S9S-  -  nUR                  US-  SXE5      nUS S 2S4   nUS S 2S4   nXxR                  SS5      -  n	Ub  UR                  US-  SU5      nUS S 2S4   R                  S5      n
US S 2S4   R                  S5      R                  SS5      nX-  nU	R                  US:H  [        R                  " U	R                  5      R                  5      n	U R                  U5      nUR                  US-  SUS5      nUS S 2S4   nUS S 2S4   n[        XU5      nU$ )Ndeviceg      ?r   r   r   r   r   )r   r  rk   tensorr*   r  r   r   	unsqueezemasked_fillfinfodtypeminr  r
  )rG   r   rh   r   r   r*   m_descriptorsm_descriptors0m_descriptors1r   mask0mask1r  matchability_0matchability_1r	  s                   rJ   r   %LightGlueMatchAssignmentLayer.forwardu  sy   4?4E4E1
>--k:%T5H5HQ^QeQe(fjn(nn%--jAoq-`&q!t,&q!t,#&>&>r2&FF
<<
aMBDAJ((,EAJ((,66r2>E=D#//	5;;zGWGW;X;\;\]J ((5#++J!OQqQ%ad+%ad+ ,JWrL   c                     U R                  U5      n[        R                  R                  U5      R	                  S5      nU$ )z0Get matchability of descriptors as a probabilityr   )r  r   r   sigmoidr  )rG   r   r  s      rJ   get_matchability.LightGlueMatchAssignmentLayer.get_matchability  s7    ((5}},,\:BB2FrL   )r*   r  r  )rQ   rR   rS   rT   r%   r   rk   r   r   r$  r\   r]   r^   s   @rJ   r  r  m  sW    I I5<< u||  4ELL U\\  rL   r  c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueTokenConfidenceLayeri  r   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g )Nr   )rE   r   r   r   r*   tokenr   s     rJ   r   &LightGlueTokenConfidenceLayer.__init__  s&    YYv44a8
rL   r   rx   c                     U R                  UR                  5       5      n[        R                  R	                  U5      R                  S5      nU$ )Nr   )r)  detachr   r   r#  r  )rG   r   r)  s      rJ   r   %LightGlueTokenConfidenceLayer.forward  s=    

;--/0%%e,44R8rL   )r)  r   r^   s   @rJ   r'  r'    s/    9 9
5<< ELL  rL   r'  c                   <    \ rS rSr% Sr\\S'   SrSrSr	Sr
SrSrS	rg
)LightGluePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
r   r'   pixel_values)imageFTr?   N)rQ   rR   rS   rT   rU   r%   rW   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_supports_flash_attn_supports_sdpar\   r?   rL   rJ   r/  r/    s1    
 #$O!&+#NrL   r/  r	  rw   c                 (   U R                   u  n  nU SS2SS2SS24   R                  S5      nU SS2SS2SS24   R                  S5      nUR                  nUR                  n[        R                  " UR                   S   UR
                  S9S   n[        R                  " UR                   S   UR
                  S9S   n	XR                  SU5      :H  n
XR                  SU5      :H  nUR                  R                  5       nUR                  S5      n[        R                  " XU5      n[        R                  " XR                  SU5      U5      nXU:  -  nXR                  SU5      -  n[        R                  " XS5      n[        R                  " UUS5      n[        R                  " Xg/5      R                  SS5      R                  US-  S5      n[        R                  " X/5      R                  SS5      R                  US-  S5      nUU4$ )z1obtain matches from a score matrix [Bx M+1 x N+1]Nr   r   r   r  r   )r   maxindicesrk   aranger  gathervaluesexp
new_tensorwherestackr   r   )r	  rw   r   _max0max1matches0matches1indices0indices1mutual0mutual1zeromatching_scores0matching_scores1valid0valid1rd   re   s                      rJ   get_matches_from_scoresrP    s   ||J1!SbS#2#+""1%D!SbS#2#+""1%D||H||H ||HNN1-hooFtLH||HNN1-hooFtLH//!X66G//!X66G ;;??D??1D{{7$7{{7,C,CAx,PRVW945F}}Q11F {{6R0H{{68R0Hkk8./99!Q?GG
UVXZ[Gkk#3"FGQQRSUVW__`jmn`nprsOO##rL   rf   heightwidthc                     [         R                  " X!/U R                  U R                  S9S   nUS-  nUR	                  S5      R
                  S-  nXSSSS24   -
  US   -  n U $ )a}  
Normalize keypoints locations based on image image_shape

Args:
    keypoints (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`):
        Keypoints locations in (x, y) format.
    height (`int`):
        Image height.
    width (`int`):
        Image width.

Returns:
    Normalized keypoints locations of shape (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`).
r  r  Nr   r   .).NN)rk   r  r  r  r9  r=  )rf   rQ  rR  sizeshiftscales         rJ   normalize_keypointsrX    sl     <<	0@0@	XY]^D1HEHHRL!#E3a<00E/4JJIrL   zV
    LightGlue model taking images as inputs and outputting the matching of them.
    c                   @  ^  \ rS rSrSrS\4U 4S jjrS\S\4S jr	 S%S\
R                  S	\
R                  S
\S-  S\\
R                  \\
R                  \
R                  4   4   4S jjrS\
R                  S\S\
R                  S\
R                  S\
R                  4
S jrS&S jrS\
R                  S\
R                  S\S\
R                  4S jrS\
R                  S	\
R                  S\
R                  S\
R                  S\
R                  S\
R                  S\4S jrS rS\
R                  S\
R                  S\
R                  S\
R                  S\\
R                  \
R                  4   4
S jr   S'S	\
R                  S\
R                  S\S\S\
R                  S-  S\S-  S
\S-  S\\
R                  \
R                  \
R                  \\4   4S  jjr\\   S'S!\
R0                  S"\
R2                  S-  S\S-  S
\S-  S\\-  4
S# jj5       5       rS$rU =r$ )(LightGlueForKeypointMatchingi  aJ  
LightGlue is a model matching keypoints in images by leveraging detections from a keypoint detector such as
SuperPoint. It is based on the SuperGlue architecture and is designed to be lightweight and efficient.
It consists of :
    1. Keypoint Encoder
    2. A Graph Neural Network with self and cross attention layers
    3. Matching Assignment layers

The correspondence ids use -1 to indicate non-matching points.

Philipp Lindenberger, Paul-Edouard Sarlin and Marc Pollefeys. LightGlue: Local Feature Matching at Light Speed.
In ICCV 2023. https://huggingface.co/papers/2306.13643
r   c           
      &  > [         TU ]  U5        [        R                  " UR                  5      U l        UR                  R                  U l        UR                  U l        UR                  U l
        UR                  U l        UR                  U l        UR                  U l        U R                  U R                  :w  a0  [        R                  " U R                  U R                  SS9U l        O[        R"                  " 5       U l        [%        U5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[-        XS9PM     sn5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[1        U5      PM     sn5      U l        [        R(                  " [+        UR                  S-
  5       Vs/ s H  n[5        U5      PM     sn5      U l        U R9                  5         g s  snf s  snf s  snf )NTr   )r   r   )rE   r   r   from_configr(   keypoint_detectordescriptor_decoder_dim keypoint_detector_descriptor_dimr*   r,   
num_layersr3   r0   r1   r   r   input_projectionIdentityr   positional_encoder
ModuleListranger   transformer_layersr  match_assignment_layersr'  token_confidence	post_init)rG   r   irB  rI   s       rJ   r   %LightGlueForKeypointMatching.__init__  s    !>!J!J6KjKj!k060O0O0f0f-$33 22 & 7 7 & 7 7 & 7 7$"G"GG$&IId.S.SUYUhUhos$tD!$&KKMD!"<V"D"$--EJ6KcKcEdeEd&v;Ede#
 (*}}<A&BZBZ<[\<[q*62<[\(
$ !#<A&BZBZ]^B^<_`<_q*62<_`!
 	 f ] as   HH	Hlayer_indexrx   c                     SS[         R                  " SU-  U R                  -  5      -  -   n[         R                  " USS5      $ )z-scaled confidence threshold for a given layerg?r2   g      r   r   )npr>  r`  clip)rG   rl  rw   s      rJ   _get_confidence_threshold6LightGlueForKeypointMatching._get_confidence_threshold  s;    #tk'9DOO'K LLL	wwy!Q''rL   r   rf   r   Nc                     UR                  5       R                  5       nU R                  U5      nU R                  X#S9nXE4$ )Nr   )r,  r   ra  rc  )rG   r   rf   r   projected_descriptorskeypoint_encoding_outputs         rJ   _keypoint_processing1LightGlueForKeypointMatching._keypoint_processing  sI     "((*557 $ 5 5k B#'#:#:9#:#p $>>rL   keypoint_confidencesrh   
num_pointsc                 l   UR                   u  pVX R                  S-
  :  ap  UR                  US:H  S5      nUR                  US-  S5      nU R	                  U5      nSX:  R                  5       R                  SS9U-  -
  nXR                  :  n	U	$ [        R                  " U[        R                  S9n	U	$ )zRevaluate whether we should stop inference based on the confidence of the keypointsr   r   r   r   g      ?r   r  )r   r`  r  r   rp  rY   sumr0   rk   onesr[   )
rG   rx  rl  rh   ry  r   rB  rw   ratio_confidentearly_stopped_pairss
             rJ   _get_early_stopped_image_pairs;LightGlueForKeypointMatching._get_early_stopped_image_pairs'  s     


1,, $8#C#CDAIq#Q #7#?#?
aQS#T 66{CI!%9%E$L$L$N$R$RWX$R$Y\f$ffO"14I4I"I
 #" #(**Zuzz"J""rL   c                 v    Ub  X   nX$   nU R                   U   " X5      n[        XPR                  5      u  pgXg4$ rz   )rg  rP  r3   )rG   r   rh   rl  early_stopsr	  rd   re   s           rJ   _get_keypoint_matching3LightGlueForKeypointMatching._get_keypoint_matching:  sI    "%2K$D--k:;M#:6CXCX#Y ''rL   confidencesr	  c                 \    USU R                   -
  :  nUb  XAU R                  U5      :*  -  nU$ )z#mask points which should be removedr   )r1   rp  )rG   r  r	  rl  keeps        rJ   _get_pruning_mask.LightGlueForKeypointMatching._get_pruning_maskB  s:    T2223"4#A#A+#NNNDrL   r:  prune_outputc                   ^ UR                   u  n  n	U R                  U   R                  U5      n
U R                  XjU5      mTR	                  US:H  [
        R                  " S5      5      mU4S jXS   US   TU4 5       u  ppn[        U5       H  nUUUU   4==   S-  ss'   M     S XX4 5       u  ppX4n[        USSS9nUUXU4$ )	zv
For a given layer, prune keypoints based on the confidence of the keypoints and the matchability of the
descriptors.
r   Fc              3   v   >#    U  H(  n[        UT5       VVs/ s H	  u  p#X#   PM     snnv   M*     g s  snnf 7frz   )zip).0r  trh   pruned_keypoints_masks       rJ   	<genexpr>JLightGlueForKeypointMatching._do_layer_keypoint_pruning.<locals>.<genexpr>]  s<      c
c %(0E$FG$FQW$FGc Hs   939r   c              3   6   #    U  H  n[        US S9v   M     g7f)T)batch_firstNr   )r  pruned_tensors     rJ   r  r  e  s      S
!j D9!js   Tr   r  padding_value)	r   rg  r$  r  r  rk   r  re  r   )rG   r   rf   rh   r:  r  rx  rl  r   rB  descriptors_matchabilitypruned_descriptorspruned_keypoints_0pruned_keypoints_1pruned_maskpruned_indicesrj  pruned_keypointsr  s                     @rJ   _do_layer_keypoint_pruning7LightGlueForKeypointMatching._do_layer_keypoint_pruningI  s    ',,
Aq#'#?#?#L#]#]^i#j  $ 6 67Kgr s 5 A A$!)U\\Z_M` ac
&!ilDY[bcc
_0BQ_ z"AN1--.!3. #S
"4J\!jS
O0B /C%n$VXY!#3^R^^^rL   c                    ^ [         R                  " T5      m[         R                  " TR                  S   5      nTU   nTU   mS XB4 5       u  pBS XS4 5       u  pSU4S jUUUU4 5       u  pEp#X#XE4$ )Nr   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   r  Nr   r  r  s     rJ   r  MLightGlueForKeypointMatching._concat_early_stopped_outputs.<locals>.<genexpr>{  s       3
C TDC   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   r  Nr   r  s     rJ   r  r    s       >
N TCNr  c              3   .   >#    U  H
  nUT   v   M     g 7frz   r?   )r  r  early_stops_indicess     rJ   r  r    s#      g
 &'s   )rk   rA  r;  r   )rG   r  final_pruned_keypoints_indices!final_pruned_keypoints_iterationsrd   re   idsorder_indicess    `      rJ   _concat_early_stopped_outputs:LightGlueForKeypointMatching._concat_early_stopped_outputsn  s     $kk*=>ll.44Q78+C01-@3
"C3
/>
*N>
:g
 .1	g
c"@ .RYjjrL   rd   re   r   c                   ^ UR                   u  mnU4S jXU4 5       u  pnUS S 2S4   nUS S 2S4   nUS S 2S4   nUS S 2S4   n	US S 2S4   n
US S 2S4   n[        R                  " TS-  SU4SUR                  UR                  S9n[        R
                  " TS-  SU4UR                  UR                  S9n[        TS-  5       H  n[        R                  " X   S:H  SX~   R                  SX   R                  SS95      5      XSXn   4'   [        R                  " X   S:H  SXn   R                  SX   R                  SS95      5      XSX~   4'   X   XSXn   4'   X   XSX~   4'   M     X4$ )Nc              3   N   >#    U  H  oR                  TS -  S S5      v   M     g7f)r   r   N)r   )r  r  r   s     rJ   r  JLightGlueForKeypointMatching._do_final_keypoint_pruning.<locals>.<genexpr>  s'      -
AdvNN:?Ar22Ads   "%r   r   r   r   rT  )r  )
r   rk   fullr  r  zerosre  r@  r<  clamp)rG   r:  rd   re   r   rB  rG  rH  rE  rF  rL  rM  _matches_matching_scoresrj  r   s                  @rJ   _do_final_keypoint_pruning7LightGlueForKeypointMatching._do_final_keypoint_pruning  s     
A-
BITcAd-
)/ 1a4=1a4=1a4=1a4=*1a40*1a40 ::zQ=A2gnndkdqdqr ;;1_a/oNcNc
 zQ'A*/++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' +0++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' 3C2E8;./2B2E8;./ ( ))rL   rQ  rR  r   c           
      	  ^' U(       a  SOS nU(       a  SOS n	UR                   S   S:X  aQ  UR                   S S n
UR                  U
S[        R                  S9UR	                  U
5      UR	                  U
5      UU	4$ UR
                  nUR                   u  pp[        R                  " UR                  US5      SS9nUR                  US-  US5      nUb  UR                  US-  U5      OS nUR                  US-  XR                  5      n[        R                  " US-  US9n[        XU5      nU R                  X!US	9u  nnUS   nU R                  S:  nU R                  S:  n/ n/ n/ n/ n/ n[        R                  " SXS9R                  US-  S5      n[        R                  " U5      n[!        U R"                  5       GH`  nUb$  [%        U R&                  US S 2SS2S S 24   US
 S9nO1[        R(                  " XR+                  5       S   4UR
                  S9nU R,                  U   " UUUUUS9nUu  nnnU(       a  UU-   nU(       a  U	U-   n	U(       Ga  UU R"                  S-
  :  a'  U R.                  U   " U5      n U R1                  U UX_S9n!O#[        R(                  " U[        R2                  S9n![        R4                  " U!5      (       Ga  U!R7                  S5      m'UT'   n"U R9                  X%UT'S9u  n#n$UR;                  [=        U"5      5        UR;                  [=        U#5      5        UR;                  [=        U$5      5        U(       a:  UR;                  [=        UT'   5      5        UR;                  [=        UT'   5      5        UU!)    n[?        U'4S jX!S   US   UU4 5       5      u  nn%n&nnU%U&4nU(       a  [?        U'4S jUUW 4 5       5      u  nnn [        R@                  " U!5      (       a    O+U(       d  GMD  U RC                  UUUUUW U5      u  p!nnnGMc     U(       a9  U(       a2  U RE                  UUUUU5      u  nnnnU RG                  UUUU5      u  nnOEU R9                  X%U R"                  S-
  5      u  nn[        R                  " U5      U R"                  -  nUR                  USU5      nUUUUU	4$ )Nr?   r   r   r   r{  r   r   r  rs  c                  H    [         R                  " S[         R                  S9$ )NTr{  )rk   r  r[   )argss    rJ   <lambda>@LightGlueForKeypointMatching._match_image_pair.<locals>.<lambda>  s    ELLUZZ4XrL   )r   inputs_embedsr   and_mask_functionr   )r   r   r   )ry  )r  c              3   0   >#    U  H  nUT)    v   M     g 7frz   r?   r  r  r  s     rJ   r  ALightGlueForKeypointMatching._match_image_pair.<locals>.<genexpr>#  s       V&dF |,&d   c              3   0   >#    U  H  nUT)    v   M     g 7frz   r?   r  s     rJ   r  r  )  s$      l+ #K<0+r  )$r   r  rk   rX   	new_zerosr  r|  r   r_  r;  rX  rv  r0   r1   expand	ones_likere  r`  r   r   r}  rU  rf  rh  r  r[   anyr   r  extendr   rn   allr  r  r  )(rG   rf   r   rQ  rR  rh   r   r   r   r   r   r  r   rB  initial_num_keypointsnum_points_per_pairimage_indicesru  do_early_stopdo_keypoint_pruningr  rd   re   r  r  pruned_keypoints_indicespruned_keypoints_iterationsrl  extended_attention_masklayer_outputri   	attentionrx  r  early_stopped_image_indicesearly_stopped_matchesearly_stopped_matching_scoreskeypoints_0
keypoint_1r  s(                                          @rJ   _match_image_pair.LightGlueForKeypointMatching._match_image_pair  s^    #7BD0d??1"OOCR(E""5"EII">##E*##E*!  !!2;///
,#iiZ(D!L%%j1n6KQO	FJFVt||JN,AB\`!))*q.:OQvQvwZ!^FC'	5A	040I0I9M 1J 1
-- -Q/	 --1 #33a7 )+&,.)#(<<3H#X#_#_`jmn`npr#s &+oo6N&O# 1K*C;;"-a1ai"8#'&X+' +0**jBRBRBTUWBX5Ybkbrbr*s'22;?6%9"3L 5A1K	#$5$E! !/)!;1!44+/+@+@+Mk+Z( +/*M*M,k4 +N +'
 +0**Zuzz*R'99011 #6"G"G"JK2?2L/KOKfKf#;K Lg LH)+H (..t4O/PQNN4(=#>?#**40M+NO*6==dC[\gCh>ij9@@FabmFnAop +>?R>R*S'PU V'2aL)A,PTVc&dV QMKj$ "-j 9I*fk l !9 ; 4+l gc02MOc 99011"" 33#!03,# d(@$Hc] 2t 0 22'25# h*,MwXg (,'F'F.%	($G_ (,'B'B;VZVeVehiVi'j$G_050PSWSbSb0b-,M,U,U0-
)
 -
 	
rL   r0  labelsc                 $   S nUb  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUR                  S:w  d  UR                  S5      S:w  a  [        S5      eUR                  u  pxpnUR                  US-  XU5      nU R                  U5      nUS S u  ppUR                  USSS5      R                  U5      nUR                  USSU R                  5      R                  U5      nUR                  USS5      nUR                  5       nUS S 2S S 2S S 2S4   U-  US S 2S S 2S S 2S4'   US S 2S S 2S S 2S4   U
-  US S 2S S 2S S 2S4'   U R                  UUU
UUUUS	9u  nnnnn[        UUUUUUUUS
9$ )Nz9LightGlue is not trainable, no labels should be provided.   r   r   zOInput must be a 5D tensor of shape (batch_size, 2, num_channels, height, width)r-   r   r   )rh   r   r   )rc   rd   re   rf   rg   rh   ri   rj   )rN   r   r   r   ndimrU  r   r   r]  tor_  cloner  ra   )rG   r0  r  r   r   rH   rc   r   rB  channelsrQ  rR  keypoint_detectionsrf   r   rh   absolute_keypointsrd   re   rg   ri   rj   s                         rJ   r   $LightGlueForKeypointMatching.forwardf  s    XYY1B1N-TXT_T_TqTq$8$D $++JjJj 	 !\%6%6q%9Q%>noo1=1C1C.
x#++JNHeT"44\B*=bq*A'	k%%j!R;>>|L	!))*aT=b=bcffgst||J2.&__.);Aq!QJ)G%)O1aA:&);Aq!QJ)G&)P1aA:&EIE[E[/!5 F\ F
B%
 /+'!	
 		
rL   )r0   r*   r3   ra  r]  r_  rg  r`  rc  rh  rf  r1   r   rz   )NNN)rQ   rR   rS   rT   rU   r%   r   rX   rY   rp  rk   r   r[   rn   rv  r  r  r  r  r  r  r  r   r   rl   
LongTensorra   r   r\   r]   r^   s   @rJ   rZ  rZ    s    <(S (U ( gl? <<?49LL?X\_cXc?	u||U5<<#=>>	??#$)LL#?B#JO,,#didpdp#	#&(U\\ 5<< ^a fkfrfr #_\\#_ <<#_ ll	#_
 #_ ll#_ $ll#_ #_Jk@#*#* #* 	#*
 ||#* 
u||U\\)	*#*V %))-,0q
<<q
 \\q
 	q

 q
 llT!q
  $;q
 #Tkq
 
u||U\\5<<E	Fq
f  +/)-,04
''4
   4'4
  $;	4

 #Tk4
 
0	04
  4
rL   rZ  )r/  rZ  r%   rs   r   )Mcollections.abcr   dataclassesr   numpyrn  rk   huggingface_hub.dataclassesr   r   torch.nn.utils.rnnr   configuration_utilsr
   masking_utilsr   modeling_flash_attention_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   utils.import_utilsr   autor   r   auto.modeling_autor   clip.modeling_clipr   cohere.modeling_coherer   llama.modeling_llamar   r   (superglue.image_processing_pil_supergluer   $superglue.image_processing_supergluer    r!   r<   r"   
get_loggerrQ   loggerr%   ra   rp   rs   r   Moduler   r   r   r   r   r
  r  r'  r/  rY   rn   rP  rX   rX  rZ  __all__r?   rL   rJ   <module>r      sL   % !   .  + 3 6 B F & W W 1 * - > ( 9 J Q i ) 
		H	% 9:AR& AR  ;ARH   7k  7  7F	$A 	X5 X 
:X!; X X " /) /) /)d7 P>		 P>f-2\\JO,,
\\&BII &R	BII 	   $ELL $U $uU\\[`[g[gMgGh $@5<<  S U\\ , 
o
#; o

o
drL   