+
    3-jm$                      a  0 t $ ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIHu H	t
 ^ RIt^ RIHt ^ RIHt ^ RIHtHtHt ^ RItRt. R:Ot/ RR;bRR<bR	R=bR
R>bRR?bRR@bRRAbRRBbRRCbRRDbRREbRRFbRRGbRRHbRRIbRRJbRRKbtR]R&   R. RLORR.RR./t]! RR7       ! R R 4      4       t] ! R! R"4      4       tR# R$ ltR% R& ltR' R( ltRMR) R* lltR+ R, lt]P@                  ! R-4      t!R. R/ lt"R0 R1 lt#R2 R3 lt$RNR4 R5 llt%R6 R7 lt&ROR8 R9 llt'R# )P    )annotationsN)	dataclass)Iterable)parse_qsunquoteurlparsez!https://html.duckduckgo.com/html/MontpellierzCastelnau-le-LezLattesu   PérolsMauguiou   Saint-Jean-de-VédasJuvignacClapiersGrabelsu   Villeneuve-lès-Magueloneu   Le Crèsu
   Fabrèguesu   Saint-Gély-du-FesczPalavas-les-FlotsCastriesBaillarguesMirevalzdict[str, tuple[float, float]]NEARBY_TOWNSmaisonappartementterrainT)frozenc                  6    ] tR t^5t$ R]R&   R]R&   R]R&   RtR# )
NearbyTownstrnamefloatlatlon N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       >/home/wildlama/real-estate-finder/real_estate_agent/scraper.pyr   r   5   s    
I	J	Jr%   r   c                      ] tR t^<t$ R]R&   R]R&   R]R&   R]R&   ]P                  ! ]P                  R7      tR]R&   R	t	R
]R&   R	t
R]R&   R	tR]R&   RtR]R&   RtR	# )SearchResultr   titleurlsnippetsource)default_factoryr   
fetched_atN
int | Nonepricez
str | Nonetownzfloat | Nonedistance_km        scorer   )r   r    r!   r"   r#   dataclassesfieldtimer.   r0   r1   r2   r4   r$   r   r%   r&   r(   r(   <   sW    J	HLK#))$))DJDE:D* $K$E5r%   r(   c          
     ,    V ^8  d   QhRRRRRRRRRR/# )   lat1r   lon1lat2lon2returnr   )formats   "r&   __annotate__r@   I   s/     + + +U +% +u + +r%   c                <   R p\         P                  ! W ,
          4      p\         P                  ! W1,
          4      p\         P                  ! V^,          4      ^,          \         P                  ! \         P                  ! V 4      4      \         P                  ! \         P                  ! V4      4      ,          \         P                  ! V^,          4      ^,          ,          ,           p^V,          \         P                  ! \         P
                  ! V4      4      ,          # )g     @)mathradianssincosasinsqrt)r:   r;   r<   r=   rdlatdlonas   &&&&    r&   _haversine_kmrL   I   s    A<<$D<<$Da
((4<<%
&$,,t2D)E
EQUXYQYHZ^_H_
_	`  q5499TYYq\***r%   c                    V ^8  d   QhRRRR/# )r9   r*   r   r>   r   )r?   s   "r&   r@   r@   T   s        r%   c                   \         P                  ! V 4      P                  4       p V P                  R 4      '       d   \	        RV ,           4      pM\	        V 4      pVP
                  P                  R4      '       dd   VP                  P                  R4      '       dC   \        VP                  4      pVP                  RR.4      ^ ,          pV'       d   \        V4      # V # )z//duckduckgo.com/l/?zhttps:zduckduckgo.comz/l/uddgN)htmlunescapestrip
startswithr   netlocendswithpathr   querygetr   )r*   parsedqsrO   s   &   r&   _normalise_urlr[   T   s    
--

"
"
$C
~~,--(S.)#}}.//FKK4J4J54Q4Qfll#vvftf%a(4= Jr%   c                    V ^8  d   QhRRRR/# )r9   	radius_kmr   r>   zlist[NearbyTown]r   )r?   s   "r&   r@   r@   b   s     \ \5 \-= \r%   c           	        aa \         R ,          w  oo. p\         P                  4        F@  w  pw  r4\        SSW44      pWPR,           8:  g   K$  VP                  \	        W#VR7      4       KB  	  \        VVV3R lR7      # )r	   g      ?)r   r   r   c                H   < \        SSV P                  V P                  4      # N)rL   r   r   )t
center_lat
center_lons   &r&   <lambda>%nearby_towns_within.<locals>.<lambda>i   s    }ZQUUTUTYTY'Zr%   )key)r   itemsrL   appendr   sorted)r]   townsr   r   r   distrb   rc   s   &     @@r&   nearby_towns_withinrl   b   sm    )-8J
E(..0jsZS>t##LLC@A 1 %Z[[r%   c               0    V ^8  d   QhRRRRRRRRRR	R
R/# )r9   centerr   r]   r   	max_priceintproperty_typesourcesIterable[str] | Noner>   z	list[str]r   )r?   s   "r&   r@   r@   l   sD        	
 " r%   c           
     4   \        V4      pY Uu0 uF  qfP                  kK  	  up9  d,   V \        9   d!   \        V .\        V ,          O5!  .V,           pVP	                  4       P                  4       ;'       g    R pV\        9  d   R p. pV FQ  pTP                  RP                  TRRVP                  9   d   RVP                   R2MVP                  .4      4       KS  	  RP                  R V 4       4      p	VP                  V RV	 R24       V# u upi )r    u	   à vendre"z OR c              3  |   "   T F2  pR VP                   9   d   RVP                    R2MVP                   x  K4  	  R# 5i)ru   rv   N)r   ).0r1   s   & r&   	<genexpr> build_queries.<locals>.<genexpr>   s4     e_dW[3$))3C499+Q/R_ds   :<u    à vendre ())	rl   r   r   r   rR   lowerPROPERTY_HINTSrh   join)
rn   r]   ro   rq   rr   rj   ra   queriesr1   commune_blobs
   &&&&&     r&   build_queriesr   l   s     	*Ee,effe,,<1GF:\&%9:;eC!'')//1==XMN* GHH!(+tyy(8a		{!$dii	
  ;;e_deeLNNm_La@AN- -s   Dc                    V ^8  d   QhRRRR/# )r9   	html_textr   r>   list[dict[str, str]]r   )r?   s   "r&   r@   r@      s       0D r%   c                   . p\         P                  ! R V \         P                  R7       F[  w  r#VP                  R\	        V4      R\         P
                  ! RR\        P                  ! V4      4      P                  4       /4       K]  	  \         P                  ! RV \         P                  R7       Uu. uF=  p\         P
                  ! RR\        P                  ! V4      4      P                  4       NK?  	  pp. p\        V4       FD  w  rxRVR,          RVR,          RV\        V4      8  d	   WW,          MR/p	VP                  V	4       KF  	  V# u upi )z8<a[^>]*class="result__a"[^>]*href="(.*?)"[^>]*>(.*?)</a>)flagsr*   r)   z<.*?> z-<a[^>]*class="result__snippet"[^>]*>(.*?)</a>r+   )refindallSrh   r[   subrP   rQ   rR   	enumeratelen)
r   linkshrefr)   r+   snippetsresultsidxlinkresults
   &         r&   parse_duckduckgo_resultsr      s    Ezz"]_hprptptue^D17BFF8RQUQ^Q^_dQe<f<l<l<nop v
 zz"RT]egeieijjG 	xT]]734::<j  
 %'Gu%	4=%eiZ]`cdl`mZmsuvv & Ns   2AEuU   (?<!\d)(\d{1,3}(?:[\s.,]\d{3})+|\d{4,6})\s*(?:€|eur|euros)|(?<!\d)(\d{2,3})\s*(k|K)c                    V ^8  d   QhRRRR/# )r9   textr   r>   r/   r   )r?   s   "r&   r@   r@      s       
 r%   c                d   V P                  4       pR V9   g   RV9   d   R# \        P                  V 4      pV'       g   R# VP                  4       w  r4pV'       d   \	        V4      R,          # V'       g   R# \
        P                  ! RRV4      pVP                  4       '       g   R# \	        V4      # )zprix sur demandezsur demandeNi  z[\s.,]r   )r|   PRICE_WITH_CURRENCY_REsearchgroupsrp   r   r   isdigit)r   loweredmatch
millionishshort_ksuffixcleaneds   &      r&   extract_pricer      s    jjlGW$(@"))$/E"',,.J7|d""ffYJ/G??w<r%   c                    V ^8  d   QhRRRR/# )r9   r   r   r>   ztuple[str | None, float | None]r   )r?   s   "r&   r@   r@      s     	 	c 	= 	r%   c                   V P                  4       p\        R ,          w  r#Rp\        P                  4        FS  w  pw  rgVP                  4       V9   g   K  \        W#Wg4      pV^,          e   W^,          8  g   KE  V\	        V^4      3pKU  	  V# )r	   )NN)r|   r   rg   rL   round)	r   r   rb   rc   bestr   r   r   rk   s	   &        r&   _guess_townr      s{    jjlG)-8J,8D(..0js::<7" BDAw$a.eD!n-	 1
 Kr%   c               0    V ^8  d   QhRRRRRRRRR	RR
R/# )r9   r   zdict[str, str]rn   r   r]   r   ro   rp   rq   r>   r   )r?   s   "r&   r@   r@      s9         SV gj ot r%   c                @  a
 V P                  R R4       RV P                  RR4       2P                  4       o
RpVP                  4       S
9   d
   V^,          p\        P                  VP                  4       VP                  4       .4       F  pVS
9   g   K  V^,          pK  	  \        S
4      pVe8   Ws8:  d   V^,          pM'V\	        ^Ws,
          R,          ^,          4      ,          p\        S
4      w  rV'       dN   V^,          pV	eA   V\        ^ ^V	,
          4      ,          pW8:  d   V^
,          pMWYV,
          R,          ,          p\        ;QJ d    V
3R lR
 4       F  '       g   K   RM	  R	M! V
3R lR
 4       4      '       d
   V^,          p\        V^4      # )r)   r   ru   r+   r3   i g      ?c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir`   r   )rx   tokenhaystacks   & r&   ry   score_result.<locals>.<genexpr>   s     
V*UH*Us   TF)jardinterrassegaragepiscine)	rX   r|   r}   r   minr   maxanyr   )r   rn   r]   ro   rq   r4   hintr0   r1   distancer   s   &&&&&     @r&   score_resultr      s\   **Wb)*!FJJy",E+FGMMOHE(""=#6#6#8=;N;N;P:QR8QJE S (#ERKESe/69A=>>E *NDSBM**E$Y.#55
s
V*U
Vsss
V*U
VVV
?r%   c               $    V ^8  d   QhRRRRRR/# )r9   rW   r   timeoutrp   r>   r   )r?   s   "r&   r@   r@      s!      #  S r%   c                z    \         P                  ! R RRRV /RR/VR7      pVP                  4        VP                  # )zhttps://www.bing.com/searchr?   rssqz
User-AgentzaMozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36)paramsheadersr   )requestsrX   raise_for_statusr   )rW   r   responses   && r&   fetch_bing_rssr      sI    ||%%e,   C  D	H ==r%   c                    V ^8  d   QhRRRR/# )r9   xml_textr   r>   r   r   )r?   s   "r&   r@   r@      s      S -A r%   c           	        \         P                  ! V 4      p. pVP                  R 4       F  pVP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pVP                  RVRVRV/4       K  	  V# )z./channel/itemr)   r   r   descriptionr*   r+   )ET
fromstringr   findtextrR   rh   )r   rootr   itemr)   r   r   s   &      r&   parse_bing_rss_resultsr      s    =="D$&G-.w'--2446f%++224}}]399r@@BtYLM	 /
 Nr%   c               4    V ^8  d   QhRRRRRRRRRR	R
RRR/# )r9   rn   r   r]   r   ro   rp   rq   rr   rs   limitr>   zlist[SearchResult]r   )r?   s   "r&   r@   r@      sN     " """ " 	"
 "" " "r%   c                0   \        4       p. p\        WW#VR 7       F  p\        V4      p	\        V	4       F  p
V
R,          pV'       d   W9   d   K  VP	                  V4       V
R,           RV
R,           2p\        V4      p\        V4      w  r\        V4      P                  P                  RR4      p\        V
R,          VV
R,          VVVVR7      p\        WWVR7      Vn        VP                  V4       K  	  K  	  VP                  R	 R
R7       VRV # ))rn   r]   ro   rq   rr   r*   r)   ru   r+   zwww.r   )r)   r*   r+   r,   r0   r1   r2   )rn   r]   ro   rq   c                H    V P                   V P                  ;'       g    ^ ) 3# )r   )r4   r0   )rH   s   &r&   rd   #search_properties.<locals>.<lambda>  s    !''QWW\\?!;r%   T)rf   reverseN)setr   r   r   addr   r   r   rT   replacer(   r   r4   rh   sort)rn   r]   ro   rq   rr   r   seen	collectedrW   r   r   r*   blobr0   r1   r   r,   r   s   &&&&&&            r&   search_propertiesr      s    UD$&IfY  ~E  F!%(*84Du+C#+HHSM7m_Ad9o%67D!$'E(.NDc]))11&"=F!7mY$F (y  vC  DFLV$' 5 F. NN;TNJVer%   )zsite:pap.frzsite:seloger.comzsite:leboncoin.frzsite:bienici.comzsite:green-acres.fr)g2ı.E@g9#J{@)gEJE@g8m4@)gm4E@g333333@)gsh|E@g@)gE@      @)gS:E@g[Ӽ@)gfffffE@gQk@)GzE@gQ@)gGzE@ffffff@)gA`E@gB`"@)gRE@q=
ףp@)gfffffE@g)\(@)gQE@r   )gp=
E@r   )gףp=
E@gףp=
@)r   r   )g(\E@g=
ףp=@)r   villademeurer`   )   )r	   r   i r   Nr   )(__conditional_annotations__
__future__r   r5   rP   rB   r   xml.etree.ElementTreeetreeElementTreer   r7   r   typingr   urllib.parser   r   r   r   DUCKDUCKGO_HTMLDEFAULT_SOURCESr   r#   r}   r   r(   rL   r[   rl   r   r   compiler   r   r   r   r   r   r   )r   s   @r&   <module>r      s   " "    	 " "  !  4 4 50$0)0 o0 	0
 ~0 -0 !0 0 }0  0 0 -0 =0 0 0  <!0" }#0, * ,M?	{ $   	 	 	+\B& \ 
(	B" "r%   