o n0j@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl ZddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,dd l-m.Z.d d l/m0Z0m1Z1e2d Z3e2d ej4Z5e2dZ6e2dej4j7Z8d9Z:gdZ;dZdj>ej?edZ@ddZAddZBddZCdDddZDdDddZEdDd d!ZFdedfd"d#ZGd$d%ZHe2d&ej4ZI eHd'd(ZJGd)d*d*ZKGd+d,d,eKZLGd-d.d.eZMe2d/jNZOd0d1ZPd2d3ZQdEd4d5ZRd6d7ZSGd8d9d9ZTGd:d;d;e jUZVejWjXfdd?ZZeRe<eYZYd@dAZ[dBdCZ\dS)Fz$PyPI and direct package downloading.Nwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST parse_version)log)DistutilsError) translate)Wheelunique_everseen)_read_utf8_with_fallback_cfg_read_utf8_with_fallbackz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezsz(interpret_distro_name..Nr) py_versionrDrU)r:anyrangerRisdigitrjoin)rLrSrIr_rDrUr<r\r%r%r&rs$  rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr%r&wrapperszunique_values..wrapperr)rhrir%rgr& unique_valuessrjz3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c cst|D]2}|\}}tttj|d}d|vs#d|vr8t |D]}t j |t |dVq(qdD] }||}|dkr[t ||}|r[t j |t |dVq;dS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager4r)z Home PagezDownload URLr3N)RELfinditergroupssetmapstrstripr+r:HREFr7r!urljoin htmldecoderHfindsearch)r;pagerGtagrelZrelsposr%r%r&find_external_linkss    r|c@(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCdS)z3 Feed a block of data to the hash. Nr%selfblockr%r%r&feedzContentChecker.feedcCr)zC Check the hash. Return False if validation fails. Tr%rr%r%r&is_validrzContentChecker.is_validcCr)zu Call reporter with information about the checker (hash name) substituted into the template. Nr%)rreportertemplater%r%r&reportszContentChecker.reportN)__name__ __module__ __qualname____doc__rrrr%r%r%r&r~s  r~c@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSrd) hash_namehashlibnewhashexpected)rrrr%r%r&__init__s  zHashChecker.__init__cCsBtj|d}|s tS|j|}|stS|di|S)z5Construct a (possibly null) ContentChecker from a URLr3Nr%)r7r!r8r~patternrw groupdict)clsr;rBrGr%r%r&from_urls zHashChecker.from_urlcCs|j|dSrd)rupdaterr%r%r&r zHashChecker.feedcCs|j|jkSrd)r hexdigestrrr%r%r&r#rzHashChecker.is_validcCs||j}||Srd)r)rrrmsgr%r%r&r&s zHashChecker.reportN) rrrrZcompilerr classmethodrrrrr%r%r%r&r s   rcsfeZdZdZ    dNfdd Zfdd ZdOd d ZdOd dZdOddZddZ ddZ ddZ ddZ ddZ dPddZddZdPfdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Z dQd+d,ZdRd-d.Zd/d0Zd1Zd2d3Zd4d5ZdPd6d7Zd8d9Zed:d;Zdd?Z!d@dAZ"dBdCZ#dDdEZ$edFdGZ%dHdIZ&dJdKZ'dLdMZ(Z)S)Srz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcsltj|i||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr2|)superrr, index_url scanned_urls fetched_urls package_pagesrZrrcrprrGallowsto_scanr7requesturlopenopener)rrhostsZ ca_bundleZ verify_sslrekw __class__r%r&r.s zPackageIndex.__init__cs0zt|jWn tyYdSwt|Srd)rrN Exceptionradd)rdistrr%r&r@s   zPackageIndex.addFc Cs||jvr |s dSd|j|<t|s||dStt|}|r.||s(dS|d||s7|r7||jvrAtt|j |dS||sMd|j|<dS| d|d|j|<d}| |||}|durhdSt |t jjr||jdkr|| d|jd|j|j<d|jd d vr|dS|j}|}t |tst |t jjrd }n|jd pd }||d }|t|D]} t j|t| d} |!| q|"|j#rt$|dddkr|%||}dSdSdS)z.)filterrWr?rr itertoolsstarmap scan_egg_link)r search_pathdirsZ egg_linksr%r%r&scan_egg_linkss zPackageIndex.scan_egg_linkscCsttj||}ttdttj| }t |dkrdS|\}}t tj||D]}tjj|g|R|_ t |_||q+dS)Nr^)rrWr?rcrrrprqrr splitlinesrRr rLrrDr)rr?rcontentlinesZegg_pathZ setup_pathrr%r%r&rs  zPackageIndex.scan_egg_linkcCsd}||js |Stttjj|t|jdd}t|dks)d|dvr+|St |d}t |d}d|j | i|<t|t|fS)N)NNr2r^r6rrT)r-rrrpr7r!r9rRr:r r r setdefaultr+r )rrZNO_MATCH_SENTINELr<pkgverr%r%r&_scans &  zPackageIndex._scanc Cst|D]}z|tj|t|dWqty!Yqw||\}}|s-dSt ||D]$}t |\}}| drQ|sQ|rL|d||f7}n| || |q2tdd|S)z#Process the contents of a PyPI pagerr.pyz #egg=%s-%scSsd|dddS)Nz%srr^)rH)mr%r%r&sz,PackageIndex.process_index..)rsrmrr7r!rtrurHr"r|rCr,need_version_infoscan_urlPYPI_MD5sub) rr;rxrGrrnew_urlr/fragr%r%r&rs&"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allrr;r%r%r&rszPackageIndex.need_version_infocGs<|j|jvr|r|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrrrrer%r%r&rs  zPackageIndex.scan_allcCsz||j|jd|j|js||j|jd|j|js)||t|j|jdD]}||q3dS)Nr2r%) rr unsafe_namerrkeyrMnot_found_in_indexr)r requirementr;r%r%r& find_packagess  zPackageIndex.find_packagescsN|||||jD]}||vr|S|d||qt||S)Nz%s does not match %s)prescanrrrrobtain)rr installerrrr%r&rs zPackageIndex.obtaincCsL||jd||s$|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrrrrWunlinkrrr.r?rS)rcheckerrXtfpr%r%r& check_hash s zPackageIndex.check_hashcCsN|D]"}|jdust|r|dstt|r||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr-rrrappend)rurlsr;r%r%r&add_find_linkss   zPackageIndex.add_find_linkscCs"|jr tt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrprrr%r%r&r(s zPackageIndex.prescancCs<||jr |jd}}n|jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)rrmethrr%r%r&r.s   zPackageIndex.not_found_in_indexcCsvt|ts1t|}|r%|||}t|\}}|dr#||||}|Stj |r-|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. rrLN) rrr _download_urlrCr, gen_setuprWr?rr'rfetch_distribution)rr#tmpdirr=foundr/rBr%r%r&r49s     zPackageIndex.downloadc sd|td}d fdd }|r%|||}|s0|dur0|||}|durAjdur=||}|durP|sP|||}|duradr[dp\d|dSd||j|jd S) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur}||jD]<}|jtkr$s$|vr#d||q ||vo/|jtkp/ }|rG|j}||_t j |jrG|Sq dS)Nz&Skipping development or system egg: %s) rrDrrrrr4rLdownload_locationrWr?r)reqenvrtestloc develop_okrZskippedsourcerr%r&rvys& z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rLrd)rrorrrrcloner) rrr force_scanrrZ local_indexrrvr%rr&r[s4         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)rrL)rrrrrrr%r%r&fetchszPackageIndex.fetchc Cst|}|rddt||ddDpg}t|dkr|tj|}tj||krGtj ||}tj |r?tj ||sGt |||}ttj |dddd}|d|d j|d jtj|d fWd|S1suwY|S|rtd ||ftd ) NcSsg|]}|jr|qSr%)rN)r[dr%r%r& s z*PackageIndex.gen_setup..rzsetup.pywzutf-8)encodingzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) rzCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rFrGrrHrRrWr?rSdirnamercrsamefileshutilcopy2openwriterMrNsplitextr) rrXrBrrGrrSdstrr%r%r&r sJ      zPackageIndex.gen_setupi c Cs2|d|d}zt|}||}t|tjjr%td||j |j f|}d}|j }d}d|vrI| d} t tt| }||||||t|d1} ||} | ro|| | | |d 7}||||||nnqP|||| Wdn1swY|W|r|SS|r|ww) NzDownloading %szCan't download %s: %s %srr3zcontent-lengthzContent-LengthwbTr)rrrrrr7rrrrr dl_blocksizeget_allmaxrpint reporthookr$rrr%rr) rr;rXfprrblocknumbssizesizesrrr%r%r& _download_tosF            zPackageIndex._download_tocCsdSrdr%)rr;rXr/Zblksizer1r%r%r&r-zPackageIndex.reporthookc Cs|dr t|Szt||jWSttjjfyD}z$ddd|j D}|r0| ||n t d||f|WYd}~dSd}~wt j jyY}z|WYd}~Sd}~wt j jy}z|rl| ||jn t d||jf|WYd}~dSd}~wtjjy}z|r| ||jn t d||jf|WYd}~dSd}~wtjjtfy}z|r| ||n t d||f|WYd}~dSd}~ww)Nr cSsg|]}t|qSr%)rq)r[argr%r%r&rsz)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r- local_openopen_with_authrr"httpclient InvalidURLrcrerrr7rrURLErrorreason BadStatusLineline HTTPExceptionOSError)rr;warningvrr%r%r&rsV  zPackageIndex.open_urlcCsvt|\}}|rd|vr|dddd}d|vs nd}|dr(|dd}tj||}|||p:|||S)Nz...\_Z__downloaded__rJr*)rCreplacer,rWr?rc _download_vcs_download_other)rr;rr.rBrXr%r%r&r 4s   zPackageIndex._download_urlcCsLtj|j}|d\}}}tddgdgt|}tt|h|@dS)a  >>> rvcs = PackageIndex._resolve_vcs >>> rvcs('git+http://foo/bar') 'git' >>> rvcs('hg+https://foo/bar') 'hg' >>> rvcs('git:myhost') 'git' >>> rvcs('hg:myhost') >>> rvcs('http://foo/bar') +svngithgN) r7r!urlsplitr= partitionroboolnextiter)r;r=preseppostallowedr%r%r& _resolve_vcsEs zPackageIndex._resolve_vcsc Cs||}|s dS|dkrtd||d\}}}||\}}|d|d|d|t|dd||gt|d |d d|g|d |d d d |dgd}|durb|d|t|||S)NrKz/Invalid config, SVN download is not supported: r6zDoing z clone from z to rz--quietz-Ccheckoutz--cwdupz-rz-q)rLrMz Checking out )rWrrO_vcs_split_rev_from_urlr subprocess check_calldict)rr;Z spec_filenamevcsrXrFrevZ co_commandsr%r%r&rHXs& zPackageIndex._download_vcscCsDtj|j}|dkrtjtj|jS||d| ||S)NrT) r7r!rNr=r url2pathnamer8r?r_attempt_download)rr;rXr=r%r%r&rIqs   zPackageIndex._download_othercCs||ddS)NT)rrr%r%r&ryrzPackageIndex.scan_urlcCs2|||}d|ddvr||||S|S)Nrrr)r3rr+_invalid_download_html)rr;rXrr%r%r&ra|s zPackageIndex._attempt_downloadcCst|td|)NzUnexpected HTML page found at )rWrr)rr;rrXr%r%r&rbs z#PackageIndex._invalid_download_htmlc Csttj|}|jddd}|jd\}}}|d\}}}|r&||fn|df\}} |j||dd } | | fS) a Given a possible VCS URL, return a clean URL and resolved revision if any. >>> vsrfu = PackageIndex._vcs_split_rev_from_url >>> vsrfu('git+https://github.com/pypa/setuptools@v69.0.0#egg-info=setuptools') ('https://github.com/pypa/setuptools', 'v69.0.0') >>> vsrfu('git+https://github.com/pypa/setuptools#egg-info=setuptools') ('https://github.com/pypa/setuptools', None) >>> vsrfu('http://foo/bar') ('http://foo/bar', None) rJrr3r6@Nr)r=r?rB) r7r!rNr=r:r?rO rpartition_replacegeturl) r;r<Z clean_schemeZno_fragment_pathrFrSrTrUZ clean_pathr_resolvedr%r%r&rZs z$PackageIndex._vcs_split_rev_from_urlcGtj|g|RdSrd)rrrr%r%r&rzPackageIndex.debugcGrhrd)rrrr%r%r&rrizPackageIndex.infocGrhrd)rrrr%r%r&rrizPackageIndex.warn)rrNT)Frd)FFFN)FF)*rrrrrrrrrrrrrrrrrrr rrr4rrr r)r3r-rr  staticmethodrWrHrIrrarbrZrrr __classcell__r%r%rr&r+s`  5     & R .$ "  rz!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?cCs|d}t|S)Nr)rHrunescape)rGwhatr%r%r& decode_entitys  rncCs tt|S)a Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subrn)textr%r%r&rus rucsfdd}|S)Ncsfdd}|S)Nc s:t}tz |i|Wt|St|wrd)socketgetdefaulttimeoutsetdefaulttimeout)rerfZ old_timeout)rhtimeoutr%r&_socket_timeouts  z@socket_timeout.._socket_timeout.._socket_timeoutr%)rhrurtrgr&rusz'socket_timeout.._socket_timeoutr%)rtrur%rvr&socket_timeouts  rwcCs2tj|}|}t|}|}|ddS)a9 Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r)r7r!r9encodebase64 b64encoderrG)authZauth_sZ auth_bytesZ encoded_bytesencodedr%r%r& _encode_auths   r~c@r}) Credentialz: A username/password pair. Use like a namedtuple. cCs||_||_dSrdusernamepassword)rrrr%r%r&rs zCredential.__init__ccs|jV|jVdSrdrrr%r%r&__iter__s zCredential.__iter__cCs dt|S)Nz%(username)s:%(password)s)varsrr%r%r&__str__s zCredential.__str__N)rrrrrrrr%r%r%r&rs  rcs8eZdZfddZeddZddZddZZS) PyPIConfigcsPtgdd}t|tjtjdd}tj|r&t ||dSdS)z% Load from ~/.pypirc )rr repositoryr~z.pypircN) r]fromkeysrrrWr?rc expanduserrr)rdefaultsrcrr%r&rs   zPyPIConfig.__init__cs&fddD}ttj|S)Ncs g|] }|dr|qS)r)rrr)r[sectionrr%r&r s z2PyPIConfig.creds_by_repository..)sectionsr]rp_get_repo_cred)rZsections_with_repositoriesr%rr&creds_by_repositorys zPyPIConfig.creds_by_repositorycCs6||d}|t||d||dfS)Nrrr)rrrr)rrrepor%r%r&rs zPyPIConfig._get_repo_credcCs*|jD] \}}||r|SqdS)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)ritemsr-)rr;rcredr%r%r&find_credentials  zPyPIConfig.find_credential) rrrrpropertyrrrrkr%r%rr&rs    rcCs:tj|}|\}}}}}}|drtjd|dvr$t|\} } nd} | sBt |} | rBt | } | j |f} t j dg| R| redt| } || ||||f} tj| }tj|}|d| ntj|}|dt||}| rtj|j\}}}}}}||kr|| kr||||||f} tj| |_|S) z4Open a urllib2 request, handling HTTP authentication:znonnumeric port: '')r9httpsNz*Authenticating as %s for %s (from .pypirc)zBasic Authorizationz User-Agent)r7r!r8r,r9r:r; _splituserrrrqrrrr~ urlunparserRequest add_header user_agentr;)r;rparsedr=netlocr?paramsrArr|addressrrr<rrr.s2h2Zpath2Zparam2Zquery2Zfrag2r%r%r&r8$s8          r8cCs$|d\}}}|r||fSd|fS)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.rcN)rd)hostuserdelimr%r%r&rSsrcCs|Srdr%)r;r%r%r& fix_sf_url^r4rcCstj|\}}}}}}tj|}tj|rtj|S| drhtj |rhg}t |D]&} tj || } | dkrCt | } n tj | rM| d7} |dj| dq0d} | j|d |d} d\} }nd \} }} d d i}t| }tj|| |||S) z7Read a local path, with special support for directoriesr2z index.htmlz{name})r.zB{url}{files}rx)r;files)OK)rzPath not foundz Not foundrz text/html)r7r!r8rr`rWr?isfilerr,rrrcrrformatioStringIOrr)r;r=r>r?paramrArrXrrfilepathbodyrstatusmessagerZ body_streamr%r%r&r7bs*       r7rd)r)]rsysr[rWrZrr"rqrzrr configparserr http.clientr9 urllib.parser7urllib.request urllib.error functoolsrr pkg_resourcesrrrrrr r r r r rrrr distutilsrZdistutils.errorsrfnmatchrZsetuptools.wheelrZ setuptools.extern.more_itertoolsrZ unicode_utilsrrrrFIrsrrGrr:rQ__all__Z_SOCKET_TIMEOUTZ_tmplr version_inforr'rrCrrErYrrjrlr|r~rrrrornrurwr~rRawConfigParserrrrr8rrr7r%r%r%r&s @         # "  !  (/