U cUh?@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddlmZddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlmZddl m!Z!dd l"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7m8Z8e9e:Z;eeGddde?Z@e!ddddZAGddde?ZBed?ZOGd@dAdAeZPGdBdCdCZQdS)FzO The main purpose of this module is to expose LinkCollector.collect_sources(). N) dataclass) HTMLParser)Values) CallableDictIterableListMutableMapping NamedTupleOptionalProtocolSequenceTupleUnion)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_sourceurlreturncCs6tjD]*}||r|t|dkr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rZschemeslower startswithlen)r"schemer(]/opt/bot/vods_filmes/script/venv/lib/python3.8/site-packages/pip/_internal/index/collector.py_match_vcs_scheme3s  r*cs&eZdZeeddfdd ZZS)_NotAPIContentN) content_type request_descr#cst||||_||_dSN)super__init__r,r-)selfr,r- __class__r(r)r0?sz_NotAPIContent.__init__)__name__ __module__ __qualname__strr0 __classcell__r(r(r2r)r+>sr+)responser#cCs6|jdd}|}|dr$dSt||jjdS)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr$r%r+requestmethod)r9r,content_type_lr(r(r)_ensure_api_headerEsrBc@s eZdZdS)_NotHTTPN)r4r5r6r(r(r(r)rC[srC)r"sessionr#cCsFtj|\}}}}}|dkr$t|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httpshttpT)Zallow_redirectsN)urllibparseurlsplitrCheadrrB)r"rDr'netlocpathqueryfragmentrespr(r(r)_ensure_api_response_s rPcCsztt|jrt||dtdt||j|ddddgddd }t |t |td t||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rDzGetting page %sz, r<z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r=zFetched page %s as %sr:r;) rrfilenamerPloggerdebugrr>joinrrBr=)r"rDrOr(r(r)_get_simple_responseqs,   rW)r=r#cCs<|r8d|kr8tj}|d|d<|d}|r8t|SdS)z=Determine if we have any encoding information in our headers.r:z content-typecharsetN)emailmessageMessage get_paramr7)r=mrXr(r(r)_get_encoding_from_headerss    r^c@s:eZdZdddddZeedddZed d d ZdS) CacheablePageContent IndexContentNpager#cCs|js t||_dSr.)cache_link_parsingAssertionErrorrbr1rbr(r(r)r0s zCacheablePageContent.__init__)otherr#cCst|t|o|jj|jjkSr.) isinstancetyperbr")r1rfr(r(r)__eq__szCacheablePageContent.__eq__r#cCs t|jjSr.)hashrbr"r1r(r(r)__hash__szCacheablePageContent.__hash__) r4r5r6r0objectboolriintrmr(r(r(r)r_sr_c@s eZdZdeedddZdS) ParseLinksr`racCsdSr.r(rer(r(r)__call__zParseLinks.__call__N)r4r5r6rrrrr(r(r(r)rqsrq)fnr#csLtjddtttdfdd tdttdfdd }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. N)maxsize)cacheable_pager#cst|jSr.)listrb)rv)rtr(r)wrappersz*with_cached_index_content..wrapperr`racs|jrt|St|Sr.)rcr_rw)rbrtrxr(r)wrapper_wrappers z2with_cached_index_content..wrapper_wrapper) functools lru_cacher_rrwraps)rtrzr(ryr)with_cached_index_contents  r~r`rac cs|j}|drTt|j}|dgD]"}t||j }|dkrHq,|Vq,dSt |j }|j pfd}| |j ||j }|jp|}|jD]$} tj| ||d}|dkrq|VqdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r<filesNzutf-8)Zpage_urlbase_url)r,r$r%jsonloadscontentr>r from_jsonr"HTMLLinkParserencodingfeeddecoderanchorsZ from_element) rbrAdatafilelinkparserrr"ranchorr(r(r) parse_linkss&       rT)frozenc@sPeZdZUdZeed<eed<eeed<eed<dZe ed<edd d Z d S) r`aRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. rr,rr"TrcrjcCs t|jSr.)rr"rlr(r(r)__str__ szIndexContent.__str__N) r4r5r6__doc__bytes__annotations__r7r rcrorr(r(r(r)r`s    csneZdZdZeddfdd ZeeeeeefddddZ eeeeefeed d d Z Z S) rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. Nr!cs$tjdd||_d|_g|_dS)NT)Zconvert_charrefs)r/r0r"rr)r1r"r2r(r)r0szHTMLLinkParser.__init__)tagattrsr#cCsH|dkr,|jdkr,||}|dk rD||_n|dkrD|jt|dS)Nbasea)rget_hrefrappenddict)r1rrhrefr(r(r)handle_starttags  zHTMLLinkParser.handle_starttag)rr#cCs"|D]\}}|dkr|SqdS)Nrr()r1rnamevaluer(r(r)r&s  zHTMLLinkParser.get_href) r4r5r6rr7r0rrr rrr8r(r(r2r)rs"r).N)rreasonmethr#cCs|dkrtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rTrU)rrrr(r(r)_handle_get_simple_fail-sr)r9rcr#cCs&t|j}t|j|jd||j|dS)Nr:)rr"rc)r^r=r`rr")r9rcrr(r(r)_make_index_content7s r)rrDr#c Cs|jddd}t|}|r0td||dStj|\}}}}}}|dkrtj tj |r| dsv|d7}tj|d}td|zt||d }WnDtk rtd |Yn2tk r}ztd ||j|jW5d}~XYntk r$}zt||W5d}~XYntk rP}zt||W5d}~XYntk r}z$d } | t|7} t|| tjd W5d}~XYndtjk r}zt|d|W5d}~XYn0tjk rt|dYnXt||jdSdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srQz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)rc) r"splitr*rTwarningrGrHurlparseosrLisdirr? url2pathnameendswithurljoinrUrWrCr+r-r,rrrrr7inforConnectionErrorZTimeoutrrc) rrDr"Z vcs_schemer'_rLrOexcrr(r(r)_get_index_contentDsV      rc@s.eZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r4r5r6r r rrr(r(r(r)rs rc@sxeZdZdZeeddddZedeee dddd Z e e e d d d Zeeed ddZe eedddZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. N)rD search_scoper#cCs||_||_dSr.)rrD)r1rDrr(r(r)r0szLinkCollector.__init__F)rDoptionssuppress_no_indexr#cCsd|jg|j}|jr8|s8tdddd|Dg}|jp@g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr.r).0r"r(r(r) sz'LinkCollector.create..)rrno_index)rDr) index_urlextra_index_urlsrrTrUrVrrcreater)clsrDrrrrrlink_collectorr(r(r)rs$   zLinkCollector.createrjcCs|jjSr.)rrrlr(r(r)rszLinkCollector.find_links)locationr#cCst||jdS)z> Fetch an HTML page containing package links. rQ)rrD)r1rr(r(r)fetch_responseszLinkCollector.fetch_response) project_namecandidates_from_pager#cstfddjD}tfddjD}ttj rddt ||D}t |ddg|}t d|tt|t|d S) Nc 3s&|]}t|jjdddVqdS)FrZpage_validatorZ expand_dirrcrNr rDZis_secure_originrlocrrr1r(r)rs z0LinkCollector.collect_sources..c 3s&|]}t|jjdddVqdS)TrNrrrr(r)rs cSs*g|]"}|dk r|jdk rd|jqS)Nz* )r)rsr(r(r) s z1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrZget_index_urls_locationsvaluesrrT isEnabledForloggingDEBUG itertoolschainr&rUrVrrw)r1rrZindex_url_sourcesZfind_links_sourceslinesr(rr)collect_sourcess&    zLinkCollector.collect_sources)F)r4r5r6rrrr0 classmethodrrorpropertyrr7rrr r`rrrrr(r(r(r)rs(  !r)N)T)Rrr email.messagerYr{rrrr urllib.parserGurllib.request dataclassesrZ html.parserroptparsertypingrrrrr r r r r rrZ pip._vendorrZpip._vendor.requestsrZpip._vendor.requests.exceptionsrrpip._internal.exceptionsrZpip._internal.models.linkrZ!pip._internal.models.search_scoperpip._internal.network.sessionrZpip._internal.network.utilsrZpip._internal.utils.filetypesrpip._internal.utils.miscrZpip._internal.vcsrsourcesrrr getLoggerr4rTr7ZResponseHeadersr* Exceptionr+rBrCrPrWr^r_rqr~rr`rrrorrrrr(r(r(r)st   4             ?      =