aVdZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZddlmZmZddlmZmZmZdZ ej!de d Z"d Z#e"e#zZ$ej%ej&zd z'd Z(d Z)e$e(ze)zdzZ*ej+dZ,d9dZ-ej+dZ.d9dZ/dZ0d:dZ1d;dZ2dZ3dZ4dZ5dZ6d Z7d!Z8e9e:e;eej?d$@e=e9e:e;eej?d$@e=hd(z ej?d$@e=ZBej+d)>eA*'ZCej+d+>eAeB,'ZDe d-d.ZEd/ZFgd0ZGd9d1ZH dd6ZPd?d8ZQdS)@zW This module contains general purpose URL functions not found in the standard library. N) namedtuple OrderedDict) urljoinurlsplit urlunsplit urldefrag urlencodeurlparsequoteparse_qs parse_qsl ParseResultunquote urlunparse) pathname2url url2pathname)to_bytes to_native_str to_unicodecvtt|j|j|j|jfSN)rr objectstartend)errors )lib/python3.11/site-packages/w3lib/url.py _quote_byters/ uU\%+ei*?@AA B BEI NN percentencodes:/?#[]@s !$&'()*+,;=z-._~ascii|%z[\t\n\r]utf8c Lt||d}ttd|} |jd}n#t $r |j}YnwxYwtt|j t| dtt|j |ttt|j|ttt|j|tfS)aConvert the given URL into a legal URL by escaping unsafe characters according to RFC-3986. Also, ASCII tabs and newlines are removed as per https://url.spec.whatwg.org/#url-parsing. If a bytes URL is given, it is first converted to `str` using the given encoding (which defaults to 'utf-8'). 'utf-8' encoding is used for URL path component (unless overriden by path_encoding), and given encoding is used for query string or form data. When passing an encoding, you should use the encoding of the original page (the page from which the URL was extracted from). Calling this function on an already "safe" URL will return the URL unmodified. Always returns a native `str` (bytes in Python2, unicode in Python3). r)encodingerrorsidna:)rr_ascii_tab_newline_resubnetlocencode UnicodeErrorrrschemerstripr rpath _safe_charsqueryfragment)urlr% path_encodingdecodedpartsr,s rsafe_url_stringr9's.xHHHG *..r7;; < >    sAA+*A+z /?(\.\./)+c8t|||}t|\}}}}}|r]tdt j|}|dr|ds|dz }nd}t||||dfS)z Make a url for download. This will call safe_url_string and then strip the fragment, if one exists. The path will be normalised. If the path is outside the document root, it will be changed to be within the document root. r'/)r9r _parent_dirsr+ posixpathnormpathendswithr) r5r%r6safe_urlr/r,r1r3_s rsafe_download_urlrBZssHm<>%  #'@ @@rcttt|d|}|||gdS)aReturn the value of a url parameter, given the url and parameter name General case: >>> import w3lib.url >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "id") '200' >>> Return a default value if the parameter is not found: >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault") 'mydefault' >>> Returns None if `keep_blank_values` not set or 0 (default): >>> w3lib.url.url_query_parameter("product.html?id=", "id") >>> Returns an empty string if `keep_blank_values` set to 1: >>> w3lib.url.url_query_parameter("product.html?id=", "id", keep_blank_values=1) '' >>> keep_blank_valuesr)r rstrget)r5 parameterdefaultrM queryparamss rurl_query_parameterrSqsN:S1+K ??9wi 0 0 33r&=FTct|tjtfr|g}t |\}}|d\}} } t } g} | |D]]} | s| |\}} } |r|| vr%|r||vr,|s||vr3| | | |^| r*d || | gn|}|r|d|zz }|S)aClean URL arguments leaving only those passed in the parameterlist keeping order >>> import w3lib.url >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ('id',)) 'product.html?id=200' >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']) 'product.html?id=200&name=wired' >>> If `unique` is ``False``, do not remove duplicated keys >>> w3lib.url.url_query_cleaner("product.html?d=1&e=b&d=2&d=3&other=other", ['d'], unique=False) 'product.html?d=1&d=2&d=3' >>> If `remove` is ``True``, leave only those **not in parameterlist**. >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'], remove=True) 'product.html?foo=bar&name=wired' >>> w3lib.url.url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True) 'product.html?name=wired' >>> By default, URL fragments are removed. If you need to preserve fragments, pass the ``keep_fragments`` argument as ``True``. >>> w3lib.url.url_query_cleaner('http://domain.tld/?bla=123#123123', ['bla'], remove=True, keep_fragments=True) 'http://domain.tld/#123123' ?#) isinstancesix text_typebytesrrGsetsplitappendaddjoin)r5 parameterlistsepkvsepremoveuniquekeep_fragmentsr4baserAr3seen querylistksvks rurl_query_cleanerrns?@-#-!788(& cNNMC]]3''ND!U 55DI{{3    --&&1a  a4ii   ]**  A]22    S ! ! ! HHQKKKK3< F#((D#((9--. / / /$C sX~ Jrct|}t|jd}t|}||t |}t ||S)NTrL)r3)rr r3rupdater r_replace)r5paramsparsedargsnew_argsr3s r_add_or_replace_parametersrvsi c]]F V\T : : :D4  H OOF h  E fooEo22 3 33rc&t|||iS)aNAdd or remove a parameter to a given url >>> import w3lib.url >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php', 'arg', 'v') 'http://www.example.com/index.php?arg=v' >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg4', 'v4') 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3&arg4=v4' >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg3', 'v3new') 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new' >>> rv)r5name new_values radd_or_replace_parameterr{s &cD)+< = ==rc"t||S)aAdd or remove a parameters to a given url >>> import w3lib.url >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php', {'arg': 'v'}) 'http://www.example.com/index.php?arg=v' >>> args = {'arg4': 'v4', 'arg3': 'v3new'} >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', args) 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new&arg4=v4' >>> rx)r5new_parameterss radd_or_replace_parametersr~s &c> : ::rcttj|}tjdkr|dd}d|dzS)zwConvert local filesystem path to legal File URIs as described in: http://en.wikipedia.org/wiki/File_URI_scheme nt|r)z file:///%sr;)rosr1abspathryreplacelstrip)r1xs rpath_to_file_urirsR RW__T**++A w$ IIc3   !((3-- ''rcHt|j}t|S)zmConvert File URI to local filesystem path according to: http://en.wikipedia.org/wiki/File_URI_scheme )r r1r)uriuri_paths rfile_uri_to_pathrs }}!H  ! !!rctj|drt|St |}|jr|nt|S)zRIf given a path name, return its File URI, otherwise return it unmodified r)rr1 splitdriverr r/) uri_or_pathus r any_to_urir sV w+&&q)- ,,,A( E;;(8(E(EErz[{}]+r' z()<>@,;:\"/[]?= z(?:[{}]|(?:\\[{}]))*> "\z{token}/{token})tokenz%;({token})=(?:({token})|"({quoted})"))rquotedParseDataURIResultz%media_type media_type_parameters datacft|ts"t|d} |dd\}}n#t $rt dwxYw|dkrt dtjrt|}nt|}d}i}t |}|rC| }||d}nd |d < t |}|rv|\}}}|rt%jd d |}|||<||d}nn |dd\}} n#t $rt dwxYw|r)|dkrt dt)j| } t-||| S)zt Parse a data: URI, returning a 3-tuple of media type, dictionary of media type parameters, and data. r :z invalid URIsdataznot a data URIz text/plainNzUS-ASCIIcharsetTs\\(.)z\1,zinvalid data URIs;base64)rZr]r9r-r_ ValueErrorlowerr[PY2runquote_to_bytes_mediatype_patternmatchgroupdecoder_mediatype_parameter_patterngroupsrer+base64 b64decode_ParseDataURIResult) rr/ media_typemedia_type_paramsm attributevalue value_quoted is_base64datas rparse_data_urir6s; c5 ! !3c""))'22(iia((  ((('''( ||~~  )*** w$clls##J  %%A2WWYY%%'' !%%''((m'1)$  ( . .s 3 3  -.XXZZ *Iul ?y%>>49LLNN i..00 1aeegghh-CC  -))D!,, 44 ---+,,,-&  " "/00 0%% z+>  s 11c4 tt||\}}}}}} n<#t$r/} tt|d\}}}}}} Yd} ~ nd} ~ wwxYwtjrt ||} nt ||} | t| }t|} t| tpd}|sdn| } t|| d|||| fS)aCanonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value - percent encode paths ; non-ASCII characters are percent-encoded using UTF-8 (RFC-3986) - percent encode query arguments ; non-ASCII characters are percent-encoded using passed `encoding` (UTF-8 by default) - normalize all spaces (in query arguments) '+' (plus symbol) - normalize percent encodings case (%2f -> %2F) - remove query arguments with blank values (unless `keep_blank_values` is True) - remove fragments (unless `keep_fragments` is True) The url passed can be bytes or unicode, while the url returned is always a native str (bytes in Python 2, unicode in Python 3). >>> import w3lib.url >>> >>> # sorting query arguments >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3&b=5&b=2&a=50') 'http://www.example.com/do?a=50&b=2&b=5&c=3' >>> >>> # UTF-8 conversion + percent-encoding of non-ASCII characters >>> w3lib.url.canonicalize_url(u'http://www.example.com/r\u00e9sum\u00e9') 'http://www.example.com/r%C3%A9sum%C3%A9' >>> For more examples, see the tests in `tests/test_url.py`. )r%r#Nr;r'r))r parse_urlUnicodeEncodeErrorr[rr parse_qsl_to_bytessortr _unquotepathr r2rrr0) r5rMrhr%r/r,r1rrr3r4ekeyvalsuqps rrrsFF-8I cNNX9/9/9/5feXX ---8I cNNV9-9-9-5feXXXXXX- w?E#4552%U,=>> LLNNN g  E t  C k " " )cD'5rrXH v||~~,,S11 ! " ""s%( A!%AA!cdD]0}|d|zd|z}1tjrt |St |S)N)2f2F3f3F%z%25)rupperr[rrr)r1reserveds rrrsa,FF||C(NEHNN4D4D,DEE w &t}} %%%rcjt|tr|Stt||S)z\Return urlparsed url from the given argument (which could be an already parsed url) )rZrr r)r5r%s rrr s3#{## JsH-- . ..r) _coerce_argsrc4t|\}}d|dD}g}|D]}|s|dd}t|dkr|r|dnGt|ds|r|ddd }t |}||}|ddd }t |}||}|||f|S) a-Parse a query given as a string argument. Data are returned as a list of name, value pairs as bytes. Arguments: qs: percent-encoded query string to be parsed keep_blank_values: flag indicating whether blank values in percent-encoded queries should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. cBg|]}|dD]}|S);)r_).0s1s2s r z&parse_qsl_to_bytes..*s/DDDbhhsmmDDDDDDrrUrVrr'r+ )rr_lenr`rr) qsrM_coerce_resultpairsr name_valuenvryrs rrrs.(*"--NDDbhhsmmDDD  ( (J !!#q))B2ww!||$IIbMMMM2a5zz (. (!u}}S#..'--%~d++1 c3//(//&u--$'''rutf-8ctjdtt||}t||}t ||S)a1 .. warning:: This function is deprecated and will be removed in future. It is not supported with Python 3. Please use ``urlparse.urljoin`` instead. Same as urlparse.urljoin but supports unicode values in base and ref parameters (in which case they will be converted to str using the given encoding). Always returns a str. >>> import w3lib.url >>> w3lib.url.urljoin_rfc('http://www.example.com/path/index.html', u'/otherpath/index2.html') 'http://www.example.com/otherpath/index2.html' >>> >>> # Note: the following does not work in Python 3 >>> w3lib.url.urljoin_rfc(b'http://www.example.com/path/index.html', u'fran\u00e7ais/d\u00e9part.htm') # doctest: +SKIP 'http://www.example.com/path/fran\xc3\xa7ais/d\xc3\xa9part.htm' >>> zAw3lib.url.urljoin_rfc is deprecated, use urlparse.urljoin instead)warningswarnDeprecationWarningrr)rirefr%str_basestr_refs rrrAsN6 MUh''HsH%%G 8W % %%r)r#r#)Nr)rTrUrVFTF)TFNr)F)r)R__doc__rcodecsrrr=rstring collectionsrrr[six.moves.urllib.parserrrrr r r r r rrrsix.moves.urllib.requestrr w3lib.utilrrrrregister_errorRFC3986_GEN_DELIMSRFC3986_SUB_DELIMSRFC3986_RESERVED ascii_lettersdigitsr-RFC3986_UNRESERVEDEXTRA_SAFE_CHARSr2compiler*r9r<rBrIrSrnrvr{r~rrrr^mapchrrange_charformatescaperb_token_quoted_stringrrrr__all__rrrrr urllib.parserrrrrTrrrs  //////// FFFFFFFFFFFFFFFFFFFFFFFFFFFF@???????::::::::::OOOo{333 #%(::*V]:VCKKGTT!336FFM " ;//....brz-(( 9999&AAA!4!4!4!4H6666r444 > > > ; ; ;((("""FFF CCUU3ZZ !! 2775+.3ss3a /E/E+F+F,G,/3/B+C+C ,D$E$EFF G G)// BIbgge////0011 BIbggenn RZF++2244 *rz,33&;I4666