a0( dZddlZddlZddlZddlmZejdejZdZ dZ dZ e dzZ e d zZ e d zZe d zZd e d e de deded ZejeejejzZejedejejzZdZidddddddddddddddd d!dd"d#d$d%d&dd'd(d)d%d*d%d+dd,dd%dd-Zd.Zd/Zejd0fejd1fejd2fejd3fejd4fgZed5eDZ d6Z!ej"d7d8d9Z# dt|}|r$t|dSdSdS)zExtract the encoding in the content-type header >>> import w3lib.encoding >>> w3lib.encoding.http_content_type_encoding("Content-Type: text/html; charset=ISO-8859-4") 'iso8859-4' N)_HEADER_ENCODING_REsearchresolve_encodinggroup) content_typematchs .lib/python3.11/site-packages/w3lib/encoding.pyhttp_content_type_encodingr sS4#**<88  4#EKKNN33 344 4 4z%s\s*=\s*["']?\s*%s\s*["']?a(?:\s+ [^=<>/\s"'-]+ # Attribute name (?:\s*=\s* (?: # ' and " are entity encoded (', "), so no need for ', " '[^']*' # attr in ' | "[^"]*" # attr in " | [^'"\s]+ # attr having no ' nor " ))? )*?)z http-equivz Content-Type)contentz.(?P[^;]+);\s*charset=(?P[\w-]+))charsetz(?P[\w-]+))encodingz(?P[\w-]+)z <\s*(?:metaz (?:(?:\s+z|\s+z){2}|\s+z)|\?xml\s[^>]+z|body)asciicV|dd}t|trt|}nt|}|rP|dp)|dp|d}|rt |SdSdS)aReturn the encoding specified in meta tags in the html body, or ``None`` if no suitable encoding was found >>> import w3lib.encoding >>> w3lib.encoding.html_body_declared_encoding( ... """ ... ... ... Some title ... ... ... ... ... ... ... """) 'utf-8' >>> Nircharset2 xmlcharset) isinstancebytes_BODY_ENCODING_BYTES_REr_BODY_ENCODING_STR_REr r) html_body_strchunkr rs r html_body_declared_encodingr1s. %4% E%4'..u55%,,U33 .;;y))-U[[-D-D-;;|,,   .#H-- - .. . .rcp1252big5 big5hkscseuc_krcp949gb2312gb18030 gb_2312_80gbk iso8859_11cp874 iso8859_9cp1254latin_1 macintosh mac_roman shift_jiscp932tis_620win_1251cp1251 windows_31jwin_31j windows_874win_874)x_sjiszh_cnctj|}tjj||S)zCannonicalize an encoding name This performs normalization and translates aliases using python's encoding aliases ) encodingsnormalize_encodingloweraliasesget)rnormeds r _c18n_encodingr?os<  )( 3 3 9 9 ; ;F   $ ( ( 8 88rct|}t||} tj|jS#t $rYdSwxYw)aReturn the encoding that `encoding_alias` maps to, or ``None`` if the encoding cannot be interpreted >>> import w3lib.encoding >>> w3lib.encoding.resolve_encoding('latin1') 'cp1252' >>> w3lib.encoding.resolve_encoding('gb_2312-80') 'gb18030' >>> N)r?DEFAULT_ENCODING_TRANSLATIONr=codecslookupname LookupError)encoding_alias c18n_encoding translateds r rrxs`#>22M-11-OOJ}Z((-- ttsA AAz utf-32-bez utf-32-lez utf-16-bez utf-16-lezutf-8c#,K|]\}}|dVdS)rN).0c_s r rNs*11FQ1Q4111111rcx|r7|dtvr(tD] \}}||r||fcS!dS)aRead the byte order mark in the text, if present, and return the encoding represented by the BOM and the BOM. If no BOM can be detected, ``(None, None)`` is returned. >>> import w3lib.encoding >>> w3lib.encoding.read_bom(b'\xfe\xff\x6c\x34') ('utf-16-be', '\xfe\xff') >>> w3lib.encoding.read_bom(b'\xff\xfe\x34\x6c') ('utf-16-le', '\xff\xfe') >>> w3lib.encoding.read_bom(b'\x00\x00\xfe\xff\x00\x00\x6c\x34') ('utf-32-be', '\x00\x00\xfe\xff') >>> w3lib.encoding.read_bom(b'\xff\xfe\x00\x00\x34\x6c\x00\x00') ('utf-32-le', '\xff\xfe\x00\x00') >>> w3lib.encoding.read_bom(b'\x01\x02\x03\x04') (None, None) >>> r)NN) _FIRST_CHARS _BOM_TABLE startswith)databomrs r read_bomrUs[, %Q<''' % %MCs## %}$$$ % :r w3lib_replacecd|jfS)Nu�)end)excs r rZs Isw3GrcX||tdddkrdndS)zConvert a str object to unicode using the encoding given Characters that cannot be converted will be converted to ``\ufffd`` (the unicode replacement character). r)r]replacerV)decoder)data_strrs r to_unicoderas0 ??8,qs2Cv2M2MYYSb c ccrutf8ct|}t|\}}|r||kr|t|d}nB|dks|dkr6|/||r|}|t|d}n|dz }|t ||fS|'|t |t|d|fSt |}| | ||}||}|t ||fS)a Convert raw html bytes to unicode This attempts to make a reasonable guess at the content encoding of the html body, following a similar process to a web browser. It will try in order: * http content type header * BOM (byte-order mark) * meta or xml tag declarations * auto-detection, if the `auto_detect_fun` keyword argument is not ``None`` * default encoding in keyword arg (which defaults to utf8) If an encoding other than the auto-detected or default encoding is used, overrides will be applied, converting some character encodings to more suitable alternatives. If a BOM is found matching the encoding, it will be stripped. The `auto_detect_fun` argument can be used to pass a function that will sniff the encoding of the text. This function must take the raw text as an argument and return the name of an encoding that python can process, or None. To use chardet, for example, you can define the function as:: auto_detect_fun=lambda x: chardet.detect(x).get('encoding') or to use UnicodeDammit (shipped with the BeautifulSoup library):: auto_detect_fun=lambda x: UnicodeDammit(x).originalEncoding If the locale of the website or user language preference is known, then a better default encoding can be supplied. If `content_type_header` is not present, ``None`` can be passed signifying that the header was not present. This method will not fail, if characters cannot be converted to unicode, ``\\ufffd`` (the unicode replacement character) will be inserted instead. Returns a tuple of ``(, )`` Examples: >>> import w3lib.encoding >>> w3lib.encoding.html_to_unicode(None, ... b""" ... ... ... ... Creative Commons France ... ... ...

Creative Commons est une organisation \xc3\xa0 but non lucratif ... qui a pour dessein de faciliter la diffusion et le partage des oeuvres ... tout en accompagnant les nouvelles pratiques de cr\xc3\xa9ation \xc3\xa0 l\xe2\x80\x99\xc3\xa8re numerique.

... ... """) ('utf-8', u'\n\n\n\nCreative Commons France\n\n\n

Creative Commons est une organisation \xe0 but non lucratif\nqui a pour dessein de faciliter la diffusion et le partage des oeuvres\ntout en accompagnant les nouvelles pratiques de cr\xe9ation \xe0 l\u2019\xe8re numerique.

\n\n') >>> Nzutf-16zutf-32z-be)r rUlenrRrar)content_type_headerrdefault_encodingauto_detect_funencbom_encrTs r html_to_unicoderjs@ %%8 9 9CM**LGS  '>>)#c(())4MM H__x"w'9'9#'>'>" -c#hhii 8 u J}c2222 =S#;WEEEE %m 4 4C {3om,, {  =#.. ..r)rbN)%__doc__rerBr9sysrcompileIrr _TEMPLATE _SKIP_ATTRS _HTTPEQUIV_RE _CONTENT_RE _CONTENT2_RE_XML_ENCODING_RE_BODY_ENCODING_PATTERNVERBOSErencoderrrAr?r BOM_UTF32_BE BOM_UTF32_LE BOM_UTF16_BE BOM_UTF16_LEBOM_UTF8rQsetrPrUregister_errorrarjrJrr rs bj!4bd;; 4 4 4 /   :: XX ?? FFKK \\\;K;K;KM" #924"*;LMM$"*%;%B%B7%K%K%'TBJ%688!.!.!.P  X  K  g  i   )    9  '  x   w  7 w  7! "w# $ '   ,999( +& +& +& +& _g  s11j11111 <o'G'GHHHddd26V/V/V/V/V/V/r