jUddlmZddlZddlmZddlmZddlmZm Z m Z m Z ddl m Z mZmZmZmZddlmZdd lmZmZdd lmZmZmZmZmZmZejd Zej Z!e!"ej#d gZ$d e%d<gZ&d e%d<e D]YZ' ee're$(e'ne&(e':#e)$re&(e'YVwxYwe$e&zZ*d e%d< d2d3d(Z+ d2d4d+Z, d2d5d.Z- d6d7d1Z.dS)8) annotationsN)PathLike)BinaryIO)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sz list[str] _mb_supported _sb_supportedIANA_SUPPORTED_MB_FIRST皙?TF皙? sequencesbytes | bytearraystepsint chunk_size thresholdfloat cp_isolationlist[str] | None cp_exclusionpreemptive_behaviourboolexplainlanguage_thresholdenable_fallbackreturnrc R$t|ttfs/tdt ||rJt j} t tt tt|} | dkrwt d|r9t tt | tt!|dddgdgS|At td d |d |D}ng}|At td d |d |D}ng}| ||zkr't td||| d}| }|dkr| |z |krt'| |z }t|t(k} t|t*k} | r4t td| n5| r3t td| g}|rt-|nd}|6||t td|t1}g}g}t1}t1}i}d}t1}d}d}d}d}d}d}t}t}t3|\} }!| D|| t tdt|!| |dd|vr|d|t4zD ]}"|r|"|vr |r|"|vr|"|vr||"d}#| |"k}$|$ot9|"}%|"dvr$|$s"t td|"l|"dvr$|$s"t td|"|"|vr"t td|"|"|vr"t td|" t;|"}&n9#t<t>f$r%t td|"Y&wxYw|ru|&st1tA|"}'nt1tC|"}'|'"|s%t td|"|'||r-|&s+||kr%t td|"|||r%|&s#t td|" | rS|&durOtG|%dur|dt'd n#|t|!t'd |"!n^|"d"kr,|$r*tG||"!}#|#r|#dd#kr |#dd}#n,tG|%dur|n|t|!d|"!}#nx#tHtJf$rd}(t|(tJs/t td$|"tG|(||"Yd}(~(d}(~(wwxYwtM|$sdnt|!| t'| |z })|&o|#duot|#| k}*|*r!t td%|"|#V|&sStO|#}+|(|+},|,,|,\}-}.}/|/rt!||"|-|$|.| dus|"|ddfvr|#nd|&}0||0||"t td'|"tS|-d(zd)*|"|ddfvr|-d+kr|-dkrmt d,|0j*|r9t tt | t|0gcS||0t|r|||vrd|vrd|vr|+}1t d,|1j*|r9t tt | t|1gcS~||"t td-|"| r3|"dd|d.d/fvr*t!||"||$g|#|&}2|"|kr|2}n |"dkr|2}n|2}t't|)d0z }3tY|3d1}3d}4d}5g}6g}7 t[||"|)||$|%|!|&|# D]y}8|6|8|7t]|8||d2uodt|cxkod1knc|7d3|kr|4dz }4|4|3ks|$r|%durnznJ#tH$r=}(t td4|"tG|(|3}4d2}5Yd}(~(nd}(~(wwxYw|5s| r|&s |t'd5d/|"d67n\#tH$rO}(t td8|"tG|(||"Yd}(~(d}(~(wwxYw|7rta|7t|7z nd}9|9|ks|4|3kr||"|"tbvr |2tb|"|#(|&s&|3tO|#|9gdft td9|"|4tS|9d(zd)*| r5|"dd|d.d/fvr,|5s*t!||"||$g|#|&}2|"|kr|2}n |"dkr|2}n|2}t td:|"tS|9d(zd)*|&stA|"}:ntC|"}:|:rAt td;|"tG|:g};|"dkrR|6D]?}8ti|8||:rd<|:nd}<|;|<@tk|;}=ntk|;}=|=r4t td=|=|"t!||"|9|$|=| dus|"|ddfvr|#nd|&}>||>|#(|&s&|3tO|#|9|=d2f|r |&s |9d>kr|dz }|"|ddfvr|9d+kr|9dkrmt d,|>j*|r9t tt | t|>gcS||>t|r|||vrd|vrd|vr|+}1t d,|1j*|r9t tt | t|1gcS|si|&sg|=rtYd?|=Dd@nd}?|?dAkrBd|vr>d|vr:d2}|2|:t tdB|"|9|?|so|&rm|*rk|#it|#| dCzkrS|"dDvrOd|vrKd|vrGd2}t tdE|"|9t|#| t|#| z d(z|"| krnt dF|"|r9t tt | t||"gcS t|dkr|s|s|r t tdG|r6t dH|j*||n{|r||r|r|j6|j6ks|0t dI||n1|r/t dJ|||rDt dK|+j*t|dz nt dL|r9t tt | |S)Maf Given a raw bytes sequence, return the best possibles charset usable to render str objects. If there is no results, it is a strong indicator that the source is binary/not text. By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence. And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page but never take it for granted. Can improve the performance. You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that purpose. This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. Custom logging format and handler can be set manually. z3Expected object of type bytes or bytearray, got: {}rzzfrom_bytes..s"DDD "e,,DDDzacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.c.g|]}t|dSr5r6r7s r:r;zfrom_bytes..~r<r=z^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.rz>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_16utf_32z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!zESkipping %s: already fast-tracked from a similar successful encoding.z2Encoding %s does not provide an IncrementalDecoderzbSkipping %s: definitive match already found, this encoding targets different languages (%s vs %s).zXSkipping %s: already accumulated %d same-family results after definitive match (cap=%d).zCSkipping single-byte %s: multi-byte definitive match already found.gA)encodingrCuz9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.)preemptive_declarationzM%s fast-tracked (identical decoded payload to a prior encoding, chaos=%f %%).d)ndigitsr z.Encoding detection: %s is most likely the one.zZ%s fast-skipped (identical decoded payload to a prior encoding that failed chaos probing).rArBTzaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sgj@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.z=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {},z We detected language {} using {}g{Gz?c3 K|] \}}|V dS)N)r8_vs r: zfrom_bytes..s&4441aQ444444r=)defaultg?zyDefinitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.g\(\?> rCr2rArB utf_16_be utf_16_le utf_32_be utf_32_le utf_8_sigzjMulti-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)7 isinstance bytearraybytes TypeErrorformattypeloggerlevel addHandlerexplain_handlersetLevelrlendebug removeHandlerrrlogjoinr$rr rappendsetrraddrrModuleNotFoundError ImportErrorrr intersectionstrUnicodeDecodeError LookupErrorrangehashgetroundrDbestmaxrrdecodesumr update setdefaultrr fingerprint)@r!r#r%r&r(r*r+r-r.r/previous_logger_levellengthis_too_small_sequenceis_too_large_sequenceprioritized_encodingsspecified_encodingtestedtested_but_hard_failuretested_but_soft_failuresoft_failure_skipsuccess_fast_trackedpayload_result_cachedefinitive_match_founddefinitive_target_languages post_definitive_sb_success_countPOST_DEFINITIVE_SB_CAPmb_definitive_match_foundfallback_ascii fallback_u8fallback_specifiedresultsearly_stop_results sig_encoding sig_payload encoding_ianadecoded_payloadbom_or_sig_availablestrip_sig_or_bomis_multi_byte_decoder enc_languageser_multi_byte_bonus payload_hashcached cached_mess cached_cd cached_passed fast_matchprobable_resultfallback_entrymax_chunk_gave_upearly_stop_countlazy_str_hard_failure md_chunks md_ratioschunkmean_mess_ratiotarget_languages cd_ratioschunk_languagescd_ratios_merged current_matchbest_coherences@ r: from_bytesr9s< i)U!3 4 4  A H HY     %+\/***i..F {{ STTT  3   1 1 1 OO1 2 2 2|IwUBPRSSTUUU  5 IIl # #    ED|DDD    6 IIl # #    ED|DDD   *u$%%  l         qyyVe^j00%(( "%i..3E"E"%i..4D"D    L S S         W ^ ^     (*.BKy)))t%$$%7888  N    uuF)+)+"%%%%(UU TV$),/EE -.$"#',*.N'+K.2,..G)7)9)9 3I > >L+$$\222  W          )))+++$$W---.1HHD <D <  M==   M\99  F " "  =!!!&*%1]%B!5" :Q ; ;  0 0 09M 0 JJn     I % %.B % JJd     - - - JJk     0 0 0 JJW     *@*O*O ! !#[1    JJD    H   " ( J #$6}$E$E F F #$9-$H$H I I  --.IJJ  x!!/  # ) 14JJJ JJj0&      % -B  JJU    * $ )>%)G)G,u44"+CII+..&s;'7'7#d))'CD* !G++0D+&)!!.'''O'>?1+=+I+I*9!""*=&) 0588&I!*3{+;+;+=+=!>!. '''O#K0   a--  O!FF  $ * *= 9 9 9 HHHH ) ?AAs;/?/?       " .t+ .O$$v-    JJ-      &/D & $_ 5 5L)--l;;F!8>5 Y  [!-!%#,!!6 > >#0$6#I$J$J,O "&/A"""J"NN:...(,,];;;JJg%kC/;;; &*>>1?..*g55-;NN*8K!$SWWq[!1!1 1155 ! %!  ' ),$ %      '''  !4GA\1B1B,G,G,G,Ga,G,G,G,GR=I--$)$$(999(:-=-F-FE  ) ) ) JJsA     1 $( ! ! ! ! ! ! )& % *  #d))++&--mH-MMMM%    t!FF  (..}=== EN!VY#i..!@!@SV i ' '+;?P+P+P # * *= 9 9 9 666!(()? )NOOO*3H*$//))OR+G JJ0 o+Q777      1!W&8(HMNN-N".!(#+="""!$666)7&&"g--%3NN"0K   K  /C' 3 3 3    % D*<]*K*K  4]CC    JJ8??!3'7#8#8     G # ## 2 2"1&2BLCHH-...##   11115i@@  5i@@    JJ299$m   %    *U22$);Wg(NNN  #5   " }%%%  &/D & + +_%% "2D9    # 2) 2 $&& , 1 , 0'7C C C#%%#%% D!*;((999OO$9:::%}o66666  % %m 4 4 4 " # # 5#+/AV/K/K6!!6!!05577O LL@(    7$$_555 5666!?"344 4 4 4& .C $44#3444cBBBB  $$F):):w&?P?P)-&+223CDDD P!#" * % !  +O$$v}44   6!!6!!(, % JJ|O$$O$$v-3    L ( ( LL1     7$$_555 5666!7=#9":;; ; ; ; ) 7||q  . ,>  JJa     + LLI"+    NN- . . . .  ++34#4 +~/III' LLU V V V NN; ' ' ' '  + LLU V V V NN> * * *V k LLNN # LL1      TUUU/_----... Ns^S2TTB3Z  \A[;;\Bj k3kk%,l m+Am&&m+fprc Vt|||||||||| S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. )rread) rr#r%r&r(r*r+r-r.r/s r:from_fpr`s<       r=pathstr | bytes | PathLikec t|d5} t| ||||||||| cdddS#1swxYwYdS)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openr) rr#r%r&r(r*r+r-r.r/rs r: from_pathr~s dD    R                               s 6::fp_or_path_or_payload!PathLike | str | BinaryIO | bytesc t|ttfrt||||||||||  } nOt|tt frt ||||||||||  } nt||||||||||  } | S)a) Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match are disabled to be stricter around ASCII-compatible but unlikely to be a string. ) r#r%r&r(r*r+r-r.r/)rZrprrr\r[rr) rr#r%r&r(r*r+r-r.r/guessess r: is_binaryrs"'#x99,  !!%%!51+            !!%%!51+     !!%%!51+    ;r=) rrrNNTFr T)r!r"r#r$r%r$r&r'r(r)r*r)r+r,r-r,r.r'r/r,r0r)rrr#r$r%r$r&r'r(r)r*r)r+r,r-r,r.r'r/r,r0r)rrr#r$r%r$r&r'r(r)r*r)r+r,r-r,r.r'r/r,r0r) rrrNNTFr F)rrr#r$r%r$r&r'r(r)r*r)r+r,r-r,r.r'r/r,r0r,)/ __future__rloggingosrtypingrcdrrr r constantr r r rrmdrmodelsrrutilsrrrrrr getLoggerr` StreamHandlerrc setFormatter Formatterr__annotations__r_supported_encrjrnrrrrrrPr=r:rsB""""""" 00000000  / 0 0'''))GABB  $--N- ! !. 1 1 1   0 0 0 0   0 0 0 ---^,,,,,-&3]%BBBBB %)%)!% # d d d d d R%)%)!% # @%)%)!% #      B%)%)!% #!???????s%6CC98C9