§
    £”jF;  ã                  óŠ  — d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	mZ
 ddlmZmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZ d-d„Zd.d„Z e¦   «         d-d„¦   «         Z e¦   «         d-d„¦   «         Z  ee¬¦  «        d/d„¦   «         Z!	 d0d1d„Z"d2d„Z#d3d!„Z$d4d%„Z%d5d&„Z& ed'¬¦  «        	 d6d7d,„¦   «         Z'dS )8é    )ÚannotationsN)ÚIncrementalDecoder)ÚCounter)Ú	lru_cacheé   )ÚFREQUENCIESÚKO_NAMESÚLANGUAGE_SUPPORTED_COUNTÚTOO_SMALL_SEQUENCEÚZH_NAMESÚ_FREQUENCIES_SETÚ_FREQUENCIES_RANK)Ú is_suspiciously_successive_range)ÚCoherenceMatches)Úis_accentuatedÚis_latinÚis_multi_byte_encodingÚis_unicode_range_secondaryÚunicode_rangeÚ	iana_nameÚstrÚreturnú	list[str]c                óÄ  ‡‡— t          | ¦  «        rt          d¦  «        ‚t          j        d| › �¦  «        j        } |d¬¦  «        }i ŠdŠt          dd¦  «        D ]h}|                     t          |g¦  «        ¦  «        }|rAt          |¦  «        }|€Œ9t          |¦  «        d	u r|‰vrd‰|<   ‰|xx         d
z  cc<   ‰d
z  ŠŒit          ˆˆfd„‰D ¦   «         ¦  «        S )zF
    Return associated unicode ranges in a single byte code page.
    z.Function not supported on multi-byte code pagez
encodings.Úignore)Úerrorsr   é@   éÿ   NFr   c                ó2   •— g | ]}‰|         ‰z  d k    ¯|‘ŒS )g333333Ã?© )Ú.0Úcharacter_rangeÚcharacter_countÚseen_rangess     €€úd/var/www/finuniver-perm.ru/html/portfolio/venv/lib/python3.11/site-packages/charset_normalizer/cd.pyú
<listcomp>z*encoding_unicode_range.<locals>.<listcomp><   s8   ø€ ð 	
ð 	
ð 	
àØ˜?Ô+¨oÑ=ÀÒEÐEð àEÐEÐEó    )r   ÚOSErrorÚ	importlibÚimport_moduler   ÚrangeÚdecodeÚbytesr   r   Úsorted)r   ÚdecoderÚpÚiÚchunkr"   r#   r$   s         @@r%   Úencoding_unicode_ranger3      sA  øø€ õ ˜iÑ(Ô(ð 
ÝØ<ñ
ô 
ð 	
õ Ô%Ð&>°9Ð&>Ð&>Ñ?Ô?ÔR€Gà#˜G¨8Ð4Ñ4Ô4€AØ"$€KØ€Oå�4˜ÑÔð %ð %ˆØ—X’X�e Q C™jœjÑ)Ô)ˆàð 
	%Ý*7¸Ñ*>Ô*>ˆOàÐ&Øå)¨/Ñ:Ô:¸eÐCÐCØ"¨+Ð5Ð5Ø34�K Ñ0Ø˜OÐ,Ð,Ô,°Ñ1Ð,Ð,Ñ,Ø 1Ñ$�øåð	
ð 	
ð 	
ð 	
ð 	
à#.ð	
ñ 	
ô 	
ñô ð r'   Úprimary_rangec                óš   — g }t          j        ¦   «         D ]4\  }}|D ],}t          |¦  «        | k    r|                     |¦  «          nŒ-Œ5|S )z>
    Return inferred languages used with a unicode range.
    )r   Úitemsr   Úappend)r4   Ú	languagesÚlanguageÚ
charactersÚ	characters        r%   Úunicode_range_languagesr<   D   so   € ð €Iå +Ô 1Ñ 3Ô 3ð ð Ñˆ�*Ø#ð 	ð 	ˆIÝ˜YÑ'Ô'¨=Ò8Ð8Ø× Ò  Ñ*Ô*Ð*Ø�ð 9øð Ðr'   c                óf   — t          | ¦  «        }d}|D ]
}d|vr|} nŒ|€dgS t          |¦  «        S )zœ
    Single-byte encoding language association. Some code page are heavily linked to particular language(s).
    This function does the correspondence.
    NÚLatinúLatin Based)r3   r<   )r   Úunicode_rangesr4   Úspecified_ranges       r%   Úencoding_languagesrB   S   s]   € õ !7°yÑ AÔ A€NØ $€Mà)ð ð ˆØ˜/Ð)Ð)Ø+ˆMØˆEð *ð ÐØˆÐå" =Ñ1Ô1Ð1r'   c                ó  — |                       d¦  «        s0|                       d¦  «        s|                       d¦  «        s| dk    rdgS |                       d¦  «        s	| t          v rdgS |                       d¦  «        s	| t          v rd	gS g S )
z›
    Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
    This function does the correspondence.
    Úshift_Ú
iso2022_jpÚeuc_jÚcp932ÚJapaneseÚgbÚChineseÚ
iso2022_krÚKorean)Ú
startswithr   r	   )r   s    r%   Úmb_encoding_languagesrN   g   s²   € ð 	×Ò˜XÑ&Ô&ðà×Ò Ñ-Ô-ðð ×Ò Ñ(Ô(ðð ˜ÒÐàˆ|ÐØ×Ò˜DÑ!Ô!ð  Yµ(Ð%:Ð%:Øˆ{ÐØ×Ò˜LÑ)Ô)ð ¨Y½(Ð-BÐ-BØˆzÐà€Ir'   )Úmaxsizer9   útuple[bool, bool]c                ó‚   — d}d}t           |          D ]*}|st          |¦  «        rd}|rt          |¦  «        du rd}Œ+||fS )zg
    Determine main aspects from a supported language if it contains accents and if is pure Latin.
    FT)r   r   r   )r9   Útarget_have_accentsÚtarget_pure_latinr;   s       r%   Úget_target_featuresrT   |   sn   € ð
 !&ÐØ"Ðå  Ô*ð &ð &ˆ	Ø"ð 	'¥~°iÑ'@Ô'@ð 	'Ø"&ÐØð 	&¥¨)Ñ!4Ô!4¸Ð!=Ð!=Ø %ÐøàÐ 1Ð1Ð1r'   Fr:   Úignore_non_latinÚboolc                ó¢  — g }t          | ¦  «        }t          d„ | D ¦   «         ¦  «        }t          j        ¦   «         D ]s\  }}t	          |¦  «        \  }}|r|du rŒ|du r|rŒ%t          |¦  «        }	t          t          |         |z  ¦  «        }
|
|	z  }|dk    r|                     ||f¦  «         Œtt          |d„ d¬¦  «        }d„ |D ¦   «         S )zE
    Return associated languages associated to given characters.
    c              3  ó4   K  — | ]}t          |¦  «        V — Œd S )N)r   )r!   r;   s     r%   ú	<genexpr>z%alphabet_languages.<locals>.<genexpr>–   s*   è è € ÐTÐT¸I�n¨YÑ7Ô7ÐTÐTÐTÐTÐTÐTr'   Fgš™™™™™É?c                ó   — | d         S ©Nr   r    ©Úxs    r%   ú<lambda>z$alphabet_languages.<locals>.<lambda>ª   s
   € °°!´€ r'   T©ÚkeyÚreversec                ó   — g | ]
}|d          ‘ŒS ©r   r    )r!   Úcompatible_languages     r%   r&   z&alphabet_languages.<locals>.<listcomp>¬   s   € ÐHÐHÐHÐ':Ð Ô"ÐHÐHÐHr'   )	Ú	frozensetÚanyr   r6   rT   Úlenr   r7   r.   )r:   rU   r8   Úcharacters_setÚsource_have_accentsr9   Úlanguage_charactersrR   rS   r#   Úcharacter_match_countÚratios               r%   Úalphabet_languagesrm   �   s  € ð *,€Iå%.¨zÑ%:Ô%:€NÝÐTÐTÈÐTÑTÔTÑTÔTÐå)4Ô):Ñ)<Ô)<ð 0ð 0Ñ%ˆÐ%Ý1DÀXÑ1NÔ1NÑ.ÐÐ.àð 	Ð 1°UÐ :Ð :Øà %Ð'Ð'Ð,?Ð'Øå"Ð#6Ñ7Ô7ˆå%(Õ)9¸(Ô)CÀnÑ)TÑ%UÔ%UÐà,¨Ñ>ˆà�CŠ<ˆ<Ø×Ò˜h¨Ð.Ñ/Ô/Ð/øå�y n n¸dÐCÑCÔC€IàHÐH¸iÐHÑHÔHÐHr'   Úordered_charactersÚfloatc                óà  ‡— | t           vrt          | › d�¦  «        ‚d}t          |          }t          |          }t	          |¦  «        }t	          t           |          ¦  «        }|dk    }||z  }d„ t          |¦  «        D ¦   «         Šˆfd„|                     ¦   «         D ¦   «         }	t	          |	¦  «        }
d„ |	D ¦   «         }d„ |	D ¦   «         }t          |t          d|¦  «        ¦  «        D ]ò\  }}||vrŒ
||         }t          ||z  ¦  «        }|du rt          ||z
  ¦  «        d	k    rŒ?|d
u rt          ||z
  ¦  «        |dz  k     r|dz  }Œbd}d}t          |
¦  «        D ]/}||         }||         }||k     r||k     r|dz  }Œ$||k    r|dz  }Œ0||z
  }|dk    r|d	k    r|dz  }Œ¼|dk    r|d	k    r|dz  }ŒÎ|dk    r	||z  dk    s|dk    r||z  dk    r|dz  }ŒòŒó|t	          |¦  «        z  S )aN  
    Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
    The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
    Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
    z not availabler   é   c                ó   — i | ]\  }}||“Œ	S r    r    )r!   ÚrankÚchars      r%   ú
<dictcomp>z1characters_popularity_compare.<locals>.<dictcomp>È   s+   € ð $ð $ð $Ù!�t˜Tˆˆdð$ð $ð $r'   c                ó2   •— g | ]\  }}|‰v ¯	|‰|         f‘ŒS r    r    )r!   ÚcÚlrÚordered_ranks      €r%   r&   z1characters_popularity_compare.<locals>.<listcomp>Î   s7   ø€ ð +ð +ð +Ù"' ! RÀÀ\Ð@QÐ@Qˆˆ\˜!Œ_ÐÐ@QÐ@QÐ@Qr'   c                ó   — g | ]
}|d          ‘ŒS rc   r    ©r!   r0   s     r%   r&   z1characters_popularity_compare.<locals>.<listcomp>Ö   s   € Ð7Ð7Ð7 Q˜A˜aœDÐ7Ð7Ð7r'   c                ó   — g | ]
}|d          ‘ŒS )r   r    r{   s     r%   r&   z1characters_popularity_compare.<locals>.<listcomp>×   s   € Ð8Ð8Ð8 a˜Q˜qœTÐ8Ð8Ð8r'   Fé   Té   r   gš™™™™™Ù?)r   Ú
ValueErrorr   r   rg   Ú	enumerater6   Úzipr+   ÚintÚabs)r9   rn   Úcharacter_approved_countÚfrequencies_language_setÚ	lang_rankÚordered_characters_countÚ target_language_characters_countÚlarge_alphabetÚexpected_projection_ratioÚcommon_charsÚcommon_countÚ	common_lrÚ
common_orrr;   Úcharacter_rankÚcharacter_rank_in_languageÚcharacter_rank_projectionÚbefore_match_countÚafter_match_countr1   Úlr_iÚorr_iÚ	after_lenry   s                          @r%   Úcharacters_popularity_comparer—   ¯   sñ  ø€ ð •{Ð"Ð"Ý˜HÐ4Ð4Ð4Ñ5Ô5Ð5à$%ÐÝ/?ÀÔ/IÐÝ 1°(Ô ;€Iå$'Ð(:Ñ$;Ô$;ÐÝ,/µ¸HÔ0EÑ,FÔ,FÐ$à;¸bÒ@€Nð 	)Ð+CÑCð ð
$ð $Ý%.Ð/AÑ%BÔ%Bð$ñ $ô $€Lð+ð +ð +ð +Ø+4¯?ª?Ñ+<Ô+<ð+ñ +ô +€Lõ ˜LÑ)Ô)€LØ7Ð7¨,Ð7Ñ7Ô7€IØ8Ð8¨<Ð8Ñ8Ô8€Jå%(Ø�E !Ð%=Ñ>Ô>ñ&ô &ð 6ð 6Ñ!ˆ	�>ð Ð4Ð4Ð4Øà*3°IÔ*>Ð"Ý),¨^Ð>WÑ-WÑ)XÔ)XÐ!ð ˜eÐ#Ð#ÝÐ-Ð0JÑJÑKÔKÈaÒOÐOàð ˜dÐ"Ð"ÝÐ-Ð0JÑJÑKÔKØ.°Ñ2ò3ð 3ð %¨Ñ)Ð$Øð #$ÐØ!"ÐÝ�|Ñ$Ô$ð 	+ð 	+ˆAØ! !œˆDØ# AœˆEØÐ0Ò0Ð0Ø˜>Ò)Ð)Ø&¨!Ñ+Ð&øà˜NÒ*Ð*Ø%¨Ñ*Ð%øà9Ð<VÑVˆ	à%¨Ò*Ð*Ð/AÀQÒ/FÐ/FØ$¨Ñ)Ð$Øà˜Š>ˆ>Ð/°1Ò4Ð4Ø$¨Ñ)Ð$Øð '¨Ò*Ð*Ø"Ð%?Ñ?À3ÒFÐFØ˜!ŠmˆmÐ 1°IÑ =ÀÒ DÐ DØ$¨Ñ)Ð$Øøà#¥cÐ*<Ñ&=Ô&=Ñ=Ð=r'   Údecoded_sequencec                óø  — i }d}d}d}d}| D ]Ð}|                      ¦   «         du rŒt          |¦  «        }|dk     rd}nt          |¦  «        }|€ŒC||k    r|�||                              |¦  «         Œgd}	|r|D ]}
t	          |
|¦  «        du r|
}	 nŒn|�t	          ||¦  «        du r|}	|	€|}	|	|vrg ||	<   |€|	}nd}||	                              |¦  «         |}|	}ŒÑd„ |                     ¦   «         D ¦   «         S )a  
    Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
    Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
    One containing the latin letters and the other hebrew.
    NFé€   zBasic LatinTc                ó\   — g | ])}d                       |¦  «                             ¦   «         ‘Œ*S )Ú )ÚjoinÚlower)r!   Úcharss     r%   r&   z'alpha_unicode_split.<locals>.<listcomp>[  s.   € Ð@Ð@Ð@ uˆB�GŠG�E‰NŒN× Ò Ñ"Ô"Ð@Ð@Ð@r'   )ÚisalphaÚordr   r7   r   Úvalues)r˜   ÚlayersÚsingle_layer_keyÚmulti_layerÚprev_character_rangeÚprev_layer_targetr;   Úcharacter_ordr"   Úlayer_target_rangeÚdiscovered_ranges              r%   Úalpha_unicode_splitr«     s§  € ð $&€Fð $(ÐØ€Kð (,ÐØ$(Ðà%ð 4/ð 4/ˆ	Ø×ÒÑÔ %Ð'Ð'Øõ ! ™^œ^ˆØ˜3ÒÐØ*7ˆOˆOå+¨IÑ6Ô6ˆOàÐ"Øð Ð2Ò2Ð2Ø Ð,ØÐ(Ô)×0Ò0°Ñ;Ô;Ð;Øà)-Ðàð 	6Ø$*ð ð Ð å4Ð5EÀÑWÔWØðð ð *:Ð&Ø�Eð	øð
 Ð)å0Ð1AÀ?ÑSÔSØðð ð &6Ð"àÐ%Ø!0Ðà VÐ+Ð+Ø)+ˆFÐ%Ñ&ØÐ'Ø#5Ð Ð à"�àÐ!Ô"×)Ò)¨)Ñ4Ô4Ð4ð  /ÐØ.ÐÐà@Ð@°·²±´Ð@Ñ@Ô@Ð@r'   Úresultsúlist[CoherenceMatches]r   c                ó´   ‡— i Š| D ]2}|D ]-}|\  }}|‰vr|g‰|<   Œ‰|                               |¦  «         Œ.Œ3ˆfd„‰D ¦   «         }t          |d„ d¬¦  «        S )z‹
    This function merge results previously given by the function coherence_ratio.
    The return type is the same as coherence_ratio.
    c           
     ó†   •— g | ]=}|t          t          ‰|         ¦  «        t          ‰|         ¦  «        z  d ¦  «        f‘Œ>S )r}   )ÚroundÚsumrg   )r!   r9   Úper_language_ratioss     €r%   r&   z*merge_coherence_ratios.<locals>.<listcomp>l  sd   ø€ ð 	ð 	ð 	ð ð ÝÝÐ'¨Ô1Ñ2Ô2µSÐ9LÈXÔ9VÑ5WÔ5WÑWØñô ð	
ð	ð 	ð 	r'   c                ó   — | d         S r[   r    r\   s    r%   r^   z(merge_coherence_ratios.<locals>.<lambda>w  s
   €  q¨¤t€ r'   Tr_   )r7   r.   )r¬   ÚresultÚ
sub_resultr9   rl   Úmerger²   s         @r%   Úmerge_coherence_ratiosr·   ^  s²   ø€ ð
 35ÐØð 8ð 8ˆØ ð 	8ð 	8ˆJØ(‰OˆH�eØÐ2Ð2Ð2Ø16°Ð# HÑ-ØØ Ô)×0Ò0°Ñ7Ô7Ð7Ð7ð	8ð	ð 	ð 	ð 	ð ,ð	ñ 	ô 	€Eõ �%˜^˜^°TÐ:Ñ:Ô:Ð:r'   c                óH  ‡— t          ¦   «         Š| D ]A}|\  }}|                     dd¦  «        }|‰vrg ‰|<   ‰|                              |¦  «         ŒBt          ˆfd„‰D ¦   «         ¦  «        r3g }‰D ],}|                     |t	          ‰|         ¦  «        f¦  «         Œ-|S | S )u³   
    We shall NOT return "Englishâ€”" in CoherenceMatches because it is an alternative
    of "English". This function only keeps the best match and remove the em-dash in it.
    u   â€”rœ   c              3  óJ   •K  — | ]}t          ‰|         ¦  «        d k    V — ŒdS )r   N)rg   )r!   ÚeÚindex_resultss     €r%   rY   z/filter_alt_coherence_matches.<locals>.<genexpr>Š  s5   øè è € Ð
<Ð
<¨�3ˆ}˜QÔÑ Ô  1Ò$Ð
<Ð
<Ð
<Ð
<Ð
<Ð
<r'   )ÚdictÚreplacer7   rf   Úmax)r¬   r´   r9   rl   Ú
no_em_nameÚfiltered_resultsr»   s         @r%   Úfilter_alt_coherence_matchesrÁ   z  sÜ   ø€ õ
 -1©F¬F€Màð 0ð 0ˆØ ‰ˆ�%Ø"×*Ò*¨5°"Ñ5Ô5ˆ
à˜]Ð*Ð*Ø(*ˆM˜*Ñ%à�jÔ!×(Ò(¨Ñ/Ô/Ð/Ð/å
Ð
<Ð
<Ð
<Ð
<¨mÐ
<Ñ
<Ô
<Ñ<Ô<ð  Ø-/Ðà%ð 	Nð 	NˆHØ×#Ò# X­s°=ÀÔ3JÑ/KÔ/KÐ$LÑMÔMÐMÐMàÐà€Nr'   i   çš™™™™™¹?Ú	thresholdÚlg_inclusionú
str | Nonec           	     ó6  — g }d}d}|�|                      d¦  «        ng }d|v rd}|                     d¦  «         t          | ¦  «        D ]°}t          |¦  «        }|                     ¦   «         }	t          |¦  «        }
|
t          k    rŒ@d„ |	D ¦   «         }|pt          ||¦  «        D ]Q}t          ||¦  «        }||k     rŒ|dk    r|d	z  }| 	                    |t          |d
¦  «        f¦  «         |dk    r nŒRŒ±t          t          |¦  «        d„ d¬¦  «        S )z¨
    Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
    A layer = Character extraction by alphabets/ranges.
    Fr   Nú,r?   Tc                ó   — g | ]\  }}|‘ŒS r    r    )r!   rw   Úos      r%   r&   z#coherence_ratio.<locals>.<listcomp>±  s   € Ð/JÐ/JÐ/J±d°a¸°Ð/JÐ/JÐ/Jr'   gš™™™™™é?r   r}   r~   c                ó   — | d         S r[   r    r\   s    r%   r^   z!coherence_ratio.<locals>.<lambda>Å  s
   € ¸Q¸q¼T€ r'   r_   )ÚsplitÚremover«   r   Úmost_commonrg   r   rm   r—   r7   r°   r.   rÁ   )r˜   rÃ   rÄ   r¬   rU   Úsufficient_match_countÚlg_inclusion_listÚlayerÚsequence_frequenciesrÍ   r#   Úpopular_character_orderedr9   rl   s                 r%   Úcoherence_ratiorÓ   •  s~  € ð (*€GØ"Ðà"#Ðà3?Ð3K˜×*Ò*¨3Ñ/Ô/Ð/ÐQSÐØÐ)Ð)Ð)ØÐØ× Ò  Ñ/Ô/Ð/å$Ð%5Ñ6Ô6ð ð ˆÝ18¸±´ÐØ*×6Ò6Ñ8Ô8ˆå" 5™zœzˆàÕ0Ò0Ð0Øà/JÐ/J¸kÐ/JÑ/JÔ/JÐ!à)ð 
Õ-?Ø%Ð'7ñ.
ô .
ð 	ð 	ˆHõ 9ØÐ3ñô ˆEð �yÒ Ð ØØ˜#’�Ø&¨!Ñ+Ð&à�NŠN˜H¥e¨E°1¡o¤oÐ6Ñ7Ô7Ð7à%¨Ò*Ð*Ø�ð +øõ Ý$ WÑ-Ô-°>°>È4ðñ ô ð r'   )r   r   r   r   )r4   r   r   r   )r9   r   r   rP   )F)r:   r   rU   rV   r   r   )r9   r   rn   r   r   ro   )r˜   r   r   r   )r¬   r­   r   r   )r¬   r   r   r   )rÂ   N)r˜   r   rÃ   ro   rÄ   rÅ   r   r   )(Ú
__future__r   r)   Úcodecsr   Úcollectionsr   Ú	functoolsr   ÚtypingÚTypeCounterÚconstantr   r	   r
   r   r   r   r   Úmdr   Úmodelsr   Úutilsr   r   r   r   r   r3   r<   rB   rN   rT   rm   r—   r«   r·   rÁ   rÓ   r    r'   r%   ú<module>rÞ      s—  ðØ "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 1Ð 0Ð 0Ð 0Ð 0Ð 0Ø $Ð $Ð $Ð $Ð $Ð $ðð ð ð ð ð ð ð ð ð ð ð ð ð ð$ð $ð $ð $ðNð ð ð ð €�„ð2ð 2ð 2ñ „ð2ð& €�„ðð ð ñ „ðð( €Ð+Ð,Ñ,Ô,ð2ð 2ð 2ñ -Ô,ð2ð" 5:ðIð Ið Ið Ið IðDb>ð b>ð b>ð b>ðJGAð GAð GAð GAðT;ð ;ð ;ð ;ð8ð ð ð ð6 €�4ÐÑÔàNRð0ð 0ð 0ð 0ñ Ôð0ð 0ð 0r'   