U
    ÌZjêD  ã                   @   sJ  d dl mZ d dlmZmZ ddlmZmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd„ dƒZG dd	„ d	eƒZG d
d„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!edd�ee" ee" e#dœdd„ƒZ$edd�d$e"e%e#e%d œd!d"„ƒZ&d#S )%é    )Ú	lru_cache)ÚListÚOptionalé   )ÚCOMMON_SAFE_ASCII_CHARACTERSÚUNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚis_asciiÚis_case_variableÚis_cjkÚis_emoticonÚ	is_hangulÚis_hiraganaÚis_katakanaÚis_latinÚis_punctuationÚis_separatorÚ	is_symbolÚis_thaiÚis_unprintableÚremove_accentÚunicode_rangec                   @   sP   e Zd ZdZeedœdd„Zeddœdd„Zddœd	d
„Ze	e
dœdd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    ©Ú	characterÚreturnc                 C   s   t ‚dS )z@
        Determine if given character should be fed in.
        N©ÚNotImplementedError©Úselfr   © r    úX/var/www/html/TRUCKING_PROJECT/venv/lib/python3.8/site-packages/charset_normalizer/md.pyÚeligible   s    zMessDetectorPlugin.eligibleNc                 C   s   t ‚dS )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        Nr   r   r    r    r!   Úfeed%   s    zMessDetectorPlugin.feed©r   c                 C   s   t ‚dS )zB
        Permit to reset the plugin to the initial state.
        Nr   ©r   r    r    r!   Úreset,   s    zMessDetectorPlugin.resetc                 C   s   t ‚dS )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nr   r%   r    r    r!   Úratio2   s    zMessDetectorPlugin.ratio)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚboolr"   r#   r&   ÚpropertyÚfloatr'   r    r    r    r!   r      s   r   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )Ú TooManySymbolOrPunctuationPluginNr$   c                 C   s"   d| _ d| _d| _d | _d| _d S )Nr   F)Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr%   r    r    r!   Ú__init__<   s
    z)TooManySymbolOrPunctuationPlugin.__init__r   c                 C   s   |  ¡ S ©N©Úisprintabler   r    r    r!   r"   D   s    z)TooManySymbolOrPunctuationPlugin.eligiblec                 C   sp   |  j d7  _ || jkrf|tkrft|ƒr8|  jd7  _n.| ¡ dkrft|ƒrft|ƒdkrf|  jd7  _|| _d S )Nr   Fé   )	r3   r4   r   r   r1   Úisdigitr   r   r2   r   r    r    r!   r#   G   s    ÿþ
ÿþ
ýz%TooManySymbolOrPunctuationPlugin.feedc                 C   s   d| _ d| _d| _d S ©Nr   )r1   r3   r2   r%   r    r    r!   r&   Y   s    z&TooManySymbolOrPunctuationPlugin.resetc                 C   s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr   ç        g333333Ó?)r3   r1   r2   )r   Zratio_of_punctuationr    r    r!   r'   ^   s    

þz&TooManySymbolOrPunctuationPlugin.ratio©r(   r)   r*   r5   r,   r-   r"   r#   r&   r.   r/   r'   r    r    r    r!   r0   ;   s   r0   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚTooManyAccentuatedPluginNr$   c                 C   s   d| _ d| _d S r;   ©r3   Ú_accentuated_countr%   r    r    r!   r5   k   s    z!TooManyAccentuatedPlugin.__init__r   c                 C   s   |  ¡ S r6   )Úisalphar   r    r    r!   r"   o   s    z!TooManyAccentuatedPlugin.eligiblec                 C   s(   |  j d7  _ t|ƒr$|  jd7  _d S ©Nr   )r3   r   r@   r   r    r    r!   r#   r   s    zTooManyAccentuatedPlugin.feedc                 C   s   d| _ d| _d S r;   r?   r%   r    r    r!   r&   x   s    zTooManyAccentuatedPlugin.resetc                 C   s*   | j dkrdS | j| j  }|dkr&|S dS )Nr   r<   gffffffÖ?r?   )r   Zratio_of_accentuationr    r    r!   r'   |   s    
zTooManyAccentuatedPlugin.ratior=   r    r    r    r!   r>   j   s   r>   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚUnprintablePluginNr$   c                 C   s   d| _ d| _d S r;   )Ú_unprintable_countr3   r%   r    r    r!   r5   …   s    zUnprintablePlugin.__init__r   c                 C   s   dS ©NTr    r   r    r    r!   r"   ‰   s    zUnprintablePlugin.eligiblec                 C   s(   t |ƒr|  jd7  _|  jd7  _d S rB   )r   rD   r3   r   r    r    r!   r#   Œ   s    zUnprintablePlugin.feedc                 C   s
   d| _ d S r;   )rD   r%   r    r    r!   r&   ‘   s    zUnprintablePlugin.resetc                 C   s   | j dkrdS | jd | j  S )Nr   r<   é   )r3   rD   r%   r    r    r!   r'   ”   s    
zUnprintablePlugin.ratior=   r    r    r    r!   rC   „   s   rC   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚSuspiciousDuplicateAccentPluginNr$   c                 C   s   d| _ d| _d | _d S r;   ©Ú_successive_countr3   Ú_last_latin_characterr%   r    r    r!   r5   �   s    z(SuspiciousDuplicateAccentPlugin.__init__r   c                 C   s   |  ¡ ot|ƒS r6   )rA   r   r   r    r    r!   r"   £   s    z(SuspiciousDuplicateAccentPlugin.eligiblec                 C   st   |  j d7  _ | jd k	rjt|ƒrjt| jƒrj| ¡ rJ| j ¡ rJ|  jd7  _t|ƒt| jƒkrj|  jd7  _|| _d S rB   )r3   rJ   r   ÚisupperrI   r   r   r    r    r!   r#   ¦   s    ÿþýz$SuspiciousDuplicateAccentPlugin.feedc                 C   s   d| _ d| _d | _d S r;   rH   r%   r    r    r!   r&   ´   s    z%SuspiciousDuplicateAccentPlugin.resetc                 C   s   | j dkrdS | jd | j  S )Nr   r<   r9   )r3   rI   r%   r    r    r!   r'   ¹   s    
z%SuspiciousDuplicateAccentPlugin.ratior=   r    r    r    r!   rG   œ   s   rG   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚSuspiciousRangeNr$   c                 C   s   d| _ d| _d | _d S r;   )Ú"_suspicious_successive_range_countr3   Ú_last_printable_seenr%   r    r    r!   r5   Â   s    zSuspiciousRange.__init__r   c                 C   s   |  ¡ S r6   r7   r   r    r    r!   r"   Ç   s    zSuspiciousRange.eligiblec                 C   sx   |  j d7  _ | ¡ s&t|ƒs&|tkr0d | _d S | jd krD|| _d S t| jƒ}t|ƒ}t||ƒrn|  jd7  _|| _d S rB   )r3   Úisspacer   r   rN   r   Ú is_suspiciously_successive_rangerM   )r   r   Úunicode_range_aÚunicode_range_br    r    r!   r#   Ê   s"    ÿþý


zSuspiciousRange.feedc                 C   s   d| _ d| _d | _d S r;   )r3   rM   rN   r%   r    r    r!   r&   á   s    zSuspiciousRange.resetc                 C   s.   | j dkrdS | jd | j  }|dk r*dS |S )Nr   r<   r9   gš™™™™™¹?)r3   rM   )r   Zratio_of_suspicious_range_usager    r    r!   r'   æ   s    
þzSuspiciousRange.ratior=   r    r    r    r!   rL   Á   s   rL   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚSuperWeirdWordPluginNr$   c                 C   s:   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr   FÚ )	Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr3   Ú_bad_character_countÚ_bufferÚ_buffer_accent_countr%   r    r    r!   r5   ö   s    zSuperWeirdWordPlugin.__init__r   c                 C   s   dS rE   r    r   r    r    r!   r"     s    zSuperWeirdWordPlugin.eligiblec                 C   sÚ  |  ¡ r�|  j|7  _t|ƒr,|  jd7  _| jdkrŒt|ƒdksJt|ƒrŒt|ƒdkrŒt|ƒdkrŒt|ƒdkrŒt	|ƒdkrŒt
|ƒdkrŒd| _d S | jsšd S | ¡ s´t|ƒs´t|ƒ�r | j�r |  jd7  _t| jƒ}|  j|7  _|dk�r6| j| dk�rd| _t| jd ƒ�r6| jd  ¡ �r6|  jd7  _d| _|dk�r\| j�r\|  jd7  _d| _| j�rŒ|  jd7  _|  jt| jƒ7  _d| _d| _d| _d	| _n6|d
k�rÖ| ¡ dk�rÖt|ƒ�rÖd| _|  j|7  _d S )Nr   FTé   gÃõ(\�ÂÕ?éÿÿÿÿé   rT   r   >   ú|ú<ú-Ú_ú>ú~ú=)rA   r[   r   r\   rY   r   r   r   r   r   r   rO   r   r   rU   Úlenr3   rX   rK   rW   rV   rZ   r:   r   )r   r   Zbuffer_lengthr    r    r!   r#     sv    ÿ
þþ
ý
ü
û
ú
ù	ÿÿÿþ

 ÿ
þýzSuperWeirdWordPlugin.feedc                 C   s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )NrT   Fr   )r[   rX   rY   rV   rU   r3   rZ   rW   r%   r    r    r!   r&   =  s    zSuperWeirdWordPlugin.resetc                 C   s$   | j dkr| jdkrdS | j| j S )Né
   r   r<   )rU   rW   rZ   r3   r%   r    r    r!   r'   G  s    zSuperWeirdWordPlugin.ratior=   r    r    r    r!   rS   õ   s   6
rS   c                   @   s^   e Zd ZdZddœdd„Zeedœdd„Zeddœd	d
„Zddœdd„Z	e
edœdd„ƒZdS )ÚCjkInvalidStopPluginu²   
    GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
    can be easily detected. Searching for the overuse of 'ä¸…' and 'ä¸„'.
    Nr$   c                 C   s   d| _ d| _d S r;   ©Ú_wrong_stop_countÚ_cjk_character_countr%   r    r    r!   r5   U  s    zCjkInvalidStopPlugin.__init__r   c                 C   s   dS rE   r    r   r    r    r!   r"   Y  s    zCjkInvalidStopPlugin.eligiblec                 C   s4   |dkr|  j d7  _ d S t|ƒr0|  jd7  _d S )N>   õ   ä¸„õ   ä¸…r   )rk   r   rl   r   r    r    r!   r#   \  s
    zCjkInvalidStopPlugin.feedc                 C   s   d| _ d| _d S r;   rj   r%   r    r    r!   r&   c  s    zCjkInvalidStopPlugin.resetc                 C   s   | j dk rdS | j| j  S )Né   r<   )rl   rk   r%   r    r    r!   r'   g  s    
zCjkInvalidStopPlugin.ratio)r(   r)   r*   r+   r5   r,   r-   r"   r#   r&   r.   r/   r'   r    r    r    r!   ri   O  s   ri   c                   @   sZ   e Zd Zddœdd„Zeedœdd„Zeddœdd	„Zddœd
d„Ze	e
dœdd„ƒZdS )ÚArchaicUpperLowerPluginNr$   c                 C   s.   d| _ d| _d| _d| _d| _d | _d| _d S )NFr   T)Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalr3   Ú_last_alpha_seenÚ_current_ascii_onlyr%   r    r    r!   r5   o  s    z ArchaicUpperLowerPlugin.__init__r   c                 C   s   dS rE   r    r   r    r    r!   r"   |  s    z ArchaicUpperLowerPlugin.eligiblec                 C   s$  |  ¡ ot|ƒ}|dk}|r†| jdkr†| jdkrV| ¡ dkrV| jdkrV|  j| j7  _d| _d| _d | _d| _|  j	d7  _	d| _d S | jdkr¢t
|ƒdkr¢d| _| jd k	rþ| ¡ r¾| j ¡ sÐ| ¡ rø| j ¡ rø| jdkrð|  jd7  _d| _qþd| _nd| _|  j	d7  _	|  jd7  _|| _d S )NFr   é@   r   Tr9   )rA   r
   rr   r:   rv   rt   rs   ru   rq   r3   r	   rK   Úislower)r   r   Zis_concernedZ	chunk_sepr    r    r!   r#     sF    ÿ
þýÿ
ÿÿ
zArchaicUpperLowerPlugin.feedc                 C   s.   d| _ d| _d| _d| _d | _d| _d| _d S )Nr   FT)r3   rr   rs   rt   ru   rq   rv   r%   r    r    r!   r&   ©  s    zArchaicUpperLowerPlugin.resetc                 C   s   | j dkrdS | j| j  S )Nr   r<   )r3   rt   r%   r    r    r!   r'   ²  s    
zArchaicUpperLowerPlugin.ratior=   r    r    r    r!   rp   n  s   *	rp   é   )Úmaxsize)rQ   rR   r   c                 C   s|  | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS |   d¡|  d¡ }}|D ]}|tkr”q†||kr† dS q†| dk|dk }}|s¾|rÒd	| ksÎd	|krÒdS |rÞ|rÞdS d
| ksðd
|k�r d	| k�sd	|k�rdS | dk�s|dk�r dS d	| k�sHd	|k�sH| dk�rx|dk�rxd| k�s\d|k�r`dS d| k�std|k�rxdS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaZCJKZHangulzBasic Latin)r}   r|   ÚPunctuationZForms)Úsplitr   )rQ   rR   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr    r    r!   rP   º  sh    ÿÿÿþÿúÿÿÿþrP   i   çš™™™™™É?F)Údecoded_sequenceÚmaximum_thresholdÚdebugr   c                 C   sà   dd„ t  ¡ D ƒ}t| ƒd }d}|dk r0d}n|dkr>d}nd	}t| d
 t|ƒƒD ]d\}}|D ]}	|	 |¡r`|	 |¡ q`|dkrŽ|| dksš||d krTtdd„ |D ƒƒ}||krT qºqT|rÖ|D ]}
t|
j	|
j
ƒ qÂt|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c                 S   s   g | ]
}|ƒ ‘qS r    r    )Ú.0Zmd_classr    r    r!   Ú
<listcomp>	  s    zmess_ratio.<locals>.<listcomp>r   r<   i   é    ry   rw   é€   Ú
r   c                 s   s   | ]}|j V  qd S r6   )r'   )r…   Údtr    r    r!   Ú	<genexpr>   s     zmess_ratio.<locals>.<genexpr>é   )r   Ú__subclasses__rg   ÚzipÚranger"   r#   ÚsumÚprintÚ	__class__r'   Úround)r‚   rƒ   r„   Z	detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr   ÚindexZdetectorrŠ   r    r    r!   Ú
mess_ratio  s6    ÿ
ÿ
ÿ
þr–   N)r�   F)'Ú	functoolsr   Útypingr   r   Zconstantr   r   Úutilsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r0   r>   rC   rG   rL   rS   ri   rp   r,   r-   rP   r/   r–   r    r    r    r!   Ú<module>   s4   H"/%4ZL þF   ÿ  þ