ó
    ´OÂf¬  ã                   ó|   • S r SSKJr  SSKJr   SSKr " S S5      rS r	\
S:X  a  \	" 5         gg! \ a    Sr N&f = f)	a˜  
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle,
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created separately to read
those files.

For details regarding the algorithm, see:
https://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
https://borel.slu.edu/crubadan/index.html
é    )Úmaxsize)ÚtrigramsNc                   óL   • \ rS rSrSr0 rSrSr0 rS r	S r
S rS rS	 rS
 rSrg)ÚTextCaté/   NÚ<Ú>c                 óÀ   • [         (       d  [        S5      eSSKJn  Xl        U R                  R                  5        H  nU R                  R                  U5        M      g )Nz›classify.textcat requires the regex module that supports unicode. Try '$ pip install regex' and see https://pypi.python.org/pypi/regex for further details.r   )Úcrubadan)ÚreÚOSErrorÚnltk.corpusr   Ú_corpusÚlangsÚ	lang_freq)Úselfr   Úlangs      Ú7/usr/lib/python3/dist-packages/nltk/classify/textcat.pyÚ__init__ÚTextCat.__init__7   sL   € ßŠrÜð#óð õ 	)àŒà—L‘L×&Ñ&Ö(ˆDØ�L‰L×"Ñ" 4Ö(ò )ó    c                 ó2   • [         R                  " SSU5      $ )z)Get rid of punctuation except apostrophesz[^\P{P}\']+Ú )r   Úsub©r   Útexts     r   Úremove_punctuationÚTextCat.remove_punctuationG   s   € ä�vŠv�n b¨$Ó/Ð/r   c                 ó>  • SSK JnJn  U R                  U5      nU" U5      nU" 5       nU Hi  n[	        U R
                  U-   U R                  -   5      nU V	s/ s H  n	SR                  U	5      PM     n
n	U
 H  nX¶;   a  Xk==   S-  ss'   M  SXk'   M     Mk     U$ s  sn	f )z'Create FreqDist of trigrams within textr   )ÚFreqDistÚword_tokenizer   é   )Únltkr    r!   r   r   Ú_START_CHARÚ	_END_CHARÚjoin)r   r   r    r!   Ú
clean_textÚtokensÚfingerprintÚtÚtoken_trigram_tuplesÚtriÚtoken_trigramsÚcur_trigrams               r   ÚprofileÚTextCat.profileK   s¡   € ç0à×,Ñ,¨TÓ2ˆ
Ù˜zÓ*ˆá“jˆÛˆAÜ#+¨D×,<Ñ,<¸qÑ,@À4Ç>Á>Ñ,QÓ#RÐ Ù6JÓKÒ6J¨s˜bŸg™g cžlÑ6JˆNÐKã-�ØÓ-ØÓ,°Ñ1Õ,à/0�KÓ,ó	  .ñ	 ð Ðùò Ls   ÁBc                 ó  • U R                   R                  U5      nSnX$;   a_  [        UR                  5       5      R	                  U5      n[        UR                  5       5      R	                  U5      n[        Xg-
  5      nU$ [        nU$ )z_Calculate the "out-of-place" measure between the
text and language profile for a single trigramr   )r   r   ÚlistÚkeysÚindexÚabsr   )r   r   ÚtrigramÚtext_profileÚlang_fdÚdistÚidx_lang_profileÚidx_texts           r   Ú	calc_distÚTextCat.calc_dist_   s€   € ð —,‘,×(Ñ(¨Ó.ˆØˆàÓÜ# G§L¡L£NÓ3×9Ñ9¸'ÓBÐÜ˜L×-Ñ-Ó/Ó0×6Ñ6°wÓ?ˆHô Ð'Ñ2Ó3ˆDð ˆô ˆDàˆr   c                 óÈ   • 0 nU R                  U5      nU R                  R                  R                  5        H&  nSnU H  nXPR	                  XFU5      -  nM     XRU'   M(     U$ )zGCalculate the "out-of-place" measure between
the text and all languagesr   )r/   r   Ú_all_lang_freqr3   r<   )r   r   Ú	distancesr/   r   Ú	lang_distr6   s          r   Ú
lang_distsÚTextCat.lang_distst   si   € ð ˆ	Ø—,‘,˜tÓ$ˆà—L‘L×/Ñ/×4Ñ4Ö6ˆDð ˆIÛ"�ØŸ^™^¨D¸7ÓCÑC’	ñ #ð (�d‹Oñ 7ð Ðr   c                 ó~   • U R                  U5      U l        [        U R                  U R                  R                  S9$ )zQFind the language with the min distance
to the text and return its ISO 639-3 code)Úkey)rB   Úlast_distancesÚminÚgetr   s     r   Úguess_languageÚTextCat.guess_language†   s4   € ð #Ÿo™o¨dÓ3ˆÔä�4×&Ñ&¨D×,?Ñ,?×,CÑ,CÑDÐDr   )r   rF   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Úfingerprintsr$   r%   rF   r   r   r/   r<   rB   rI   Ú__static_attributes__© r   r   r   r   /   s:   † Ø€GØ€LØ€KØ€Ià€Nò)ò 0òò(ò*õ$Er   r   c            
      óö  • SSK Jn   / SQnSSSSSS	S
SSS.	n[        5       nU HÎ  nU R                  U5      n[	        U5      S-
  n[        [        [        U5      5      nSn[        SU5       H=  n	SSR                  [        SXy   5       V
s/ s H
  o¥U	   U
   PM     sn
5      -   nX‹-  nM?     [        SUSS -   S-   5        UR                  U5      n[        SU SX,    S35        [        S5        MÐ     g s  sn
f )Nr   )Úudhr)	zKurdish-UTF8zAbkhaz-UTF8zFarsi_Persian-UTF8z
Hindi-UTF8zHawaiian-UTF8zRussian-UTF8zVietnamese-UTF8zSerbian_Srpski-UTF8zEsperanto-UTF8zNorthern KurdishÚ	AbkhazianzIranian PersianÚHindiÚHawaiianÚRussianÚ
VietnameseÚSerbianÚ	Esperanto)	ÚkmrÚabkÚpesÚhinÚhawÚrusÚvieÚsrpÚepor"   r   Ú zLanguage snippet: éŒ   z...zLanguage detection: z (Ú)zŒ############################################################################################################################################)r   rS   r   ÚsentsÚlenr2   ÚmapÚranger&   ÚprintrI   )rS   r   ÚfriendlyÚtcÚcur_langÚraw_sentencesÚrowsÚcolsÚsampleÚiÚjÚcur_sentÚguesss                r   Údemorw   �   s  € Ý ò
€Eð "ØØ ØØØØØØñ
€Hô 
‹€BãˆàŸ
™
 8Ó,ˆÜ�=Ó! AÑ%ˆÜ”Cœ˜]Ó+Ó,ˆàˆô �q˜$–ˆAØ˜SŸX™XÄEÈ!ÈTÉWÔDUÓ&VÒDU¸q°QÑ'7¸Ô':ÑDUÑ&VÓWÑWˆHØÑŠFñ  ô
 	Ð" V¨A¨c ]Ñ2°UÑ:Ô;Ø×!Ñ! &Ó)ˆÜÐ$ U G¨2¨h©oÐ->¸aÐ@ÔAÜˆiÖò# ùò 'Ws   ÂC6Ú__main__)Ú__doc__Úsysr   Ú	nltk.utilr   Úregexr   ÚImportErrorr   rw   rK   rQ   r   r   Ú<module>r~      sZ   ðñõ* å ðÛ÷\Eñ \Eò@.ðb ˆzÓÙ…Fð øðq ó Ø	‚Bðús   �0 °;º;