ó
    ´OÂf2  ã                   ó   • S SK r S SKJr  S SKJr  S SKJr  \" SS/S9r\ R                  " \S9\ R                  " 5       S	 5       5       r
\
R                  S
5      \ R                  " SSSSS9\ R                  " SSSSSS9\ R                  " SSSSS9\ R                  " SSSSS9\ R                  " SSSSS9S  5       5       5       5       5       5       rg)!é    N)Útqdm)Úword_tokenize)Úparallelize_preprocessz-hz--help)Úhelp_option_names)Úcontext_settingsc                  ó   • g )N© r	   ó    Ú*/usr/lib/python3/dist-packages/nltk/cli.pyÚclir      s   € ð 	r
   Útokenizez
--languagez-lÚenz1The language for the Punkt sentence tokenization.)ÚdefaultÚhelpz--preserve-lineTzIAn option to keep the preserve the sentence and not sentence tokenize it.)r   Úis_flagr   z--processesz-jé   zNo. of processes.z
--encodingz-eÚutf8zSpecify encoding of file.z--delimiterz-dÚ z%Specify delimiter to join the tokens.c           
      óÒ  • [         R                  " SUS9 n[         R                  " SUS9 nUS:X  aD  [        UR                  5       5       H&  n[	        UR                  [        U5      5      SUS9  M(     O?[        [        UR                  5       USS9 H  n[	        UR                  U5      SUS9  M     S	S	S	5        S	S	S	5        g	! , (       d  f       N= f! , (       d  f       g	= f)
z;This command tokenizes text stream using nltk.word_tokenizeÚstdin)ÚencodingÚstdoutr   Ú
)ÚendÚfileT)Úprogress_barN)ÚclickÚget_text_streamr   Ú	readlinesÚprintÚjoinr   r   )	ÚlanguageÚpreserve_lineÚ	processesr   Ú	delimiterÚfinÚfoutÚlineÚoutlines	            r   Útokenize_filer*      sµ   € ô* 
×	Ò	˜w°Ò	:¸cÜ×"Ò" 8°hÒ?À4ð ˜A‹~Ü  §¡£Ö1�DÜ˜)Ÿ.™.¬°tÓ)<Ó=À4ÈdÔSò 2ô  6Ü! 3§=¡=£?°IÈDô �Gô ˜)Ÿ.™.¨Ó1°tÀ$ÔGñ ÷ @÷ 
;Ð	:ß?Õ?ú÷ 
;Õ	:ús#   –C¬B
CÂ6CÃ
C	ÃCÃ
C&)r   r   Únltkr   Ú	nltk.utilr   ÚdictÚCONTEXT_SETTINGSÚgroupÚversion_optionr   ÚcommandÚoptionr*   r	   r
   r   Ú<module>r3      sÿ   ðó Ý å Ý ,á¨4°Ð*:Ñ;Ð ð ‡‚Ð.Ñ/Ø×ÒÓñ	ó ó 0ð	ð ‡�ˆZÓØ‡‚ØØØØ	<ñ	ð ‡‚ØØØØØ	Tñð ‡‚ˆm˜T¨1Ð3FÑGØ‡‚ˆl˜D¨&Ð7RÑSØ‡‚Ø�4 Ð+RññHóó Tó Hóóó ñ&Hr
   