ó
    ´OÂfâ  ã                   óp   • S r SSKJrJr  SSKJrJrJr  SSKJ	r	  SSK
Jr   " S S\5      r " S S	\5      rg
)z
Tokenizer Interface
é    )ÚABCÚabstractmethod)ÚIteratorÚListÚTuple)Ú
overridden)Ústring_span_tokenizec                   ó®   • \ rS rSrSr\S\S\\   4S j5       rS\S\	\
\\4      4S jrS\\   S\\\      4S jrS\\   S\	\\
\\4         4S	 jrS
rg)Ú
TokenizerIé   zz
A processing interface for tokenizing a string.
Subclasses must define ``tokenize()`` or ``tokenize_sents()`` (or both).
ÚsÚreturnc                 ób   • [        U R                  5      (       a  U R                  U/5      S   $ g)z4
Return a tokenized copy of *s*.

:rtype: List[str]
r   N)r   Útokenize_sents©Úselfr   s     Ú3/usr/lib/python3/dist-packages/nltk/tokenize/api.pyÚtokenizeÚTokenizerI.tokenize   s2   € ô �d×)Ñ)×*Ñ*Ø×&Ñ&¨ sÓ+¨AÑ.Ð.ð +ó    c                 ó   • [        5       e)z›
Identify the tokens using integer offsets ``(start_i, end_i)``,
where ``s[start_i:end_i]`` is the corresponding token.

:rtype: Iterator[Tuple[int, int]]
©ÚNotImplementedErrorr   s     r   Úspan_tokenizeÚTokenizerI.span_tokenize$   s   € ô "Ó#Ð#r   Ústringsc                 óN   • U Vs/ s H  o R                  U5      PM     sn$ s  snf )z‹
Apply ``self.tokenize()`` to each element of ``strings``.  I.e.:

    return [self.tokenize(s) for s in strings]

:rtype: List[List[str]]
)r   ©r   r   r   s      r   r   ÚTokenizerI.tokenize_sents-   s#   € ñ +2Ó2ª' Q—‘˜aÖ ©'Ñ2Ð2ùÒ2s   …"c              #   óV   #   • U H  n[        U R                  U5      5      v •  M!     g7f)z›
Apply ``self.span_tokenize()`` to each element of ``strings``.  I.e.:

    return [self.span_tokenize(s) for s in strings]

:yield: List[Tuple[int, int]]
N)Úlistr   r   s      r   Úspan_tokenize_sentsÚTokenizerI.span_tokenize_sents7   s'   é € ó ˆAÜ�t×)Ñ)¨!Ó,Ó-Ô-ò ùs   ‚')© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   Ústrr   r   r   r   Úintr   r   r"   Ú__static_attributes__r$   r   r   r   r      s—   † ñð
 ð/˜#ð / $ s¡)ó /ó ð/ð$˜sð $ x°°c¸3°h±Ñ'@ô $ð3 d¨3¡ið 3°D¸¸c¹±Oô 3ð.Ø˜C‘yð.à	�$�u˜S #˜X‘Ñ'Ñ	(÷.r   r   c                   ó>   • \ rS rSrSr\\S 5       5       rS rS r	Sr
g)ÚStringTokenizeréE   zpA tokenizer that divides a string into substrings by splitting
on the specified string (defined in subclasses).
c                 ó   • [         e©Nr   )r   s    r   Ú_stringÚStringTokenizer._stringJ   s
   € ô "Ð!r   c                 ó8   • UR                  U R                  5      $ r1   )Úsplitr2   r   s     r   r   ÚStringTokenizer.tokenizeO   s   € Ø�w‰w�t—|‘|Ó$Ð$r   c              #   óJ   #   • [        XR                  5       S h  v•N   g  N7fr1   )r	   r2   r   s     r   r   ÚStringTokenizer.span_tokenizeR   s   é € Ü'¨¯<©<Ó8×8Ó8ùs   ‚#›!œ#r$   N)r%   r&   r'   r(   r)   Úpropertyr   r2   r   r   r,   r$   r   r   r.   r.   E   s-   † ñð Øñ"ó ó ð"ò%õ9r   r.   N)r)   Úabcr   r   Útypingr   r   r   Únltk.internalsr   Únltk.tokenize.utilr	   r   r.   r$   r   r   Ú<module>r>      s4   ðñ÷ $ß (Ñ (å %Ý 3ô..�ô ..ôb9�jõ 9r   