ó
    ´OÂf  ã                   ór   • S SK Jr  S SKJr  S SKJrJr  \R                  r\" \SSSSS9r	S\	l
        S	 rS
 rg)é    )Úpartial)Úchain)Ú
everygramsÚpad_sequenceTz<s>z</s>)Úpad_leftÚleft_pad_symbolÚ	pad_rightÚright_pad_symbolz‘Pads both ends of a sentence to length specified by ngram order.

    Following convention <s> pads the start of sentence </s> pads its end.
    c           	      ó6   • [        [        [        XS95      U S9$ )zhHelper with some useful defaults.

Applies pad_both_ends to sentence and follows it up with everygrams.
©Ún©Úmax_len)r   ÚlistÚpad_both_ends)ÚorderÚsentences     Ú7/usr/lib/python3/dist-packages/nltk/lm/preprocessing.pyÚpadded_everygramsr      s   € ô
 ”dœ=¨Ñ;Ó<ÀeÑLÐLó    c                 óf   ^ ^• [        [        T S9mU U4S jU 5       [        [        TU5      5      4$ )aÚ  Default preprocessing for a sequence of sentences.

Creates two iterators:

- sentences padded and turned into sequences of `nltk.util.everygrams`
- sentences padded as above and chained together for a flat stream of words

:param order: Largest ngram length produced by `everygrams`.
:param text: Text to iterate over. Expected to be an iterable of sentences.
:type text: Iterable[Iterable[str]]
:return: iterator over text as ngrams, iterator over text as vocabulary data
r   c              3   óV   >#   • U  H  n[        [        T" U5      5      TS 9v •  M      g7f)r   N)r   r   )Ú.0Úsentr   Ú
padding_fns     €€r   Ú	<genexpr>Ú,padded_everygram_pipeline.<locals>.<genexpr>1   s"   øé € ÐLÂt¸tŒ”D™ DÓ)Ó*°EÖ	:Âtùs   ƒ&))r   r   ÚflattenÚmap)r   Útextr   s   ` @r   Úpadded_everygram_pipeliner!   "   s2   ù€ ô œ¨%Ñ0€JåLÁtÓLÜ”�J Ó%Ó&ðð r   N)Ú	functoolsr   Ú	itertoolsr   Ú	nltk.utilr   r   Úfrom_iterabler   r   Ú__doc__r   r!   © r   r   Ú<module>r(      sI   ðõ Ý ç .à
×
Ñ
€ÙØØØØØñ€ð€Ô òMór   