ó
    ´OÂf
  ã                   ót   • S r SSKJr  SSKJr  SSKJr  S r " S S\5      r " S S	\5      r	 " S
 S\5      r
g)z…Smoothing algorithms for language modeling.

According to Chen & Goodman 1995 these should work with both Backoff and
Interpolation.
é    )Úmethodcaller)Ú	Smoothing)ÚConditionalFreqDistc                 ó�   ^• [        U [        5      (       a  [        S5      OS m[        U4S jU R	                  5        5       5      $ )z©Count values that are greater than zero in a distribution.

Assumes distribution is either a mapping with counts as values or
an instance of `nltk.ConditionalFreqDist`.
ÚNc                 ó   • U $ ©N© )Úcounts    Ú3/usr/lib/python3/dist-packages/nltk/lm/smoothing.pyÚ<lambda>Ú'_count_values_gt_zero.<locals>.<lambda>   s   € ™5ó    c              3   óD   >#   • U  H  nT" U5      S :”  d  M  Sv •  M     g7f)r   é   Nr
   )Ú.0Údist_or_countÚas_counts     €r   Ú	<genexpr>Ú(_count_values_gt_zero.<locals>.<genexpr>   s$   øé € ð Ú4ˆm¹ÀÓ8OÐRSÑ8S�‰Ò4ùs   ƒ —	 )Ú
isinstancer   r   ÚsumÚvalues)Údistributionr   s    @r   Ú_count_values_gt_zeror      sJ   ø€ ô �lÔ$7×8Ñ8ô 	�SÔá ð ô ô Ø+×2Ñ2Ô4óó ð r   c                   ó>   ^ • \ rS rSrSrU 4S jrS rS rS rSr	U =r
$ )Ú
WittenBellé$   zWitten-Bell smoothing.c                 ó(   >• [         TU ]  " X40 UD6  g r	   )ÚsuperÚ__init__)ÚselfÚ
vocabularyÚcounterÚkwargsÚ	__class__s       €r   r!   ÚWittenBell.__init__'   s   ø€ Ü‰Ò˜Ñ7°Ó7r   c                 ót   • U R                   U   R                  U5      nU R                  U5      nSU-
  U-  U4$ )Ng      ð?)ÚcountsÚfreqÚ_gamma©r"   ÚwordÚcontextÚalphaÚgammas        r   Úalpha_gammaÚWittenBell.alpha_gamma*   s=   € Ø—‘˜GÑ$×)Ñ)¨$Ó/ˆØ—‘˜GÓ$ˆØ�e‘˜uÑ$ eÐ+Ð+r   c                 óv   • [        U R                  U   5      nX"U R                  U   R                  5       -   -  $ r	   )r   r)   r   ©r"   r.   Ún_pluss      r   r+   ÚWittenBell._gamma/   s5   € Ü& t§{¡{°7Ñ';Ó<ˆØ $§+¡+¨gÑ"6×"8Ñ"8Ó":Ñ:Ñ;Ð;r   c                 óL   • U R                   R                  R                  U5      $ r	   ©r)   Úunigramsr*   ©r"   r-   s     r   Úunigram_scoreÚWittenBell.unigram_score3   ó   € Ø�{‰{×#Ñ#×(Ñ(¨Ó.Ð.r   r
   ©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r!   r1   r+   r;   Ú__static_attributes__Ú__classcell__©r&   s   @r   r   r   $   s   ø† Ù õ8ò,ò
<÷/ð /r   r   c                   óB   ^ • \ rS rSrSrSU 4S jjrS rS rS rSr	U =r
$ )	ÚAbsoluteDiscountingé7   z!Smoothing with absolute discount.c                 ó4   >• [         TU ]  " X40 UD6  X0l        g r	   )r    r!   Údiscount)r"   r#   r$   rK   r%   r&   s        €r   r!   ÚAbsoluteDiscounting.__init__:   s   ø€ Ü‰Ò˜Ñ7°Ò7Ø �r   c                 ó¸   • [        U R                  U   U   U R                  -
  S5      U R                  U   R                  5       -  nU R	                  U5      nX44$ )Nr   )Úmaxr)   rK   r   r+   r,   s        r   r1   ÚAbsoluteDiscounting.alpha_gamma>   sZ   € ä�—‘˜GÑ$ TÑ*¨T¯]©]Ñ:¸AÓ>Ø�k‰k˜'Ñ"×$Ñ$Ó&ñ'ð 	ð —‘˜GÓ$ˆØˆ|Ðr   c                 óŒ   • [        U R                  U   5      nU R                  U-  U R                  U   R                  5       -  $ r	   )r   r)   rK   r   r4   s      r   r+   ÚAbsoluteDiscounting._gammaF   s;   € Ü& t§{¡{°7Ñ';Ó<ˆØ—‘ Ñ&¨$¯+©+°gÑ*>×*@Ñ*@Ó*BÑBÐBr   c                 óL   • U R                   R                  R                  U5      $ r	   r8   r:   s     r   r;   Ú!AbsoluteDiscounting.unigram_scoreJ   r=   r   )rK   )g      è?r>   rF   s   @r   rH   rH   7   s    ø† Ù+÷!òòC÷/ð /r   rH   c                   óR   ^ • \ rS rSrSrSU 4S jjrS rS r\" 5       4S jr	Sr
U =r$ )	Ú	KneserNeyéN   aÑ  Kneser-Ney Smoothing.

This is an extension of smoothing with a discount.

Resources:
- https://pages.ucsd.edu/~rlevy/lign256/winter2008/kneser_ney_mini_example.pdf
- https://www.youtube.com/watch?v=ody1ysUTD7o
- https://medium.com/@dennyc/a-simple-numerical-example-for-kneser-ney-smoothing-nlp-4600addf38b8
- https://www.cl.uni-heidelberg.de/courses/ss15/smt/scribe6.pdf
- https://www-i6.informatik.rwth-aachen.de/publications/download/951/Kneser-ICASSP-1995.pdf
c                 ó@   >• [         TU ]  " X40 UD6  X@l        X0l        g r	   )r    r!   rK   Ú_order)r"   r#   r$   ÚorderrK   r%   r&   s         €r   r!   ÚKneserNey.__init__[   s   ø€ Ü‰Ò˜Ñ7°Ò7Ø ŒØ�r   c                 ó0   • U R                  U5      u  p#X#-  $ r	   )Ú_continuation_counts)r"   r-   Úword_continuation_countÚtotal_counts       r   r;   ÚKneserNey.unigram_score`   s   € Ø/3×/HÑ/HÈÓ/NÑ,ÐØ&Ñ4Ð4r   c                 ó  • U R                   U   n[        U5      S-   U R                  :X  a  X1   UR                  5       4OU R	                  X5      u  pE[        X@R                  -
  S5      U-  nU R                  [        U5      -  U-  nXg4$ )Nr   g        )r)   ÚlenrX   r   r\   rN   rK   r   )r"   r-   r.   Úprefix_countsr]   r^   r/   r0   s           r   r1   ÚKneserNey.alpha_gammad   s�   € ØŸ™ GÑ,ˆô �7‹|˜aÑ 4§;¡;Ó.ð Ñ  -§/¡/Ó"3Ñ4à×*Ñ*¨4Ó9ñ 	-Ðô
 Ð+¯m©mÑ;¸SÓAÀKÑOˆØ—‘Ô 5°mÓ DÑDÀ{ÑRˆØˆ|Ðr   c                 óÌ   ^• U4S jU R                   [        T5      S-      R                  5        5       nSu  pEU H$  nU[        Xa   S:„  5      -  nU[	        U5      -  nM&     XE4$ )zèCount continuations that end with context and word.

Continuations track unique ngram "types", regardless of how many
instances were observed for each "type".
This is different than raw ngram counts which track number of instances.
c              3   óB   >#   • U  H  u  pUS S T:X  d  M  Uv •  M     g7f)r   Nr
   )r   Úprefix_ngramr)   r.   s      €r   r   Ú1KneserNey._continuation_counts.<locals>.<genexpr>v   s,   øé € ð ,
â(MÑ$�Ø˜A˜BÐ 7Ñ*÷ ‰FÚ(Mùs   ƒ–	é   )r   r   r   )r)   ra   ÚitemsÚintr   )r"   r-   r.   Ú higher_order_ngrams_with_contextÚ#higher_order_ngrams_with_word_countÚtotalr)   s     `    r   r\   ÚKneserNey._continuation_countso   su   ø€ ô,
à(,¯©´C¸³LÀ1Ñ4DÑ(E×(KÑ(KÔ(Mó,
Ð(ð
 6:Ñ2Ð+Û6ˆFØ/´3°v±|ÀaÑ7GÓ3HÑHÐ/ØÔ*¨6Ó2Ñ2ŠEñ 7ð 3Ð9Ð9r   )rX   rK   )gš™™™™™¹?)r?   r@   rA   rB   rC   r!   r;   r1   Útupler\   rD   rE   rF   s   @r   rU   rU   N   s(   ø† ñ
÷ò
5ò	ñ 27³÷ :ò :r   rU   N)rC   Úoperatorr   Únltk.lm.apir   Únltk.probabilityr   r   r   rH   rU   r
   r   r   Ú<module>rs      s>   ðñõ
 "å !Ý 0òô"/�ô /ô&/˜)ô /ô.1:�	õ 1:r   