ó
    ´OÂfP/  ã                   óŒ   • S r SSKrSSKrSSKJr  SS jrS rS rS r	 " S S	5      r
S
 rS r\4S jr\4S jr0 qSS jrS rg)z0
Utility functions and classes for classifiers.
é    N)ÚLazyMapc                 ó–   ^ • Uc"  U=(       a    [        US   [        [        45      nU(       a  U 4S jn[        X15      $ [        T U5      $ )an  
Use the ``LazyMap`` class to construct a lazy list-like
object that is analogous to ``map(feature_func, toks)``.  In
particular, if ``labeled=False``, then the returned list-like
object's values are equal to::

    [feature_func(tok) for tok in toks]

If ``labeled=True``, then the returned list-like object's values
are equal to::

    [(feature_func(tok), label) for (tok, label) in toks]

The primary purpose of this function is to avoid the memory
overhead involved in storing all the featuresets for every token
in a corpus.  Instead, these featuresets are constructed lazily,
as-needed.  The reduction in memory overhead can be especially
significant when the underlying list of tokens is itself lazy (as
is the case with many corpus readers).

:param feature_func: The function that will be applied to each
    token.  It should return a featureset -- i.e., a dict
    mapping feature names to feature values.
:param toks: The list of tokens to which ``feature_func`` should be
    applied.  If ``labeled=True``, then the list elements will be
    passed directly to ``feature_func()``.  If ``labeled=False``,
    then the list elements should be tuples ``(tok,label)``, and
    ``tok`` will be passed to ``feature_func()``.
:param labeled: If true, then ``toks`` contains labeled tokens --
    i.e., tuples of the form ``(tok, label)``.  (Default:
    auto-detect based on types.)
r   c                 ó$   >• T" U S   5      U S   4$ )Nr   é   © )Úlabeled_tokenÚfeature_funcs    €Ú4/usr/lib/python3/dist-packages/nltk/classify/util.pyÚ	lazy_funcÚ!apply_features.<locals>.lazy_funcA   s   ø€ Ù  ¨qÑ!1Ó2°MÀ!Ñ4DÐEÐEó    )Ú
isinstanceÚtupleÚlistr   )r	   ÚtoksÚlabeledr   s   `   r
   Úapply_featuresr      sG   ø€ ðB �Ø×=œ: d¨1¡g´´t¨}Ó=ˆÞõ	Fô �yÓ'Ð'ä�| TÓ*Ð*r   c                 óN   • [        U  VVs1 s H  u  pUiM	     snn5      $ s  snnf )a  
:return: A list of all labels that are attested in the given list
    of tokens.
:rtype: list of (immutable)
:param tokens: The list of classified tokens from which to extract
    labels.  A classified token has the form ``(token, label)``.
:type tokens: list
)r   )ÚtokensÚtokÚlabels      r
   Úattested_labelsr   I   s#   € ô ©FÔ3ªF™L˜S“%©FÒ3Ó4Ð4ùÓ3s   ‹!
c                 ó"  • U R                  U VVs/ s H  u  p#UPM	     snn5      n[        X5       VVVs/ s H  u  u  p#oUR                  U5      PM     nnnn[        R                  " [        U5      [        U5      -  5      $ s  snnf s  snnnf ©N)Úprob_classify_manyÚzipÚprobÚmathÚlogÚsumÚlen)Ú
classifierÚgoldÚfsÚlÚresultsÚpdistÚlls          r
   Úlog_likelihoodr)   U   sp   € Ø×+Ñ+¹tÔ,Dºt±G°R«R¹tÒ,DÓE€GÜ03°DÔ0BÕ	CÒ0BÑ,™W˜b e�*‰*�QŽ-Ñ0B€BÒ	CÜ�8Š8”C˜“Gœc "›gÑ%Ó&Ð&ùó -EùÜ	Cs
   ‘B
¶B
c                 óò   • U R                  U VVs/ s H  u  p#UPM	     snn5      n[        X5       VVVs/ s H  u  u  p#oSU:H  PM     nnnnU(       a  [        U5      [        U5      -  $ gs  snnf s  snnnf )Nr   )Úclassify_manyr   r    r!   )r"   r#   r$   r%   r&   ÚrÚcorrects          r
   Úaccuracyr.   [   sh   € Ø×&Ñ&¹$Ô'?º$©w°«¹$Ò'?Ó@€GÜ*-¨dÔ*<Õ=Ò*<™,™7˜B A�AŒvÑ*<€GÒ=ÞÜ�7‹|œc '›lÑ*Ð*àùó (@ùÜ=s
   ‘A,
¶A2c                   ó$   • \ rS rSrSrS rS rSrg)ÚCutoffCheckeréd   zµ
A helper class that implements cutoff checks based on number of
iterations and log likelihood.

Accuracy cutoffs are also implemented, but they're almost never
a good idea to use.
c                 ó¶   • UR                  5       U l        SU;   a  [        US   5      * US'   SU;   a  [        US   5      US'   S U l        S U l        SU l        g )NÚmin_llÚmin_lldeltar   )ÚcopyÚcutoffsÚabsr(   ÚaccÚiter)Úselfr6   s     r
   Ú__init__ÚCutoffChecker.__init__m   s_   € Ø—|‘|“~ˆŒØ�wÓÜ!$ W¨XÑ%6Ó!7Ð 7ˆG�HÑØ˜GÓ#Ü%(¨°Ñ)?Ó%@ˆG�MÑ"ØˆŒØˆŒØˆ�	r   c                 ó´  • U R                   nU =R                  S-  sl        SU;   a  U R                  US   :¼  a  g[        R                  R                  R                  X5      n[        R                  " U5      (       a  gSU;   d  SU;   aK  SU;   a	  XCS   :¼  a  gSU;   a0  U R                  (       a  X@R                  -
  [        US   5      ::  a  gX@l        SU;   d  SU;   au  [        R                  R                  R                  X5      nSU;   a	  XSS   :¼  a  gSU;   a0  U R                  (       a  XPR                  -
  [        US   5      ::  a  gXPl
        gg )	Nr   Úmax_iterTr3   r4   Úmax_accÚmin_accdeltaF)r6   r9   ÚnltkÚclassifyÚutilr)   r   Úisnanr(   r7   r8   )r:   r"   Ú
train_toksr6   Únew_llÚnew_accs         r
   ÚcheckÚCutoffChecker.checkw   s)  € Ø—,‘,ˆØ�	Š	�Q‰�	Ø˜Ó  T§Y¡Y°'¸*Ñ2EÓ%EØä—‘×#Ñ#×2Ñ2°:ÓJˆÜ�:Š:�f×ÑØà�wÓ -°7Ó":Ø˜7Ó" v¸Ñ1BÓ'BØà Ó(Ø—G—GØŸw™wÑ&¬3¨w°}Ñ/EÓ+FÓFàØŒGà˜Ó >°WÓ#<Ü—m‘m×(Ñ(×7Ñ7¸
ÓOˆGØ˜GÓ#¨¸9Ñ3EÓ(EØà 'Ó)Ø—H—HØ§¡Ñ(¬S°¸Ñ1HÓ-IÓIàØŒHàð $=r   )r8   r6   r9   r(   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r;   rH   Ú__static_attributes__r   r   r
   r0   r0   d   s   † ñòõ!r   r0   c                 óø   • 0 nSUS'   U S   R                  5       US'   U S   R                  5       US'   S H@  nU R                  5       R                  U5      USU-  '   X R                  5       ;   US	U-  '   MB     U$ )
NTÚalwaysonr   Ú
startswithéÿÿÿÿÚendswithÚabcdefghijklmnopqrstuvwxyzú	count(%s)úhas(%s)©ÚlowerÚcount©ÚnameÚfeaturesÚletters      r
   Únames_demo_featuresr_       sƒ   € Ø€HØ€HˆZÑØ! !™WŸ]™]›_€Hˆ\ÑØ ™8Ÿ>™>Ó+€HˆZÑÛ.ˆØ)-¯©«×);Ñ);¸FÓ)Cˆ�˜vÑ%Ñ&Ø'-·±³Ñ'=ˆ�˜VÑ#Ó$ñ /ð €Or   c                 óp  • 0 nSUS'   U S   R                  5       S;   US'   U S   R                  5       S;   US'   S Hv  nU R                  5       R                  U5      US	U-  '   X R                  5       ;   US
U-  '   X S   R                  5       :H  USU-  '   X S   R                  5       :H  USU-  '   Mx     U$ )NTrQ   r   Úaeiouyzstartswith(vowel)rS   zendswith(vowel)rU   rV   rW   zstartswith(%s)zendswith(%s)rX   r[   s      r
   Úbinary_names_demo_featuresrb   «   sÉ   € Ø€HØ€HˆZÑØ$(¨¡G§M¡M£O°xÑ$?€HÐ Ñ!Ø"& r¡(§.¡.Ó"2°hÑ">€HÐÑÛ.ˆØ)-¯©«×);Ñ);¸FÓ)Cˆ�˜vÑ%Ñ&Ø'-·±³Ñ'=ˆ�˜VÑ#Ñ$Ø.4¸Q¹¿¹»Ñ.GˆÐ! FÑ*Ñ+Ø,2¸2±h·n±nÓ6FÑ,Fˆ� &Ñ(Ó)ñ	 /ð
 €Or   c           
      ó4  • SS K nSSKJn  UR                  S5       Vs/ s H  oDS4PM     snUR                  S5       Vs/ s H  oDS4PM     sn-   nUR	                  S5        UR                  U5        US S nUSS	 n[        S
5        U " U VV	s/ s H  u  p‰U" U5      U	4PM     sn	n5      n
[        S5        [        X§ VV	s/ s H  u  p‰U" U5      U	4PM     sn	n5      n[        SU-  5         U VV	s/ s H  u  p‰U" U5      PM     nnn	U
R                  U5      n[        X}5       VVVs/ s H  u  u  pNoÿR                  U5      PM     nnnn[        S[        U5      [        U5      -  -  5        [        5         [        S5        [        [        X}5      5      S S  HB  u  u  nnnUS:X  a  SnOSn[        UXOR                  S5      UR                  S5      4-  5        MD     U
$ s  snf s  snf s  sn	nf s  sn	nf s  sn	nf s  snnnf ! [         a     U
$ f = f)Nr   ©Únamesúmale.txtÚmaleú
female.txtÚfemaleé@â iˆ  i|  úTraining classifier...úTesting classifier...úAccuracy: %6.4fúAvg. log likelihood: %6.4fúMUnseen Names      P(Male)  P(Female)
----------------------------------------é   ú  %-15s *%6.4f   %6.4fú  %-15s  %6.4f  *%6.4f)ÚrandomÚnltk.corpusre   ÚwordsÚseedÚshuffleÚprintr.   r   r   Úlogprobr    r!   r   r   ÚNotImplementedError)Útrainerr]   rs   re   r\   ÚnamelistÚtrainÚtestÚnÚgr"   r8   Útest_featuresetsÚpdistsr#   r'   r(   ÚgenderÚfmts                      r
   Ú
names_demor…   ¸   s  € Ûå!ð ,1¯;©;°zÔ+BÓCÒ+B 4�v“Ñ+BÑCØ%*§[¡[°Ô%>óGÚ%>˜TˆxÓÑ%>ñGñ €Hð
 ‡K�K�ÔØ
‡N�N�8ÔØ�U�dˆO€EØ�D˜Ð€Dô 
Ð
"Ô#Ù¹Ô?º©v°™8 A›;¨Ó*¹Ò?Ó@€Jô 
Ð
!Ô"Ü
�:¸tÔDºt±V°a¡¨!£¨aÓ 0¹tÒDÓ
E€CÜ	Ð
˜cÑ
!Ô"ðÙ6:Ô;²d©F¨Q™H QžK±dÐÑ;Ø×.Ñ.Ð/?Ó@ˆÜ?BÀ4Ô?PÕQÒ?PÑ&;¡|¨°U�m‰m˜DÖ!Ñ?PˆÒQÜÐ*¬c°"«g¼¸D»	Ñ.AÑBÔCÜŒÜÐAÔBÜ%)¬#¨dÓ*;Ó%<¸R¸aÓ%@Ñ!‰NˆT�6˜EØ˜ÓØ.‘à.�Ü�#˜Ÿz™z¨&Ó1°5·:±:¸hÓ3GÐHÑHÖIñ &Að ÐùòM Dùò Gùó @ùó  Eùó <ùäQøô ó Øð Ðð	úsH   žG&Á G+ÂG0
ÃG6Ã0H	 Ã5G<Ä	#H	 Ä,HÅBH	 Ç<H	 È	
HÈHc           
      ój  • SS K nSSKJn  UR                  S5      nUR                  S5      nUR	                  S5        UR                  U5        UR                  U5        [        XS S 5      n[        XSS US S -   5      nUSS	  Vs/ s H  oˆS
4PM     snUSS  Vs/ s H  oˆS4PM     sn-   n	UR                  U	5        [        S5        U " Xg5      n
[        S5        [        X© VVs/ s H  u  p¼U" U5      U4PM     snn5      n[        SU-  5         U	 VVs/ s H  u  p¼U" U5      PM     nnnU
R                  U5      n[        XŸ5       VVVs/ s H  u  u  nnnUR                  U5      PM     nnnn[        S[        U5      [        U	5      -  -  5        [        5         [        S5        [        XŸ5      S S  HC  u  u  nnnUS
:X  a  SnOSn[        UUUR                  S
5      UR                  S5      4-  5        ME     U
$ s  snf s  snf s  snnf s  snnf s  snnnf ! [         a     U
$ f = f)Nr   rd   rf   rh   iñû	 iÐ  iÄ	  iô  i¾
  Tiî  Frk   rl   rm   rn   ro   rp   rq   rr   )rs   rt   re   ru   rv   rw   Úmaprx   r.   r   r   ry   r    r!   r   rz   )r{   r]   rs   re   Ú
male_namesÚfemale_namesÚpositiveÚ	unlabeledr\   r~   r"   r   Úmr8   r�   r‚   r#   r'   r(   Úis_maler„   s                        r
   Úpartial_names_demorŽ   ç   s=  € Ûå!à—‘˜ZÓ(€JØ—;‘;˜|Ó,€Là
‡K�K�ÔØ
‡N�N�:ÔØ
‡N�N�<Ô ô �8¨¨Ð.Ó/€Hô �H¨¨dÐ3°lÀ4ÀCÐ6HÑHÓI€Ið &0°°TÑ%:Ó;Ò%:˜T�4‹LÑ%:Ñ;Ø".¨s°3Ñ"7ó?Ú"7˜$ˆu‹Ñ"7ñ?ñ €Dð ‡N�N�4Ôô 
Ð
"Ô#Ù˜Ó-€Jô 
Ð
!Ô"Ü
�:¸tÔDºt±V°a¡¨!£¨aÓ 0¹tÒDÓ
E€CÜ	Ð
˜cÑ
!Ô"ðÙ6:Ô;²d©F¨Q™H QžK±dÐÑ;Ø×.Ñ.Ð/?Ó@ˆÜ?BÀ4Ô?PÕQÒ?PÑ&;¡|¨¨d°Uˆe�m‰m˜DÖ!Ñ?PˆÒQÜÐ*¬c°"«g¼¸D»	Ñ.AÑBÔCÜŒÜÐAÔBÜ&)¨$Ó&7¸¸Ó&;Ñ"‰OˆT�7˜UØ˜$‹Ø.‘à.�Ü�#˜˜uŸz™z¨$Ó/°·±¸EÓ1BÐCÑCÖDñ '<ð ÐùòE <ùò ?ùó  Eùó <ùäQøô ó Øð Ðð	úsC   Â	HÂHÃ*HÄH$ ÄHÄ0#H$ Å!HÅ4BH$ ÈH$ È$
H2È1H2c           
      ól  • SS K nSSKJn  [        S5        U[        ;  a7  UR                  U5       Vs/ s H  ofUR                  S   4PM     sn[        U'   [        U   S S  nU[        U5      :”  a  [        U5      n[        U VVs1 s H  u  phUiM	     snn5      n	[        SSR                  U	5      -   5        [        S5        UR                  S5        UR                  U5        US [        SU-  5       n
U[        SU-  5      U n[        S	5        U " U
 VVs/ s H  u  phU" U5      U4PM     snn5      n[        S
5        [        XË VVs/ s H  u  phU" U5      U4PM     snn5      n[        SU-  5         U VVs/ s H  u  pcU" U5      PM     nnnUR                  U5      n[        X¿5       VVVs/ s H  u  u  nnnUR!                  U5      PM     nnnn[        S[#        U5      [        U5      -  -  5        U$ s  snf s  snnf s  snnf s  snnf s  snnf s  snnnf ! [$         a     U$ f = f)Nr   )ÚsensevalzReading data...z
  Senses: Ú zSplitting into test & train...rj   gš™™™™™é?rk   rl   rm   rn   )rs   rt   r�   rx   Ú_inst_cacheÚ	instancesÚsensesr!   r   Újoinrv   rw   Úintr.   r   r   ry   r    rz   )r{   Úwordr]   r   rs   r�   Úir“   r%   r”   r}   r~   r"   r8   r�   r‚   r\   r#   r'   r(   s                       r
   Úwsd_demor™   "  s  € Ûå$ô 
Ð
Ôà”;ÓØ7?×7IÑ7IÈ$Ô7OÓPÒ7O°! §¡¨!¡Ó-Ñ7OÑPŒ�DÑÜ˜DÑ!¡!Ð$€IØŒ3ˆy‹>ÓÜ�	‹NˆÜ¡9Ô-¢9™˜!“1¡9Ò-Ó.€FÜ	ˆ,˜Ÿ™ &Ó)Ñ
)Ô*ô 
Ð
*Ô+Ø
‡K�K�ÔØ
‡N�N�9ÔØ�nœ˜C !™G›Ð%€EØ”S˜˜q™“\ AÐ&€Dô 
Ð
"Ô#Ù¹Ô?º©v°™8 A›;¨Ó*¹Ò?Ó@€Jô 
Ð
!Ô"Ü
�:¸tÔDºt±V°a¡¨!£¨aÓ 0¹tÒDÓ
E€CÜ	Ð
˜cÑ
!Ô"ðÙ6:Ô;²d©F¨Q™H QžK±dÐÑ;Ø×.Ñ.Ð/?Ó@ˆÜ?BÀ4Ô?PÕQÒ?PÑ&;¡|¨¨d°Uˆe�m‰m˜DÖ!Ñ?PˆÒQÜÐ*¬c°"«g¼¸D»	Ñ.AÑBÔCð
 ÐùòE Qùó .ùó @ùó  Eùó <ùäQøäó Øð Ðð	úsG   ³HÂH
ÄH
ÅHÅ;H% Æ HÆ#H% Æ7!HÇ'H% ÈH% È%
H3È2H3c                  ód   •  [           g! [         a  n [        S5      n[        U5      U eSn A ff = f)z0
Checks whether the MEGAM binary is configured.
z\Please configure your megam binary first, e.g.
>>> nltk.config_megam('/usr/bin/local/megam')N)Ú
_megam_binÚ	NameErrorÚstr)ÚeÚerr_msgs     r
   Úcheck_megam_configr    P  s8   € ð(ÞøÜó (Üð<ó
ˆô ˜Ó  aÐ'ûð(ús   ‚	 ‰
/“*ª/r   )iè  )rN   r   Únltk.classify.utilrA   Ú	nltk.utilr   r   r   r)   r.   r0   r_   rb   r…   rŽ   r’   r™   r    r   r   r
   Ú<module>r£      si   ðñó ó Ý ô*+òZ	5ò'ò÷4ñ 4òxò
ð "5ô ,ð^ *=ô 5ðp €ô+ó\(r   