Of5jSrSSKrSSKJr SSKJrJr S SjrS SjrSr S Sjr S r SS jr SS jr g)zRIBES score implementation N)islice)choosengramsc SnUH}n[XQ5n[U5n[S[R"S[ U5[ U5- - 55n[ U5[ U5- n XyU--X--n X:dM{U nM U$)a/ The RIBES (Rank-based Intuitive Bilingual Evaluation Score) from Hideki Isozaki, Tsutomu Hirao, Kevin Duh, Katsuhito Sudoh and Hajime Tsukada. 2010. "Automatic Evaluation of Translation Quality for Distant Language Pairs". In Proceedings of EMNLP. https://www.aclweb.org/anthology/D/D10/D10-1092.pdf The generic RIBES scores used in shared task, e.g. Workshop for Asian Translation (WAT) uses the following RIBES calculations: RIBES = kendall_tau * (alpha**p1) * (beta**bp) Please note that this re-implementation differs from the official RIBES implementation and though it emulates the results as describe in the original paper, there are further optimization implemented in the official RIBES script. Users are encouraged to use the official RIBES script instead of this implementation when evaluating your machine translation system. Refer to https://www.kecl.ntt.co.jp/icl/lirg/ribes/ for the official script. :param references: a list of reference sentences :type references: list(list(str)) :param hypothesis: a hypothesis sentence :type hypothesis: list(str) :param alpha: hyperparameter used as a prior for the unigram precision. :type alpha: float :param beta: hyperparameter used as a prior for the brevity penalty. :type beta: float :return: The best ribes score from one of the references. :rtype: float gg?)word_rank_alignment kendall_tauminmathexplen) references hypothesisalphabeta best_ribes referencewordernktbpp1_ribess >> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', ... 'ensures', 'that', 'the', 'military', 'always', ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', ... 'ensures', 'that', 'the', 'military', 'will', 'forever', ... 'heed', 'Party', 'commands'] >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', ... 'guarantees', 'the', 'military', 'forces', 'always', ... 'being', 'under', 'the', 'command', 'of', 'the', 'Party'] >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', ... 'army', 'always', 'to', 'heed', 'the', 'directions', ... 'of', 'the', 'party'] >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', ... 'interested', 'in', 'world', 'history'] >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', ... 'because', 'he', 'read', 'the', 'book'] >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] >>> hypotheses = [hyp1, hyp2] >>> round(corpus_ribes(list_of_references, hypotheses),4) 0.3597 :param references: a corpus of lists of reference sentences, w.r.t. hypotheses :type references: list(list(list(str))) :param hypotheses: a list of hypothesis sentences :type hypotheses: list(list(str)) :param alpha: hyperparameter used as a prior for the unigram precision. :type alpha: float :param beta: hyperparameter used as a prior for the brevity penalty. :type beta: float :return: The best ribes score from one of the references. :rtype: float g)ziprr )list_of_references hypothesesrrcorpus_best_ribesr rs r corpus_ribesr Fs@Z"%&8"E ^JEPP#F s: ..rcd[[U[U555Hup#X:XdM Us $ g)a^ This function returns the position of the first instance of the ngram appearing in a sentence. Note that one could also use string as follows but the code is a little convoluted with type casting back and forth: char_pos = ' '.join(sent)[:' '.join(sent).index(' '.join(ngram))] word_pos = char_pos.count(' ') Another way to conceive this is: return next(i for i, ng in enumerate(ngrams(sentence, len(ngram))) if ng == ngram) :param ngram: The ngram that needs to be searched :type ngram: tuple :param sentence: The list of tokens to search from. :type sentence: list(str) N) enumeraterr )ngramsentenceisublists rposition_of_ngramr'zs+, xU <=   H>rc /n[U5n/n/n[S[U5S-5HIn[X5HnURU5 M [X5HnURU5 M MK [ U5GHwupX;aM UR U 5UR U 5s=:XaS:Xa%O O"URUR U 55 M][XU - S-5n [SU 5Hn X-U:aj[[XX-S-55n UR U 5nUR U 5nXs=:XaS:Xa"O O[X5nURU5 MX::dM|[[XU - U S-55nUR U5nUR U5nXs=:XaS:XdMO M[UU5nURU[U5-S- 5 GMu GMz U$)a This is the word rank alignment algorithm described in the paper to produce the *worder* list, i.e. a list of word indices of the hypothesis word orders w.r.t. the list of reference words. Below is (H0, R0) example from the Isozaki et al. 2010 paper, note the examples are indexed from 1 but the results here are indexed from 0: >>> ref = str('he was interested in world history because he ' ... 'read the book').split() >>> hyp = str('he read the book because he was interested in world ' ... 'history').split() >>> word_rank_alignment(ref, hyp) [7, 8, 9, 10, 6, 0, 1, 2, 3, 4, 5] The (H1, R1) example from the paper, note the 0th index: >>> ref = 'John hit Bob yesterday'.split() >>> hyp = 'Bob hit John yesterday'.split() >>> word_rank_alignment(ref, hyp) [2, 1, 0, 3] Here is the (H2, R2) example from the paper, note the 0th index here too: >>> ref = 'the boy read the book'.split() >>> hyp = 'the book was read by the boy'.split() >>> word_rank_alignment(ref, hyp) [3, 4, 2, 0, 1] :param reference: a reference sentence :type reference: list(str) :param hypothesis: a hypothesis sentence :type hypothesis: list(str) ) r rangerappendr"countindexmaxtuplerr')rrcharacter_basedrhyp_len ref_ngrams hyp_ngramsnngr%h_wordmax_window_sizewindowright_context_ngramnum_times_in_refnum_times_in_hypposleft_context_ngrams rrrsFF*oGJJ 1c)nq( )&B   b !''B   b !(* z*   "   f %)@ EA E MM)//&1 2!!q[1_5O?3:'*/zajSTn0U*V''1'7'78K'L$'1'7'78K'L$'@q@/0CO c*;).vjf*aRSe/T)U&'1'7'78J'K$'1'7'78J'K$'@q@@/0BIN cC0B,C&Ca&GH-4+B Mrc## [U5nS[US5p2U/nUbPU[US5p2UbUS-U:XaURU5 O[U5S:a [ U5v U/nUbMOgg7f)a+ Given the *worder* list, this function groups monotonic +1 sequences. >>> worder = [7, 8, 9, 10, 6, 0, 1, 2, 3, 4, 5] >>> list(find_increasing_sequences(worder)) [(7, 8, 9, 10), (0, 1, 2, 3, 4, 5)] :param worder: The worder list output from word_rank_alignment :param type: list(int) Nr))iternextr+r r/)ritemsabresults rfind_increasing_sequencesrEsv LE eT"qSF -$ud#1 =QUaZ MM! 6{QFm#SF -s A,A20A2c[U5nUS:aSnO4[U5n[SU55n[US5nSU-U- S- nU(aUS-S- $U$)a] Calculates the Kendall's Tau correlation coefficient given the *worder* list of word alignments from word_rank_alignment(), using the formula: tau = 2 * num_increasing_pairs / num_possible_pairs -1 Note that the no. of increasing pairs can be discontinuous in the *worder* list and each each increasing sequence can be tabulated as choose(len(seq), 2) no. of increasing pairs, e.g. >>> worder = [7, 8, 9, 10, 6, 0, 1, 2, 3, 4, 5] >>> number_possible_pairs = choose(len(worder), 2) >>> round(kendall_tau(worder, normalize=False),3) -0.236 >>> round(kendall_tau(worder),3) 0.382 :param worder: The worder list output from word_rank_alignment :type worder: list(int) :param normalize: Flag to indicate normalization to between 0.0 and 1.0. :type normalize: boolean :return: The Kendall's Tau correlation coefficient. :rtype: float c3L# UHn[[U5S5v M g7frGN)rr ).0seqs r kendall_tau..#s "WBV36#c(A#6#6BVs"$r))r rEsumr)r normalize worder_lentauincreasing_sequencesnum_increasing_pairsnum_possible_pairss rrrss2VJA~ 9@""WBV"WW#J2&&);;a?a1} rc [U5n[S[U[U5555nSU[ US-S5- - nU(aUS-S- $U$)a Calculates the Spearman's Rho correlation coefficient given the *worder* list of word alignment from word_rank_alignment(), using the formula: rho = 1 - sum(d**2) / choose(len(worder)+1, 3) Given that d is the sum of difference between the *worder* list of indices and the original word indices from the reference sentence. Using the (H0,R0) and (H5, R5) example from the paper >>> worder = [7, 8, 9, 10, 6, 0, 1, 2, 3, 4, 5] >>> round(spearman_rho(worder, normalize=False), 3) -0.591 >>> round(spearman_rho(worder), 3) 0.205 :param worder: The worder list output from word_rank_alignment :param type: list(int) c34# UHupX- S-v M g7frJ)rKwir%s rrMspearman_rho..DsQ2P1}2Psr)rG)r rOrr*r)rrPrQ sum_d_squarerhos r spearman_rhor^.sY*VJQ#feJ>O2PQQL lVJNA66 6Ca1} r)g?g?)F)T)__doc__r itertoolsr nltk.utilrrrr r'rrErr^rXrrrbs;# $3l1/h8Od0+\r