diff --git a/pyNTCIREVAL/metrics/__init__.py b/pyNTCIREVAL/metrics/__init__.py index faf8250..b49b6cb 100644 --- a/pyNTCIREVAL/metrics/__init__.py +++ b/pyNTCIREVAL/metrics/__init__.py @@ -16,4 +16,5 @@ from .ms_ndcg import MSnDCG from .precision import Precision from .hit import Hit +from .recall import Recall diff --git a/pyNTCIREVAL/metrics/recall.py b/pyNTCIREVAL/metrics/recall.py new file mode 100644 index 0000000..a7c79d1 --- /dev/null +++ b/pyNTCIREVAL/metrics/recall.py @@ -0,0 +1,18 @@ +from .metric import Metric + +class Recall(Metric): + ''' + Recall + + Args: + xrelnum: the number of judged X-rel docs (including 0-rel=judged nonrel). + labeled_ranked_list: a list of tuples, where each tuple contains a document ID and its corresponding relevance score. + ''' + def __init__(self, xrelnum): + self.total_positives = sum(xrelnum[1:]) + + def compute(self, labeled_ranked_list): + true_positives = sum(grade for docid, grade in labeled_ranked_list if grade is not None and grade > 0) + if true_positives == 0: + return 0.0 + return true_positives / self.total_positives diff --git a/tests/test_recall.py b/tests/test_recall.py new file mode 100644 index 0000000..d4059d0 --- /dev/null +++ b/tests/test_recall.py @@ -0,0 +1,22 @@ +# -*- coding:utf-8 -*- +import pytest + +class TestRecall(object): + + def test_precision(self): + from pyNTCIREVAL import Labeler + from pyNTCIREVAL.metrics import Recall + + qrels = {0: 1, 1: 0, 2: 0, 3: 0, 4: 1, 5: 0, 6: 0, 7: 1, 8: 0, 9: 0} + ranked_list = [0, 1, 3, 5, 8, 9] # a list of document IDs + + # labeling: [doc_id] -> [(doc_id, rel_level)] + labeler = Labeler(qrels) + labeled_ranked_list = labeler.label(ranked_list) + + xrelnum = labeler.compute_per_level_doc_num(2) + metric = Recall(xrelnum) + assert xrelnum == [7, 3] + + result = metric.compute(labeled_ranked_list) + assert result == 0.3333333333333333 \ No newline at end of file