From 4947c83db9c9ba1a8adb9b5795eaec51d366e875 Mon Sep 17 00:00:00 2001 From: tagaihahimazin Date: Fri, 21 Jul 2023 22:39:40 +0900 Subject: [PATCH 1/3] ADD Recall metrics --- pyNTCIREVAL/metrics/__init__.py | 1 + pyNTCIREVAL/metrics/recall.py | 11 +++++++++++ tests/test_recall.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 pyNTCIREVAL/metrics/recall.py create mode 100644 tests/test_recall.py diff --git a/pyNTCIREVAL/metrics/__init__.py b/pyNTCIREVAL/metrics/__init__.py index faf8250..b49b6cb 100644 --- a/pyNTCIREVAL/metrics/__init__.py +++ b/pyNTCIREVAL/metrics/__init__.py @@ -16,4 +16,5 @@ from .ms_ndcg import MSnDCG from .precision import Precision from .hit import Hit +from .recall import Recall diff --git a/pyNTCIREVAL/metrics/recall.py b/pyNTCIREVAL/metrics/recall.py new file mode 100644 index 0000000..500ea15 --- /dev/null +++ b/pyNTCIREVAL/metrics/recall.py @@ -0,0 +1,11 @@ +from .metric import Metric + +class Recall(Metric): + def __init__(self, total_positive): + self.total_positives = total_positive + + def compute(self, labeled_ranked_list): + true_positives = sum(item[1] if item[1] is not None else 0 for item in labeled_ranked_list) + if true_positives == 0: + return 0.0 + return true_positives / self.total_positives \ No newline at end of file diff --git a/tests/test_recall.py b/tests/test_recall.py new file mode 100644 index 0000000..0b9b911 --- /dev/null +++ b/tests/test_recall.py @@ -0,0 +1,22 @@ +# -*- coding:utf-8 -*- +import pytest + +class TestRecall(object): + + def test_precision(self): + from pyNTCIREVAL import Labeler + from pyNTCIREVAL.metrics import Recall + + qrels = {0: 1, 1: 0, 2: 0, 3: 0, 4: 1, 5: 0, 6: 0, 7: 1, 8: 0, 9: 0} + ranked_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # a list of document IDs + + # labeling: [doc_id] -> [(doc_id, rel_level)] + labeler = Labeler(qrels) + labeled_ranked_list = labeler.label(ranked_list) + + total_positive = len([{docid, grade} for docid, grade in qrels.items() if grade == 1]) + metric = Recall(total_positive) + assert total_positive == 3 + + result = metric.compute(labeled_ranked_list) + assert result == 1.0 \ No newline at end of file From 6104d1a1f9a6b65c5befde8dac01cfa39c0c3bfe Mon Sep 17 00:00:00 2001 From: tagaihahimazin Date: Mon, 31 Jul 2023 23:35:43 +0900 Subject: [PATCH 2/3] FIX: Recall Program --- pyNTCIREVAL/metrics/recall.py | 6 +++--- tests/test_recall.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyNTCIREVAL/metrics/recall.py b/pyNTCIREVAL/metrics/recall.py index 500ea15..1aab077 100644 --- a/pyNTCIREVAL/metrics/recall.py +++ b/pyNTCIREVAL/metrics/recall.py @@ -1,11 +1,11 @@ from .metric import Metric class Recall(Metric): - def __init__(self, total_positive): - self.total_positives = total_positive + def __init__(self, xrelnum): + self.total_positives = sum(xrelnum[1:]) def compute(self, labeled_ranked_list): - true_positives = sum(item[1] if item[1] is not None else 0 for item in labeled_ranked_list) + true_positives = sum(grade for docid, grade in labeled_ranked_list if grade is not None and grade > 0) if true_positives == 0: return 0.0 return true_positives / self.total_positives \ No newline at end of file diff --git a/tests/test_recall.py b/tests/test_recall.py index 0b9b911..d4059d0 100644 --- a/tests/test_recall.py +++ b/tests/test_recall.py @@ -8,15 +8,15 @@ def test_precision(self): from pyNTCIREVAL.metrics import Recall qrels = {0: 1, 1: 0, 2: 0, 3: 0, 4: 1, 5: 0, 6: 0, 7: 1, 8: 0, 9: 0} - ranked_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # a list of document IDs + ranked_list = [0, 1, 3, 5, 8, 9] # a list of document IDs # labeling: [doc_id] -> [(doc_id, rel_level)] labeler = Labeler(qrels) labeled_ranked_list = labeler.label(ranked_list) - total_positive = len([{docid, grade} for docid, grade in qrels.items() if grade == 1]) - metric = Recall(total_positive) - assert total_positive == 3 + xrelnum = labeler.compute_per_level_doc_num(2) + metric = Recall(xrelnum) + assert xrelnum == [7, 3] result = metric.compute(labeled_ranked_list) - assert result == 1.0 \ No newline at end of file + assert result == 0.3333333333333333 \ No newline at end of file From 4efc918f2cfb3b013e790e862ef006dae8786d07 Mon Sep 17 00:00:00 2001 From: tagaihahimazin Date: Mon, 11 Sep 2023 21:21:17 +0900 Subject: [PATCH 3/3] ADD: Comments to Recall.py --- pyNTCIREVAL/metrics/recall.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyNTCIREVAL/metrics/recall.py b/pyNTCIREVAL/metrics/recall.py index 1aab077..a7c79d1 100644 --- a/pyNTCIREVAL/metrics/recall.py +++ b/pyNTCIREVAL/metrics/recall.py @@ -1,6 +1,13 @@ from .metric import Metric class Recall(Metric): + ''' + Recall + + Args: + xrelnum: the number of judged X-rel docs (including 0-rel=judged nonrel). + labeled_ranked_list: a list of tuples, where each tuple contains a document ID and its corresponding relevance score. + ''' def __init__(self, xrelnum): self.total_positives = sum(xrelnum[1:]) @@ -8,4 +15,4 @@ def compute(self, labeled_ranked_list): true_positives = sum(grade for docid, grade in labeled_ranked_list if grade is not None and grade > 0) if true_positives == 0: return 0.0 - return true_positives / self.total_positives \ No newline at end of file + return true_positives / self.total_positives