v0.6.0:

Add 3 metrics for regression tasks: Spearman's r, Pearson's r, and Kendall's tau.

Author: Xiangyan93

molalkit/__init__.py

@@ -2,4 +2,4 @@
 # -*- coding: utf-8 -*-
 
 
-__version__ = '0.5.2'
+__version__ = '0.6.0'

molalkit/al/learner.py

@@ -9,6 +9,7 @@
 import numpy as np
 from logging import Logger
 from sklearn.metrics import *
+import scipy
 from ..args import Metric
 from .selection_method import BaseSelectionMethod, RandomSelectionMethod
 from .forgetter import BaseForgetter, RandomForgetter, FirstForgetter
@@ -40,9 +41,15 @@ def eval_metric_func(y, y_pred, metric: str) -> float:
     elif metric == 'mse':
         return mean_squared_error(y, y_pred)
     elif metric == 'rmse':
-        return np.sqrt(eval_metric_func(y, y_pred, 'mse'))
+        return mean_squared_error(y, y_pred, squared=False)
     elif metric == 'max':
         return np.max(abs(y - y_pred))
+    elif metric == 'spearman':
+        return scipy.stats.spearmanr(y, y_pred)[0]
+    elif metric == 'kendall':
+        return scipy.stats.kendalltau(y, y_pred)[0]
+    elif metric == 'pearson':
+        return scipy.stats.pearsonr(y, y_pred)[0]
     else:
         raise RuntimeError(f'Unsupported metrics {metric}')
 
@@ -254,7 +261,7 @@ def evaluate(self, alr: ActiveLearningResult):
                     pd.DataFrame({'true': self.dataset_val_selector.y.ravel(), 'pred': y_pred}).to_csv(
                         os.path.join(self.save_dir, f'selector_{self.current_iter}.csv'), index=False)
                 for metric in self.metrics:
-                    metric_value = eval_metric_func(self.dataset_val_selector.y, y_pred, metric=metric)
+                    metric_value = eval_metric_func(self.dataset_val_selector.y.ravel(), y_pred, metric=metric)
                     alr.results[f'{metric}_selector'] = metric_value
             # evaluate the percentage of top-k data selected in the training set
             if self.top_k_id is not None:
@@ -268,7 +275,7 @@ def evaluate(self, alr: ActiveLearningResult):
                     pd.DataFrame({'true': self.dataset_val_selector.y.ravel(), 'pred': y_pred}).to_csv(
                         os.path.join(self.save_dir, f'evaluator_{i}_{self.current_iter}.csv'), index=False)
                 for metric in self.metrics:
-                    metric_value = eval_metric_func(self.dataset_val_evaluators[i].y, y_pred, metric=metric)
+                    metric_value = eval_metric_func(self.dataset_val_evaluators[i].y.ravel(), y_pred, metric=metric)
                     alr.results[f'{metric}_evaluator_{i}'] = metric_value
         self.info('evaluating model performance finished.')
 

molalkit/args.py

@@ -11,15 +11,13 @@
 import numpy as np
 from mgktools.features_mol import FeaturesGenerator
 from mgktools.data.split import data_split_index
+from mgktools.evaluators.metric import Metric
 from molalkit.logging import create_logger
 from molalkit.utils import get_data, get_model, get_kernel
 from molalkit.data.datasets import DATA_DIR
 from molalkit.al.selection_method import *
 from molalkit.al.forgetter import *
-
 CWD = os.path.dirname(os.path.abspath(__file__))
-Metric = Literal['roc-auc', 'accuracy', 'precision', 'recall', 'f1_score', 'mcc',
-'rmse', 'mae', 'mse', 'r2', 'max']
 
 
 class CommonArgs(Tap):

test/test_metric.py

@@ -31,7 +31,7 @@ def test_classification(metric):
     al_results_check(save_dir)
 
 
-@pytest.mark.parametrize('metric', ['rmse', 'mae', 'mse', 'r2', 'max'])
+@pytest.mark.parametrize('metric', ['rmse', 'mae', 'mse', 'r2', 'max', 'spearman', 'kendall', 'pearson'])
 def test_regression(metric):
     save_dir = os.path.join(CWD, 'test')
     arguments = [