Merge pull request #13 from stdiff/dev

Dev

Author: stdiff

.travis.yml

@@ -10,11 +10,14 @@ install:
   - sudo apt-get install graphviz
   - pip install -r requirements.txt
 script:
-  - python -m unittest test/test_processing.py
-  - python -m unittest test/test_utilities.py
-  - python -m unittest test/test_modeling.py
+  - coverage run -m unittest test/test_processing.py
+  - coverage run -m unittest test/test_utilities.py
+  - coverage run -m unittest test/test_modeling.py
+  - bash <(curl -s https://codecov.io/bash)
   - python setup.py sdist bdist_wheel
   - pip install dist/adhoc-0.2-py3-none-any.whl
   - cd notebooks
   - python usage-processing.py
   - python usage-modeling.py
+
+

adhoc/modeling.py

@@ -15,6 +15,7 @@
 ## for show_tree
 from io import StringIO
 from sklearn.tree import export_graphviz
+from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
 from IPython.display import Image
 import pydot
 
@@ -178,17 +179,18 @@ def show_coefficients(grid:GridSearchCV, columns:List[str]) -> pd.DataFrame:
     return df_coef
 
 
-def show_tree(grid:GridSearchCV, columns:List[str]) -> Image:
+def show_tree(model:Union[DecisionTreeRegressor, DecisionTreeClassifier, GridSearchCV],
+              columns:Union[List[str],pd.Index]) -> Image:
     """
     Visualize the given trained DecisionTree model on Jupyter.
     This function requires also pydot.
 
-    :param grid: GridSearchCV instance with a Tree model
+    :param model: DecisionTree model or GridSearchCV instance with a Tree model
     :param columns: names of columns
     :return: Image instance (for Jupyter)
     """
-
-    model = pick_the_last_estimator(grid)
+    if isinstance(model, GridSearchCV):
+        model = pick_the_last_estimator(model)
 
     dot_data = StringIO()
     export_graphviz(model, out_file=dot_data, feature_names=columns,
@@ -197,17 +199,20 @@ def show_tree(grid:GridSearchCV, columns:List[str]) -> Image:
     return Image(graph.create_png())
 
 
-def show_feature_importance(grid:GridSearchCV, columns:List[str]) -> pd.Series:
+def show_feature_importance(estimator:BaseEstimator,
+                            columns:Union[List[str],pd.Index]) -> pd.Series:
     """
     Return the series of feature importance of given random forest model.
     XGB model and DecisionTree model can be accepted as well.
 
-    :param grid: fitted GridSearchCV instance with `feature_importances_` attribute
-    :param columns: list of column names
+    :param estimator: fitted Estimator with `feature_importances_` attribute
+    :param columns: list (or pandas.Index) of column names
     :return: Series of feature importance
     """
-    ## TODO: change the name of the method. (Do not use show_)
-    model = pick_the_last_estimator(grid)
+    if isinstance(estimator, GridSearchCV):
+        model = pick_the_last_estimator(estimator)
+    else:
+        model = estimator
 
     if hasattr(model,"feature_importances_"):
         s = pd.Series(model.feature_importances_, index=columns, name="importance")
@@ -216,8 +221,6 @@ def show_feature_importance(grid:GridSearchCV, columns:List[str]) -> pd.Series:
         raise AttributeError("Your model does not have an attribute 'feature_importances_'.")
 
 
-
-
 def recover_label(data:pd.DataFrame, field2columns:Dict[str,List[str]],
                   sep:str=None, other:str="other", inplace=True) -> pd.DataFrame:
     """

adhoc/utilities.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 import tempfile
 import shutil
+import re
 
 import numpy as np
 import pandas as pd
@@ -156,10 +157,18 @@ def fetch_adult_dataset(csv_path:Path):
         raise Exception("You seem to have downloaded a wrong file")
 
 
+def grep_data(data:pd.DataFrame, column:str, expr:str) -> pd.DataFrame:
+    """
+    Pick the rows with a specified expression and return them as
+    the subset of the given DataFrame.
 
-
-
-    pass
+    :param data: panda's DataFrame
+    :param column: column to check
+    :param expr: expression to find (passed to re.search)
+    :return: copy of the matched rows
+    """
+    s_matched = data[column].apply(lambda s: True if re.search(expr,str(s)) else False)
+    return data[s_matched].copy()
 
 
 def facet_grid_scatter_plot(data:pd.DataFrame, row:str, col:str,