Cross Validation Added

gramex/handlers/mlhandler.py

@@ -20,6 +20,9 @@
 from slugify import slugify
 from tornado.gen import coroutine
 from tornado.web import HTTPError
+from sklearn.metrics import get_scorer
+from sklearn.model_selection import cross_val_predict, cross_val_score
+from sklearn.model_selection import cross_val_predict, cross_val_score
 
 op = os.path
 MLCLASS_MODULES = [
@@ -39,7 +42,7 @@
     'pipeline': True,
     'nums': [],
     'cats': [],
-    'target_col': None,
+    'target_col': None
 }
 ACTIONS = ['predict', 'score', 'append', 'train', 'retrain']
 DEFAULT_TEMPLATE = op.join(op.dirname(__file__), '..', 'apps', 'mlhandler', 'template.html')
@@ -103,7 +106,6 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs):
 
         cls.set_opt('class', model.get('class'))
         cls.set_opt('params', model.get('params', {}))
-
         if op.exists(cls.model_path):  # If the pkl exists, load it
             cls.model = joblib.load(cls.model_path)
         elif data is not None:
@@ -112,14 +114,26 @@ def setup(cls, data=None, model={}, config_dir='', **kwargs):
             data = cls._filtercols(data)
             data = cls._filterrows(data)
             cls.model = cls._assemble_pipeline(data, mclass=mclass, params=params)
-
             # train the model
             target = data[target_col]
             train = data[[c for c in data if c != target_col]]
+            # cross validation
+            mod = cls.modelFunction()
+            CVscore = cross_val_score(mod, train, target)    
+            CV = sum(CVscore)/len(CVscore)
+            print('CV score: ', CV)
             gramex.service.threadpool.submit(
                 _fit, cls.model, train, target, cls.model_path, cls.name)
         cls.config_store.flush()
 
+    @classmethod
+    def modelFunction(cls, mclass = ''):
+        model_kwargs = cls.config_store.load('model', {})
+        mclass = model_kwargs.get('class', False)
+        if mclass:
+            model = search_modelclass(mclass)(**model_kwargs.get('params', {}))
+            return model
+
     @classmethod
     def load_data(cls, default=pd.DataFrame()):
         try:
@@ -268,6 +282,10 @@ def _predict(self, data=None, score_col=''):
         self.model = cache.open(self.model_path, joblib.load)
         try:
             target = data.pop(score_col)
+            metric = self.get_argument('_metric', False)
+            if metric:
+                scorer = get_scorer(metric)
+                return scorer(self.model, data, target)
             return self.model.score(data, target)
         except KeyError:
             # Set data in the same order as the transformer requests

tests/test_mlhandler.py

@@ -266,6 +266,11 @@ def test_get_bulk_score(self):
             data=self.df.to_json(orient='records'),
             headers={'Content-Type': 'application/json'})
         self.assertGreaterEqual(resp.json()['score'], self.ACC_TOL)
+        resp = self.get(
+            '/mlhandler?_action=score&_metric=f1_weighted', method='post',
+            data=self.df.to_json(orient='records'),
+            headers={'Content-Type': 'application/json'})
+        self.assertGreaterEqual(resp.json()['score'], self.ACC_TOL)
 
     def test_get_cache(self):
         df = pd.DataFrame.from_records(self.get('/mlhandler?_cache=true').json())