From 41f7d3712a019c07ba03b55ff9ebd8ae1f6c7f90 Mon Sep 17 00:00:00 2001 From: Manoj K Date: Sat, 17 Feb 2018 02:06:36 +0100 Subject: [PATCH 1/4] fix: exception is not imported sklearn.exceptions.NotFittedError currently the error is "NameError: name 'sklearn' is not defined" --- tffm/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tffm/base.py b/tffm/base.py index 4e72626..30818cf 100644 --- a/tffm/base.py +++ b/tffm/base.py @@ -1,5 +1,6 @@ import tensorflow as tf from .core import TFFMCore +import sklearn from sklearn.base import BaseEstimator from abc import ABCMeta, abstractmethod import six From aca995de119e9b6adfa0c98668665d9de8d7de19 Mon Sep 17 00:00:00 2001 From: Manoj K Date: Sat, 17 Feb 2018 07:47:52 +0100 Subject: [PATCH 2/4] Save and load entire model as a pickled object 1. Added option to save init parameters 2. n_features from core (more can be added) --- a | 0 tffm/base.py | 31 +++++++++++++++++++++++++++++++ tffm/models.py | 4 +++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 a diff --git a/a b/a new file mode 100644 index 0000000..e69de29 diff --git a/tffm/base.py b/tffm/base.py index 30818cf..52f8f35 100644 --- a/tffm/base.py +++ b/tffm/base.py @@ -7,6 +7,7 @@ from tqdm import tqdm import numpy as np import os +import pickle def batcher(X_, y_=None, w_=None, batch_size=-1): @@ -278,3 +279,33 @@ def destroy(self): """Terminates session and destroyes graph.""" self.session.close() self.core.graph = None + + def save_model(self, path): + """Saves the entire model""" + ## Additional core attributes that are to be pickled! + CORE_ATTRS = ['n_features'] + self.save_state(path) + pickle_params ={ + 'init': self._init_params_copy, + 'core': {k:getattr(self.core, k) for k in CORE_ATTRS} + } + pickle.dump(pickle_params, open(path+'.tffm', 'wb')) + + @staticmethod + def load_model(klass, path): + """ + Restores the TFFM model, along with tensorflow state + + Parameters: + ----------- + klass: TFFMRegressor or TFFMClassifier + path: path to save pickled model file with extension .tffm + """ + if klass.__name__.rpartition('.')[2] not in ['TFFMRegressor', 'TFFMClassifier']: + raise Exception('klass is not supported: %s'%klass) + _unpickled_obj = pickle.load(open(path+'.tffm', 'rb')) + _new_model = klass(**_unpickled_obj['init']) + for k in _unpickled_obj['core']: + setattr(_new_model.core, k, _unpickled_obj['core'][k]) + _new_model.load_state(path) + return _new_model diff --git a/tffm/models.py b/tffm/models.py index b37e98e..5f8a5bb 100644 --- a/tffm/models.py +++ b/tffm/models.py @@ -3,7 +3,7 @@ import numpy as np from .base import TFFMBaseModel from .utils import loss_logistic, loss_mse, sigmoid - +import copy class TFFMClassifier(TFFMBaseModel): @@ -24,6 +24,7 @@ def __init__(self, **init_params): base class TFFMBaseModel.""" init_params['loss_function'] = loss_logistic + self._init_params_copy = copy.deepcopy(init_params) self.init_basemodel(**init_params) def _preprocess_sample_weights(self, sample_weight, pos_class_weight, used_y): @@ -118,6 +119,7 @@ def __init__(self, **init_params): not supported for TFFMRegressor. For custom loss function, extend the base class TFFMBaseModel.""" + self._init_params_copy = copy.deepcopy(init_params) init_params['loss_function'] = loss_mse self.init_basemodel(**init_params) From 6832e6a59d100d0cfbc6590fc4ce8af9f74b00de Mon Sep 17 00:00:00 2001 From: mk Date: Tue, 20 Feb 2018 11:26:28 +0100 Subject: [PATCH 3/4] Create base directory for save_model --- tffm/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tffm/base.py b/tffm/base.py index 52f8f35..5034f81 100644 --- a/tffm/base.py +++ b/tffm/base.py @@ -289,6 +289,9 @@ def save_model(self, path): 'init': self._init_params_copy, 'core': {k:getattr(self.core, k) for k in CORE_ATTRS} } + if not os.path.isdir(os.path.dirname(path)): + os.makedirs(path, exist_ok=True) + pickle.dump(pickle_params, open(path+'.tffm', 'wb')) @staticmethod From 10a1d836a4a3cfae2b0416b3dc80a25855322469 Mon Sep 17 00:00:00 2001 From: mk Date: Mon, 19 Mar 2018 01:13:33 +0100 Subject: [PATCH 4/4] Fix to save and load TFFMClassifier --- tffm/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tffm/base.py b/tffm/base.py index 5034f81..ed62227 100644 --- a/tffm/base.py +++ b/tffm/base.py @@ -282,9 +282,14 @@ def destroy(self): def save_model(self, path): """Saves the entire model""" + self.save_state(path) + ## Additional core attributes that are to be pickled! CORE_ATTRS = ['n_features'] - self.save_state(path) + + # remove loss_function if classifier + if self.__class__.__name__.rpartition('.')[2] == 'TFFMClassifier': + del self._init_params_copy['loss_function'] pickle_params ={ 'init': self._init_params_copy, 'core': {k:getattr(self.core, k) for k in CORE_ATTRS} @@ -305,7 +310,7 @@ def load_model(klass, path): path: path to save pickled model file with extension .tffm """ if klass.__name__.rpartition('.')[2] not in ['TFFMRegressor', 'TFFMClassifier']: - raise Exception('klass is not supported: %s'%klass) + raise TypeError('klass is not supported: %s'%klass) _unpickled_obj = pickle.load(open(path+'.tffm', 'rb')) _new_model = klass(**_unpickled_obj['init']) for k in _unpickled_obj['core']: