tensorflow_dnn_model.py

#!/usr/bin/env python3
 
 
 
 
import os
import sys
import time
import numpy
import pickle
 
import tensorflow as tf
 
 
class Layer:
    """
    definition of a layer obj
    """
 
    def __init__(self, name, tf_activation_str, dim_input, dim_output, p_bias, p_w,
                 operation_seed=None):
        """
        :param name:
        :param tf_activation: string, name of an available tensorflow activations function
        :param dim_input:
        :param dim_output:
        :param p_bias: initial constant
        :param p_w: stddev of uniform distribution to initialize
        :return:
        """
        
        self.namename = name
 
        tf_activation_dict = {
            'tanh': tf.tanh,
            'sigmoid': tf.sigmoid,
            'relu': tf.nn.relu,
            'leaky_relu': tf.nn.leaky_relu,
        }
 
        if tf_activation_str not in tf_activation_dict:
            raise ValueError
 
        
        self.tf_activationtf_activation = tf_activation_dict[tf_activation_str]
 
        
        self.seedseed = None
 
        
        self.shapeshape = [dim_input, dim_output]
 
        
        self.ww = self._init_weight_init_weight(self.shapeshape, p_w, operation_seed)
 
        
        self.bb = self._init_bias_init_bias(self.shapeshape[1], p_bias)
 
        
        self.inputinput = None
 
        
        self.outputoutput = None
 
        
        self.is_initializedis_initialized = False
 
        
        self._add_all_to_summary_add_all_to_summary()
 
    def _init_bias(self, width, init_val, name=None):
        """
        define bias variables
        """
        if name is None:
            name = self.namename + '_b'
        initial = tf.constant(init_val, shape=[width], name=name)
        return tf.Variable(initial, name=name)
 
    def _init_weight(self, shape, stddev, operation_seed, name=None):
        """
        define weight variables
        """
        if name is None:
            name = self.namename + '_w'
        initial = tf.truncated_normal(shape, stddev=stddev, seed=operation_seed, name=name)
        return tf.Variable(initial, name=name)
 
    def initialize(self, layer_input):
        """
        initialization, requires layer input
        """
        if self.is_initializedis_initialized:
            raise RuntimeError('Layer %s is already initialized.' % self.namename)
 
        self.inputinput = layer_input
        name = self.namename + '_output'
        with tf.name_scope('output'):
            self.outputoutput = self.tf_activationtf_activation(tf.matmul(layer_input, self.ww) + self.bb, name)
            tf.add_to_collection('output', self.outputoutput)
 
        # set initialized status
        self.is_initializedis_initialized = True
 
    def _add_var_to_summary(self, var):
        """
        add variables of this layer to tensorboard
        :param var: tensorflow variable
        :return:
        """
        with tf.name_scope('summaries'):
            mean = tf.reduce_mean(var)
            tf.summary.scalar('mean', mean)
 
            with tf.name_scope('stddev'):
                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
 
            tf.summary.scalar('stddev', stddev)
            tf.summary.scalar('max', tf.reduce_max(var))
            tf.summary.scalar('min', tf.reduce_min(var))
 
            tf.summary.histogram('histogram', var)
 
    def _add_all_to_summary(self):
        """
        adds tuneable parameters to summary
        """
        with tf.name_scope(self.namename):
            with tf.name_scope('weights'):
                self._add_var_to_summary_add_var_to_summary(self.ww)
            with tf.name_scope('biases'):
                self._add_var_to_summary_add_var_to_summary(self.bb)
 
 
class MultilayerPerceptron:
    """
    class combine layer obj
    """
 
    def __init__(self, layers):
        """
        initialization
        """
 
        
        self.layerslayers = layers
 
        
        self.inputinput = None
 
        
        self.outputoutput = None
 
        
        self.ww = None
 
        
        self.bb = None
 
        
        self.is_initializedis_initialized = False
 
    @classmethod
    def from_list(cls, layers):
        """
        define layers from list
        """
        layer_obj = []
        for layer in layers:
            layer_obj.append(Layer(*layer))
 
        mlp = cls(layer_obj)
        return mlp
 
    def _set_weights_and_biases(self):
        """
        collect tunable parameters
        """
        self.ww = []
        self.bb = []
        for layer in self.layerslayers:
            self.ww.append(layer.w)
            self.bb.append(layer.b)
 
    def _connect_layers(self, layer_input):
        """
        connect layers
        """
        for layer in self.layerslayers:
            layer.initialize(layer_input)
            layer_input = layer.output
 
    def initialize(self, layer_input):
        """
        initialize
        """
        if self.is_initializedis_initialized:
            raise RuntimeError
 
        self.is_initializedis_initialized = True
        # check shape
        for _idx in range(len(self.layerslayers) - 1):
            assert self.layerslayers[_idx].shape[1] == self.layerslayers[_idx + 1].shape[0]
 
        self._connect_layers_connect_layers(layer_input)
        self._set_weights_and_biases_set_weights_and_biases()
 
        self.inputinput = self.layerslayers[0].input
        self.outputoutput = self.layerslayers[-1].output
 
 
class DefaultModel:
    """
    define the default model
    """
 
    def __init__(self, mlp, mom_init=.9, mom_max=.99, mom_epochs=200, lr_init=.05, lr_min=1e-6,
                 lr_dec_rate=.976,
                 stop_epochs=10, min_epochs=200, max_epochs=1000, wd_coeffs=None, change_optimizer=None,
                 staircase=True, smooth_cross_entropy=False):
        """
        initialization function
        """
 
        if wd_coeffs is not None:
            assert len(wd_coeffs) == len(mlp.layers)
 
        
        self.wd_coeffswd_coeffs = wd_coeffs
 
        
        self.mlpmlp = mlp
 
        
        self.outputoutput = None
 
        with tf.name_scope('global_step'):
            
            self.global_stepglobal_step = tf.Variable(0, trainable=False)
 
        # optimizer params
        
        self.c_mom_initc_mom_init = mom_init
        
        self.c_mom_maxc_mom_max = mom_max
        
        self.c_mom_epochsc_mom_epochs = mom_epochs
 
        
        self.c_lr_initc_lr_init = lr_init
        
        self.c_lr_minc_lr_min = lr_min
        
        self.c_lr_dec_ratec_lr_dec_rate = lr_dec_rate
 
        
        self.c_stop_epochsc_stop_epochs = stop_epochs
 
        
        self.c_staircasec_staircase = staircase
 
        
        self.batches_per_epochbatches_per_epoch = None
 
        
        self.optimizersoptimizers = []
        # list with epochs in which optimizers will be changed, if None is given, only the default optimizer will be
        
        self.optimizer_change_epochsoptimizer_change_epochs = change_optimizer
        if change_optimizer is not None:
            self.optimizer_change_epochsoptimizer_change_epochs.insert(0, 0)
            self.optimizer_change_epochsoptimizer_change_epochs.append(sys.maxsize)
 
        
        self.monitoring_paramsmonitoring_params = None
        
        self.monitoring_labelsmonitoring_labels = None
 
        
        self.mon_dictmon_dict = dict()
 
        
        self.xx = None
        
        self.y_y_ = None
        
        self.weightsweights = None
 
        
        self.max_epochsmax_epochs = max_epochs
 
        # termination criterion
        
        self.min_epochsmin_epochs = min_epochs
 
        
        self.termination_criteriontermination_criterion = None
        
        self.recent_paramsrecent_params = []
        
        self.best_valuebest_value = numpy.inf
 
        
        self.step_countdownstep_countdown = self.c_stop_epochsc_stop_epochs
 
        
        self.smooth_cross_entropysmooth_cross_entropy = smooth_cross_entropy
 
        
        self.is_initializedis_initialized = False
 
    def _set_optimizer(self):
        """
        set optimizer
        """
 
        t_learning_rate = tf.train.exponential_decay(self.c_lr_initc_lr_init, self.global_stepglobal_step, self.batches_per_epochbatches_per_epoch,
                                                     self.c_lr_dec_ratec_lr_dec_rate, staircase=self.c_staircasec_staircase)
 
        t_lr_min = tf.Variable(self.c_lr_minc_lr_min, trainable=False)
 
        with tf.name_scope('learning_rate'):
            t_limited_lr = tf.maximum(t_learning_rate, t_lr_min)
            tf.summary.scalar('learning rate', t_limited_lr)
 
        # linear increasing momentum
        c_mom_dec_rate = (self.c_mom_maxc_mom_max - self.c_mom_initc_mom_init) / self.c_mom_epochsc_mom_epochs
 
        t_mom_dec_rate = tf.Variable(c_mom_dec_rate, trainable=False, dtype=tf.float32)
 
        t_momentum = tf.Variable(self.c_mom_initc_mom_init, trainable=False, dtype=tf.float32)
 
        t_mom_max = tf.Variable(self.c_mom_maxc_mom_max, trainable=False, dtype=tf.float32)
 
        t_batches_per_epoch = tf.Variable(self.batches_per_epochbatches_per_epoch, trainable=False, dtype=tf.float32)
 
        with tf.name_scope('momentum'):
            # tf.floor -- 'staircase = true'
            global_step = tf.cast(self.global_stepglobal_step, tf.float32)
            if self.c_staircasec_staircase:
                t_limited_mom = tf.minimum(t_momentum + t_mom_dec_rate *
                                           tf.floor(global_step / t_batches_per_epoch), t_mom_max)
            else:
                t_limited_mom = tf.minimum(t_momentum + t_mom_dec_rate *
                                           (global_step / t_batches_per_epoch), t_mom_max)
            tf.summary.scalar('momentum', t_limited_mom)
 
        self.mon_dictmon_dict['learning_rate'] = t_limited_lr
        self.mon_dictmon_dict['momentum'] = t_limited_mom
 
        with tf.name_scope('optimizer'):
            self.optimizersoptimizers.append(tf.train.MomentumOptimizer(t_limited_lr, t_limited_mom))
            self.optimizersoptimizers.append(tf.train.GradientDescentOptimizer(t_limited_lr))
 
    def _set_loss(self):
        """
        set loss
        """
 
        _y = self.mlpmlp.output
        _y_ = self.y_y_
 
        epsilon = 1e-10
        t_epsilon = tf.constant(epsilon)
 
        # sum over batch
        with tf.name_scope('cross_entropy'):
            if self.smooth_cross_entropysmooth_cross_entropy:
                cross_entropy = -tf.reduce_mean(tf.reduce_sum((_y_ * tf.log(_y + t_epsilon) + (1 - _y_) *
                                                               tf.log(1 - _y + t_epsilon)), 1))
            else:
                cross_entropy = -tf.reduce_mean(tf.reduce_sum((_y_ * tf.log(tf.clip_by_value(_y, epsilon, 1))) +
                                                              ((1 - _y_) * tf.log(tf.clip_by_value((1 - _y),
                                                                                                   epsilon, 1))), 1))
            tf.summary.scalar('cross entropy', cross_entropy)
 
        loss = cross_entropy
 
        if self.wd_coeffswd_coeffs:
            wd_coeffs = self.wd_coeffswd_coeffs
            weights = self.mlpmlp.w
 
            wd = [tf.constant(coeff) * tf.nn.l2_loss(w) for coeff, w in zip(wd_coeffs, weights)]
            loss += sum(wd)
 
        with tf.name_scope('loss_function'):
            
            self.lossloss = loss
            tf.summary.scalar('loss', loss)
 
    def _set_monitoring_params(self):
        """
        sets monitoring params
        """
 
        _y = self.mlpmlp.output
        _y_ = self.y_y_
 
        epsilon = 1e-10
        t_epsilon = tf.constant(epsilon)
 
        with tf.name_scope('mean_cross_entropy'):
            if self.smooth_cross_entropysmooth_cross_entropy:
                mean_cross_entropy = -tf.reduce_mean(tf.reduce_sum((_y_ * tf.log(_y + t_epsilon) + (1 - _y_) *
                                                                    tf.log(1 - _y + t_epsilon)), 1))
            else:
                mean_cross_entropy = -tf.reduce_mean(tf.reduce_sum((_y_ * tf.log(tf.clip_by_value(_y, epsilon, 1))) +
                                                                   ((1 - _y_) * tf.log(tf.clip_by_value((1 - _y),
                                                                                                        epsilon, 1))), 1))
            tf.summary.scalar('mean cross entropy', mean_cross_entropy)
        loss = self.lossloss
 
        self.monitoring_labelsmonitoring_labels = ['mean_cross_entropy', 'loss']
        self.monitoring_paramsmonitoring_params = [mean_cross_entropy, loss]
 
    def _default_termination_criterion(self, monitoring_params, epoch, label_name='mean_cross_entropy',
                                       prop_dec=1e-5):
        """
        :param monitoring_params:
        :param epoch:
        :param label_name:
        :param steps:
        :param prop_dec:
        :return:
        """
 
        loss_idx = self.monitoring_labelsmonitoring_labels.index(label_name)
        mon_param = monitoring_params[loss_idx]
 
        if epoch < self.min_epochsmin_epochs:
            return False
 
        if mon_param < self.best_valuebest_value * (1. - prop_dec):
            self.step_countdownstep_countdown = self.c_stop_epochsc_stop_epochs
            self.best_valuebest_value = mon_param
        else:
            self.step_countdownstep_countdown -= 1
 
        if self.step_countdownstep_countdown > 0:
            return False
 
        return True
 
    def get_minimizer(self, epoch=0):
        """
        get minimizer
        """
 
        if self.optimizer_change_epochsoptimizer_change_epochs is None:
            return self.optimizersoptimizers[0].minimize(self.lossloss, self.global_stepglobal_step)
 
        if len(self.optimizer_change_epochsoptimizer_change_epochs) > len(self.optimizersoptimizers) + 1:
            raise RuntimeError
 
        # switch optimizer for given epoch
        for i in range(1, len(self.optimizer_change_epochsoptimizer_change_epochs)):
            if self.optimizer_change_epochsoptimizer_change_epochs[i - 1] <= epoch < self.optimizer_change_epochsoptimizer_change_epochs[i]:
                return self.optimizersoptimizers[i - 1].minimize(self.lossloss, self.global_stepglobal_step)
 
    # needs to be called since parameters depends on number of batches
    def initialize(self, data_set, input_placeholders=None):
        """
        initialized
        """
        if self.is_initializedis_initialized:
            raise RuntimeError
        self.is_initializedis_initialized = True
 
        self.batches_per_epochbatches_per_epoch = data_set.batches
        # placeholders
        if input_placeholders is None:
            with tf.name_scope('input'):
                self.xx = tf.placeholder(tf.float32, shape=[None, data_set.feature_number], name='x-input')
                self.y_y_ = tf.placeholder(tf.float32, shape=[None, 1], name="y-input")
                # TODO: implement weights
                self.weightsweights = tf.placeholder(tf.float32, shape=[None, 1], name="weight-input")
        else:
            self.xx = input_placeholders[0]
            self.y_y_ = input_placeholders[1]
 
        # layer input, calls cascade initialization
        self.mlpmlp.initialize(self.xx)
 
        self._set_optimizer_set_optimizer()
        self._set_loss_set_loss()
        self._set_monitoring_params_set_monitoring_params()
 
        # termination criterion
        self.termination_criteriontermination_criterion = self._default_termination_criterion_default_termination_criterion
 
 
class Trainer:
    """
    handling the training of the network model
    """
 
    def __init__(self, model, data_set, sess, log_dir=None, save_name=None, monitoring_size=100000,
                 input_placeholders=None):
        """
        class to train a predefined model
        :param model: DefaultModel obj
        :param data_set: TFData obj
        :param sess: tensorflow.Session obj
        :param log_dir: str, directory name of tensorboard logging
        :param save_name: str, path and name for saving the weightfiles
        :param monitoring_size: int, number of events of training fraction used for monitoring
        :param input_placeholders: list of tf.placeholders, [features, targets]
        """
 
        
        self._time_time = time.time()
        
        self.modelmodel = model
        
        self.data_setdata_set = data_set
        
        self.monitoring_sizemonitoring_size = monitoring_size
 
        
        self.sesssess = sess
 
        
        self.log_dirlog_dir = log_dir
 
        if input_placeholders is None:
            with tf.name_scope('input'):
                
                self.xx = tf.placeholder(tf.float32, shape=[None, data_set.feature_number], name='x-input')
 
                
                self.y_y_ = tf.placeholder(tf.float32, shape=[None, 1], name="y-input")
        else:
            
            self.xx = input_placeholders[0]
            
            self.y_y_ = input_placeholders[1]
 
        self.modelmodel.initialize(data_set, input_placeholders=[self.xx, self.y_y_])
 
        
        self.monitoring_paramsmonitoring_params = self.modelmodel.monitoring_params
        
        self.termination_criteriontermination_criterion = self.modelmodel.termination_criterion
 
        
        self.max_epochsmax_epochs = self.modelmodel.max_epochs
 
        
        self.current_epochcurrent_epoch = 0
 
        
        self.minimizerminimizer = self.modelmodel.get_minimizer()
 
        
        self.train_log_dicttrain_log_dict = {}
        for label in self.modelmodel.monitoring_labels:
            self.train_log_dicttrain_log_dict['train_' + label] = []
            self.train_log_dicttrain_log_dict['valid_' + label] = []
 
        for label in self.modelmodel.mon_dict.keys():
            self.train_log_dicttrain_log_dict[label] = []
 
        self._add_to_basf2_collections_add_to_basf2_collections()
 
        if log_dir is not None:
            self._prepare_tensorboard_prepare_tensorboard(log_dir)
 
        
        self.saversaver = tf.train.Saver()
 
        init_op = tf.global_variables_initializer()
 
        self.sesssess.run(init_op)
 
        if save_name is None:
            time_str = time.strftime("%Y%m%d-%H%M%S")
            
            self.save_namesave_name = os.path.join(os.getcwd(), '_'.join([self.data_setdata_set.selection, time_str,
                                                                 'model.ckpt']))
 
        else:
            
            self.save_namesave_name = save_name
 
        self._prepare_monitoring_prepare_monitoring()
 
    def _prepare_monitoring(self):
        """
        checking dataset sizes for evaluation
        """
        
        self.train_monitortrain_monitor = -1
        
        self.valid_monitorvalid_monitor = -1
        if self.data_setdata_set.train_events > self.monitoring_sizemonitoring_size:
            self.train_monitortrain_monitor = self.monitoring_sizemonitoring_size
 
        if self.data_setdata_set.valid_events > self.monitoring_sizemonitoring_size:
            self.valid_monitorvalid_monitor = self.monitoring_sizemonitoring_size
 
    def _prepare_tensorboard(self, log_dir):
        """
        prepare tensorboard
        """
        log_dir_train = os.path.join(log_dir, 'train')
        log_dir_test = os.path.join(log_dir, 'test')
 
        
        self.train_writertrain_writer = tf.summary.FileWriter(log_dir_train, self.sesssess.graph)
        
        self.test_writertest_writer = tf.summary.FileWriter(log_dir_test, self.sesssess.graph)
 
        
        self.merged_summarymerged_summary = tf.summary.merge_all()
 
    def _add_to_basf2_collections(self):
        """
        add to basf2 collection
        """
        tf.add_to_collection('x', self.xx)
        tf.add_to_collection('y', self.y_y_)
        tf.add_to_collection('activation', self.modelmodel.mlp.output)
        tf.add_to_collection('cost', self.modelmodel.loss)
        tf.add_to_collection('optimizer', self.minimizerminimizer)
 
    def _save_best_state(self, monitoring_params, label_name='mean_cross_entropy'):
        """
        save model only if a global minimum is reached on validation set
        :return:
        """
        # current last state
        self.saversaver.save(self.sesssess, self.save_namesave_name.replace('.ckpt', '_current.ckpt'))
 
        # check for a not set best value
        if self.modelmodel.best_value == numpy.inf:
            return
 
        loss_idx = self.modelmodel.monitoring_labels.index(label_name)
        mon_param = monitoring_params[loss_idx]
 
        if mon_param < self.modelmodel.best_value:
            self.train_log_dicttrain_log_dict['best_epoch'] = self.current_epochcurrent_epoch
            self.saversaver.save(self.sesssess, self.save_namesave_name)
 
    def _closing_ops(self):
        """
        closing ops
        """
        if self.log_dirlog_dir is not None:
            self.train_writertrain_writer.close()
            self.test_writertest_writer.close()
            with open(os.path.join(self.log_dirlog_dir, 'training_params.pkl'), 'wb') as f:
                pickle.dump(self.train_log_dicttrain_log_dict, f)
 
        # FIXME: raises an exception when calling session._del -> session.TF_DeleteStatus() !
        # FIXME: not closing a session can cause memory leaks!
        # self.sess.close()
        # del self.sess
 
    def _train_epoch(self, current_epoch):
        """
        train epoch
        """
 
        # set minimizer for this epoch
        self.minimizerminimizer = self.modelmodel.get_minimizer(current_epoch)
 
        batch_iter = self.data_setdata_set.batch_iterator()
 
        for i in range(self.data_setdata_set.batches):
            batch = next(batch_iter)
            feed = {self.xx: batch[0], self.y_y_: batch[1]}
            self.sesssess.run(self.minimizerminimizer, feed_dict=feed)
            if i % int(.2 * self.data_setdata_set.batches) == 0:
                print('Epoch status: %1.2f' % (i / self.data_setdata_set.batches))
 
        train_mon_dict = {self.xx: self.data_setdata_set.train_x[:self.train_monitortrain_monitor],
                          self.y_y_: self.data_setdata_set.train_y[:self.train_monitortrain_monitor]}
 
        valid_dict = {self.xx: self.data_setdata_set.valid_x[:self.valid_monitorvalid_monitor],
                      self.y_y_: self.data_setdata_set.valid_y[:self.valid_monitorvalid_monitor]}
 
        # depending on the fact, if the log should be monitored or not
        if self.log_dirlog_dir is not None:
            # todo raise exception if not empty
 
            summary = self.sesssess.run(self.merged_summarymerged_summary, feed_dict=train_mon_dict)
            self.train_writertrain_writer.add_summary(summary, current_epoch)
 
            
            self.epoch_parametersepoch_parameters = self.sesssess.run(self.monitoring_paramsmonitoring_params + [self.merged_summarymerged_summary],
                                                  feed_dict=valid_dict)
            epoch_parameters_train = self.sesssess.run(
                self.monitoring_paramsmonitoring_params + [self.merged_summarymerged_summary], feed_dict=train_mon_dict)
 
            # merged summary will be at last position
            self.test_writertest_writer.add_summary(self.epoch_parametersepoch_parameters.pop(-1), current_epoch)
        else:
            self.epoch_parametersepoch_parameters = self.sesssess.run(self.monitoring_paramsmonitoring_params,
                                                  feed_dict=valid_dict)
            epoch_parameters_train = self.sesssess.run(self.monitoring_paramsmonitoring_params, feed_dict=train_mon_dict)
 
        model_mon_params = []
        for key, entry in self.modelmodel.mon_dict.items():
            model_mon_params.append((key, self.sesssess.run(entry)))
 
        print('Current Epoch: %d' % self.current_epochcurrent_epoch)
        for label, param in zip(self.modelmodel.monitoring_labels, self.epoch_parametersepoch_parameters):
            self.train_log_dicttrain_log_dict['valid_' + label].append(param)
            print(f"valid: {label}: {param:1.5f}")
        for label, param in zip(self.modelmodel.monitoring_labels, epoch_parameters_train):
            print(f"train: {label}: {param:1.5f}")
            self.train_log_dicttrain_log_dict['train_' + label].append(param)
 
        for val in model_mon_params:
            print("%s: %f" % val)
            self.train_log_dicttrain_log_dict[val[0]].append(val[1])
 
        print('Epoch training time: %.1f' % (time.time() - self._time_time))
        self._time_time = time.time()
        print('\n')
        self.current_epochcurrent_epoch += 1
 
    def train_model(self):
        """
        train model
        """
        for epoch in range(self.max_epochsmax_epochs):
            self._train_epoch_train_epoch(epoch)
 
            self._save_best_state_save_best_state(self.epoch_parametersepoch_parameters)
 
            if self.termination_criteriontermination_criterion(self.epoch_parametersepoch_parameters, epoch):
                break
 
        self._closing_ops_closing_ops()