DefaultModel Class Reference

Inheritance diagram for DefaultModel:

Public Member Functions
	__init__ (self, mlp, mom_init=.9, mom_max=.99, mom_epochs=200, lr_init=.05, lr_min=1e-6, lr_dec_rate=.976, stop_epochs=10, min_epochs=200, max_epochs=1000, wd_coeffs=None, change_optimizer=None, staircase=True, smooth_cross_entropy=False)
	initialize (self, data_set)
	__call__ (self, x)
	get_optimizer (self, epoch=0)
	loss (self, predict_y, true_y)

Public Attributes
	mlp = mlp
	mlp net
	wd_coeffs = wd_coeffs
	weight decay coefficients
	global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int64)
	global step
	c_mom_init = tf.constant(mom_init, dtype=tf.float32)
	initial momentum
	c_mom_max = tf.constant(mom_max, dtype=tf.float32)
	maximum momentum
	c_mom_epochs = tf.constant(mom_epochs, dtype=tf.float32)
	momentum epochs
tuple	c_mom_dec_rate = (self.c_mom_max - self.c_mom_init) / tf.cast(self.c_mom_epochs, tf.float32)
	momentum decay rate
	c_lr_init = tf.constant(lr_init, dtype=tf.float32)
	initial learning rate
	c_lr_min = tf.constant(lr_min, dtype=tf.float32)
	minimum learning rate
	c_lr_dec_rate = tf.constant(lr_dec_rate, dtype=tf.float32)
	learning rate decay rate
	c_stop_epochs = stop_epochs
	number of epochs without improvement for early termination
	c_staircase = staircase
	use staircase
	batches_per_epoch = None
	batches per epoch unknown.
list	optimizers = []
	define multiple optimizers
	optimizer_change_epochs = change_optimizer
	used optimizers
	min_epochs = min_epochs
	min epochs
	max_epochs = max_epochs
	max epochs
	termination_criterion = None
	termination criterion
list	recent_params = []
	recent params
	best_value = np.inf
	the best value will be set a default start value, then updated with the termination criterion
	step_countdown = self.c_stop_epochs
	step countdown
	smooth_cross_entropy = smooth_cross_entropy
	True for a small epsilon addition, false for a clipped network output.
bool	is_initialized = False
	check if initialized

Protected Member Functions
	_default_termination_criterion (self, monitoring_param, epoch, prop_dec=1e-5)
	_get_learning_rate (self)
	_get_momentum (self)
	_set_optimizer (self)

Detailed Description

define the default model

Definition at line 198 of file tensorflow_dnn_model.py.

Constructor & Destructor Documentation

__init__()

__init__	(		self,
			mlp,
			mom_init = .9,
			mom_max = .99,
			mom_epochs = 200,
			lr_init = .05,
			lr_min = 1e-6,
			lr_dec_rate = .976,
			stop_epochs = 10,
			min_epochs = 200,
			max_epochs = 1000,
			wd_coeffs = None,
			change_optimizer = None,
			staircase = True,
			smooth_cross_entropy = False )

initialization function
:param mlp: network model.
:param mom_init: initial momentum
:param mom_max: maximum momentum
:param mom_epochs: momentum epochs
:param lr_init: initial learning rate
:param lr_min: minimum learning rate
:param lr_dec_rate: learning rate decay factor
:param stop_epochs: number of epochs without improvement required for early termination
:param min_epochs: minimum number of epochs for training
:param max_epochs: maximum number of epochs for training
:param wd_coeffs: weight decay coefficients. If not None must have one per mlp layer.
:param change_optimizer:
:param staircaise:
"param smooth_cross_entropy:

Definition at line 203 of file tensorflow_dnn_model.py.

                 smooth_cross_entropy=False):
        """
        initialization function
        :param mlp: network model.
        :param mom_init: initial momentum
        :param mom_max: maximum momentum
        :param mom_epochs: momentum epochs
        :param lr_init: initial learning rate
        :param lr_min: minimum learning rate
        :param lr_dec_rate: learning rate decay factor
        :param stop_epochs: number of epochs without improvement required for early termination
        :param min_epochs: minimum number of epochs for training
        :param max_epochs: maximum number of epochs for training
        :param wd_coeffs: weight decay coefficients. If not None must have one per mlp layer.
        :param change_optimizer:
        :param staircaise:
        "param smooth_cross_entropy:
        """
 
        
        self.mlp = mlp
 
        if wd_coeffs is not None:
            assert len(wd_coeffs) == len(mlp.layers)
 
        
        self.wd_coeffs = wd_coeffs
 
        
        self.global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int64)
 
        # --optimizer params--
        
        self.c_mom_init = tf.constant(mom_init, dtype=tf.float32)
 
        
        self.c_mom_max = tf.constant(mom_max, dtype=tf.float32)
 
        
        self.c_mom_epochs = tf.constant(mom_epochs, dtype=tf.float32)
 
        
        self.c_mom_dec_rate = (self.c_mom_max - self.c_mom_init) / tf.cast(self.c_mom_epochs, tf.float32)
 
        
        self.c_lr_init = tf.constant(lr_init, dtype=tf.float32)
 
        
        self.c_lr_min = tf.constant(lr_min, dtype=tf.float32)
 
        
        self.c_lr_dec_rate = tf.constant(lr_dec_rate, dtype=tf.float32)
 
        
        self.c_stop_epochs = stop_epochs
 
        
        self.c_staircase = staircase
 
        
        self.batches_per_epoch = None
 
        
        self.optimizers = []
        # list with epochs in which optimizers will be changed, if None is given, only the default optimizer will be
        
        self.optimizer_change_epochs = change_optimizer
        if change_optimizer is not None:
            self.optimizer_change_epochs.insert(0, 0)
            self.optimizer_change_epochs.append(sys.maxsize)
 
        # termination criterion
        
        self.min_epochs = min_epochs
 
        
        self.max_epochs = max_epochs
 
        
        self.termination_criterion = None
 
        
        self.recent_params = []
 
        
        self.best_value = np.inf
 
        
        self.step_countdown = self.c_stop_epochs
 
        
        self.smooth_cross_entropy = smooth_cross_entropy
 
        
        self.is_initialized = False
 
        return
 

Member Function Documentation

__call__()

__call__	(		self,
			x )

Call the mlp

Definition at line 336 of file tensorflow_dnn_model.py.

    def __call__(self, x):
        """
        Call the mlp
        """
        return self.mlp(x)
 

_default_termination_criterion()

_default_termination_criterion ( self, monitoring_param, epoch, prop_dec = 1e-5 )

protected

early stopping criterion

:param monitoring_param: the parameter to monitor for early termination
:param epoch: the current epoch
:param prop_dec:
:return:

Definition at line 342 of file tensorflow_dnn_model.py.

                                       prop_dec=1e-5):
        """
        early stopping criterion
 
        :param monitoring_param: the parameter to monitor for early termination
        :param epoch: the current epoch
        :param prop_dec:
        :return:
        """
        if epoch < self.min_epochs:
            return False
 
        if monitoring_param < self.best_value * (1. - prop_dec):
            self.step_countdown = self.c_stop_epochs
            self.best_value = monitoring_param
        else:
            self.step_countdown -= 1
 
        if self.step_countdown > 0:
            return False
        return True
 

_get_learning_rate()

_get_learning_rate ( self )

protected

Returns the learning rate at the current global step.

Definition at line 367 of file tensorflow_dnn_model.py.

    def _get_learning_rate(self):
        """
        Returns the learning rate at the current global step.
        """
        p = tf.cast(self.global_step, tf.float32) / tf.cast(self.batches_per_epoch, tf.float32)
        if self.c_staircase:
            p = tf.floor(p)
        return tf.maximum(tf.multiply(self.c_lr_init, tf.pow(self.c_lr_dec_rate, p)), self.c_lr_min)
 

_get_momentum()

_get_momentum ( self )

protected

returns the momentum at the current global step.

Definition at line 376 of file tensorflow_dnn_model.py.

    def _get_momentum(self):
        """
        returns the momentum at the current global step.
        """
        t_batches_per_epoch = tf.constant(self.batches_per_epoch, dtype=tf.float32)
        global_step = tf.cast(self.global_step, tf.float32)
 
        if self.c_staircase:
            t_limited_mom = tf.minimum(self.c_mom_init + self.c_mom_dec_rate *
                                       tf.floor(global_step / t_batches_per_epoch), self.c_mom_max)
        else:
            t_limited_mom = tf.minimum(self.c_mom_init + self.c_mom_dec_rate *
                                       (global_step / t_batches_per_epoch), self.c_mom_max)
        return t_limited_mom
 

_set_optimizer()

_set_optimizer ( self )

protected

set optimizers

Definition at line 391 of file tensorflow_dnn_model.py.

    def _set_optimizer(self):
        """
        set optimizers
        """
        self.optimizers.append(tf.optimizers.SGD(learning_rate=self._get_learning_rate,
                                                 momentum=self._get_momentum))
        self.optimizers.append(tf.optimizers.SGD(learning_rate=self._get_learning_rate))
        return
 

get_optimizer()

get_optimizer	(		self,
			epoch = 0 )

get the optimizer. If multiple optimizers are booked gets the one appropriate for the epoch.

:param epoch: current epoch.

Definition at line 400 of file tensorflow_dnn_model.py.

    def get_optimizer(self, epoch=0):
        """
        get the optimizer. If multiple optimizers are booked gets the one appropriate for the epoch.
 
        :param epoch: current epoch.
        """
 
        if self.optimizer_change_epochs is None:
            return self.optimizers[0]
 
        if len(self.optimizer_change_epochs) > len(self.optimizers) + 1:
            raise RuntimeError
 
        # switch optimizer for given epoch
        for i in range(1, len(self.optimizer_change_epochs)):
            if self.optimizer_change_epochs[i - 1] <= epoch < self.optimizer_change_epochs[i]:
                return self.optimizers[i - 1]
 

initialize()

initialize	(		self,
			data_set )

Finalises initialization based of data_set specific information (number of batches per epoch)

Definition at line 316 of file tensorflow_dnn_model.py.

    def initialize(self, data_set):
        """
        Finalises initialization based of data_set specific information (number of batches per epoch)
        """
        if self.is_initialized:
            raise RuntimeError
 
        self.batches_per_epoch = data_set.batches
 
        # check layer dimensions align properly
        if not self.mlp.is_initialized:
            self.mlp.initialize()
 
        self._set_optimizer()
        self.termination_criterion = self._default_termination_criterion
 
        self.is_initialized = True
        return
 

loss()

loss	(		self,
			predict_y,
			true_y )

calculate the loss

:param predict_y: predicted labels
:param true_y: true labels

Definition at line 418 of file tensorflow_dnn_model.py.

    def loss(self, predict_y, true_y):
        """
        calculate the loss
 
        :param predict_y: predicted labels
        :param true_y: true labels
        """
 
        epsilon = 1e-10
        t_epsilon = tf.constant(epsilon)
 
        # sum over batch
        with tf.name_scope('cross_entropy'):
            if self.smooth_cross_entropy:
                cross_entropy = -tf.reduce_mean(tf.reduce_sum((true_y * tf.math.log(predict_y + t_epsilon) + (1 - true_y) *
                                                               tf.math.log(1 - predict_y + t_epsilon)), 1))
            else:
                cross_entropy = -tf.reduce_mean(tf.reduce_sum((true_y * tf.math.log(tf.clip_by_value(predict_y, epsilon, 1))) +
                                                              ((1 - true_y) * tf.math.log(tf.clip_by_value((1 - predict_y),
                                                                                                           epsilon, 1))), 1))
        loss = cross_entropy
 
        if self.wd_coeffs:
            wd_coeffs = self.wd_coeffs
            weights = self.mlp.w
 
            wd = [tf.constant(coeff) * tf.nn.l2_loss(w) for coeff, w in zip(wd_coeffs, weights)]
            loss += sum(wd)
 
        return loss, cross_entropy
 
 

Member Data Documentation

batches_per_epoch

batches_per_epoch = None

batches per epoch unknown.

needs to be set with initialize

Definition at line 276 of file tensorflow_dnn_model.py.

best_value

best_value = np.inf

the best value will be set a default start value, then updated with the termination criterion

Definition at line 301 of file tensorflow_dnn_model.py.

c_lr_dec_rate

c_lr_dec_rate = tf.constant(lr_dec_rate, dtype=tf.float32)

learning rate decay rate

Definition at line 267 of file tensorflow_dnn_model.py.

c_lr_init

c_lr_init = tf.constant(lr_init, dtype=tf.float32)

initial learning rate

Definition at line 261 of file tensorflow_dnn_model.py.

c_lr_min

c_lr_min = tf.constant(lr_min, dtype=tf.float32)

minimum learning rate

Definition at line 264 of file tensorflow_dnn_model.py.

c_mom_dec_rate

tuple c_mom_dec_rate = (self.c_mom_max - self.c_mom_init) / tf.cast(self.c_mom_epochs, tf.float32)

momentum decay rate

Definition at line 258 of file tensorflow_dnn_model.py.

c_mom_epochs

c_mom_epochs = tf.constant(mom_epochs, dtype=tf.float32)

momentum epochs

Definition at line 255 of file tensorflow_dnn_model.py.

c_mom_init

c_mom_init = tf.constant(mom_init, dtype=tf.float32)

initial momentum

Definition at line 249 of file tensorflow_dnn_model.py.

c_mom_max

c_mom_max = tf.constant(mom_max, dtype=tf.float32)

maximum momentum

Definition at line 252 of file tensorflow_dnn_model.py.

c_staircase

c_staircase = staircase

use staircase

Definition at line 273 of file tensorflow_dnn_model.py.

c_stop_epochs

c_stop_epochs = stop_epochs

number of epochs without improvement for early termination

Definition at line 270 of file tensorflow_dnn_model.py.

global_step

global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int64)

global step

Definition at line 245 of file tensorflow_dnn_model.py.

is_initialized

bool is_initialized = False

check if initialized

Definition at line 310 of file tensorflow_dnn_model.py.

max_epochs

max_epochs = max_epochs

max epochs

Definition at line 292 of file tensorflow_dnn_model.py.

min_epochs

min_epochs = min_epochs

min epochs

Definition at line 289 of file tensorflow_dnn_model.py.

mlp

mlp = mlp

mlp net

Definition at line 236 of file tensorflow_dnn_model.py.

optimizer_change_epochs

optimizer_change_epochs = change_optimizer

used optimizers

Definition at line 282 of file tensorflow_dnn_model.py.

optimizers

optimizers = []

define multiple optimizers

Definition at line 279 of file tensorflow_dnn_model.py.

recent_params

list recent_params = []

recent params

Definition at line 298 of file tensorflow_dnn_model.py.

smooth_cross_entropy

smooth_cross_entropy = smooth_cross_entropy

True for a small epsilon addition, false for a clipped network output.

Definition at line 307 of file tensorflow_dnn_model.py.

step_countdown

step_countdown = self.c_stop_epochs

step countdown

Definition at line 304 of file tensorflow_dnn_model.py.

termination_criterion

termination_criterion = None

termination criterion

Definition at line 295 of file tensorflow_dnn_model.py.

wd_coeffs

wd_coeffs = wd_coeffs

weight decay coefficients

Definition at line 242 of file tensorflow_dnn_model.py.

---

The documentation for this class was generated from the following file:

• analysis/scripts/dft/tensorflow_dnn_model.py