#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
import numpy as np
__author__ = ['Nico Curti']
__email__ = ['nico.curti2@unibo.it']
[docs]class BaseWeights (object):
'''
Base class for weights initialization
References
----------
- https://raw.githubusercontent.com/oujago/NumpyDL/master/npdl/initializations.py
'''
[docs] def get (self, size):
'''
Initialize the weigths matrix according to the
specialization
Parameters
----------
size : tuple
Weights matrix shape
Returns
-------
weights : array-like
Matrix of weights with the given shape
'''
raise NotImplementedError
@property
def name (self):
'''
Get the name of the weight initializer function
'''
class_name = self.__class__.__qualname__
return class_name
def __repr__ (self):
'''
Printer
'''
class_name = self.name
try:
params = super(type(self), self).__init__.__code__.co_varnames
except AttributeError:
params = self.__init__.__code__.co_varnames
params = set(params) - {'self', 'args', 'kwargs'}
args = ', '.join(['{0}={1}'.format(k, str(getattr(self, k)))
if not isinstance(getattr(self, k), str) else '{0}="{1}"'.format(k, str(getattr(self, k)))
for k in params])
return '{0}({1})'.format(class_name, args)
[docs]class Zeros (BaseWeights):
'''
Initialize weights with zero values
'''
def __init__ (self):
super(Zeros, self).__init__()
[docs] def get (self, size):
return np.zeros(shape=size, dtype=float)
[docs]class Ones (BaseWeights):
'''
Initialize weights with one values
'''
def __init__ (self):
super(Ones, self).__init__()
[docs] def get (self, size):
return np.ones(shape=size, dtype=float)
[docs]class Normal (BaseWeights):
'''
Sample initial weights from the Gaussian distribution.
Initial weight parameters are sampled from N(mean, std).
Parameters
----------
mu : float (default=0.)
Mean of initial parameters.
std : float (default=1.)
Std of initial parameters.
'''
def __init__ (self, mu=0., std=1.):
self.mu = mu
self.std = std
super(Normal, self).__init__()
[docs] def get (self, size):
return np.random.normal(loc=self.mu, scale=self.std, size=size)
[docs]class GlorotNormal (BaseWeights):
'''
Glorot normal initializer, also called Xavier normal initializer.
It draws samples from a truncated normal distribution centered on 0
with `stddev = sqrt(2 / (inputs + outputs))` [4]_
where `inputs` is the number of input units in the weight matrix
and `outputs` is the number of output units in the weight matrix.
References
----------
.. [4] Glorot & Bengio, AISTATS 2010. http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
'''
def __init__ (self):
super(GlorotNormal, self).__init__()
[docs] def get (self, size):
inputs, outputs = size
std = np.sqrt(2. / (inputs + outputs))
return np.random.normal(loc=0., scale=std, size=size)
[docs]class HeNormal (BaseWeights):
'''
He normal initializer.
It draws samples from a truncated normal distribution centered on 0
with `stddev = sqrt(2 / inputs)` [6]_
where `inputs` is the number of input units in the weight matrix.
References
----------
.. [6] He et al., http://arxiv.org/abs/1502.01852
'''
def __init__ (self):
super(HeNormal, self).__init__()
[docs] def get (self, size):
inputs = size[0]
std = np.sqrt(2. / inputs)
return np.random.normal(loc=0., scale=std, size=size)
[docs]class Orthogonal (BaseWeights):
'''
Intialize weights as Orthogonal matrix.
Orthogonal matrix initialization [7]_. For n-dimensional shapes where
n > 2, the n-1 trailing axes are flattened. For convolutional layers, this
corresponds to the fan-in, so this makes the initialization usable for
both dense and convolutional layers.
Parameters
----------
gain : float or 'relu'.
Scaling factor for the weights. Set this to ``1.0`` for linear and
sigmoid units, to 'relu' or ``sqrt(2)`` for rectified linear units, and
to ``sqrt(2/(1+alpha**2))`` for leaky rectified linear units with
leakiness ``alpha``. Other transfer functions may need different
factors.
References
----------
.. [7] Saxe, Andrew M., James L. McClelland, and Surya Ganguli.
"Exact solutions to the nonlinear dynamics of learning in deep
linear neural networks." arXiv preprint arXiv:1312.6120 (2013).
'''
def __init__ (self, gain=1.):
self.gain = np.sqrt(2) if gain == 'relu' else gain
super(Orthogonal, self).__init__()
[docs] def get (self, size):
flat_shape = (size[0], np.prod(size[1:]))
a = np.random.normal(loc=0., scale=1., size=flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
q = u if u.shape == flat_shape else v
q = q.reshape(size)
q = self.gain * q
return q
[docs]class TruncatedNormal (BaseWeights):
'''
Generate draws from a truncated normal distribution via rejection sampling.
Parameters
----------
mean : float or array_like of floats
The mean/center of the distribution
std : float or array_like of floats
Standard deviation (spread or "width") of the distribution.
out_shape : int or tuple of ints
Output shape. If the given shape is, e.g., ``(m, n, k)``, then
``m * n * k`` samples are drawn.
Notes
-----
The rejection sampling regimen draws samples from a normal distribution
with mean `mean` and standard deviation `std`, and resamples any values
more than two standard deviations from `mean`.
References
----------
- https://raw.githubusercontent.com/ddbourgin/numpy-ml/master/numpy_ml/neural_nets/utils/utils.py
'''
def __init__ (self, mu=0., std=1.):
self.mu = mu
self.std = std
super(TruncatedNormal, self).__init__()
[docs] def get (self, size):
samples = np.random.normal(loc=self.mu, scale=self.std, size=size)
reject = np.logical_or(samples >= self.mu + 2 * self.std, samples <= self.mu - 2 * self.std)
while any(reject.flatten()):
resamples = np.random.normal(loc=self.mu, scale=self.std, size=reject.sum())
samples[reject] = resamples
reject = np.logical_or(samples >= self.mu + 2 * self.std, samples <= self.mu - 2 * self.std)
return samples