classifiers.py

"""
FIXME TODO
"""
__docformat__ = 'restructedtext en'

import numpy
import theano
import theano.tensor as T


class BasicLinear(object):
    def __init__(self, rng, input, n_in, n_out, W=None, b=None):
        # rng is a required useless parameter so that we keep the same
        # __init__ signature as for hidden layers (see layers.py)
        if W != None:
            self.W = W
        else:
            W_values = numpy.zeros((n_in, n_out), dtype=theano.config.floatX)
            self.W = theano.shared(value=W_values, name='W', borrow=True)
        if b != None:
            self.b = b
        else:
            self.b = theano.shared(value=numpy.zeros((n_out,),
                dtype=theano.config.floatX),
                name='b', borrow=True)


class LogisticRegression(BasicLinear):
    """Multi-class Logistic Regression
    """

    def __init__(self, rng, input, n_in, n_out, W=None, b=None):
        super(LogisticRegression, self).__init__(rng, input, n_in, n_out,
                W, b)
        # compute vector of class-membership probabilities in symbolic form
        self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b)

        # compute prediction as class whose probability is maximal in
        # symbolic form
        self.y_pred = T.argmax(self.p_y_given_x, axis=1)

        # for a standard theano Variable output attribute
        self.output = self.y_pred

        # parameters of the model
        self.params = [self.W, self.b]

    def negative_log_likelihood(self, y):
        """Return the mean of the negative log-likelihood of the prediction
        of this model under a given target distribution.

        .. math::

            \frac{1}{|\mathcal{D}|} \mathcal{L} (\theta=\{W,b\}, \mathcal{D}) =
            \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\
                \ell (\theta=\{W,b\}, \mathcal{D})

        :type y: theano.tensor.TensorType
        :param y: corresponds to a vector that gives for each example the
                  correct label

        Note: we use the mean instead of the sum so that
              the learning rate is less dependent on the batch size
        """
        # y.shape[0] is (symbolically) the number of rows in y, i.e.,
        # number of examples (call it n) in the minibatch
        # T.arange(y.shape[0]) is a symbolic vector which will contain
        # [0,1,2,... n-1] T.log(self.p_y_given_x) is a matrix of
        # Log-Probabilities (call it LP) with one row per example and
        # one column per class LP[T.arange(y.shape[0]),y] is a vector
        # v containing [LP[0,y[0]], LP[1,y[1]], LP[2,y[2]], ...,
        # LP[n-1,y[n-1]]] and T.mean(LP[T.arange(y.shape[0]),y]) is
        # the mean (across minibatch examples) of the elements in v,
        # i.e., the mean log-likelihood across the minibatch.
        return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y])

    def negative_log_likelihood_sum(self, y):
        return -T.sum(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y])

    def training_cost(self, y):
        """ Wrapper for standard name """
        return self.negative_log_likelihood_sum(y)

    def errors(self, y):
        """Return a float representing the number of errors in the minibatch
        over the total number of examples of the minibatch ; zero one
        loss over the size of the minibatch

        :type y: theano.tensor.TensorType
        :param y: corresponds to a vector that gives for each example the
                  correct label
        """

        # check if y has same dimension of y_pred
        if y.ndim != self.y_pred.ndim:
            raise TypeError('y should have the same shape as self.y_pred',
                ('y', y.type, 'y_pred', self.y_pred.type))
        # check if y is of the correct datatype
        if y.dtype.startswith('int'):
            # the T.neq operator returns a vector of 0s and 1s, where 1
            # represents a mistake in prediction
            return T.mean(T.neq(self.y_pred, y))
        else:
            raise NotImplementedError()


class EmbdLogisticRegression(BasicLinear):
    def __init__(self, rng, input, n_in, n_out, embd_distances,
            W=None, b=None):
        super(EmbdLogisticRegression, self).__init__(rng, input, n_in, n_out,
                W, b)

    def training_cost(self, y):
        pass
        # TODO use the embeddings cost as the loss