Dropout_Sinple_6_Gaussian.py

import tensorflow as tf
import mnist_data

import tensorflow.contrib.slim as slim
import time
import seaborn as sns
from Assign_Dataset import *
from tensorflow.examples.tutorials.mnist import input_data
from keras.datasets import mnist
from Support import *
from Mnist_DataHandle import *
from HSICSupport import *
from scipy.misc import imsave as ims
from utils import *

import keras
from keras.datasets import reuters
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.preprocessing.text import Tokenizer

distributions = tf.distributions
from Mixture_Models import *


def merge(images, size):
    h, w = images.shape[1], images.shape[2]
    img = np.zeros((h * size[0], w * size[1]))

    for idx, image in enumerate(images):
        i = idx % size[1]
        j = idx / size[1]
        j = int(j)

        img[int(j * h):int(j * h + h), int(i * w):int(i * w + w)] = image

    return img


def custom_layer(input_matrix, mix, resue=False):
    # with tf.variable_scope("custom_layer",reuse=resue):
    # w_init = tf.contrib.layers.variance_scaling_initializer()
    # b_init = tf.constant_initializer(0.)

    # weights = tf.get_variable(name="mix_weights", initializer=[0.25,0.25,0.25,0.25],trainable=True)
    weights = mix
    a1 = input_matrix[:, 0, :]
    a2 = input_matrix[:, 1, :]
    a3 = input_matrix[:, 2, :]
    a4 = input_matrix[:, 3, :]
    a5 = input_matrix[:, 4, :]
    a6 = input_matrix[:, 5, :]
    a7 = input_matrix[:, 6, :]
    a8 = input_matrix[:, 7, :]
    a9 = input_matrix[:, 8, :]
    a10 = input_matrix[:, 9, :]

    w1 = mix[:, 0:1]
    w2 = mix[:, 1:2]
    w3 = mix[:, 2:3]
    w4 = mix[:, 3:4]
    w5 = mix[:, 4:5]
    w6 = mix[:, 5:6]
    w7 = mix[:, 6:7]
    w8 = mix[:, 7:8]
    w9 = mix[:, 8:9]
    w10 = mix[:, 9:10]

    outputs = w1 * a1 + w2 * a2 + w3 * a3 + w4 * a4 + w5 * a5 + w6 * a6 + w7 * a7 + w8 * a8 + w9 * a9 + w10 * a10
    return outputs


def KL_Dropout2(log_alpha):
    ab = tf.cast(log_alpha, tf.float32)
    k1, k2, k3 = 0.63576, 1.8732, 1.48695;
    C = -k1
    mdkl = k1 * tf.nn.sigmoid(k2 + k3 * ab) - 0.5 * tf.log1p(tf.exp(-ab)) + C
    return -tf.reduce_sum(mdkl)


# Gateway
def autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term, Component_Count):
    # encoding
    mu1, sigma1, mix1 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder1")
    mu2, sigma2, mix2 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder2")
    mu3, sigma3, mix3 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder3")
    mu4, sigma4, mix4 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder4")
    mu5, sigma5, mix5 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder5")
    mu6, sigma6, mix6 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder6")

    z1 = distributions.Normal(loc=mu1, scale=sigma1)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)
    z5 = distributions.Normal(loc=mu5, scale=sigma5)
    z6 = distributions.Normal(loc=mu6, scale=sigma6)

    p = 0.5
    # a = p / (1.0-p)
    ard_init = -10.
    dropout_a = tf.get_variable("dropout", shape=[1], initializer=tf.constant_initializer(ard_init))

    # Dropout of components
    m1 = np.ones(batch_size)
    s1 = np.zeros(batch_size)
    dropout_a = tf.cast(dropout_a, tf.float64)

    dropout_dis = distributions.Normal(loc=m1, scale=dropout_a)
    dropout_samples = dropout_dis.sample(sample_shape=(6))
    dropout_samples = tf.transpose(dropout_samples)
    dropout_samples = tf.cast(dropout_samples, tf.float32)
    dropout_samples = tf.clip_by_value(dropout_samples, 1e-8, 1 - 1e-8)

    sum1 = mix1 + mix2 + mix3 + mix4+mix6+mix5
    mix1 = mix1 / sum1
    mix2 = mix2 / sum1
    mix3 = mix3 / sum1
    mix4 = mix4 / sum1
    mix5 = mix5 / sum1
    mix6 = mix6 / sum1

    mix = tf.concat([mix1, mix2, mix3, mix4,mix5,mix6], 1)
    mix_parameters = mix
    dist = tf.distributions.Dirichlet(mix)
    mix_samples = dist.sample()
    mix = mix_samples

    mix_dropout1 = dropout_samples[:, 0:1] * mix_samples[:, 0:1]
    mix_dropout2 = dropout_samples[:, 1:2] * mix_samples[:, 1:2]
    mix_dropout3 = dropout_samples[:, 2:3] * mix_samples[:, 2:3]
    mix_dropout4 = dropout_samples[:, 3:4] * mix_samples[:, 3:4]
    mix_dropout5 = dropout_samples[:, 4:5] * mix_samples[:, 4:5]
    mix_dropout6 = dropout_samples[:, 5:6] * mix_samples[:, 5:6]

    sum1 = mix_dropout1 + mix_dropout2 + mix_dropout3 + mix_dropout4+mix_dropout5+mix_dropout6
    mix_dropout1 = mix_dropout1 / sum1
    mix_dropout2 = mix_dropout2 / sum1
    mix_dropout3 = mix_dropout3 / sum1
    mix_dropout4 = mix_dropout4 / sum1
    mix_dropout5 = mix_dropout5 / sum1
    mix_dropout6 = mix_dropout6 / sum1

    # sampling by re-parameterization technique
    # z = mu + sigma * tf.random_normal(tf.shape(mu), 0, 1, dtype=tf.float32)

    z1_samples = z1.sample()
    z2_samples = z2.sample()
    z3_samples = z3.sample()
    z4_samples = z4.sample()
    z5_samples = z5.sample()
    z6_samples = z6.sample()

    ttf = []
    ttf.append(z1_samples)
    ttf.append(z2_samples)
    ttf.append(z3_samples)
    ttf.append(z4_samples)
    ttf.append(z5_samples)
    ttf.append(z6_samples)

    dHSIC_Value = dHSIC(ttf)


    # decoding
    y1 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder1")
    y2 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder2")
    y3 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder3")
    y4 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder4")
    y5 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder5")
    y6 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder6")

    # dropout out
    y1 = y1 * mix_dropout1
    y2 = y2 * mix_dropout2
    y3 = y3 * mix_dropout3
    y4 = y4 * mix_dropout4
    y5 = y5 * mix_dropout5
    y6 = y6 * mix_dropout6

    y = y1 + y2 + y3 + y4+y5+y6
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final")
    y = output

    m1 = np.zeros(dim_z, dtype=np.float32)
    m1[:] = 0
    v1 = np.zeros(dim_z, dtype=np.float32)
    v1[:] = 1

    # p_z1 = distributions.Normal(loc=np.zeros(dim_z, dtype=np.float32),
    #                           scale=np.ones(dim_z, dtype=np.float32))
    p_z1 = distributions.Normal(loc=m1,
                                scale=v1)

    m2 = np.zeros(dim_z, dtype=np.float32)
    m2[:] = 0
    v2 = np.zeros(dim_z, dtype=np.float32)
    v2[:] = 1
    p_z2 = distributions.Normal(loc=m2,
                                scale=v2)

    m3 = np.zeros(dim_z, dtype=np.float32)
    m3[:] = 0
    v3 = np.zeros(dim_z, dtype=np.float32)
    v3[:] = 1
    p_z3 = distributions.Normal(loc=m3,
                                scale=v3)

    m4 = np.zeros(dim_z, dtype=np.float32)
    m4[:] = 0
    v4 = np.zeros(dim_z, dtype=np.float32)
    v4[:] = 1
    p_z4 = distributions.Normal(loc=m4,
                                scale=v4)

    kl1 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z1, p_z1), 1))
    kl2 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z2, p_z2), 1))
    kl3 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z3, p_z3), 1))
    kl4 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z4, p_z4), 1))
    kl5 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z5, p_z4), 1))
    kl6 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z6, p_z4), 1))

    KL_divergence = (kl1 + kl2 + kl3 + kl4+kl5+kl6) / 6.0

    # loss
    marginal_likelihood = tf.reduce_sum(x * tf.log(y) + (1 - x) * tf.log(1 - y), 1)

    marginal_likelihood = tf.reduce_mean(marginal_likelihood)

    # KL divergence between two Dirichlet distributions
    a1 = tf.clip_by_value(mix_parameters, 0.1, 0.8)
    a2 = tf.constant((0.17, 0.17, 0.17, 0.17,0.17,0.17), shape=(batch_size, 6))

    r = tf.reduce_sum((a1 - a2) * (tf.polygamma(0.0, a1) - tf.polygamma(0.0, 1)), axis=1)
    a = tf.lgamma(tf.reduce_sum(a1, axis=1)) - tf.lgamma(tf.reduce_sum(a2, axis=1)) + tf.reduce_sum(tf.lgamma(a2),
                                                                                                    axis=-1) - tf.reduce_sum(
        tf.lgamma(a1), axis=1) + r
    kl = a
    kl = tf.reduce_mean(kl)

    p1 = 1
    p2 = 1
    p4 = 1
    ELBO = marginal_likelihood - KL_divergence * p2

    loss = -ELBO + kl * p1 + p4 * dHSIC_Value + KL_Dropout2(dropout_a)

    z = z1_samples
    return y, z, loss, -marginal_likelihood, KL_divergence,dropout_samples


def decoder(z, dim_img, n_hidden):
    y = bernoulli_MLP_decoder(z, n_hidden, dim_img, 1.0, reuse=True)
    return y

def HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term):
    # encoding
    mu1, sigma1, mix1 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder1", True)
    mu2, sigma2, mix2 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder2", True)
    mu3, sigma3, mix3 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder3", True)
    mu4, sigma4, mix4 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder4", True)
    mu5, sigma5, mix5 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder5", True)
    mu6, sigma6, mix6 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder6", True)

    z1 = distributions.Normal(loc=mu1, scale=sigma1)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)
    z5 = distributions.Normal(loc=mu5, scale=sigma5)
    z6 = distributions.Normal(loc=mu6, scale=sigma6)

    z1_samples = z1.sample()
    z2_samples = z2.sample()
    z3_samples = z3.sample()
    z4_samples = z4.sample()
    z5_samples = z5.sample()
    z6_samples = z6.sample()

    sum1 = mix1 + mix2 + mix3 + mix4+mix5+mix6
    mix1 = mix1 / sum1
    mix2 = mix2 / sum1
    mix3 = mix3 / sum1
    mix4 = mix4 / sum1
    mix5 = mix5 / sum1
    mix6 = mix6 / sum1

    mix = tf.concat([mix1, mix2, mix3, mix4,mix5,mix6], 1)
    mix_parameters = mix
    dist = tf.distributions.Dirichlet(mix)
    mix_samples = dist.sample()
    mix = mix_samples

    return z1_samples, z2_samples, z3_samples, z4_samples,z5_samples, z6_samples, mix


def Final_Output(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term, Component_Count):
    # encoding
    mu1, sigma1, mix1 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder1",True)
    mu2, sigma2, mix2 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder2",True)
    mu3, sigma3, mix3 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder3",True)
    mu4, sigma4, mix4 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder4",True)
    mu5, sigma5, mix5 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder5",True)
    mu6, sigma6, mix6 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder6",True)

    z1 = distributions.Normal(loc=mu1, scale=sigma1)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)
    z5 = distributions.Normal(loc=mu5, scale=sigma5)
    z6 = distributions.Normal(loc=mu6, scale=sigma6)

    sum1 = mix1 + mix2 + mix3 + mix4+mix6+mix5
    mix1 = mix1 / sum1
    mix2 = mix2 / sum1
    mix3 = mix3 / sum1
    mix4 = mix4 / sum1
    mix5 = mix5 / sum1
    mix6 = mix6 / sum1

    mix = tf.concat([mix1, mix2, mix3, mix4,mix5,mix6], 1)
    mix_parameters = mix
    dist = tf.distributions.Dirichlet(mix)
    mix_samples = dist.sample()
    mix = mix_samples

    mix_dropout1 = mix_samples[:, 0:1]
    mix_dropout2 = mix_samples[:, 1:2]
    mix_dropout3 = mix_samples[:, 2:3]
    mix_dropout4 = mix_samples[:, 3:4]
    mix_dropout5 = mix_samples[:, 4:5]
    mix_dropout6 = mix_samples[:, 5:6]

    # sampling by re-parameterization technique
    # z = mu + sigma * tf.random_normal(tf.shape(mu), 0, 1, dtype=tf.float32)

    z1_samples = z1.sample()
    z2_samples = z2.sample()
    z3_samples = z3.sample()
    z4_samples = z4.sample()
    z5_samples = z5.sample()
    z6_samples = z6.sample()

    # decoding
    y1 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder1",True)
    y2 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder2",True)
    y3 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder3",True)
    y4 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder4",True)
    y5 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder5",True)
    y6 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder6",True)

    # dropout out
    y1 = y1 * mix_dropout1
    y2 = y2 * mix_dropout2
    y3 = y3 * mix_dropout3
    y4 = y4 * mix_dropout4
    y5 = y5 * mix_dropout5
    y6 = y6 * mix_dropout6

    y = y1 + y2 + y3 + y4+y5+y6
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final",True)
    y = output
    return y

def Final_Output_Confusion(z1_samples, z2_samples, z3_samples, z4_samples, dim_img, dim_z, n_hidden, keep_prob):
    ard_init = -10.
    with tf.variable_scope("", reuse=True):  # root variable scope
        dropout_a = tf.get_variable("dropout")

    # Dropout of components
    m1 = np.ones(batch_size)
    s1 = np.zeros(batch_size)
    dropout_a = tf.cast(dropout_a, tf.float64)

    dropout_dis = distributions.Normal(loc=m1, scale=dropout_a)
    dropout_samples = dropout_dis.sample(sample_shape=(10))
    dropout_samples = tf.transpose(dropout_samples)
    dropout_samples = tf.cast(dropout_samples, tf.float32)
    dropout_samples = tf.clip_by_value(dropout_samples, 1e-8, 1 - 1e-8)

    # decoding
    y1 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder1", True)
    y1 = tf.clip_by_value(y1, 1e-8, 1 - 1e-8)

    y2 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder2", True)
    y2 = tf.clip_by_value(y2, 1e-8, 1 - 1e-8)

    y3 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder3", True)
    y3 = tf.clip_by_value(y3, 1e-8, 1 - 1e-8)

    y4 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder4", True)
    y4 = tf.clip_by_value(y4, 1e-8, 1 - 1e-8)

    # dropout out
    y1 = y1 * dropout_samples[:, 0:1]
    y2 = y2 * dropout_samples[:, 1:2]
    y3 = y3 * dropout_samples[:, 2:3]
    y4 = y4 * dropout_samples[:, 3:4]

    y = y1 + y2 + y3 + y4
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final", True)
    output = tf.clip_by_value(output, 1e-8, 1 - 1e-8)
    y = output
    return y

def CalculateMSE(x1, x2):
    c = tf.square(x1 - x2)
    return tf.reduce_mean(c)

n_hidden = 500
IMAGE_SIZE_MNIST = 28
dim_img = IMAGE_SIZE_MNIST ** 2  # number of pixels for a MNIST image

dim_z = 50

# train
n_epochs = 100
batch_size = 128
learn_rate = 0.001

train_total_data, train_size, _, _, test_data, test_labels = mnist_data.prepare_MNIST_data()
n_samples = train_size
# input placeholders

# In denoising-autoencoder, x_hat == x + noise, otherwise x_hat == x
x_hat = tf.placeholder(tf.float32, shape=[None, dim_img], name='input_img')
x = tf.placeholder(tf.float32, shape=[None, dim_img], name='target_img')

# dropout
keep_prob = tf.placeholder(tf.float32, name='keep_prob')

# input for PMLR
z_in = tf.placeholder(tf.float32, shape=[None, dim_z], name='latent_variable')
last_term = tf.placeholder(tf.float32, shape=[10])
Component_Count = tf.placeholder(tf.float32)

# network architecture
y, z, loss, neg_marginal_likelihood, KL_divergence,dropout_samples = autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,
                                                                 last_term, Component_Count)
z1_samples, z2_samples, z3_samples, z4_samples,z5_samples,z6_samples, mix = HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,
                                                                   last_term)
# y1, y2, y3, y4 = Outpiuts_Component(x_hat, x, dim_img, dim_z, n_hidden, keep_prob)

# optimization
t_vars = tf.trainable_variables()
train_op = tf.train.AdamOptimizer(learn_rate).minimize(loss, var_list=t_vars)

# train

total_batch = int(n_samples / batch_size)

min_tot_loss = 1e99
ADD_NOISE = False

train_data2_ = train_total_data[:, :-mnist_data.NUM_LABELS]
train_y = train_total_data[:, 784:784 + mnist_data.NUM_LABELS]

# MNIST dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

x_fixed = train_data2_[0:128]
saver = tf.train.Saver()

isWeight = True

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer(), feed_dict={keep_prob: 0.9})
    lastvalue = np.ones((4))
    
    if isWeight:
        saver.restore(sess, 'models/Dropout_Sinmple_6_Gaussian')
        tIndex = 7
        x_fixed = x_train[batch_size * tIndex:batch_size * tIndex + batch_size]
        x_fixed = x_test[batch_size * tIndex:batch_size * tIndex + batch_size]
        x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

        reco,dropoutSamples = sess.run([y,dropout_samples], feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})
        reco = np.reshape(reco, (-1, 28, 28))
        x_fixed = np.reshape(x_fixed, (-1, 28, 28))
        ims("results/" + "MNIST__Real_Bernoulli" + str(0) + ".png", merge(x_fixed[:64], [8, 8]))
        ims("results/" + "MNIST__Reco_Bernoulli" + str(0) + ".png", merge(reco[:64], [8, 8]))

        # yy = sess.run(y,feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})
        # yy = np.reshape(yy,(-1,28,28))
        # ims("results/" + "VAE" + str(0) + ".jpg", merge(yy[:64], [8, 8]))

        z_in1 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in2 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in3 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in4 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in5 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in6 = tf.placeholder(tf.float32, shape=[None, dim_z])

        mix_in = tf.placeholder(tf.float32, shape=[None, 6])
        x_in1 = tf.placeholder(tf.float32, shape=[None, 28 * 28])
        x_in2 = tf.placeholder(tf.float32, shape=[None, 28 * 28])

        mse = CalculateMSE(x_in1, x_in2)

        yy = Final_Output(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,
                                                                 last_term, Component_Count)

        x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

        y1 = sess.run(yy, feed_dict={x_hat:x_fixed,x:x_fixed,keep_prob: 1.0})
        y1 = np.reshape(y1, (-1, 28, 28))
        ims("results/" + "AAA" + str(0) + ".png", merge(y1[:64], [8, 8]))

        sum1 = 0
        for t2 in range(batch_size):
            noCount = 0
            for t1 in range(4):
                if dropoutSamples[t2, t1] < 0.8:
                    noCount = noCount + 1
            yesCount = 4 - noCount
            sum1 = sum1 + yesCount

        sum1 = float(sum1 / batch_size)

        y1 = np.reshape(y1, (-1, 28 * 28))
        reco = np.reshape(reco, (-1, 28 * 28))
        mse1 = sess.run(mse, feed_dict={x_in1: x_fixed, x_in2: reco})
        mse2 = sess.run(mse, feed_dict={x_in1: x_fixed, x_in2: y1})
        mse1 = mse1 * 28 * 28
        mse2 = mse2 * 28 * 28

        bc = 0

    # training

    lastvalue = np.ones((10))
    valid_reco = Give_Reconstruction(x_hat, x)
    x_valid = x_test[0:batch_size]
    x_valid = np.reshape(x_valid, (-1, 28 * 28))
    bestScore = 100000
    x_train = np.reshape(x_train, (-1, 28 * 28))
    x_fixed = x_train[0:batch_size]
    x_fixed = np.reshape(x_fixed,(-1,28*28))

    n_epochs = 100
    for epoch in range(n_epochs):
        # Random shuffling
        index = [i for i in range(np.shape(x_train)[0])]
        random.shuffle(index)
        x_train = x_train[index]

        # Loop over all batches
        for i in range(total_batch):
            # Compute the offset of the current minibatch in the data.
            batch_xs_input = x_train[i * batch_size:i * batch_size + batch_size]
            batch_xs_target = batch_xs_input

            if ADD_NOISE:
                batch_xs_input = batch_xs_input * np.random.randint(2, size=batch_xs_input.shape)
                batch_xs_input += np.random.randint(2, size=batch_xs_input.shape)

            _, tot_loss, loss_likelihood, loss_divergence = sess.run(
                (train_op, loss, neg_marginal_likelihood, KL_divergence),
                feed_dict={x_hat: batch_xs_input, x: batch_xs_target, keep_prob: 1.0})

        y_PRR = sess.run(y, feed_dict={x_hat: x_fixed, keep_prob: 1})
        y_RPR = np.reshape(y_PRR, (-1, 28, 28))
        ims("results/" + "VAE" + str(epoch) + ".jpg", merge(y_RPR[:64], [8, 8]))

        print("epoch %f: L_tot %03.2f L_likelihood %03.2f L_divergence %03.2f" % (
            epoch, tot_loss, loss_likelihood, loss_divergence))

        if epoch > 0:
            x_fixed_image = np.reshape(x_fixed, (-1, 28, 28))
            ims("results/" + "Real" + str(epoch) + ".jpg", merge(x_fixed_image[:64], [8, 8]))

        if bestScore > loss_likelihood:
            bestScore = loss_likelihood
            saver.save(sess, 'models/Dropout_Sinmple_6_Gaussian')
    #saver.save(sess, 'models/Dropout_Sinmple_6_Gaussian')