GAN_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "IOError",
     "evalue": "cannot identify image file './dataset/input/img_0.jpg'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mIOError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-24a4d855bdd0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0mtrain_input_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'./dataset/input'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0mtrain_output_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'./dataset/output'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m \u001b[0mtrain_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_output\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mload_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_input_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_output_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-1-24a4d855bdd0>\u001b[0m in \u001b[0;36mload_data\u001b[0;34m(intput_path, output_path)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mintput_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m     \u001b[0minput_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mintput_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m     \u001b[0minput_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_input_sentences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0moutput_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-24a4d855bdd0>\u001b[0m in \u001b[0;36mload_images\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mmyfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfile_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m         \u001b[0mimage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmyfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0mnum_images\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/scipy/ndimage/io.pyc\u001b[0m in \u001b[0;36mimread\u001b[0;34m(fname, flatten, mode)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflatten\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_have_pil\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_imread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflatten\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m     raise ImportError(\"Could not import the Python Imaging Library (PIL)\"\n\u001b[1;32m     26\u001b[0m                       \u001b[0;34m\" required to load image files.  Please refer to\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/scipy/misc/pilutil.pyc\u001b[0m in \u001b[0;36mimread\u001b[0;34m(name, flatten, mode)\u001b[0m\n\u001b[1;32m    154\u001b[0m     \"\"\"\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m     \u001b[0mim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfromimage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflatten\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/PIL/Image.pyc\u001b[0m in \u001b[0;36mopen\u001b[0;34m(fp, mode)\u001b[0m\n\u001b[1;32m   2450\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2451\u001b[0m     raise IOError(\"cannot identify image file %r\"\n\u001b[0;32m-> 2452\u001b[0;31m                   % (filename if filename else fp))\n\u001b[0m\u001b[1;32m   2453\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2454\u001b[0m \u001b[0;31m#\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: cannot identify image file './dataset/input/img_0.jpg'"
     ]
    }
   ],
   "source": [
    "# Import required libraries\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "import os\n",
    "import random\n",
    "import sklearn\n",
    "from scipy.ndimage import imread\n",
    "# import skipthoughts\n",
    "import nltk\n",
    "\n",
    "%matplotlib inline\n",
    "# Load cifar-10 data\n",
    "\n",
    "\n",
    "def load_images(path):\n",
    "    working_dir = path\n",
    "    file_list = []\n",
    "    for root, dirs, files in os.walk(working_dir):\n",
    "        for filename in files:\n",
    "            if filename.endswith('.jpg'):\n",
    "                file_list.append(root + \"/\" + filename) \n",
    "    images = []\n",
    "    for myfile in file_list:\n",
    "        image = imread(myfile)\n",
    "        images.append(image)\n",
    "    num_images = len(images)\n",
    "    images=np.asarray(images)\n",
    "    \n",
    "    images=images.reshape(num_images,1,28,28)\n",
    "    images=images.reshape(images.shape[0], 1, 28, 28).transpose(\n",
    "        0, 2, 3, 1).astype(\"uint8\")\n",
    "       \n",
    "    return images / 255.0\n",
    "    \n",
    "def load_input_sentences():\n",
    "    sentences = np.load('sentences_embeddings.npy')\n",
    "    solutions = np.load('solutions.npy')\n",
    "    \n",
    "    return sentences, solutions\n",
    "\n",
    "def load_data(intput_path, output_path):\n",
    "    input_images = load_images(intput_path)\n",
    "    input_sentences, labels = load_input_sentences()\n",
    "    output_images = load_images(output_path)\n",
    "    \n",
    "    inputs = list(zip(input_images, input_sentences, labels, output_images))\n",
    "    random.shuffle(inputs)\n",
    "    input_images, input_sentences, labels,output_images = zip(*inputs)\n",
    "    \n",
    "    return input_images, input_sentences, labels, output_images\n",
    "\n",
    "\n",
    "train_input_path = './dataset/input'\n",
    "train_output_path = './dataset/output'\n",
    "train_images, train_sentences, train_labels, train_output= load_data(train_input_path, train_output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def viz_grid(Xs, padding):\n",
    "    N, H, W, C = Xs.shape\n",
    "    grid_size = int(math.ceil(math.sqrt(N)))\n",
    "    grid_height = H * grid_size + padding * (grid_size + 1)\n",
    "    grid_width = W * grid_size + padding * (grid_size + 1)\n",
    "    grid = np.zeros((grid_height, grid_width, C))\n",
    "    next_idx = 0\n",
    "    y0, y1 = padding, H + padding\n",
    "    for y in range(grid_size):\n",
    "        x0, x1 = padding, W + padding\n",
    "        for x in range(grid_size):\n",
    "            if next_idx < N:\n",
    "                img = Xs[next_idx]\n",
    "                grid[y0:y1, x0:x1] = img\n",
    "                next_idx += 1\n",
    "            x0 += W + padding\n",
    "            x1 += W + padding\n",
    "        y0 += H + padding\n",
    "        y1 += H + padding\n",
    "    return grid\n",
    "\n",
    "def max_pool(input, kernel_size, stride):\n",
    "    ksize = [1, kernel_size, kernel_size, 1]\n",
    "    strides = [1, stride, stride, 1]\n",
    "    return tf.nn.max_pool(input, ksize=ksize, strides=strides, padding='SAME')\n",
    "\n",
    "def conv2d(input, kernel_size, stride, num_filter, name = 'conv2d'):\n",
    "    with tf.variable_scope(name):\n",
    "        stride_shape = [1, stride, stride, 1]\n",
    "        filter_shape = [kernel_size, kernel_size, input.get_shape()[3], num_filter]\n",
    "\n",
    "        W = tf.get_variable('w', filter_shape, tf.float32, tf.random_normal_initializer(0.0, 0.02))\n",
    "        b = tf.get_variable('b', [1, 1, 1, num_filter], initializer = tf.constant_initializer(0.0))\n",
    "        return tf.nn.conv2d(input, W, stride_shape, padding = 'SAME') + b\n",
    "\n",
    "def conv2d_transpose(input, kernel_size, stride, num_filter, name = 'conv2d_transpose'):\n",
    "    with tf.variable_scope(name):\n",
    "        stride_shape = [1, stride, stride, 1]\n",
    "        filter_shape = [kernel_size, kernel_size, num_filter, input.get_shape()[3]]\n",
    "        output_shape = tf.stack([tf.shape(input)[0], tf.shape(input)[1] * 2, tf.shape(input)[2] * 2, num_filter])\n",
    "\n",
    "        W = tf.get_variable('w', filter_shape, tf.float32, tf.random_normal_initializer(0.0, 0.02))\n",
    "        b = tf.get_variable('b', [1, 1, 1, num_filter], initializer = tf.constant_initializer(0.0))\n",
    "        return tf.nn.conv2d_transpose(input, W, output_shape, stride_shape, padding = 'SAME') + b\n",
    "\n",
    "def fc(input, num_output, name = 'fc'):\n",
    "    with tf.variable_scope(name):\n",
    "        num_input = input.get_shape()[1]\n",
    "        W = tf.get_variable('w', [num_input, num_output], tf.float32, tf.random_normal_initializer(0.0, 0.02))\n",
    "        b = tf.get_variable('b', [num_output], initializer = tf.constant_initializer(0.0))\n",
    "        return tf.matmul(input, W) + b\n",
    "\n",
    "def batch_norm(input, is_training):\n",
    "    out = tf.contrib.layers.batch_norm(input, decay = 0.99, center = True, scale = True,\n",
    "                                       is_training = is_training, updates_collections = None)\n",
    "    return out\n",
    "\n",
    "def leaky_relu(input, alpha = 0.2):\n",
    "    return tf.maximum(alpha * input, input)\n",
    "\n",
    "\n",
    "class DCGAN(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        self.num_epoch = 5\n",
    "        self.batch_size = 32\n",
    "        self.log_step = 50\n",
    "        self.visualize_step = 200\n",
    "        self.code_size = 256\n",
    "        self.learning_rate = 1e-4\n",
    "        self.vis_learning_rate = 1e-2\n",
    "        self.recon_steps = 100\n",
    "        self.actmax_steps = 100\n",
    "        \n",
    "        self._dis_called = False\n",
    "        self._gen_called = False\n",
    "\n",
    "        self.tracked_noise = np.random.normal(0, 1, [64, self.code_size])\n",
    "        self.arr = tf.placeholder(tf.int32, [10,self.batch_size])\n",
    "\n",
    "        self.real_sentences = tf.placeholder(tf.float32, [None, 4800])\n",
    "        self.real_images = tf.placeholder(tf.float32, [None, 28,28,1])\n",
    "        self.real_output_images = tf.placeholder(tf.float32, [None, 28,28,1])\n",
    "        \n",
    "        self.real_labels = tf.placeholder(tf.float32, [None, 100])\n",
    "        self.fake_labels = tf.placeholder(tf.float32, [10, None, 100])\n",
    "        \n",
    "        #self.fake_label = tf.placeholder(tf.float32, [None, 100])\n",
    "        self.noise = tf.placeholder(tf.float32, [None, 256])\n",
    "        \n",
    "        self.is_train = tf.placeholder(tf.bool)\n",
    "        \n",
    "        with tf.variable_scope('actmax'):\n",
    "            self.actmax_code = tf.get_variable('actmax_code', [1, self.code_size],\n",
    "                                               initializer = tf.constant_initializer(0.0))\n",
    "        \n",
    "        self._init_ops()\n",
    "\n",
    "    def _discriminator(self, input):\n",
    "        # We have multiple instances of the discriminator in the same computation graph,\n",
    "        # so set variable sharing if this is not the first invocation of this function.\n",
    "        with tf.variable_scope('dis', reuse = self._dis_called):\n",
    "            self._dis_called = True\n",
    "            \n",
    "            #dis_fc1 = fc(text_embedding, 784, 'fc1')\n",
    "            #dis_relu1 = leaky_relu(dis_fc1)\n",
    "            #dis_relu1_reshape = tf.reshape(dis_relu1, [-1,28,28,1])\n",
    "            #z_text = tf.concat([input, image_input, dis_relu1_reshape], 1)\n",
    "            \n",
    "            dis_conv1 = conv2d(input, 7, 1, 32, 'conv1')\n",
    "            dis_lrelu1 = leaky_relu(dis_conv1)\n",
    "            dis_maxpool1 = max_pool(dis_lrelu1,3,2 )\n",
    "            \n",
    "            dis_conv2 = conv2d(dis_maxpool1, 5, 1, 64, 'conv2')\n",
    "            dis_batchnorm2 = batch_norm(dis_conv2, self.is_train)\n",
    "            dis_lrelu2 = leaky_relu(dis_batchnorm2)\n",
    "            dis_maxpool2 = max_pool(dis_lrelu2,3,2)     \n",
    "            \n",
    "            dis_conv3 = conv2d(dis_maxpool2, 5, 1, 32, 'conv3')\n",
    "            dis_batchnorm3 = batch_norm(dis_conv3, self.is_train)\n",
    "            dis_lrelu3 = leaky_relu(dis_batchnorm3)\n",
    "            dis_mazpool3 = max_pool(dis_lrelu3,3,2) \n",
    "            \n",
    "            dis_reshape3 = tf.reshape(dis_mazpool3, [-1, 4 * 4 * 32])\n",
    "            dis_fc4 = fc(dis_reshape3, 256, 'fc4')\n",
    "            dis_lrelu3 = leaky_relu(dis_fc4)\n",
    "            dis_fc5 = fc(dis_lrelu3, 100, 'fc5')\n",
    "            # dis_sigmoid= tf.sigmoid(dis_fc5)\n",
    "            \n",
    "            return dis_fc5\n",
    "\n",
    "    def _generator(self, noise,text_embedding, image_input):\n",
    "        with tf.variable_scope('gen', reuse = self._gen_called):\n",
    "            self._gen_called = True\n",
    "            gen_fc1 = fc(text_embedding, 256, 'fc1')\n",
    "            gen_relu1 = leaky_relu(gen_fc1)\n",
    "            z_text = tf.concat([noise, gen_relu1], 1)\n",
    "            \n",
    "            gen_fc2 = fc(z_text, 784*2, 'fc2')\n",
    "            gen_relu2 = leaky_relu(gen_fc2)\n",
    "            \n",
    "            gen_reshape1 = tf.reshape(gen_relu2, [-1, 28, 28, 2])\n",
    "            print gen_reshape1.shape\n",
    "            image_reshape = tf.concat([image_input, gen_reshape1],3)\n",
    "            print image_reshape.shape\n",
    "            \n",
    "            gen_batchnorm1 = batch_norm(image_reshape, self.is_train)\n",
    "            gen_lrelu1 = leaky_relu(gen_batchnorm1)\n",
    "            print gen_lrelu1.shape\n",
    "            gen_conv2 = conv2d(gen_lrelu1, 4, 1, 64, 'conv2')\n",
    "            gen_batchnorm2 = batch_norm(gen_conv2, self.is_train)\n",
    "            gen_lrelu2 = leaky_relu(gen_batchnorm2)\n",
    "            print(gen_lrelu2.shape)\n",
    "            gen_conv3 = conv2d(gen_lrelu2, 4, 1, 32, 'conv3')\n",
    "            gen_batchnorm3 = batch_norm(gen_conv3, self.is_train)\n",
    "            gen_lrelu3 = leaky_relu(gen_batchnorm3)\n",
    "            gen_conv4 = conv2d(gen_lrelu3, 4, 1, 1, 'conv4')\n",
    "            \n",
    "            gen_sigmoid4 = tf.sigmoid(gen_conv4)\n",
    "            print(gen_sigmoid4.shape)\n",
    "            return gen_sigmoid4\n",
    "\n",
    "    def _loss(self, labels, logits):\n",
    "        loss = tf.nn.sigmoid_cross_entropy_with_logits(labels = labels, logits = logits)\n",
    "        return tf.reduce_mean(loss)\n",
    "\n",
    "    def _reconstruction_loss(self, generated, target):\n",
    "        loss = tf.nn.l2_loss(generated - target)\n",
    "        return tf.reduce_mean(loss)\n",
    "    \n",
    "    # Define operations\n",
    "    def _init_ops(self):\n",
    "        \n",
    "        ################################################################################\n",
    "        # Prob 2-1: complete the definition of these operations                        #\n",
    "        ################################################################################\n",
    "        \n",
    "        self.fake_samples_op = self._generator(self.noise, self.real_sentences,self.real_images)\n",
    "        self.dis_loss_op = self._loss(self.real_labels,self._discriminator(self.real_output_images))\n",
    "        \n",
    "        fake_label_list = []\n",
    "        \n",
    "        for i in range(10):\n",
    "            #print(self.fake_labels[i].shape, tf.one_hot(self.arr[i],100).shape )\n",
    "            fake_label_list.append(tf.one_hot(self.arr[i],100))\n",
    "        \n",
    "        fake_labels = tf.stack(fake_label_list)\n",
    "        \n",
    "        for i in range(self.fake_labels.shape[0]):\n",
    "            self.dis_loss_op = self.dis_loss_op + self._loss(self.fake_labels[i], self._discriminator(self.fake_samples_op))\n",
    "        self.gen_loss_op = self._loss(self.real_labels, self._discriminator(self.fake_samples_op))\n",
    "        #self.gen_loss_op = self.dis_loss_op\n",
    "        #self.gen_loss_op= self.dis_loss_op + self._loss(self.fake_labels[i], self._discriminator(self.fake_samples_op))       \n",
    "        \n",
    "        \n",
    "        \n",
    "        ################################################################################\n",
    "        # Prob 2-1: fix the definition of these operations                             #\n",
    "        ################################################################################\n",
    "        \n",
    "        dis_optimizer = tf.train.RMSPropOptimizer(self.learning_rate)\n",
    "        first_train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                                     \"dis\")\n",
    "        self.dis_train_op = dis_optimizer.minimize(self.dis_loss_op,var_list = first_train_vars)\n",
    "        \n",
    "        gen_optimizer = tf.train.RMSPropOptimizer(self.learning_rate)\n",
    "        second_train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                                     \"gen\")\n",
    "        self.gen_train_op = gen_optimizer.minimize(self.gen_loss_op, var_list = second_train_vars)\n",
    "\n",
    "    # Training function\n",
    "    def train(self, sess,train_images,train_sentences,train_labels ):\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        num_train = train_images.shape[0]\n",
    "        step = 0\n",
    "        \n",
    "        # smooth the loss curve so that it does not fluctuate too much\n",
    "        smooth_factor = 0.95\n",
    "        plot_dis_s = 0\n",
    "        plot_gen_s = 0\n",
    "        plot_ws = 0\n",
    "        \n",
    "        dis_losses = []\n",
    "        gen_losses = []\n",
    "        for epoch in range(self.num_epoch):\n",
    "            for i in range(num_train // (self.batch_size)):\n",
    "                step += 1\n",
    "\n",
    "                image_batch = train_images[i * self.batch_size : (i + 1) * self.batch_size]\n",
    "                output_batch = train_output[i * self.batch_size : (i + 1) * self.batch_size]\n",
    "                sentence_batch = train_sentences[i * self.batch_size : (i + 1) * self.batch_size]\n",
    "                noise = np.random.normal(0, 1, [self.batch_size, 256])\n",
    "                real_labels = np.zeros([32,100])\n",
    "                \n",
    "                for j in range(32):\n",
    "                    real_labels[j][train_labels[i * self.batch_size + j]] = 1\n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "                arr = np.zeros([10,self.batch_size],dtype=np.int32)\n",
    "                for i in range(self.batch_size):\n",
    "                    for j in range(10):\n",
    "                        num = random.randint(0,100)\n",
    "                        if num != train_labels[i]:\n",
    "                            arr[j][i] = num\n",
    "                            \n",
    "                #fake_labels = np.zeros([10,self.batch_size,100])\n",
    "                #for i in range(10):\n",
    "                    #print(fake_labels[i].shape, tf.one_hot(arr[i],100).shape )\n",
    "                    #fake_labels[i] = tf.one_hot(arr[i],100)\n",
    "                    \n",
    "                \n",
    "                #labels = np.zeros([self.batch_size,100])\n",
    "                #fake_labels = np.zeros([self.batch_size,10,100])\n",
    "                \n",
    "                #for j in range(self.batch_size ):\n",
    "                    #index = 0\n",
    "                    #for k in range(10):\n",
    "                        #index = random.randint(0,99)\n",
    "                        #if index != train_labels[i*self.batch_size + j]:\n",
    "                            #fake_labels[j][k][index] = 1\n",
    "                        #else:\n",
    "                            #k = k - 1\n",
    "\n",
    "                ################################################################################\n",
    "                # Prob 2-1: complete the feed dictionary                                       #\n",
    "                ################################################################################\n",
    "                \n",
    "                dis_feed_dict = {self.real_images:image_batch, self.real_output_images:output_batch, self.real_sentences:sentence_batch ,self.real_labels:real_labels, self.fake_labels:np.zeros([10,32,100]), self.noise:noise, self.arr:arr, self.is_train:True}\n",
    "        \n",
    "                ################################################################################\n",
    "                #                               END OF YOUR CODE                               #\n",
    "                ################################################################################\n",
    "\n",
    "                _, dis_loss = sess.run([self.dis_train_op, self.dis_loss_op], feed_dict = dis_feed_dict)\n",
    "        \n",
    "                ################################################################################\n",
    "                # Prob 2-1: complete the feed dictionary                                       #\n",
    "                ################################################################################\n",
    "                \n",
    "                gen_feed_dict = {self.noise:noise,self.real_images:image_batch, self.real_sentences:sentence_batch, self.real_labels:real_labels,self.is_train:True}\n",
    "        \n",
    "                ################################################################################\n",
    "                #                               END OF YOUR CODE                               #\n",
    "                ################################################################################\n",
    "\n",
    "                _, gen_loss = sess.run([self.gen_train_op, self.gen_loss_op], feed_dict = gen_feed_dict)\n",
    "\n",
    "                plot_dis_s = plot_dis_s * smooth_factor + dis_loss * (1 - smooth_factor)\n",
    "                plot_gen_s = plot_gen_s * smooth_factor + gen_loss * (1 - smooth_factor)\n",
    "                plot_ws = plot_ws * smooth_factor + (1 - smooth_factor)\n",
    "                dis_losses.append(plot_dis_s / plot_ws)\n",
    "                gen_losses.append(plot_gen_s / plot_ws)\n",
    "\n",
    "                if step % self.log_step == 0:\n",
    "                    print('Iteration {0}: dis loss = {1:.4f}, gen loss = {2:.4f}'.format(step, dis_loss, gen_loss))\n",
    "\n",
    "            #fig = plt.figure(figsize = (8, 8))   \n",
    "            #ax1 = plt.subplot(111)\n",
    "            #generated_image = self.generate(self.tracked_noise)\n",
    "            #print(generated_image.shape)\n",
    "            #image = viz_grid(generated_image, 1)\n",
    "            #print(image.shape)\n",
    "            #image = image.reshape(30,30)\n",
    "            #list = []\n",
    "            #for i in range(3):\n",
    "                #for j in range(image.shape[0]):\n",
    "                    #list.append(image[j])\n",
    "            #image = np.array(list)\n",
    "            #image = image.reshape(30,30,3)\n",
    "            \n",
    "            fig = plt.figure(figsize = (8, 8))   \n",
    "            ax1 = plt.subplot(111)\n",
    "            gen = self.generate(self.tracked_noise)\n",
    "            print (gen.shape)\n",
    "            images = viz_grid(gen, 1)\n",
    "            images = images.reshape(233,233)\n",
    "            ax1.imshow(images,cmap='gray')\n",
    "            plt.show()\n",
    "\n",
    "            plt.plot(dis_losses)\n",
    "            plt.title('discriminator loss')\n",
    "            plt.xlabel('iterations')\n",
    "            plt.ylabel('loss')\n",
    "            plt.show()\n",
    "\n",
    "            plt.plot(gen_losses)\n",
    "            plt.title('generator loss')\n",
    "            plt.xlabel('iterations')\n",
    "            plt.ylabel('loss')\n",
    "            plt.show()\n",
    "\n",
    "\n",
    "    # Generates a single sample from input code\n",
    "    def generate_one_sample(self, code, i):\n",
    "        \n",
    "        ################################################################################\n",
    "        # Prob 2-1: complete the feed dictionary                                       #\n",
    "        ################################################################################\n",
    "        image_batch = train_images[ i : i+1]\n",
    "        sentence_batch = train_sentences[i : i+1]\n",
    "        gen_vis_feed_dict = {self.noise:code,self.real_images:image_batch, self.real_sentences:sentence_batch, self.is_train:True}\n",
    "        \n",
    "        ################################################################################\n",
    "        #                               END OF YOUR CODE                               #\n",
    "        ################################################################################\n",
    "        \n",
    "        generated = sess.run(self.fake_samples_op, feed_dict = gen_vis_feed_dict)\n",
    "        return generated\n",
    "\n",
    "    # Generates samples from input batch of codes\n",
    "    def generate(self, codes):\n",
    "        generated = np.zeros((codes.shape[0], 28, 28, 1))\n",
    "        for i in range(codes.shape[0]):\n",
    "            generated[i:i+1] = self.generate_one_sample(codes[i:i+1],i)\n",
    "        return generated\n",
    "\n",
    "\n",
    "\n",
    "    # Perform activation maximization on a batch of different initial codes\n",
    "    def actmax(self, initial_codes):\n",
    "        actmax_results = np.zeros((initial_codes.shape[0], 32, 32, 3))\n",
    "        for i in range(initial_codes.shape[0]):\n",
    "            actmax_results[i:i+1] = self.actmax_one_sample(initial_codes[i:i+1])\n",
    "        return actmax_results.clip(0, 1)\n",
    "    \n",
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    dcgan = DCGAN()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    train_images = np.array(train_images)\n",
    "    train_sentences = np.array(train_sentences)\n",
    "    train_labels = np.array(train_labels)\n",
    "    train_images = np.array(train_images)\n",
    "    dcgan.train(sess, train_images,train_sentences,train_labels)\n",
    "    dis_var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'dis')\n",
    "    gen_var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'gen')\n",
    "    saver = tf.train.Saver(dis_var_list + gen_var_list)\n",
    "    saver.save(sess, 'model/dcgan')\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}