examples/datasets-test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets Test\n",
    "\n",
    "Test the various kind of data sets that come with TensorLight.\n",
    "\n",
    "*Remarks: The order of the image outputs can change between single executions!*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Force matplotlib to use inline rendering\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "\n",
    "# add path to libraries for ipython\n",
    "sys.path.append(os.path.expanduser(\"~/libs\"))\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorlight as light\n",
    "\n",
    "DATA_ROOT = \"data\"\n",
    "TRAIN_DIR = \"train-test/datasets\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_animation(dir_path, inputs, targets, fps, index):\n",
    "    concat_tgt = np.concatenate((inputs, targets))\n",
    "    print(concat_tgt.shape)\n",
    "\n",
    "    light.utils.video.write_multi_gif(os.path.join(dir_path, \"anim-{:02d}.gif\".format(index)),\n",
    "                                   [concat_tgt],\n",
    "                                   fps=fps, pad_value=1.0)\n",
    "\n",
    "    light.utils.video.write_multi_image_sequence(os.path.join(dir_path, \"timeline-{:02d}.png\".format(index)),\n",
    "                                              [concat_tgt],\n",
    "                                              pad_value=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mnist_train = light.datasets.mnist.MNISTTrainDataset(DATA_ROOT)\n",
    "mnist_valid = light.datasets.mnist.MNISTValidDataset(DATA_ROOT)\n",
    "mnist_test = light.datasets.mnist.MNISTTestDataset(DATA_ROOT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_mnist(dataset):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    light.visualization.display_array(x[0])\n",
    "    print('Label: {}'.format(y))\n",
    "    \n",
    "display_mnist(mnist_train)\n",
    "display_mnist(mnist_valid)\n",
    "display_mnist(mnist_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MovingMNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEQ_LEN = 10\n",
    "moving_train = light.datasets.moving_mnist.MovingMNISTTrainDataset(\n",
    "    DATA_ROOT, input_shape=[SEQ_LEN,64,64,1], target_shape=[SEQ_LEN,64,64,1])\n",
    "moving_valid = light.datasets.moving_mnist.MovingMNISTValidDataset(\n",
    "    DATA_ROOT, input_shape=[SEQ_LEN,64,64,1], target_shape=[SEQ_LEN,64,64,1])\n",
    "moving_test = light.datasets.moving_mnist.MovingMNISTTestDataset(\n",
    "    DATA_ROOT, input_seq_length=SEQ_LEN, target_seq_length=SEQ_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_moving(dataset, title):\n",
    "    x, y = dataset.get_batch(8)\n",
    "    for i in range(x.shape[0]):\n",
    "        full_seq = np.concatenate((x[i], y[i]))\n",
    "        light.visualization.display_batch(full_seq, nrows=4, ncols=5, title=title)\n",
    "        write_animation(os.path.join(TRAIN_DIR, \"mm\", \"out\", title), x[i], y[i], 5, i)\n",
    "\n",
    "display_moving(moving_train, 'Train')\n",
    "display_moving(moving_valid, 'Validation')\n",
    "display_moving(moving_test, 'Test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_single_images(dataset):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    for i in range(x.shape[1]):\n",
    "        light.utils.image.write(os.path.join(\"out\", \"mm-input-{}.png\".format(i)), x[0][i])\n",
    "    for i in range(y.shape[1]):\n",
    "        light.utils.image.write(os.path.join(\"out\", \"mm-target-{}.png\".format(i)), y[0][i])\n",
    "        \n",
    "        \n",
    "write_single_images(moving_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## UCF11\n",
    "\n",
    "*Remarks: This example uses an input queue.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ucf11_train = light.datasets.ucf11.UCF11TrainDataset(DATA_ROOT,\n",
    "                                                  input_seq_length=3, target_seq_length=3,\n",
    "                                                  image_scale_factor=0.5, gray_scale=False,\n",
    "                                                  min_examples_in_queue=128, queue_capacitiy=256,\n",
    "                                                  num_threads=16, do_distortion=False,\n",
    "                                                  crop_size=None)\n",
    "ucf11_valid = light.datasets.ucf11.UCF11ValidDataset(DATA_ROOT,\n",
    "                                                  input_seq_length=3, target_seq_length=3,\n",
    "                                                  image_scale_factor=0.5, gray_scale=False,\n",
    "                                                  crop_size=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, batch_y = ucf11_train.get_batch(32)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "    \n",
    "    try:\n",
    "        print('Starting queue runners...')\n",
    "        x, y = sess.run([batch_x, batch_y])       \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print(\"Done training -- epoch limit reached\")\n",
    "    finally:\n",
    "        # When done, ask the threads to stop\n",
    "        coord.request_stop()\n",
    "\n",
    "    coord.join(threads)\n",
    "    \n",
    "def display_ucf11_queue(x, y, title):\n",
    "    print(\"x-range: [{}, {}], y-range: [{}, {}]\".format(x.min(), x.max(), y.min(), y.max()))\n",
    "    full_seq = np.concatenate((x[0], y[0]))\n",
    "    \n",
    "    light.visualization.display_batch(full_seq, nrows=2, ncols=3, title=title)\n",
    "    \n",
    "def display_ucf11_batch(dataset, title):\n",
    "    x, y = dataset.get_batch(2)\n",
    "    print(\"x-range: [{}, {}], y-range: [{}, {}]\".format(x.min(), x.max(), y.min(), y.max()))\n",
    "    \n",
    "    light.visualization.display_batch(x[0], nrows=1, ncols=3, title=title + '-Inputs')\n",
    "    light.visualization.display_batch(y[0], nrows=1, ncols=3, title=title + '-Targets')\n",
    "\n",
    "display_ucf11_queue(x, y, 'Train')\n",
    "display_ucf11_batch(ucf11_valid, 'Validation')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## UCF101\n",
    "\n",
    "*Remarks: This example uses an input queue.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ucf101_train = light.datasets.ucf101.UCF101TrainDataset(DATA_ROOT,\n",
    "                                                     input_seq_length=10, target_seq_length=10,\n",
    "                                                     image_scale_factor=0.5, gray_scale=False,\n",
    "                                                     num_threads=16, min_examples_in_queue=32,\n",
    "                                                     queue_capacitiy=64, do_distortion=False,\n",
    "                                                     crop_size=None)\n",
    "ucf101_valid = light.datasets.ucf101.UCF101ValidDataset(DATA_ROOT,\n",
    "                                                     input_seq_length=10, target_seq_length=10,\n",
    "                                                     image_scale_factor=0.5, gray_scale=False,\n",
    "                                                     double_with_flipped=True,\n",
    "                                                     crop_size=None)\n",
    "ucf101_test = light.datasets.ucf101.UCF101TestDataset(DATA_ROOT,\n",
    "                                                   input_seq_length=10, target_seq_length=10,\n",
    "                                                   image_scale_factor=0.5, gray_scale=False,\n",
    "                                                   double_with_flipped=False,\n",
    "                                                   crop_size=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Train-size: {}\".format(ucf101_train.size))\n",
    "print(\"Valid-size: {}\".format(ucf101_valid.size))\n",
    "print(\"Test-size: {}\".format(ucf101_test.size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(\"/cpu:0\"):\n",
    "    batch_x, batch_y = ucf101_train.get_batch(8)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "    \n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "    \n",
    "    try:\n",
    "        print('Starting queue runners...')\n",
    "        x, y = sess.run([batch_x, batch_y]) \n",
    "\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print(\"Done training -- epoch limit reached\")\n",
    "    finally:\n",
    "        # When done, ask the threads to stop\n",
    "        coord.request_stop()\n",
    "\n",
    "    coord.join(threads)\n",
    "    \n",
    "def display_ucf101(x, y, title):\n",
    "    print(\"x-shape: {}, y-shape: \", x.shape, y.shape)\n",
    "    print(\"x-range: [{}, {}], y-range: [{}, {}]\".format(x.min(), x.max(), y.min(), y.max()))\n",
    "    \n",
    "    for i in range(x.shape[0]):\n",
    "        diff = 0.0\n",
    "        for j in range(x.shape[1] - 1):\n",
    "            diff += np.sum(np.square( (x[i,j+1] - x[i,j]) * 2) )\n",
    "        print(\"Diff @ {}: {}\".format(i, diff))\n",
    "        \n",
    "        full_seq = np.concatenate((x[i], y[i]))\n",
    "        light.visualization.display_batch(full_seq, nrows=4, ncols=5, title=\"{}-{:2d}\".format(title, i))\n",
    "        write_animation(os.path.join(TRAIN_DIR, \"ucf\", \"out-full\", title), x[i], y[i], 5, i)\n",
    "\n",
    "display_ucf101(x, y, 'Train')\n",
    "print('-' * 80)\n",
    "x, y = ucf101_valid.get_batch(8)\n",
    "display_ucf101(x, y, 'Validation')\n",
    "print('-' * 80)\n",
    "x, y = ucf101_test.get_batch(8)\n",
    "display_ucf101(x, y, 'Test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MsPacman"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pac_train = light.datasets.ms_pacman.MsPacmanTrainDataset(DATA_ROOT,\n",
    "                                                       input_seq_length=8,\n",
    "                                                       target_seq_length=8,\n",
    "                                                       crop_size=None)\n",
    "pac_valid = light.datasets.ms_pacman.MsPacmanValidDataset(DATA_ROOT,\n",
    "                                                       input_seq_length=8,\n",
    "                                                       target_seq_length=8,\n",
    "                                                       crop_size=None)\n",
    "pac_test = light.datasets.ms_pacman.MsPacmanTestDataset(DATA_ROOT,\n",
    "                                                   input_seq_length=8,\n",
    "                                                   target_seq_length=8,\n",
    "                                                   crop_size=None)\n",
    "print(\"Dataset-lengths:\", pac_train.size, pac_valid.size, pac_test.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_moving(dataset, title):\n",
    "    x, y = dataset.get_batch(8)\n",
    "    for i in range(x.shape[0]):\n",
    "        full_seq = np.concatenate((x[i], y[i]))\n",
    "        light.visualization.display_batch(full_seq, nrows=4, ncols=5, title=title)\n",
    "        write_animation(os.path.join(TRAIN_DIR, \"pac\", \"out-full\", title), x[i], y[i], 5, i)\n",
    "\n",
    "display_moving(pac_train, 'Train')\n",
    "display_moving(pac_valid, 'Validation')\n",
    "display_moving(pac_test, 'Test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Runtime: Train-Queue + Validation-Feeding\n",
    "\n",
    "This test tests the internal conditional switches withing the light.core.AbstractRuntime class to handle a **queue** for training and standard **feeding** für validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimplePredictionModel(light.model.AbstractModel):   \n",
    "    \"\"\"Predicts n future frames from given n frames.\"\"\"\n",
    "    def __init__(self, weight_decay=0.0):\n",
    "        super(SimplePredictionModel, self).__init__(weight_decay)\n",
    "        \n",
    "    @light.utils.attr.override\n",
    "    def inference(self, inputs, targets, feeds,\n",
    "                  is_training, device_scope, memory_device):\n",
    "        with tf.variable_scope(\"Encoder\"):\n",
    "            # stack time-dimension on channels (inputs=[bs,t,h,w,c])\n",
    "            unpacked_inputs = tf.unpack(inputs, axis=1) # unpacked_inputs=t*[bs,h,w,c]\n",
    "            stacked_inputs = tf.concat(concat_dim=3, values=unpacked_inputs) # stacked_inputs=[bs,h,w,c*t]\n",
    "            \n",
    "            # 1: Conv\n",
    "            conv1 = light.network.conv2d(\"Conv1\", stacked_inputs,\n",
    "                                      16, (5, 5), (2, 2),\n",
    "                                      weight_init=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                      bias_init=0.1,\n",
    "                                      regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),\n",
    "                                      activation=tf.nn.relu)\n",
    "\n",
    "            # 2: Conv\n",
    "            conv2 = light.network.conv2d(\"Conv2\", conv1,\n",
    "                                      16, (3, 3), (2, 2),\n",
    "                                      weight_init=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                      bias_init=0.1,\n",
    "                                      regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),\n",
    "                                      activation=tf.nn.relu)\n",
    "            encoder_out = conv2\n",
    "\n",
    "        with tf.variable_scope(\"Decoder\"):\n",
    "            # 3: Deconv\n",
    "            conv3t = light.network.conv2d_transpose(\"Deconv1\", encoder_out,\n",
    "                                                 16, (3, 3), (2, 2),\n",
    "                                                 weight_init=light.init.bilinear_initializer(),\n",
    "                                                 bias_init=0.1,\n",
    "                                                 regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),\n",
    "                                                 activation=tf.nn.relu)\n",
    "\n",
    "            # 4: Deconv\n",
    "            target_shape = targets.get_shape().as_list()\n",
    "            channels_out = target_shape[1] * target_shape[4] # t * c\n",
    "            conv4t = light.network.conv2d_transpose(\"Deconv2\", conv3t,\n",
    "                                                 channels_out, (5, 5), (2, 2),\n",
    "                                                 weight_init=light.init.bilinear_initializer(), \n",
    "                                                 bias_init=0.1,\n",
    "                                                 regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),\n",
    "                                                 activation=tf.nn.sigmoid)\n",
    "            # split channel dimensions\n",
    "            spl_out = tf.split(split_dim=3, num_split=target_shape[1], value=conv4t) # spl_out=t*[b,h,w,c]\n",
    "            decoder_out = tf.pack(spl_out, axis=1) # decoder_out=[b,t,h,w,c]\n",
    "            \n",
    "        return decoder_out\n",
    "    \n",
    "    @light.utils.attr.override\n",
    "    def loss(self, predictions, targets, device_scope):\n",
    "        return light.loss.bce(predictions, targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UCF11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runtime = light.core.DefaultRuntime(TRAIN_DIR)\n",
    "runtime.register_datasets(ucf11_train, ucf11_valid)\n",
    "runtime.register_model(SimplePredictionModel(weight_decay=0.001))\n",
    "runtime.build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runtime.train(batch_size=16, steps=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_ucf11(seq, title):\n",
    "    light.visualization.display_batch(seq, nrows=1, ncols=3, title=title)\n",
    "\n",
    "x, y = ucf11_valid.get_batch(1)\n",
    "    \n",
    "pred = runtime.predict(x)\n",
    "\n",
    "display_ucf11(x[0], \"Inputs\")\n",
    "display_ucf11(y[0], \"Targets\")\n",
    "display_ucf11(pred[0], \"Pred\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance Benchmark (Throughput)\n",
    "\n",
    "Remarks: Using too feq threads can lead to an OutOfRange exception, because we consuming batches in faster speed than the producers can create examples.\n",
    "\n",
    "The queue might be feel to be slower, because it might perform image-preprocessing, as well as a session is launched in every run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOOPS = 10000\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "# define which dataset to test\n",
    "bench_dataset_queue = ucf11_train\n",
    "bench_dataset_feeding = ucf11_valid\n",
    "\n",
    "a = tf.constant(2)\n",
    "b = tf.constant(3)\n",
    "c = tf.constant(4)\n",
    "\n",
    "# do a calcualtion to slow (especially the benchmark with the queue) a little bit down, because it can not\n",
    "# produce examples in such a high speed. Additionally, this makes both benchmarks more comparable, because\n",
    "# the feeding_queue needs to run a session as well with this calculation.\n",
    "calculation = a * b + c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For queue-datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(\"/cpu:0\"):\n",
    "    # Inputs should be done on CPU only (best performance)\n",
    "    batch_x, batch_y = bench_dataset_queue.get_batch(BATCH_SIZE)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "    \n",
    "    try:\n",
    "        this_step = 0\n",
    "        overhead = 0.0\n",
    "        while not coord.should_stop():\n",
    "            this_step += 1\n",
    "            \n",
    "            if this_step > LOOPS + 1: # extra loop because we skip the first run\n",
    "                break\n",
    "            \n",
    "            if this_step == 1:\n",
    "                print(\"Starting queue runners...\")\n",
    "\n",
    "            if this_step == 2:\n",
    "                print(\"Starting...\")\n",
    "                # skipt 1st round, because queue runners have to be filled\n",
    "                start_time = time.time()\n",
    "                \n",
    "            if this_step % 1000 == 0:\n",
    "                print(this_step)\n",
    "                \n",
    "            x, y, _ = sess.run([batch_x, batch_y, calculation])\n",
    "\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print(\"Interrupted: Queue runners are out of range. Epoch limit reached?\")\n",
    "    finally:\n",
    "        # When done, ask the threads to stop\n",
    "        duration = time.time() - start_time\n",
    "        coord.request_stop()\n",
    "\n",
    "    coord.join(threads)\n",
    "    \n",
    "print(\"Duration: {}\".format(duration))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For feeding-datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    overhead = 0.0\n",
    "    for i in xrange(LOOPS):\n",
    "        if i == 0:\n",
    "            print(\"Starting...\")\n",
    "            start_time = time.time()\n",
    "\n",
    "        start_overhead = time.time()\n",
    "        _ = sess.run(calculation)\n",
    "        overhead += time.time() - start_overhead\n",
    "        \n",
    "        batch_x, batch_y = bench_dataset_feeding.get_batch(BATCH_SIZE)\n",
    "\n",
    "    duration = time.time() - start_time\n",
    "\n",
    "print(\"Duration: {}\".format(duration))\n",
    "print(\"Overhead: {}\".format(overhead))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}