D76-optimizer_HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 作業重點:\n",
    "\n",
    "(1)以, Adam, 為例, 調整 batch_size, epoch , 觀察accurancy, loss 的變化\n",
    "\n",
    "(2)以同一模型, 分別驗證 SGD, Adam, Rmsprop 的 accurancy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 作業目標:\n",
    "    \n",
    "    取得各種優化器的運算結果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "import os\n",
    "from keras import optimizers\n",
    "\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Blas GEMM launch failed , 避免動態分配GPU / CPU, 出現問題\n",
    "import tensorflow as tf\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)\n",
    "sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# '''\n",
    "#    宣告並設定\n",
    "#    batch_size：對總的樣本數進行分組，每組包含的樣本數量\n",
    "#    epochs ：訓練次數\n",
    "   \n",
    "# ''' \n",
    "\n",
    "\n",
    "\n",
    "batch_size = 32\n",
    "num_classes = 10\n",
    "epochs = 20\n",
    "data_augmentation = True\n",
    "num_predictions = 20\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'keras_cifar10_trained_model.h5'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# The data, split between train and test sets:\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#    第一步：選擇模型, 順序模型是多個網絡層的線性堆疊\n",
    " \n",
    "model = Sequential()\n",
    "\n",
    "#   第二步：構建網絡層\n",
    "model.add(Conv2D(32, (3, 3), padding='same',\n",
    "                 input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Dense( 10)) # 輸出結果是10個類別，所以維度是10   \n",
    "model.add(Activation('softmax')) # 最後一層用softmax作為激活函數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Parameters：3702186\n"
     ]
    }
   ],
   "source": [
    "# 模型建立完成後，統計參數總量\n",
    "print(\"Total Parameters：%d\" % model.count_params())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_17 (Conv2D)           (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_25 (Activation)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_18 (Conv2D)           (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_26 (Activation)   (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 7200)              0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 512)               3686912   \n",
      "_________________________________________________________________\n",
      "activation_27 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_28 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 3,702,186\n",
      "Trainable params: 3,702,186\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 輸出模型摘要資訊\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第三步編譯\n",
    "#  '''\n",
    "#  SGD(隨機梯度下降) - Arguments\n",
    "# lr: float >= 0. Learning rate.\n",
    "# momentum: float >= 0. Parameter that accelerates SGD in the relevant direction and dampens oscillations.\n",
    "# decay: float >= 0. Learning rate decay over each update.\n",
    "# nesterov: boolean. Whether to apply Nesterov momentum.\n",
    "# '''\n",
    "\n",
    "# '''\n",
    "# RMSprop- Arguments\n",
    "# lr: float >= 0. Learning rate.\n",
    "# rho: float >= 0.\n",
    "# epsilon: float >= 0. Fuzz factor. If None, defaults to K.epsilon().\n",
    "# decay: float >= 0. Learning rate decay over each update.\n",
    "# '''\n",
    "\n",
    "# '''\n",
    "# Example:\n",
    "# opt = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "\n",
    "# '''\n",
    "opt = optimizers.Adam()\n",
    "model.compile(optimizer = opt, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 資料正規化\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "26432/50000 [==============>...............] - ETA: 2:11 - loss: 1.5887 - acc: 0.4223"
     ]
    }
   ],
   "source": [
    "# 是否要做資料處理\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    history=model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True)\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    print('')\n",
    "        \n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n",
    "        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        # randomly shift images horizontally (fraction of total width)\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically (fraction of total height)\n",
    "        height_shift_range=0.1,\n",
    "        shear_range=0.,  # set range for random shear\n",
    "        zoom_range=0.,  # set range for random zoom\n",
    "        channel_shift_range=0.,  # set range for random channel shifts\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        cval=0.,  # value used for fill_mode = \"constant\"\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False,  # randomly flip images\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for feature-wise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "    history=model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True)   \n",
    "\n",
    "'''\n",
    "   第四步：訓練\n",
    "   .fit的一些參數\n",
    "   batch_size：對總的樣本數進行分組，每組包含的樣本數量\n",
    "   epochs ：訓練次數\n",
    "   shuffle：是否把數據隨機打亂之後再進行訓練\n",
    "   validation_split：拿出百分之多少用來做交叉驗證\n",
    "   verbose：屏顯模式 - 0：不輸出, 1：輸出進度, 2：輸出每次的訓練結果\n",
    "''' \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save model and weights\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "model_path = os.path.join(save_dir, model_name)\n",
    "model.save(model_path)\n",
    "print('Saved trained model at %s ' % model_path)\n",
    "\n",
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#    第六步：輸出\n",
    "import numpy \n",
    "\n",
    "print ( \" test set \" )\n",
    "scores = model.evaluate(x_test,y_test,batch_size=200,verbose= 0)\n",
    "print ( \"\" )\n",
    "#print ( \" The test loss is %f \" % scores)\n",
    "print ( \" The test loss is %f \", scores)\n",
    "\n",
    "\n",
    "result = model.predict(x_test,batch_size=200,verbose= 0)\n",
    "\n",
    "result_max = numpy.argmax(result, axis = 1 )\n",
    "test_max = numpy.argmax(y_test, axis = 1 )\n",
    "\n",
    "result_bool = numpy.equal(result_max, test_max)\n",
    "true_num = numpy.sum(result_bool)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Plot training & validation accuracy values\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Valiidation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Valiidation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Plot training & validation accuracy values\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('Model accuracy - SGD')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Valiidation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Valiidation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}