keras-tuner-cifar100-BayesianOptimization.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"keras-tuner-cifar100-BayesianOptimization.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyONVg+e1ci08W5nRCGeTPxT"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"Uf5x_MJvIemY"},"source":["import tensorflow as tf\r\n","print(tf.__version__)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uxyFQ1AeQCgb"},"source":["!pip install keras-tuner\r\n","import kerastuner\r\n","from kerastuner.tuners import RandomSearch, Hyperband, BayesianOptimization"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"O1Yg6QmZIgYE"},"source":["from tensorflow.keras.utils import to_categorical\r\n","from tensorflow.keras.models import Sequential\r\n","from tensorflow.keras.layers import Conv2D\r\n","from tensorflow.keras.layers import MaxPooling2D\r\n","from tensorflow.keras.layers import Dense\r\n","from tensorflow.keras.layers import Flatten\r\n","from tensorflow.keras.optimizers import SGD, Adam\r\n","from tensorflow.keras.layers import BatchNormalization\r\n","from tensorflow.keras.layers import Dropout\r\n","from tensorflow.keras.preprocessing.image import ImageDataGenerator\r\n","import matplotlib.pyplot as plt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JmzNYNnaIh7b"},"source":["(trainX, trainY), (testX, testY) = tf.keras.datasets.cifar100.load_data()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"CzJPQh6sIj6E"},"source":["# one hot encode target values\r\n","trainY = to_categorical(trainY)\r\n","testY = to_categorical(testY)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_k5XuHXPIlxE"},"source":["# convert from integers to floats\r\n","#train_norm = trainX.astype('float32')\r\n","#test_norm = testX.astype('float32')\r\n","# normalize to range 0-1\r\n","trainX = trainX / 255.0\r\n","testX = testX / 255.0"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0D9N5J1iIoXU"},"source":["# define cnn model\r\n","def build_model(hp):\r\n","\tmodel = Sequential()\r\n","\thp_filters = hp.Int('filters', min_value = 32, max_value = 64, step = 32)\r\n","\tmodel.add(Conv2D(filters=hp_filters, kernel_size=(3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(MaxPooling2D((2, 2)))\r\n","\tmodel.add(Dropout(0.2))\r\n","\tmodel.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(MaxPooling2D((2, 2)))\r\n","\tmodel.add(Dropout(0.3))\r\n","\tmodel.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(MaxPooling2D((2, 2)))\r\n","\tmodel.add(Dropout(0.4))\r\n","\tmodel.add(Flatten())\r\n","\tmodel.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\r\n","\tmodel.add(BatchNormalization())\r\n","\tmodel.add(Dropout(0.5))\r\n","\tmodel.add(Dense(100, activation='softmax'))\r\n","\t# compile model\r\n","\t#opt = SGD(lr=0.001, momentum=0.9)\r\n","\thp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3])\r\n","\topt = Adam(learning_rate=hp_learning_rate)#, momentum=0.9)\r\n","\tmodel.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\r\n","\treturn model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RyxnjB27KX22"},"source":["tuner = BayesianOptimization(\r\n","  build_model,\r\n","  objective='val_accuracy',\r\n","  max_trials=4)\r\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cGzeQk1ndnMZ"},"source":["tuner.search(trainX, trainY,\r\n","             epochs=5,\r\n","             validation_data=(testX, testY))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hi-D-SqjeuOz"},"source":["tuner.results_summary()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"e48hfe-TeHcY"},"source":["# Get the optimal hyperparameters\r\n","best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]\r\n","print(best_hps.values)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XXidvqRgfD7i"},"source":["model = tuner.hypermodel.build(best_hps)\r\n","history = model.fit(trainX, trainY,\r\n","                      epochs=5,\r\n","                      validation_data=(testX, testY))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Bw-Qi-UAjqsz"},"source":["# evaluate model\r\n","_, acc = model.evaluate(testX, testY, verbose=0)\r\n","print('> %.3f' % (acc * 100.0))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"C90Zn8swjxQZ"},"source":["# Plot training & validation accuracy values\r\n","epoch_range = range(1, 6) #6 here is the number of epochs of final training\r\n","plt.plot(epoch_range, history.history['accuracy'])\r\n","plt.plot(epoch_range, history.history['val_accuracy'])\r\n","plt.title('Model accuracy')\r\n","plt.ylabel('Accuracy')\r\n","plt.xlabel('Epoch')\r\n","plt.legend(['Train', 'Val'], loc='upper left')\r\n","plt.show()\r\n","\r\n","# Plot training & validation loss values\r\n","plt.plot(epoch_range, history.history['loss'])\r\n","plt.plot(epoch_range, history.history['val_loss'])\r\n","plt.title('Model loss')\r\n","plt.ylabel('Loss')\r\n","plt.xlabel('Epoch')\r\n","plt.legend(['Train', 'Val'], loc='upper left')\r\n","plt.show()"],"execution_count":null,"outputs":[]}]}