model_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate results "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from collections import deque\n",
    "import numpy as np\n",
    "\n",
    "file_name='./data/processed_data/questions-visual_features-test.txt'\n",
    "\n",
    "with open(file_name, 'rb') as pickleFile:\n",
    "  test_deque=pickle.load(pickleFile)\n",
    "\n",
    "test_questions, test_images, test_answers = zip(*test_deque)\n",
    "\n",
    "### CONVERT FROM TUPLE TO ARRAY\n",
    "test_questions=np.array(test_questions)\n",
    "test_visual_features=np.array(test_images)\n",
    "test_answers=np.array(test_answers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "embedding_matrix=np.load('./data/embedding/glove_300d_embedding.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Aymeric\\Miniconda3\\envs\\vqa\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Aymeric\\Miniconda3\\envs\\vqa\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Aymeric\\Miniconda3\\envs\\vqa\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Aymeric\\Miniconda3\\envs\\vqa\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\Aymeric\\Miniconda3\\envs\\vqa\\lib\\site-packages\\keras\\optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from keras import optimizers\n",
    "from keras.models import load_model\n",
    "from numpy import array\n",
    "from keras.models import Sequential\n",
    "from keras.models import Model\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.layers import Input\n",
    "from keras.layers import Embedding\n",
    "from keras.initializers import Constant\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Dense, Conv2D, MaxPooling2D\n",
    "from keras.layers import Multiply, Dropout\n",
    "\n",
    "MAX_WORDS=3000\n",
    "MAX_LEN=25\n",
    "EMBED_DIM=300\n",
    "\n",
    "\n",
    "# encoder input model\n",
    "encoder_inputs = Input(shape=(MAX_LEN,))\n",
    "encoder1 = Embedding(MAX_WORDS,\n",
    "                     EMBED_DIM,\n",
    "                     embeddings_initializer=Constant(embedding_matrix),\n",
    "                     trainable=False)(encoder_inputs)\n",
    "encoder2  = Bidirectional(LSTM(512,activation='relu', trainable=False))(encoder1)\n",
    "#ENCODER MODEL\n",
    "#make sure to create an model inside your model. Because the encoder will be \n",
    "#saved as a model itself.\n",
    "encoder_model = Model(inputs=encoder_inputs,outputs=encoder2, name='Encoder')\n",
    "encoder_model.load_weights('./data/models/bidirectionnal_lstm_encoder2.h5')\n",
    "\n",
    "## IMAGES\n",
    "cnn_input=Input(shape=(14,14,512), name='CNN-Input')\n",
    "\n",
    "x = Conv2D(256, (3, 3),\n",
    "                  activation='relu',\n",
    "                  padding='same')(cnn_input)\n",
    "\n",
    "x = Conv2D(128, (3, 3),\n",
    "                  activation='relu',\n",
    "                  padding='valid')(x)\n",
    "\n",
    "x = MaxPooling2D((2, 2), strides=(2, 2))(x)\n",
    "flatten=Flatten()(x)\n",
    "\n",
    "cnn_output=Dense(1024, input_dim=4096, activation='tanh')(flatten)\n",
    "cnn=Model(inputs=cnn_input,outputs=cnn_output)\n",
    "\n",
    "multiplied = Multiply()([cnn.output, encoder_model.output])\n",
    "dropout_1 = Dropout(0.5)(multiplied)\n",
    "fully_connected=Dense(1000, activation='tanh')(dropout_1)\n",
    "dropout_2 = Dropout(0.5)(fully_connected)\n",
    "fully_connected=Dense(MAX_WORDS, activation='softmax')(dropout_2)\n",
    "\n",
    "question_answering=Model(inputs=[cnn.input,encoder_model.input], outputs=fully_connected)\n",
    "\n",
    "question_answering.compile(optimizer='adam', loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_answering.load_weights('./data/models/question_answering_model2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#make predicitons on the test set\n",
    "predictions=question_answering.predict([test_visual_features,test_questions])\n",
    "#use argmax to get the index with the highest probability\n",
    "predictions_argmax=np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# loading\n",
    "with open('./data/tokenizer/tokenizer.pickle', 'rb') as handle:\n",
    "    tokenizer = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "reverse_word_map[0]='not in vocab'\n",
    "\n",
    "predicted_answers=[]\n",
    "real_answers=[]\n",
    "\n",
    "for word in predictions_argmax:\n",
    "  predicted_answers.append(reverse_word_map[word])\n",
    "for word in np.squeeze(test_answers):\n",
    "  real_answers.append(reverse_word_map[word])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy :0.21871138570167697\n"
     ]
    }
   ],
   "source": [
    "### accuracy on test set\n",
    "correct=0\n",
    "\n",
    "for i in range(len(predicted_answers)):\n",
    "  if predicted_answers[i]==real_answers[i]:\n",
    "    correct+=1\n",
    "\n",
    "accuracy=correct/len(predicted_answers)\n",
    "print('accuracy :' + str(accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### WUPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import WUPS\n",
    "from WUPS import wup_measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_answers=np.array(list(map(lambda x: x.replace(\"'\",''), predicted_answers)))\n",
    "real_answers=np.array(list(map(lambda x: x.replace(\"'\",''), real_answers)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### wup score @0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_wup=0\n",
    "\n",
    "for i in range(len(predicted_answers)):\n",
    "    single_wup=wup_measure(predicted_answers[i],real_answers[i],similarity_threshold=0)\n",
    "    total_wup+=single_wup\n",
    "\n",
    "wup_score=total_wup/len(predicted_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7625313402551096"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wup_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### wup score @0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_wup=0\n",
    "\n",
    "for i in range(len(predicted_answers)):\n",
    "    single_wup=wup_measure(predicted_answers[i],real_answers[i],similarity_threshold=0.9)\n",
    "    total_wup+=single_wup\n",
    "\n",
    "wup_score=total_wup/len(predicted_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3031790144457981"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wup_score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vqa",
   "language": "python",
   "name": "vqa"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}