Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")\n",
    "import os\n",
    "import numpy as np\n",
    "import zipfile\n",
    "from tqdm import tqdm\n",
    "from tempfile import TemporaryDirectory\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('ERROR') # only show error messages\n",
    "\n",
    "from iterator import MINDIterator\n",
    "from utils import prepare_hparams\n",
    "from utils import get_mind_data_set\n",
    "from model import NRMSModel\n",
    "from model import LSTURModel\n",
    "from layer import cal_metric\n",
    "iterator = MINDIterator\n",
    "seed=40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams_nrms = prepare_hparams('data/utils/nrms.yaml', \n",
    "                          wordEmb_file='data/utils/embedding.npy',\n",
    "                          wordDict_file='data/utils/word_dict.pkl', \n",
    "                          userDict_file='data/utils/uid2index.pkl',\n",
    "                          batch_size=32,\n",
    "                          show_step=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams_lstur = prepare_hparams('data/utils/lstur.yaml', \n",
    "                          wordEmb_file='data/utils/embedding_all.npy',\n",
    "                          wordDict_file='data/utils/word_dict_all.pkl', \n",
    "                          userDict_file='data/utils/uid2index.pkl',\n",
    "                          batch_size=32,\n",
    "                          show_step=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"conv1d_1/Relu:0\", shape=(?, 30, 400), dtype=float32)\n",
      "Tensor(\"att_layer2_5/Sum_1:0\", shape=(?, 400), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "model_nrms = NRMSModel(hparams_nrms, iterator, seed=seed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nrms.model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nrms.scorer.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:02, 222.40it/s]\n",
      "236it [00:40,  5.80it/s]\n",
      "7538it [00:01, 4032.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'group_auc': 0.517, 'mean_mrr': 0.2221, 'ndcg@5': 0.2296, 'ndcg@10': 0.2914}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pretrianed_metric,pretrianed_labels, pretrianed_preds=model_nrms.run_eval('data/valid/news.tsv', 'data/valid/behaviors.tsv')\n",
    "print(pretrianed_metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7fc1386fc358>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status_nrms=model_nrms.scorer.load_weights(tf.train.latest_checkpoint(\"./data/nrms_3e-4/\")).assert_existing_objects_matched()\n",
    "status_nrms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:02, 290.14it/s]\n",
      "236it [00:38,  6.11it/s]\n",
      "7538it [00:01, 4088.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'group_auc': 0.6005, 'mean_mrr': 0.2733, 'ndcg@5': 0.2977, 'ndcg@10': 0.3595}\n"
     ]
    }
   ],
   "source": [
    "trained_metrices_nrms, group_labels_nrms, group_preds_nrms=model_nrms.run_eval('data/valid/news.tsv', 'data/valid/behaviors.tsv')\n",
    "print(trained_metrices_nrms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"conv1d/Relu:0\", shape=(?, 30, 400), dtype=float32)\n",
      "Tensor(\"att_layer2_2/Sum_1:0\", shape=(?, 400), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:02, 214.75it/s]\n",
      "236it [00:27,  8.54it/s]\n",
      "7538it [00:01, 3852.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'group_auc': 0.519, 'mean_mrr': 0.2209, 'ndcg@5': 0.2286, 'ndcg@10': 0.2908}\n"
     ]
    }
   ],
   "source": [
    "model_lstur = LSTURModel(hparams_lstur, iterator, seed=seed)\n",
    "\n",
    "pretrianed_metric_lstur,pretrianed_labels_lstur, pretrianed_preds_lstur, =model_lstur.run_eval('data/valid/news.tsv', 'data/valid/behaviors.tsv')\n",
    "print(pretrianed_metric_lstur)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7fc0f47425f8>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status_lstur=model_lstur.model.load_weights(tf.train.latest_checkpoint(\"./data/lstur_3e-4/\")).assert_existing_objects_matched()\n",
    "status_lstur\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:01, 334.21it/s]\n",
      "236it [00:26,  8.79it/s]\n",
      "7538it [00:01, 4078.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'group_auc': 0.6108, 'mean_mrr': 0.2798, 'ndcg@5': 0.3026, 'ndcg@10': 0.3654}\n"
     ]
    }
   ],
   "source": [
    "trained_metrices_lstur, group_labels_lstur, group_preds_lstur=model_lstur.run_eval('data/valid/news.tsv', 'data/valid/behaviors.tsv')\n",
    "print(trained_metrices_lstur)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}