NRMS_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.15.4'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")\n",
    "import os\n",
    "import numpy as np\n",
    "import zipfile\n",
    "from tqdm import tqdm\n",
    "# import scrapbook as sb\n",
    "from tempfile import TemporaryDirectory\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('ERROR') # only show error messages\n",
    "\n",
    "from utils import download_deeprec_resources \n",
    "from utils import prepare_hparams\n",
    "from utils import get_mind_data_set\n",
    "from iterator import MINDIterator\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (c) Microsoft Corporation. All rights reserved.\n",
    "# Licensed under the MIT License.\n",
    "\n",
    "\n",
    "from os.path import join\n",
    "import abc\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras import layers\n",
    "from layer import cal_metric\n",
    "from layer import SelfAttention\n",
    "\n",
    "from layer import (\n",
    "    AttLayer2,\n",
    "    ComputeMasking,\n",
    "    OverwriteMasking,\n",
    ")\n",
    "\n",
    "class BaseModel:\n",
    "    \"\"\"Basic class of models\n",
    "\n",
    "    Attributes:\n",
    "        hparams (obj): A tf.contrib.training.HParams object, hold the entire set of hyperparameters.\n",
    "        iterator_creator_train (obj): An iterator to load the data in training steps.\n",
    "        iterator_creator_train (obj): An iterator to load the data in testing steps.\n",
    "        graph (obj): An optional graph.\n",
    "        seed (int): Random seed.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        hparams,\n",
    "        iterator_creator,\n",
    "        seed=None,\n",
    "    ):\n",
    "        \"\"\"Initializing the model. Create common logics which are needed by all deeprec models, such as loss function,\n",
    "        parameter set.\n",
    "\n",
    "        Args:\n",
    "            hparams (obj): A tf.contrib.training.HParams object, hold the entire set of hyperparameters.\n",
    "            iterator_creator_train (obj): An iterator to load the data in training steps.\n",
    "            iterator_creator_train (obj): An iterator to load the data in testing steps.\n",
    "            graph (obj): An optional graph.\n",
    "            seed (int): Random seed.\n",
    "        \"\"\"\n",
    "        self.seed = seed\n",
    "        tf.compat.v1.set_random_seed(seed)\n",
    "        np.random.seed(seed)\n",
    "\n",
    "        self.train_iterator = iterator_creator(\n",
    "            hparams,\n",
    "            hparams.npratio,\n",
    "            col_spliter=\"\\t\",\n",
    "        )\n",
    "        self.test_iterator = iterator_creator(\n",
    "            hparams,\n",
    "            col_spliter=\"\\t\",\n",
    "        )\n",
    "\n",
    "        self.hparams = hparams\n",
    "        self.support_quick_scoring = hparams.support_quick_scoring\n",
    "\n",
    "        # set GPU use with on demand growth\n",
    "        gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)\n",
    "        sess = tf.compat.v1.Session(\n",
    "            config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)\n",
    "        )\n",
    "\n",
    "        # set this TensorFlow session as the default session for Keras\n",
    "        tf.compat.v1.keras.backend.set_session(sess)\n",
    "\n",
    "        # IMPORTANT: models have to be loaded AFTER SETTING THE SESSION for keras!\n",
    "        # Otherwise, their weights will be unavailable in the threads after the session there has been set\n",
    "        self.model, self.scorer = self._build_graph()\n",
    "\n",
    "        self.loss = self._get_loss()\n",
    "        self.train_optimizer = self._get_opt()\n",
    "\n",
    "        self.model.compile(loss=self.loss, optimizer=self.train_optimizer)\n",
    "\n",
    "    def _init_embedding(self, file_path):\n",
    "        \"\"\"Load pre-trained embeddings as a constant tensor.\n",
    "\n",
    "        Args:\n",
    "            file_path (str): the pre-trained glove embeddings file path.\n",
    "\n",
    "        Returns:\n",
    "            np.array: A constant numpy array.\n",
    "        \"\"\"\n",
    "\n",
    "        return np.load(file_path)\n",
    "\n",
    "    @abc.abstractmethod\n",
    "    def _build_graph(self):\n",
    "        \"\"\"Subclass will implement this.\"\"\"\n",
    "        pass\n",
    "\n",
    "    @abc.abstractmethod\n",
    "    def _get_input_label_from_iter(self, batch_data):\n",
    "        \"\"\"Subclass will implement this\"\"\"\n",
    "        pass\n",
    "\n",
    "    def _get_loss(self):\n",
    "        \"\"\"Make loss function, consists of data loss and regularization loss\n",
    "\n",
    "        Returns:\n",
    "            obj: Loss function or loss function name\n",
    "        \"\"\"\n",
    "        if self.hparams.loss == \"cross_entropy_loss\":\n",
    "            data_loss = \"categorical_crossentropy\"\n",
    "        elif self.hparams.loss == \"log_loss\":\n",
    "            data_loss = \"binary_crossentropy\"\n",
    "        else:\n",
    "            raise ValueError(\"this loss not defined {0}\".format(self.hparams.loss))\n",
    "        return data_loss\n",
    "\n",
    "    def _get_opt(self):\n",
    "        \"\"\"Get the optimizer according to configuration. Usually we will use Adam.\n",
    "        Returns:\n",
    "            obj: An optimizer.\n",
    "        \"\"\"\n",
    "        lr = self.hparams.learning_rate\n",
    "        optimizer = self.hparams.optimizer\n",
    "\n",
    "        if optimizer == \"adam\":\n",
    "            train_opt = keras.optimizers.Adam(lr=lr)\n",
    "\n",
    "        return train_opt\n",
    "\n",
    "    def _get_pred(self, logit, task):\n",
    "        \"\"\"Make final output as prediction score, according to different tasks.\n",
    "\n",
    "        Args:\n",
    "            logit (obj): Base prediction value.\n",
    "            task (str): A task (values: regression/classification)\n",
    "\n",
    "        Returns:\n",
    "            obj: Transformed score\n",
    "        \"\"\"\n",
    "        if task == \"regression\":\n",
    "            pred = tf.identity(logit)\n",
    "        elif task == \"classification\":\n",
    "            pred = tf.sigmoid(logit)\n",
    "        else:\n",
    "            raise ValueError(\n",
    "                \"method must be regression or classification, but now is {0}\".format(\n",
    "                    task\n",
    "                )\n",
    "            )\n",
    "        return pred\n",
    "\n",
    "    def train(self, train_batch_data):\n",
    "        \"\"\"Go through the optimization step once with training data in feed_dict.\n",
    "\n",
    "        Args:\n",
    "            sess (obj): The model session object.\n",
    "            feed_dict (dict): Feed values to train the model. This is a dictionary that maps graph elements to values.\n",
    "\n",
    "        Returns:\n",
    "            list: A list of values, including update operation, total loss, data loss, and merged summary.\n",
    "        \"\"\"\n",
    "        train_input, train_label = self._get_input_label_from_iter(train_batch_data)\n",
    "        rslt = self.model.train_on_batch(train_input, train_label)\n",
    "        return rslt\n",
    "\n",
    "    def eval(self, eval_batch_data):\n",
    "        \"\"\"Evaluate the data in feed_dict with current model.\n",
    "\n",
    "        Args:\n",
    "            sess (obj): The model session object.\n",
    "            feed_dict (dict): Feed values for evaluation. This is a dictionary that maps graph elements to values.\n",
    "\n",
    "        Returns:\n",
    "            list: A list of evaluated results, including total loss value, data loss value,\n",
    "                predicted scores, and ground-truth labels.\n",
    "        \"\"\"\n",
    "        eval_input, eval_label = self._get_input_label_from_iter(eval_batch_data)\n",
    "        imp_index = eval_batch_data[\"impression_index_batch\"]\n",
    "\n",
    "        pred_rslt = self.scorer.predict_on_batch(eval_input)\n",
    "\n",
    "        return pred_rslt, eval_label, imp_index\n",
    "\n",
    "    def fit(\n",
    "        self,\n",
    "        train_news_file,\n",
    "        train_behaviors_file,\n",
    "        valid_news_file,\n",
    "        valid_behaviors_file,\n",
    "        test_news_file=None,\n",
    "        test_behaviors_file=None,\n",
    "    ):\n",
    "        \"\"\"Fit the model with train_file. Evaluate the model on valid_file per epoch to observe the training status.\n",
    "        If test_news_file is not None, evaluate it too.\n",
    "\n",
    "        Args:\n",
    "            train_file (str): training data set.\n",
    "            valid_file (str): validation set.\n",
    "            test_news_file (str): test set.\n",
    "\n",
    "        Returns:\n",
    "            obj: An instance of self.\n",
    "        \"\"\"\n",
    "        \n",
    "        train_losses=[]\n",
    "        val_losses=[]\n",
    "        val_result=[]\n",
    "        for epoch in range(1, self.hparams.epochs + 1):\n",
    "            step = 0\n",
    "            self.hparams.current_epoch = epoch\n",
    "            epoch_loss = 0\n",
    "            train_start = time.time()\n",
    "\n",
    "            tqdm_util = tqdm(\n",
    "                self.train_iterator.load_data_from_file(\n",
    "                    train_news_file, train_behaviors_file\n",
    "                )\n",
    "            )\n",
    "\n",
    "            for batch_data_input in tqdm_util:\n",
    "\n",
    "                step_result = self.train(batch_data_input)\n",
    "                step_data_loss = step_result\n",
    "\n",
    "                epoch_loss += step_data_loss\n",
    "                step += 1\n",
    "                if step % self.hparams.show_step == 0:\n",
    "                    tqdm_util.set_description(\n",
    "                        \"step {0:d} , total_loss: {1:.4f}, data_loss: {2:.4f}\".format(\n",
    "                            step, epoch_loss / step, step_data_loss\n",
    "                        )\n",
    "                    )\n",
    "                    \n",
    "            train_losses.append(epoch_loss/step)\n",
    "            train_end = time.time()\n",
    "            train_time = train_end - train_start\n",
    "\n",
    "            eval_start = time.time()\n",
    "\n",
    "            train_info = \",\".join(\n",
    "                [\n",
    "                    str(item[0]) + \":\" + str(item[1])\n",
    "                    for item in [(\"logloss loss\", epoch_loss / step)]\n",
    "                ]\n",
    "            )\n",
    "\n",
    "            eval_res = self.run_eval(valid_news_file, valid_behaviors_file)\n",
    "            \n",
    "            val_result.append(eval_res)\n",
    "            \n",
    "            eval_info = \", \".join(\n",
    "                [\n",
    "                    str(item[0]) + \":\" + str(item[1])\n",
    "                    for item in sorted(eval_res.items(), key=lambda x: x[0])\n",
    "                ]\n",
    "            )\n",
    "            if test_news_file is not None:\n",
    "                test_res = self.run_eval(test_news_file, test_behaviors_file)\n",
    "                test_info = \", \".join(\n",
    "                    [\n",
    "                        str(item[0]) + \":\" + str(item[1])\n",
    "                        for item in sorted(test_res.items(), key=lambda x: x[0])\n",
    "                    ]\n",
    "                )\n",
    "            eval_end = time.time()\n",
    "            eval_time = eval_end - eval_start\n",
    "\n",
    "            if test_news_file is not None:\n",
    "                print(\n",
    "                    \"at epoch {0:d}\".format(epoch)\n",
    "                    + \"\\ntrain info: \"\n",
    "                    + train_info\n",
    "                    + \"\\neval info: \"\n",
    "                    + eval_info\n",
    "                    + \"\\ntest info: \"\n",
    "                    + test_info\n",
    "                )\n",
    "            else:\n",
    "                print(\n",
    "                    \"at epoch {0:d}\".format(epoch)\n",
    "                    + \"\\ntrain info: \"\n",
    "                    + train_info\n",
    "                    + \"\\neval info: \"\n",
    "                    + eval_info\n",
    "                )\n",
    "            print(\n",
    "                \"at epoch {0:d} , train time: {1:.1f} eval time: {2:.1f}\".format(\n",
    "                    epoch, train_time, eval_time\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            if epoch%5==0:\n",
    "                self.model.save_weights(os.path.join('data/model_newlr', \"nmrs_3e-4_{}\".format(epoch)))\n",
    "        return train_losses, val_result\n",
    "\n",
    "    def group_labels(self, labels, preds, group_keys):\n",
    "        \"\"\"Devide labels and preds into several group according to values in group keys.\n",
    "\n",
    "        Args:\n",
    "            labels (list): ground truth label list.\n",
    "            preds (list): prediction score list.\n",
    "            group_keys (list): group key list.\n",
    "\n",
    "        Returns:\n",
    "            all_labels: labels after group.\n",
    "            all_preds: preds after group.\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        all_keys = list(set(group_keys))\n",
    "        all_keys.sort()\n",
    "        group_labels = {k: [] for k in all_keys}\n",
    "        group_preds = {k: [] for k in all_keys}\n",
    "\n",
    "        for l, p, k in zip(labels, preds, group_keys):\n",
    "            group_labels[k].append(l)\n",
    "            group_preds[k].append(p)\n",
    "\n",
    "        all_labels = []\n",
    "        all_preds = []\n",
    "        for k in all_keys:\n",
    "            all_labels.append(group_labels[k])\n",
    "            all_preds.append(group_preds[k])\n",
    "\n",
    "        return all_keys, all_labels, all_preds\n",
    "\n",
    "    def run_eval(self, news_filename, behaviors_file):\n",
    "        \"\"\"Evaluate the given file and returns some evaluation metrics.\n",
    "\n",
    "        Args:\n",
    "            filename (str): A file name that will be evaluated.\n",
    "\n",
    "        Returns:\n",
    "            dict: A dictionary contains evaluation metrics.\n",
    "        \"\"\"\n",
    "\n",
    "        if self.support_quick_scoring:\n",
    "            _, group_labels, group_preds = self.run_fast_eval(\n",
    "                news_filename, behaviors_file\n",
    "            )\n",
    "        else:\n",
    "            _, group_labels, group_preds = self.run_slow_eval(\n",
    "                news_filename, behaviors_file\n",
    "            )\n",
    "        res = cal_metric(group_labels, group_preds, self.hparams.metrics)\n",
    "\n",
    "        return res\n",
    "\n",
    "    def user(self, batch_user_input):\n",
    "        user_input = self._get_user_feature_from_iter(batch_user_input)\n",
    "        user_vec = self.userencoder.predict_on_batch(user_input)\n",
    "        user_index = batch_user_input[\"impr_index_batch\"]\n",
    "\n",
    "        return user_index, user_vec\n",
    "\n",
    "    def news(self, batch_news_input):\n",
    "        news_input = self._get_news_feature_from_iter(batch_news_input)\n",
    "        news_vec = self.newsencoder.predict_on_batch(news_input)\n",
    "        news_index = batch_news_input[\"news_index_batch\"]\n",
    "\n",
    "        return news_index, news_vec\n",
    "\n",
    "    def run_user(self, news_filename, behaviors_file):\n",
    "        if not hasattr(self, \"userencoder\"):\n",
    "            raise ValueError(\"model must have attribute userencoder\")\n",
    "\n",
    "        user_indexes = []\n",
    "        user_vecs = []\n",
    "        for batch_data_input in tqdm(\n",
    "            self.test_iterator.load_user_from_file(news_filename, behaviors_file)\n",
    "        ):\n",
    "            user_index, user_vec = self.user(batch_data_input)\n",
    "            user_indexes.extend(np.reshape(user_index, -1))\n",
    "            user_vecs.extend(user_vec)\n",
    "\n",
    "        return dict(zip(user_indexes, user_vecs))\n",
    "\n",
    "    def run_news(self, news_filename):\n",
    "        if not hasattr(self, \"newsencoder\"):\n",
    "            raise ValueError(\"model must have attribute newsencoder\")\n",
    "\n",
    "        news_indexes = []\n",
    "        news_vecs = []\n",
    "        for batch_data_input in tqdm(\n",
    "            self.test_iterator.load_news_from_file(news_filename)\n",
    "        ):\n",
    "            news_index, news_vec = self.news(batch_data_input)\n",
    "            news_indexes.extend(np.reshape(news_index, -1))\n",
    "            news_vecs.extend(news_vec)\n",
    "\n",
    "        return dict(zip(news_indexes, news_vecs))\n",
    "\n",
    "    def run_slow_eval(self, news_filename, behaviors_file):\n",
    "        preds = []\n",
    "        labels = []\n",
    "        imp_indexes = []\n",
    "\n",
    "        for batch_data_input in tqdm(\n",
    "            self.test_iterator.load_data_from_file(news_filename, behaviors_file)\n",
    "        ):\n",
    "            step_pred, step_labels, step_imp_index = self.eval(batch_data_input)\n",
    "            preds.extend(np.reshape(step_pred, -1))\n",
    "            labels.extend(np.reshape(step_labels, -1))\n",
    "            imp_indexes.extend(np.reshape(step_imp_index, -1))\n",
    "\n",
    "        group_impr_indexes, group_labels, group_preds = self.group_labels(\n",
    "            labels, preds, imp_indexes\n",
    "        )\n",
    "        return group_impr_indexes, group_labels, group_preds\n",
    "\n",
    "    def run_fast_eval(self, news_filename, behaviors_file):\n",
    "        news_vecs = self.run_news(news_filename)\n",
    "        user_vecs = self.run_user(news_filename, behaviors_file)\n",
    "\n",
    "        self.news_vecs = news_vecs\n",
    "        self.user_vecs = user_vecs\n",
    "\n",
    "        group_impr_indexes = []\n",
    "        group_labels = []\n",
    "        group_preds = []\n",
    "\n",
    "        for (\n",
    "            impr_index,\n",
    "            news_index,\n",
    "            user_index,\n",
    "            label,\n",
    "        ) in tqdm(self.test_iterator.load_impression_from_file(behaviors_file)):\n",
    "            pred = np.dot(\n",
    "                np.stack([news_vecs[i] for i in news_index], axis=0),\n",
    "                user_vecs[impr_index],\n",
    "            )\n",
    "            group_impr_indexes.append(impr_index)\n",
    "            group_labels.append(label)\n",
    "            group_preds.append(pred)\n",
    "\n",
    "        return group_impr_indexes, group_labels, group_preds\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NRMSModel(BaseModel):\n",
    "    \"\"\"NRMS model(Neural News Recommendation with Multi-Head Self-Attention)\n",
    "\n",
    "    Chuhan Wu, Fangzhao Wu, Suyu Ge, Tao Qi, Yongfeng Huang,and Xing Xie, \"Neural News\n",
    "    Recommendation with Multi-Head Self-Attention\" in Proceedings of the 2019 Conference \n",
    "    on Empirical Methods in Natural Language Processing and the 9th International Joint Conference \n",
    "    on Natural Language Processing (EMNLP-IJCNLP)\n",
    "\n",
    "    Attributes:\n",
    "        word2vec_embedding (numpy.array): Pretrained word embedding matrix.\n",
    "        hparam (obj): Global hyper-parameters.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, hparams, iterator_creator, seed=None,\n",
    "    ):\n",
    "        \"\"\"Initialization steps for NRMS.\n",
    "        Compared with the BaseModel, NRMS need word embedding.\n",
    "        After creating word embedding matrix, BaseModel's __init__ method will be called.\n",
    "        \n",
    "        Args:\n",
    "            hparams (obj): Global hyper-parameters. Some key setttings such as head_num and head_dim are there.\n",
    "            iterator_creator_train(obj): NRMS data loader class for train data.\n",
    "            iterator_creator_test(obj): NRMS data loader class for test and validation data\n",
    "        \"\"\"\n",
    "        self.word2vec_embedding = self._init_embedding(hparams.wordEmb_file)\n",
    "\n",
    "        super().__init__(\n",
    "            hparams, iterator_creator, seed=seed,\n",
    "        )\n",
    "\n",
    "    def _get_input_label_from_iter(self, batch_data):\n",
    "        \"\"\" get input and labels for trainning from iterator\n",
    "\n",
    "        Args: \n",
    "            batch data: input batch data from iterator\n",
    "\n",
    "        Returns:\n",
    "            list: input feature fed into model (clicked_title_batch & candidate_title_batch)\n",
    "            array: labels\n",
    "        \"\"\"\n",
    "        input_feat = [\n",
    "            batch_data[\"clicked_title_batch\"],\n",
    "            batch_data[\"candidate_title_batch\"],\n",
    "        ]\n",
    "        input_label = batch_data[\"labels\"]\n",
    "        return input_feat, input_label\n",
    "\n",
    "    def _get_user_feature_from_iter(self, batch_data):\n",
    "        \"\"\" get input of user encoder \n",
    "        Args:\n",
    "            batch_data: input batch data from user iterator\n",
    "        \n",
    "        Returns:\n",
    "            array: input user feature (clicked title batch)\n",
    "        \"\"\"\n",
    "        return batch_data[\"clicked_title_batch\"]\n",
    "\n",
    "    def _get_news_feature_from_iter(self, batch_data):\n",
    "        \"\"\" get input of news encoder\n",
    "        Args:\n",
    "            batch_data: input batch data from news iterator\n",
    "        \n",
    "        Returns:\n",
    "            array: input news feature (candidate title batch)\n",
    "        \"\"\"\n",
    "        return batch_data[\"candidate_title_batch\"]\n",
    "\n",
    "    def _build_graph(self):\n",
    "        \"\"\"Build NRMS model and scorer.\n",
    "\n",
    "        Returns:\n",
    "            obj: a model used to train.\n",
    "            obj: a model used to evaluate and inference.\n",
    "        \"\"\"\n",
    "        hparams = self.hparams\n",
    "        model, scorer = self._build_nrms()\n",
    "        return model, scorer\n",
    "\n",
    "    def _build_userencoder(self, titleencoder):\n",
    "        \"\"\"The main function to create user encoder of NRMS.\n",
    "\n",
    "        Args:\n",
    "            titleencoder(obj): the news encoder of NRMS. \n",
    "\n",
    "        Return:\n",
    "            obj: the user encoder of NRMS.\n",
    "        \"\"\"\n",
    "        hparams = self.hparams\n",
    "        his_input_title = keras.Input(\n",
    "            shape=(hparams.his_size, hparams.title_size), dtype=\"int32\"\n",
    "        )\n",
    "\n",
    "        click_title_presents = layers.TimeDistributed(titleencoder)(his_input_title)\n",
    "        y = SelfAttention(hparams.head_num, hparams.head_dim, seed=self.seed)(\n",
    "            [click_title_presents] * 3\n",
    "        )\n",
    "        user_present = AttLayer2(hparams.attention_hidden_dim, seed=self.seed)(y)\n",
    "\n",
    "        model = keras.Model(his_input_title, user_present, name=\"user_encoder\")\n",
    "        return model\n",
    "\n",
    "    def _build_newsencoder(self, embedding_layer):\n",
    "        \"\"\"The main function to create news encoder of NRMS.\n",
    "\n",
    "        Args:\n",
    "            embedding_layer(obj): a word embedding layer.\n",
    "        \n",
    "        Return:\n",
    "            obj: the news encoder of NRMS.\n",
    "        \"\"\"\n",
    "        hparams = self.hparams\n",
    "        sequences_input_title = keras.Input(shape=(hparams.title_size,), dtype=\"int32\")\n",
    "\n",
    "        embedded_sequences_title = embedding_layer(sequences_input_title)\n",
    "\n",
    "        y = layers.Dropout(hparams.dropout)(embedded_sequences_title)\n",
    "        y = SelfAttention(hparams.head_num, hparams.head_dim, seed=self.seed)([y, y, y])\n",
    "        y = layers.Dropout(hparams.dropout)(y)\n",
    "        pred_title = AttLayer2(hparams.attention_hidden_dim, seed=self.seed)(y)\n",
    "\n",
    "        model = keras.Model(sequences_input_title, pred_title, name=\"news_encoder\")\n",
    "        return model\n",
    "\n",
    "    def _build_nrms(self):\n",
    "        \"\"\"The main function to create NRMS's logic. The core of NRMS\n",
    "        is a user encoder and a news encoder.\n",
    "        \n",
    "        Returns:\n",
    "            obj: a model used to train.\n",
    "            obj: a model used to evaluate and inference.\n",
    "        \"\"\"\n",
    "        hparams = self.hparams\n",
    "\n",
    "        his_input_title = keras.Input(\n",
    "            shape=(hparams.his_size, hparams.title_size), dtype=\"int32\"\n",
    "        )\n",
    "        pred_input_title = keras.Input(\n",
    "            shape=(hparams.npratio + 1, hparams.title_size), dtype=\"int32\"\n",
    "        )\n",
    "        pred_input_title_one = keras.Input(\n",
    "            shape=(1, hparams.title_size,), dtype=\"int32\"\n",
    "        )\n",
    "        pred_title_one_reshape = layers.Reshape((hparams.title_size,))(\n",
    "            pred_input_title_one\n",
    "        )\n",
    "\n",
    "        embedding_layer = layers.Embedding(\n",
    "            self.word2vec_embedding.shape[0],\n",
    "            hparams.word_emb_dim,\n",
    "            weights=[self.word2vec_embedding],\n",
    "            trainable=True,\n",
    "        )\n",
    "\n",
    "        titleencoder = self._build_newsencoder(embedding_layer)\n",
    "        self.userencoder = self._build_userencoder(titleencoder)\n",
    "        self.newsencoder = titleencoder\n",
    "\n",
    "        user_present = self.userencoder(his_input_title)\n",
    "        news_present = layers.TimeDistributed(self.newsencoder)(pred_input_title)\n",
    "        news_present_one = self.newsencoder(pred_title_one_reshape)\n",
    "\n",
    "        preds = layers.Dot(axes=-1)([news_present, user_present])\n",
    "        preds = layers.Activation(activation=\"softmax\")(preds)\n",
    "\n",
    "        pred_one = layers.Dot(axes=-1)([news_present_one, user_present])\n",
    "        pred_one = layers.Activation(activation=\"sigmoid\")(pred_one)\n",
    "\n",
    "        model = keras.Model([his_input_title, pred_input_title], preds)\n",
    "        scorer = keras.Model([his_input_title, pred_input_title_one], pred_one)\n",
    "\n",
    "        return model, scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "epochs = 50\n",
    "seed = 42\n",
    "batch_size = 32\n",
    "\n",
    "# Options: demo, small, large\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "MIND_type = 'large'\n",
    "\n",
    "# tmpdir = TemporaryDirectory()\n",
    "# data_path = tmpdir.name\n",
    "\n",
    "#select a nontemporary folder to avoid re-download\n",
    "data_path = 'data'\n",
    "\n",
    "train_news_file = os.path.join(data_path, 'train', r'news.tsv')\n",
    "train_behaviors_file = os.path.join(data_path, 'train', r'behaviors.tsv')\n",
    "valid_news_file = os.path.join(data_path, 'valid', r'news.tsv')\n",
    "valid_behaviors_file = os.path.join(data_path, 'valid', r'behaviors.tsv')\n",
    "wordEmb_file = os.path.join(data_path, \"utils\", \"embedding.npy\")\n",
    "userDict_file = os.path.join(data_path, \"utils\", \"uid2index.pkl\")\n",
    "wordDict_file = os.path.join(data_path, \"utils\", \"word_dict.pkl\")\n",
    "yaml_file = os.path.join(data_path, \"utils\", r'nrms.yaml')\n",
    "\n",
    "mind_url, mind_train_dataset, mind_dev_dataset, mind_utils = get_mind_data_set(MIND_type)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_format=news,iterator_type=None,support_quick_scoring=True,wordEmb_file=data/utils/embedding.npy,wordDict_file=data/utils/word_dict.pkl,userDict_file=data/utils/uid2index.pkl,vertDict_file=None,subvertDict_file=None,title_size=30,body_size=None,word_emb_dim=300,word_size=None,user_num=None,vert_num=None,subvert_num=None,his_size=50,npratio=4,dropout=0.2,attention_hidden_dim=200,head_num=20,head_dim=20,cnn_activation=None,dense_activation=None,filter_num=200,window_size=3,vert_emb_dim=100,subvert_emb_dim=100,gru_unit=400,type=ini,user_emb_dim=50,learning_rate=0.0003,loss=cross_entropy_loss,optimizer=adam,epochs=50,batch_size=32,show_step=10,metrics=['group_auc', 'mean_mrr', 'ndcg@5;10']\n"
     ]
    }
   ],
   "source": [
    "hparams = prepare_hparams(yaml_file, \n",
    "                          wordEmb_file=wordEmb_file,\n",
    "                          wordDict_file=wordDict_file, \n",
    "                          userDict_file=userDict_file,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          show_step=10)\n",
    "hparams.learning_rate=3e-4\n",
    "\n",
    "print(hparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterator = MINDIterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = NRMSModel(hparams, iterator, seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:02, 208.26it/s]\n",
      "236it [00:05, 44.28it/s]\n",
      "7538it [00:02, 3318.67it/s]\n"
     ]
    }
   ],
   "source": [
    "pre_trained_metric=model.run_eval(valid_news_file, valid_behaviors_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.4681, data_loss: 1.1930: : 1086it [01:23, 12.97it/s]\n",
      "586it [00:01, 305.84it/s]\n",
      "236it [00:04, 50.29it/s]\n",
      "7538it [00:02, 3097.64it/s]\n",
      "2it [00:00, 12.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 1\n",
      "train info: logloss loss:1.467323507612801\n",
      "eval info: group_auc:0.6046, mean_mrr:0.2601, ndcg@10:0.3529, ndcg@5:0.2788\n",
      "at epoch 1 , train time: 83.7 eval time: 18.7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.3596, data_loss: 1.2187: : 1086it [01:19, 13.58it/s]\n",
      "586it [00:01, 297.16it/s]\n",
      "236it [00:04, 50.96it/s]\n",
      "7538it [00:02, 2738.69it/s]\n",
      "2it [00:00, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 2\n",
      "train info: logloss loss:1.3600232077566958\n",
      "eval info: group_auc:0.6216, mean_mrr:0.2744, ndcg@10:0.3688, ndcg@5:0.2952\n",
      "at epoch 2 , train time: 80.0 eval time: 17.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.3110, data_loss: 1.1483: : 1086it [01:20, 13.53it/s]\n",
      "586it [00:01, 295.27it/s]\n",
      "236it [00:04, 50.14it/s]\n",
      "7538it [00:02, 3406.53it/s]\n",
      "2it [00:00, 12.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 3\n",
      "train info: logloss loss:1.310909848461274\n",
      "eval info: group_auc:0.6207, mean_mrr:0.2781, ndcg@10:0.3725, ndcg@5:0.3019\n",
      "at epoch 3 , train time: 80.3 eval time: 17.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.2723, data_loss: 1.1987: : 1086it [01:20, 13.53it/s]\n",
      "586it [00:01, 299.31it/s]\n",
      "236it [00:04, 49.56it/s]\n",
      "7538it [00:02, 3740.79it/s]\n",
      "2it [00:00, 12.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 4\n",
      "train info: logloss loss:1.2723403669084195\n",
      "eval info: group_auc:0.6257, mean_mrr:0.2791, ndcg@10:0.3748, ndcg@5:0.3054\n",
      "at epoch 4 , train time: 80.2 eval time: 17.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.2280, data_loss: 1.1777: : 1086it [01:20, 13.54it/s]\n",
      "586it [00:01, 306.34it/s]\n",
      "236it [00:04, 50.09it/s]\n",
      "7538it [00:02, 3107.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 5\n",
      "train info: logloss loss:1.2284527074675973\n",
      "eval info: group_auc:0.6175, mean_mrr:0.2784, ndcg@10:0.3704, ndcg@5:0.3008\n",
      "at epoch 5 , train time: 80.2 eval time: 17.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.1916, data_loss: 1.2666: : 1086it [01:20, 13.54it/s]\n",
      "586it [00:01, 296.50it/s]\n",
      "236it [00:04, 50.67it/s]\n",
      "7538it [00:02, 3183.77it/s]\n",
      "2it [00:00, 12.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 6\n",
      "train info: logloss loss:1.192296288729153\n",
      "eval info: group_auc:0.6353, mean_mrr:0.2883, ndcg@10:0.3827, ndcg@5:0.3148\n",
      "at epoch 6 , train time: 80.2 eval time: 17.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.1567, data_loss: 1.2623: : 1086it [01:20, 13.51it/s]\n",
      "586it [00:01, 304.53it/s]\n",
      "236it [00:04, 51.45it/s]\n",
      "7538it [00:02, 2952.73it/s]\n",
      "2it [00:00, 12.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 7\n",
      "train info: logloss loss:1.1566130410241817\n",
      "eval info: group_auc:0.6337, mean_mrr:0.2916, ndcg@10:0.3848, ndcg@5:0.3157\n",
      "at epoch 7 , train time: 80.4 eval time: 18.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.1144, data_loss: 1.1861: : 1086it [01:19, 13.72it/s]\n",
      "586it [00:01, 310.00it/s]\n",
      "236it [00:04, 53.20it/s]\n",
      "7538it [00:02, 3369.48it/s]\n",
      "2it [00:00, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 8\n",
      "train info: logloss loss:1.11420575086144\n",
      "eval info: group_auc:0.6291, mean_mrr:0.2874, ndcg@10:0.3797, ndcg@5:0.3127\n",
      "at epoch 8 , train time: 79.2 eval time: 17.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.0748, data_loss: 1.1230: : 1086it [01:19, 13.72it/s]\n",
      "586it [00:01, 297.34it/s]\n",
      "236it [00:04, 50.37it/s]\n",
      "7538it [00:02, 3634.17it/s]\n",
      "2it [00:00, 13.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 9\n",
      "train info: logloss loss:1.0749805745279373\n",
      "eval info: group_auc:0.6281, mean_mrr:0.2878, ndcg@10:0.38, ndcg@5:0.3133\n",
      "at epoch 9 , train time: 79.2 eval time: 17.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 1.0340, data_loss: 0.9224: : 1086it [01:19, 13.74it/s]\n",
      "586it [00:01, 306.68it/s]\n",
      "236it [00:04, 50.59it/s]\n",
      "7538it [00:02, 3509.74it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 10\n",
      "train info: logloss loss:1.0338402505737643\n",
      "eval info: group_auc:0.6251, mean_mrr:0.2877, ndcg@10:0.3802, ndcg@5:0.3136\n",
      "at epoch 10 , train time: 79.0 eval time: 17.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.9868, data_loss: 0.8007: : 1086it [01:19, 13.70it/s]\n",
      "586it [00:01, 298.11it/s]\n",
      "236it [00:04, 52.67it/s]\n",
      "7538it [00:02, 3766.46it/s]\n",
      "2it [00:00, 13.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 11\n",
      "train info: logloss loss:0.9872817896567216\n",
      "eval info: group_auc:0.6221, mean_mrr:0.2868, ndcg@10:0.3794, ndcg@5:0.3119\n",
      "at epoch 11 , train time: 79.3 eval time: 17.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.9471, data_loss: 0.9331: : 1086it [01:19, 13.72it/s]\n",
      "586it [00:01, 294.34it/s]\n",
      "236it [00:04, 50.15it/s]\n",
      "7538it [00:02, 3440.34it/s]\n",
      "2it [00:00, 13.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 12\n",
      "train info: logloss loss:0.9472576936305557\n",
      "eval info: group_auc:0.6107, mean_mrr:0.2805, ndcg@10:0.3703, ndcg@5:0.3016\n",
      "at epoch 12 , train time: 79.1 eval time: 17.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.8992, data_loss: 0.9839: : 1086it [01:19, 13.74it/s]\n",
      "586it [00:01, 309.29it/s]\n",
      "236it [00:04, 50.90it/s]\n",
      "7538it [00:01, 3843.52it/s]\n",
      "2it [00:00, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 13\n",
      "train info: logloss loss:0.8994593631551631\n",
      "eval info: group_auc:0.6125, mean_mrr:0.2835, ndcg@10:0.374, ndcg@5:0.307\n",
      "at epoch 13 , train time: 79.0 eval time: 17.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.8574, data_loss: 0.9976: : 1086it [01:21, 13.27it/s]\n",
      "586it [00:01, 396.20it/s]\n",
      "236it [00:04, 48.36it/s]\n",
      "7538it [00:02, 3349.36it/s]\n",
      "2it [00:00, 12.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 14\n",
      "train info: logloss loss:0.8578004978089482\n",
      "eval info: group_auc:0.6103, mean_mrr:0.2823, ndcg@10:0.372, ndcg@5:0.3041\n",
      "at epoch 14 , train time: 81.8 eval time: 19.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.8139, data_loss: 0.6784: : 1086it [01:23, 12.97it/s]\n",
      "586it [00:01, 297.87it/s]\n",
      "236it [00:04, 48.50it/s]\n",
      "7538it [00:02, 3364.73it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 15\n",
      "train info: logloss loss:0.8143713253088858\n",
      "eval info: group_auc:0.6228, mean_mrr:0.2885, ndcg@10:0.3792, ndcg@5:0.3132\n",
      "at epoch 15 , train time: 83.7 eval time: 19.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.7714, data_loss: 0.9177: : 1086it [01:22, 13.22it/s]\n",
      "586it [00:01, 298.57it/s]\n",
      "236it [00:04, 49.12it/s]\n",
      "7538it [00:02, 3021.54it/s]\n",
      "2it [00:00, 12.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 16\n",
      "train info: logloss loss:0.7718944720251565\n",
      "eval info: group_auc:0.6118, mean_mrr:0.2833, ndcg@10:0.3717, ndcg@5:0.3067\n",
      "at epoch 16 , train time: 82.1 eval time: 17.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.7334, data_loss: 0.9780: : 1086it [01:22, 13.13it/s]\n",
      "586it [00:01, 300.81it/s]\n",
      "236it [00:04, 49.04it/s]\n",
      "7538it [00:02, 2621.10it/s]\n",
      "2it [00:00, 12.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 17\n",
      "train info: logloss loss:0.7333167729072588\n",
      "eval info: group_auc:0.6216, mean_mrr:0.287, ndcg@10:0.3781, ndcg@5:0.3114\n",
      "at epoch 17 , train time: 82.7 eval time: 18.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.6958, data_loss: 0.6309: : 1086it [01:23, 13.07it/s]\n",
      "586it [00:01, 302.77it/s]\n",
      "236it [00:04, 49.07it/s]\n",
      "7538it [00:02, 2647.20it/s]\n",
      "2it [00:00, 12.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 18\n",
      "train info: logloss loss:0.6957906975271952\n",
      "eval info: group_auc:0.6117, mean_mrr:0.2815, ndcg@10:0.3699, ndcg@5:0.3051\n",
      "at epoch 18 , train time: 83.1 eval time: 18.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.6530, data_loss: 1.1214: : 1086it [01:23, 13.06it/s]\n",
      "586it [00:01, 302.88it/s]\n",
      "236it [00:04, 49.25it/s]\n",
      "7538it [00:02, 2593.97it/s]\n",
      "2it [00:00, 12.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 19\n",
      "train info: logloss loss:0.6531660534290539\n",
      "eval info: group_auc:0.6109, mean_mrr:0.2811, ndcg@10:0.3695, ndcg@5:0.3058\n",
      "at epoch 19 , train time: 83.2 eval time: 19.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.6196, data_loss: 0.8200: : 1086it [01:22, 13.20it/s]\n",
      "586it [00:01, 306.76it/s]\n",
      "236it [00:04, 49.46it/s]\n",
      "7538it [00:02, 2562.21it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 20\n",
      "train info: logloss loss:0.6195797041662271\n",
      "eval info: group_auc:0.6094, mean_mrr:0.2797, ndcg@10:0.3673, ndcg@5:0.3046\n",
      "at epoch 20 , train time: 82.3 eval time: 19.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.5895, data_loss: 0.6522: : 1086it [01:21, 13.31it/s]\n",
      "586it [00:02, 292.03it/s]\n",
      "236it [00:04, 50.33it/s]\n",
      "7538it [00:02, 2961.60it/s]\n",
      "2it [00:00, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 21\n",
      "train info: logloss loss:0.5894609831177507\n",
      "eval info: group_auc:0.6059, mean_mrr:0.2798, ndcg@10:0.3668, ndcg@5:0.3024\n",
      "at epoch 21 , train time: 81.6 eval time: 17.7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.5576, data_loss: 0.7617: : 1086it [01:22, 13.20it/s]\n",
      "586it [00:02, 288.13it/s]\n",
      "236it [00:04, 48.73it/s]\n",
      "7538it [00:02, 3431.85it/s]\n",
      "2it [00:00, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 22\n",
      "train info: logloss loss:0.5578092303175074\n",
      "eval info: group_auc:0.6088, mean_mrr:0.2809, ndcg@10:0.3691, ndcg@5:0.3077\n",
      "at epoch 22 , train time: 82.3 eval time: 19.7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.5325, data_loss: 0.4534: : 1086it [01:22, 13.19it/s]\n",
      "586it [00:01, 303.03it/s]\n",
      "236it [00:04, 49.21it/s]\n",
      "7538it [00:02, 2917.51it/s]\n",
      "2it [00:00, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 23\n",
      "train info: logloss loss:0.5322018436964046\n",
      "eval info: group_auc:0.6108, mean_mrr:0.2806, ndcg@10:0.3688, ndcg@5:0.3049\n",
      "at epoch 23 , train time: 82.3 eval time: 19.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.5079, data_loss: 0.6355: : 1086it [01:22, 13.17it/s]\n",
      "586it [00:01, 298.04it/s]\n",
      "236it [00:04, 49.40it/s]\n",
      "7538it [00:02, 3292.35it/s]\n",
      "2it [00:00, 12.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 24\n",
      "train info: logloss loss:0.5080855257164268\n",
      "eval info: group_auc:0.6107, mean_mrr:0.2798, ndcg@10:0.3678, ndcg@5:0.3048\n",
      "at epoch 24 , train time: 82.5 eval time: 17.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.4762, data_loss: 0.5636: : 1086it [01:21, 13.31it/s]\n",
      "586it [00:01, 305.41it/s]\n",
      "236it [00:04, 49.71it/s]\n",
      "7538it [00:02, 3750.13it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 25\n",
      "train info: logloss loss:0.4758488375206459\n",
      "eval info: group_auc:0.6015, mean_mrr:0.2761, ndcg@10:0.362, ndcg@5:0.2988\n",
      "at epoch 25 , train time: 81.6 eval time: 17.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.4576, data_loss: 0.4936: : 1086it [01:22, 13.17it/s]\n",
      "586it [00:01, 299.64it/s]\n",
      "236it [00:04, 49.87it/s]\n",
      "7538it [00:03, 2467.33it/s]\n",
      "2it [00:00, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 26\n",
      "train info: logloss loss:0.4574486224004796\n",
      "eval info: group_auc:0.6059, mean_mrr:0.278, ndcg@10:0.3649, ndcg@5:0.3009\n",
      "at epoch 26 , train time: 82.5 eval time: 20.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.4366, data_loss: 0.5970: : 1086it [01:22, 13.16it/s]\n",
      "586it [00:01, 305.88it/s]\n",
      "236it [00:04, 49.49it/s]\n",
      "7538it [00:02, 2557.57it/s]\n",
      "2it [00:00, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 27\n",
      "train info: logloss loss:0.43698586704651954\n",
      "eval info: group_auc:0.6075, mean_mrr:0.2802, ndcg@10:0.3673, ndcg@5:0.3043\n",
      "at epoch 27 , train time: 82.5 eval time: 19.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.4164, data_loss: 0.5240: : 1086it [01:23, 13.07it/s]\n",
      "586it [00:01, 307.59it/s]\n",
      "236it [00:04, 48.60it/s]\n",
      "7538it [00:02, 2790.05it/s]\n",
      "2it [00:00, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 28\n",
      "train info: logloss loss:0.4168904404709901\n",
      "eval info: group_auc:0.6013, mean_mrr:0.2758, ndcg@10:0.3625, ndcg@5:0.2982\n",
      "at epoch 28 , train time: 83.1 eval time: 20.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3984, data_loss: 0.2585: : 1086it [01:23, 13.03it/s]\n",
      "586it [00:01, 301.07it/s]\n",
      "236it [00:04, 48.56it/s]\n",
      "7538it [00:02, 2834.28it/s]\n",
      "2it [00:00, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 29\n",
      "train info: logloss loss:0.3988917582217281\n",
      "eval info: group_auc:0.6007, mean_mrr:0.2755, ndcg@10:0.3622, ndcg@5:0.2986\n",
      "at epoch 29 , train time: 83.3 eval time: 18.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3836, data_loss: 0.2501: : 1086it [01:23, 12.99it/s]\n",
      "586it [00:01, 293.74it/s]\n",
      "236it [00:04, 48.65it/s]\n",
      "7538it [00:02, 2883.18it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 30\n",
      "train info: logloss loss:0.3832172507190353\n",
      "eval info: group_auc:0.6093, mean_mrr:0.2786, ndcg@10:0.3662, ndcg@5:0.3029\n",
      "at epoch 30 , train time: 83.6 eval time: 18.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3658, data_loss: 0.5025: : 1086it [01:22, 13.11it/s]\n",
      "586it [00:01, 299.42it/s]\n",
      "236it [00:04, 49.16it/s]\n",
      "7538it [00:03, 2503.24it/s]\n",
      "2it [00:00, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 31\n",
      "train info: logloss loss:0.36613150560477165\n",
      "eval info: group_auc:0.6041, mean_mrr:0.2754, ndcg@10:0.3619, ndcg@5:0.2982\n",
      "at epoch 31 , train time: 82.9 eval time: 18.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3492, data_loss: 0.4536: : 1086it [01:22, 13.13it/s]\n",
      "586it [00:02, 288.78it/s]\n",
      "236it [00:04, 49.13it/s]\n",
      "7538it [00:03, 2391.11it/s]\n",
      "2it [00:00, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 32\n",
      "train info: logloss loss:0.3490085786424618\n",
      "eval info: group_auc:0.6053, mean_mrr:0.2772, ndcg@10:0.3637, ndcg@5:0.3006\n",
      "at epoch 32 , train time: 82.7 eval time: 18.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3411, data_loss: 0.6897: : 1086it [01:22, 13.12it/s]\n",
      "586it [00:01, 299.08it/s]\n",
      "236it [00:04, 48.83it/s]\n",
      "7538it [00:03, 2505.47it/s]\n",
      "2it [00:00, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 33\n",
      "train info: logloss loss:0.3412127620291864\n",
      "eval info: group_auc:0.6047, mean_mrr:0.276, ndcg@10:0.3628, ndcg@5:0.2983\n",
      "at epoch 33 , train time: 82.8 eval time: 20.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3224, data_loss: 0.2814: : 1086it [01:22, 13.14it/s]\n",
      "586it [00:02, 292.84it/s]\n",
      "236it [00:04, 49.26it/s]\n",
      "7538it [00:02, 2718.52it/s]\n",
      "2it [00:00, 12.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 34\n",
      "train info: logloss loss:0.3229876074109837\n",
      "eval info: group_auc:0.6048, mean_mrr:0.2765, ndcg@10:0.3649, ndcg@5:0.2992\n",
      "at epoch 34 , train time: 82.6 eval time: 18.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3205, data_loss: 0.2267: : 1086it [01:22, 13.18it/s]\n",
      "586it [00:01, 305.67it/s]\n",
      "236it [00:04, 49.07it/s]\n",
      "7538it [00:02, 2714.20it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 35\n",
      "train info: logloss loss:0.3208690561820679\n",
      "eval info: group_auc:0.6041, mean_mrr:0.2729, ndcg@10:0.3603, ndcg@5:0.295\n",
      "at epoch 35 , train time: 82.4 eval time: 18.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.3075, data_loss: 0.4599: : 1086it [01:22, 13.13it/s]\n",
      "586it [00:01, 302.24it/s]\n",
      "236it [00:04, 50.51it/s]\n",
      "7538it [00:02, 3165.66it/s]\n",
      "2it [00:00, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 36\n",
      "train info: logloss loss:0.3079007255294211\n",
      "eval info: group_auc:0.6006, mean_mrr:0.2739, ndcg@10:0.3601, ndcg@5:0.2966\n",
      "at epoch 36 , train time: 82.7 eval time: 17.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2912, data_loss: 0.3138: : 1086it [01:21, 13.30it/s]\n",
      "586it [00:01, 317.30it/s]\n",
      "236it [00:04, 49.29it/s]\n",
      "7538it [00:03, 2466.19it/s]\n",
      "2it [00:00, 11.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 37\n",
      "train info: logloss loss:0.2913656254156628\n",
      "eval info: group_auc:0.6024, mean_mrr:0.2715, ndcg@10:0.3592, ndcg@5:0.2958\n",
      "at epoch 37 , train time: 81.6 eval time: 18.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2865, data_loss: 0.3044: : 1086it [01:22, 13.12it/s]\n",
      "586it [00:01, 302.30it/s]\n",
      "236it [00:04, 49.46it/s]\n",
      "7538it [00:03, 2448.29it/s]\n",
      "2it [00:00, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 38\n",
      "train info: logloss loss:0.28653857700910906\n",
      "eval info: group_auc:0.5982, mean_mrr:0.2711, ndcg@10:0.3571, ndcg@5:0.294\n",
      "at epoch 38 , train time: 82.8 eval time: 18.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2802, data_loss: 0.2829: : 1086it [01:22, 13.16it/s]\n",
      "586it [00:01, 304.14it/s]\n",
      "236it [00:04, 50.87it/s]\n",
      "7538it [00:02, 2847.30it/s]\n",
      "2it [00:00, 13.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 39\n",
      "train info: logloss loss:0.280197961255908\n",
      "eval info: group_auc:0.5946, mean_mrr:0.2714, ndcg@10:0.3565, ndcg@5:0.294\n",
      "at epoch 39 , train time: 82.5 eval time: 17.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2665, data_loss: 0.0623: : 1086it [01:21, 13.27it/s]\n",
      "586it [00:02, 289.46it/s]\n",
      "236it [00:04, 48.86it/s]\n",
      "7538it [00:02, 2765.98it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 40\n",
      "train info: logloss loss:0.26693846728536086\n",
      "eval info: group_auc:0.5999, mean_mrr:0.2728, ndcg@10:0.3599, ndcg@5:0.2947\n",
      "at epoch 40 , train time: 81.9 eval time: 19.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2560, data_loss: 0.2162: : 1086it [01:22, 13.17it/s]\n",
      "586it [00:01, 316.46it/s]\n",
      "236it [00:04, 49.50it/s]\n",
      "7538it [00:03, 2354.64it/s]\n",
      "2it [00:00, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 41\n",
      "train info: logloss loss:0.2557736539777581\n",
      "eval info: group_auc:0.5993, mean_mrr:0.2714, ndcg@10:0.358, ndcg@5:0.2944\n",
      "at epoch 41 , train time: 82.5 eval time: 19.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2520, data_loss: 0.2166: : 1086it [01:21, 13.37it/s]\n",
      "586it [00:01, 297.15it/s]\n",
      "236it [00:04, 50.05it/s]\n",
      "7538it [00:02, 3493.33it/s]\n",
      "2it [00:00, 12.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 42\n",
      "train info: logloss loss:0.25209687639569817\n",
      "eval info: group_auc:0.6029, mean_mrr:0.2768, ndcg@10:0.3642, ndcg@5:0.301\n",
      "at epoch 42 , train time: 81.3 eval time: 17.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2541, data_loss: 0.1395: : 1086it [01:21, 13.36it/s]\n",
      "586it [00:01, 316.64it/s]\n",
      "236it [00:04, 49.76it/s]\n",
      "7538it [00:02, 3378.81it/s]\n",
      "2it [00:00, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 43\n",
      "train info: logloss loss:0.25409988030571745\n",
      "eval info: group_auc:0.5999, mean_mrr:0.2727, ndcg@10:0.3595, ndcg@5:0.296\n",
      "at epoch 43 , train time: 81.3 eval time: 18.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2448, data_loss: 0.3073: : 1086it [01:21, 13.36it/s]\n",
      "586it [00:01, 296.28it/s]\n",
      "236it [00:04, 50.73it/s]\n",
      "7538it [00:02, 2914.87it/s]\n",
      "2it [00:00, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 44\n",
      "train info: logloss loss:0.24429078970934584\n",
      "eval info: group_auc:0.5949, mean_mrr:0.2686, ndcg@10:0.3545, ndcg@5:0.291\n",
      "at epoch 44 , train time: 81.3 eval time: 18.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2338, data_loss: 0.3154: : 1086it [01:21, 13.40it/s]\n",
      "586it [00:01, 303.85it/s]\n",
      "236it [00:04, 50.18it/s]\n",
      "7538it [00:02, 3162.30it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 45\n",
      "train info: logloss loss:0.23399824352890364\n",
      "eval info: group_auc:0.5978, mean_mrr:0.2721, ndcg@10:0.3586, ndcg@5:0.2963\n",
      "at epoch 45 , train time: 81.0 eval time: 17.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2350, data_loss: 0.4026: : 1086it [01:21, 13.31it/s]\n",
      "586it [00:01, 307.14it/s]\n",
      "236it [00:04, 49.86it/s]\n",
      "7538it [00:02, 2918.80it/s]\n",
      "2it [00:00, 13.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 46\n",
      "train info: logloss loss:0.23510191723546545\n",
      "eval info: group_auc:0.596, mean_mrr:0.2696, ndcg@10:0.3564, ndcg@5:0.2939\n",
      "at epoch 46 , train time: 81.6 eval time: 17.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2239, data_loss: 0.2544: : 1086it [01:21, 13.36it/s]\n",
      "586it [00:01, 300.99it/s]\n",
      "236it [00:04, 49.86it/s]\n",
      "7538it [00:02, 3007.89it/s]\n",
      "2it [00:00, 12.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 47\n",
      "train info: logloss loss:0.22382652589581345\n",
      "eval info: group_auc:0.6021, mean_mrr:0.2724, ndcg@10:0.3596, ndcg@5:0.2984\n",
      "at epoch 47 , train time: 81.3 eval time: 19.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2225, data_loss: 0.1757: : 1086it [01:21, 13.38it/s]\n",
      "586it [00:01, 309.51it/s]\n",
      "236it [00:04, 50.04it/s]\n",
      "7538it [00:02, 2963.23it/s]\n",
      "2it [00:00, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 48\n",
      "train info: logloss loss:0.22271163778595668\n",
      "eval info: group_auc:0.6028, mean_mrr:0.2753, ndcg@10:0.3628, ndcg@5:0.3005\n",
      "at epoch 48 , train time: 81.2 eval time: 17.7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2120, data_loss: 0.2901: : 1086it [01:21, 13.30it/s]\n",
      "586it [00:01, 301.48it/s]\n",
      "236it [00:04, 49.69it/s]\n",
      "7538it [00:02, 3206.78it/s]\n",
      "2it [00:00, 12.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 49\n",
      "train info: logloss loss:0.21246073488489095\n",
      "eval info: group_auc:0.5957, mean_mrr:0.2707, ndcg@10:0.3568, ndcg@5:0.2918\n",
      "at epoch 49 , train time: 81.6 eval time: 19.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 1080 , total_loss: 0.2130, data_loss: 0.4338: : 1086it [01:22, 13.24it/s]\n",
      "586it [00:01, 308.38it/s]\n",
      "236it [00:04, 49.70it/s]\n",
      "7538it [00:02, 2809.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "at epoch 50\n",
      "train info: logloss loss:0.2131707558209423\n",
      "eval info: group_auc:0.6005, mean_mrr:0.2733, ndcg@10:0.3595, ndcg@5:0.2977\n",
      "at epoch 50 , train time: 82.0 eval time: 17.9\n",
      "CPU times: user 1h 47min 39s, sys: 49min 6s, total: 2h 36min 45s\n",
      "Wall time: 1h 23min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_results=model.fit(train_news_file, train_behaviors_file, valid_news_file, valid_behaviors_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([1.467323507612801,\n",
       "  1.3600232077566958,\n",
       "  1.310909848461274,\n",
       "  1.2723403669084195,\n",
       "  1.2284527074675973,\n",
       "  1.192296288729153,\n",
       "  1.1566130410241817,\n",
       "  1.11420575086144,\n",
       "  1.0749805745279373,\n",
       "  1.0338402505737643,\n",
       "  0.9872817896567216,\n",
       "  0.9472576936305557,\n",
       "  0.8994593631551631,\n",
       "  0.8578004978089482,\n",
       "  0.8143713253088858,\n",
       "  0.7718944720251565,\n",
       "  0.7333167729072588,\n",
       "  0.6957906975271952,\n",
       "  0.6531660534290539,\n",
       "  0.6195797041662271,\n",
       "  0.5894609831177507,\n",
       "  0.5578092303175074,\n",
       "  0.5322018436964046,\n",
       "  0.5080855257164268,\n",
       "  0.4758488375206459,\n",
       "  0.4574486224004796,\n",
       "  0.43698586704651954,\n",
       "  0.4168904404709901,\n",
       "  0.3988917582217281,\n",
       "  0.3832172507190353,\n",
       "  0.36613150560477165,\n",
       "  0.3490085786424618,\n",
       "  0.3412127620291864,\n",
       "  0.3229876074109837,\n",
       "  0.3208690561820679,\n",
       "  0.3079007255294211,\n",
       "  0.2913656254156628,\n",
       "  0.28653857700910906,\n",
       "  0.280197961255908,\n",
       "  0.26693846728536086,\n",
       "  0.2557736539777581,\n",
       "  0.25209687639569817,\n",
       "  0.25409988030571745,\n",
       "  0.24429078970934584,\n",
       "  0.23399824352890364,\n",
       "  0.23510191723546545,\n",
       "  0.22382652589581345,\n",
       "  0.22271163778595668,\n",
       "  0.21246073488489095,\n",
       "  0.2131707558209423],\n",
       " [{'group_auc': 0.6046,\n",
       "   'mean_mrr': 0.2601,\n",
       "   'ndcg@5': 0.2788,\n",
       "   'ndcg@10': 0.3529},\n",
       "  {'group_auc': 0.6216,\n",
       "   'mean_mrr': 0.2744,\n",
       "   'ndcg@5': 0.2952,\n",
       "   'ndcg@10': 0.3688},\n",
       "  {'group_auc': 0.6207,\n",
       "   'mean_mrr': 0.2781,\n",
       "   'ndcg@5': 0.3019,\n",
       "   'ndcg@10': 0.3725},\n",
       "  {'group_auc': 0.6257,\n",
       "   'mean_mrr': 0.2791,\n",
       "   'ndcg@5': 0.3054,\n",
       "   'ndcg@10': 0.3748},\n",
       "  {'group_auc': 0.6175,\n",
       "   'mean_mrr': 0.2784,\n",
       "   'ndcg@5': 0.3008,\n",
       "   'ndcg@10': 0.3704},\n",
       "  {'group_auc': 0.6353,\n",
       "   'mean_mrr': 0.2883,\n",
       "   'ndcg@5': 0.3148,\n",
       "   'ndcg@10': 0.3827},\n",
       "  {'group_auc': 0.6337,\n",
       "   'mean_mrr': 0.2916,\n",
       "   'ndcg@5': 0.3157,\n",
       "   'ndcg@10': 0.3848},\n",
       "  {'group_auc': 0.6291,\n",
       "   'mean_mrr': 0.2874,\n",
       "   'ndcg@5': 0.3127,\n",
       "   'ndcg@10': 0.3797},\n",
       "  {'group_auc': 0.6281, 'mean_mrr': 0.2878, 'ndcg@5': 0.3133, 'ndcg@10': 0.38},\n",
       "  {'group_auc': 0.6251,\n",
       "   'mean_mrr': 0.2877,\n",
       "   'ndcg@5': 0.3136,\n",
       "   'ndcg@10': 0.3802},\n",
       "  {'group_auc': 0.6221,\n",
       "   'mean_mrr': 0.2868,\n",
       "   'ndcg@5': 0.3119,\n",
       "   'ndcg@10': 0.3794},\n",
       "  {'group_auc': 0.6107,\n",
       "   'mean_mrr': 0.2805,\n",
       "   'ndcg@5': 0.3016,\n",
       "   'ndcg@10': 0.3703},\n",
       "  {'group_auc': 0.6125, 'mean_mrr': 0.2835, 'ndcg@5': 0.307, 'ndcg@10': 0.374},\n",
       "  {'group_auc': 0.6103,\n",
       "   'mean_mrr': 0.2823,\n",
       "   'ndcg@5': 0.3041,\n",
       "   'ndcg@10': 0.372},\n",
       "  {'group_auc': 0.6228,\n",
       "   'mean_mrr': 0.2885,\n",
       "   'ndcg@5': 0.3132,\n",
       "   'ndcg@10': 0.3792},\n",
       "  {'group_auc': 0.6118,\n",
       "   'mean_mrr': 0.2833,\n",
       "   'ndcg@5': 0.3067,\n",
       "   'ndcg@10': 0.3717},\n",
       "  {'group_auc': 0.6216,\n",
       "   'mean_mrr': 0.287,\n",
       "   'ndcg@5': 0.3114,\n",
       "   'ndcg@10': 0.3781},\n",
       "  {'group_auc': 0.6117,\n",
       "   'mean_mrr': 0.2815,\n",
       "   'ndcg@5': 0.3051,\n",
       "   'ndcg@10': 0.3699},\n",
       "  {'group_auc': 0.6109,\n",
       "   'mean_mrr': 0.2811,\n",
       "   'ndcg@5': 0.3058,\n",
       "   'ndcg@10': 0.3695},\n",
       "  {'group_auc': 0.6094,\n",
       "   'mean_mrr': 0.2797,\n",
       "   'ndcg@5': 0.3046,\n",
       "   'ndcg@10': 0.3673},\n",
       "  {'group_auc': 0.6059,\n",
       "   'mean_mrr': 0.2798,\n",
       "   'ndcg@5': 0.3024,\n",
       "   'ndcg@10': 0.3668},\n",
       "  {'group_auc': 0.6088,\n",
       "   'mean_mrr': 0.2809,\n",
       "   'ndcg@5': 0.3077,\n",
       "   'ndcg@10': 0.3691},\n",
       "  {'group_auc': 0.6108,\n",
       "   'mean_mrr': 0.2806,\n",
       "   'ndcg@5': 0.3049,\n",
       "   'ndcg@10': 0.3688},\n",
       "  {'group_auc': 0.6107,\n",
       "   'mean_mrr': 0.2798,\n",
       "   'ndcg@5': 0.3048,\n",
       "   'ndcg@10': 0.3678},\n",
       "  {'group_auc': 0.6015,\n",
       "   'mean_mrr': 0.2761,\n",
       "   'ndcg@5': 0.2988,\n",
       "   'ndcg@10': 0.362},\n",
       "  {'group_auc': 0.6059,\n",
       "   'mean_mrr': 0.278,\n",
       "   'ndcg@5': 0.3009,\n",
       "   'ndcg@10': 0.3649},\n",
       "  {'group_auc': 0.6075,\n",
       "   'mean_mrr': 0.2802,\n",
       "   'ndcg@5': 0.3043,\n",
       "   'ndcg@10': 0.3673},\n",
       "  {'group_auc': 0.6013,\n",
       "   'mean_mrr': 0.2758,\n",
       "   'ndcg@5': 0.2982,\n",
       "   'ndcg@10': 0.3625},\n",
       "  {'group_auc': 0.6007,\n",
       "   'mean_mrr': 0.2755,\n",
       "   'ndcg@5': 0.2986,\n",
       "   'ndcg@10': 0.3622},\n",
       "  {'group_auc': 0.6093,\n",
       "   'mean_mrr': 0.2786,\n",
       "   'ndcg@5': 0.3029,\n",
       "   'ndcg@10': 0.3662},\n",
       "  {'group_auc': 0.6041,\n",
       "   'mean_mrr': 0.2754,\n",
       "   'ndcg@5': 0.2982,\n",
       "   'ndcg@10': 0.3619},\n",
       "  {'group_auc': 0.6053,\n",
       "   'mean_mrr': 0.2772,\n",
       "   'ndcg@5': 0.3006,\n",
       "   'ndcg@10': 0.3637},\n",
       "  {'group_auc': 0.6047,\n",
       "   'mean_mrr': 0.276,\n",
       "   'ndcg@5': 0.2983,\n",
       "   'ndcg@10': 0.3628},\n",
       "  {'group_auc': 0.6048,\n",
       "   'mean_mrr': 0.2765,\n",
       "   'ndcg@5': 0.2992,\n",
       "   'ndcg@10': 0.3649},\n",
       "  {'group_auc': 0.6041,\n",
       "   'mean_mrr': 0.2729,\n",
       "   'ndcg@5': 0.295,\n",
       "   'ndcg@10': 0.3603},\n",
       "  {'group_auc': 0.6006,\n",
       "   'mean_mrr': 0.2739,\n",
       "   'ndcg@5': 0.2966,\n",
       "   'ndcg@10': 0.3601},\n",
       "  {'group_auc': 0.6024,\n",
       "   'mean_mrr': 0.2715,\n",
       "   'ndcg@5': 0.2958,\n",
       "   'ndcg@10': 0.3592},\n",
       "  {'group_auc': 0.5982,\n",
       "   'mean_mrr': 0.2711,\n",
       "   'ndcg@5': 0.294,\n",
       "   'ndcg@10': 0.3571},\n",
       "  {'group_auc': 0.5946,\n",
       "   'mean_mrr': 0.2714,\n",
       "   'ndcg@5': 0.294,\n",
       "   'ndcg@10': 0.3565},\n",
       "  {'group_auc': 0.5999,\n",
       "   'mean_mrr': 0.2728,\n",
       "   'ndcg@5': 0.2947,\n",
       "   'ndcg@10': 0.3599},\n",
       "  {'group_auc': 0.5993,\n",
       "   'mean_mrr': 0.2714,\n",
       "   'ndcg@5': 0.2944,\n",
       "   'ndcg@10': 0.358},\n",
       "  {'group_auc': 0.6029,\n",
       "   'mean_mrr': 0.2768,\n",
       "   'ndcg@5': 0.301,\n",
       "   'ndcg@10': 0.3642},\n",
       "  {'group_auc': 0.5999,\n",
       "   'mean_mrr': 0.2727,\n",
       "   'ndcg@5': 0.296,\n",
       "   'ndcg@10': 0.3595},\n",
       "  {'group_auc': 0.5949,\n",
       "   'mean_mrr': 0.2686,\n",
       "   'ndcg@5': 0.291,\n",
       "   'ndcg@10': 0.3545},\n",
       "  {'group_auc': 0.5978,\n",
       "   'mean_mrr': 0.2721,\n",
       "   'ndcg@5': 0.2963,\n",
       "   'ndcg@10': 0.3586},\n",
       "  {'group_auc': 0.596,\n",
       "   'mean_mrr': 0.2696,\n",
       "   'ndcg@5': 0.2939,\n",
       "   'ndcg@10': 0.3564},\n",
       "  {'group_auc': 0.6021,\n",
       "   'mean_mrr': 0.2724,\n",
       "   'ndcg@5': 0.2984,\n",
       "   'ndcg@10': 0.3596},\n",
       "  {'group_auc': 0.6028,\n",
       "   'mean_mrr': 0.2753,\n",
       "   'ndcg@5': 0.3005,\n",
       "   'ndcg@10': 0.3628},\n",
       "  {'group_auc': 0.5957,\n",
       "   'mean_mrr': 0.2707,\n",
       "   'ndcg@5': 0.2918,\n",
       "   'ndcg@10': 0.3568},\n",
       "  {'group_auc': 0.6005,\n",
       "   'mean_mrr': 0.2733,\n",
       "   'ndcg@5': 0.2977,\n",
       "   'ndcg@10': 0.3595}])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('nrms_loss_lr3e-4.txt', 'w') as filehandle:\n",
    "    for listitem in train_results:\n",
    "        for item in listitem:\n",
    "            filehandle.write('%s\\n' % item)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:01, 297.22it/s]\n",
      "236it [00:04, 50.03it/s]\n",
      "7538it [00:03, 2443.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'group_auc': 0.6005, 'mean_mrr': 0.2733, 'ndcg@5': 0.2977, 'ndcg@10': 0.3595}\n",
      "CPU times: user 57.7 s, sys: 55.7 s, total: 1min 53s\n",
      "Wall time: 18.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "res_syn = model.run_eval(valid_news_file, valid_behaviors_file)\n",
    "print(res_syn)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = os.path.join(data_path, \"model_newlr\")\n",
    "os.makedirs(model_path, exist_ok=True)\n",
    "\n",
    "model.model.save_weights(os.path.join(model_path, \"nrms_ckpt_3e-4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.scorer.save_weights('data/model_newlr/nrms_ckpt_503e-4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "586it [00:01, 296.08it/s]\n",
      "236it [00:04, 50.60it/s]\n",
      "7538it [00:02, 2670.96it/s]\n",
      "7538it [00:00, 30285.70it/s]\n"
     ]
    }
   ],
   "source": [
    "group_impr_indexes, group_labels, group_preds = model.run_fast_eval(valid_news_file, valid_behaviors_file)\n",
    "with open(os.path.join(data_path, 'prediction_nrms.txt'), 'w') as f:\n",
    "    for impr_index, preds in tqdm(zip(group_impr_indexes, group_preds)):\n",
    "        impr_index += 1\n",
    "        pred_rank = (np.argsort(np.argsort(preds)[::-1]) + 1).tolist()\n",
    "        pred_rank = '[' + ','.join([str(i) for i in pred_rank]) + ']'\n",
    "        f.write(' '.join([str(impr_index), pred_rank])+ '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'data/nrms_prediction.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-efd4d6cf2005>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZipFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'nrms_prediction.zip'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZIP_DEFLATED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'nrms_prediction.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marcname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'nrms_prediction.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/reco_gpu/lib/python3.6/zipfile.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, filename, arcname, compress_type)\u001b[0m\n\u001b[1;32m   1615\u001b[0m             )\n\u001b[1;32m   1616\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1617\u001b[0;31m         \u001b[0mzinfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mZipInfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marcname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1618\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1619\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mzinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/reco_gpu/lib/python3.6/zipfile.py\u001b[0m in \u001b[0;36mfrom_file\u001b[0;34m(cls, filename, arcname)\u001b[0m\n\u001b[1;32m    505\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPathLike\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m             \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfspath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 507\u001b[0;31m         \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    508\u001b[0m         \u001b[0misdir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mS_ISDIR\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mst_mode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m         \u001b[0mmtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocaltime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mst_mtime\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/nrms_prediction.txt'"
     ]
    }
   ],
   "source": [
    "f = zipfile.ZipFile(os.path.join(data_path, 'nrms_prediction.zip'), 'w', zipfile.ZIP_DEFLATED)\n",
    "f.write(os.path.join(data_path, 'nrms_prediction.txt'), arcname='nrms_prediction.txt')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}