example2.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.9 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "890a3aa13989506a4f157c210d7e888dcd9c9bfc264e152253245030dde69647"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from deepctr.feature_column import SparseFeat, DenseFeat,get_feature_names\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import tensorflow as tf\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "source": [
    "### task 1 aims to predict whether the income exceeds 50K, \n",
    "### task 2 aims to predict whether this person’s marital status is never married."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = ['age', 'class_worker', 'det_ind_code', 'det_occ_code', 'education', 'wage_per_hour', 'hs_college',\n",
    "                'marital_stat', 'major_ind_code', 'major_occ_code', 'race', 'hisp_origin', 'sex', 'union_member',\n",
    "                'unemp_reason', 'full_or_part_emp', 'capital_gains', 'capital_losses', 'stock_dividends',\n",
    "                'tax_filer_stat', 'region_prev_res', 'state_prev_res', 'det_hh_fam_stat', 'det_hh_summ',\n",
    "                'instance_weight', 'mig_chg_msa', 'mig_chg_reg', 'mig_move_reg', 'mig_same', 'mig_prev_sunbelt',\n",
    "                'num_emp', 'fam_under_18', 'country_father', 'country_mother', 'country_self', 'citizenship',\n",
    "                'own_or_self', 'vet_question', 'vet_benefits', 'weeks_worked', 'year', 'income_50k']\n",
    "df_train = pd.read_csv('./data/census-income/census-income.data',header=None,names=column_names)\n",
    "df_test = pd.read_csv('./data/census-income/census-income.test',header=None,names=column_names)\n",
    "data = pd.concat([df_train, df_test], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       " - 50000.    280717\n",
       " 50000+.      18568\n",
       "Name: income_50k, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "#task 1 label：'income_50k'\n",
    "data['income_50k'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       " Never married                      129628\n",
       " Married-civilian spouse present    126315\n",
       " Divorced                            19160\n",
       " Widowed                             15788\n",
       " Separated                            5156\n",
       " Married-spouse absent                2234\n",
       " Married-A F spouse present           1004\n",
       "Name: marital_stat, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "#task 2 label: 'marital_stat'\n",
    "data['marital_stat'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#change the label to binary classification\n",
    "data['label_income'] = data['income_50k'].map({' - 50000.':0, ' 50000+.':1})\n",
    "data['label_marital'] = data['marital_stat'].apply(lambda x: 1 if x==' Never married' else 0)\n",
    "data.drop(labels=['income_50k', 'marital_stat'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define dense and sparse features. \n",
    "#the functions used here can reference https://deepctr-torch.readthedocs.io/en/latest/Quick-Start.html\n",
    "columns = data.columns.values.tolist()\n",
    "sparse_features = ['class_worker', 'det_ind_code', 'det_occ_code', 'education', 'hs_college', 'major_ind_code',\n",
    "                        'major_occ_code', 'race', 'hisp_origin', 'sex', 'union_member', 'unemp_reason',\n",
    "                        'full_or_part_emp', 'tax_filer_stat', 'region_prev_res', 'state_prev_res', 'det_hh_fam_stat',\n",
    "                        'det_hh_summ', 'mig_chg_msa', 'mig_chg_reg', 'mig_move_reg', 'mig_same', 'mig_prev_sunbelt',\n",
    "                        'fam_under_18', 'country_father', 'country_mother', 'country_self', 'citizenship',\n",
    "                        'vet_question']\n",
    "dense_features = [col for col in columns if col not in sparse_features and col not in ['label_income', 'label_marital']]\n",
    "\n",
    "data[sparse_features] = data[sparse_features].fillna('-1', )\n",
    "data[dense_features] = data[dense_features].fillna(0, )\n",
    "mms = MinMaxScaler(feature_range=(0, 1))\n",
    "data[dense_features] = mms.fit_transform(data[dense_features])\n",
    "    \n",
    "for feat in sparse_features:\n",
    "    lbe = LabelEncoder()\n",
    "    data[feat] = lbe.fit_transform(data[feat])\n",
    "    \n",
    "fixlen_feature_columns = [SparseFeat(feat, data[feat].max()+1, embedding_dim=4)for feat in sparse_features] \\\n",
    "                        + [DenseFeat(feat, 1,) for feat in dense_features]\n",
    "\n",
    "dnn_feature_columns = fixlen_feature_columns\n",
    "feature_names = get_feature_names(dnn_feature_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the test dataset into 1:1 validation to test according to the MMOE paper\n",
    "# validation_split = n_val/len(train) = 0.2\n",
    "n_train = df_train.shape[0]\n",
    "n_val = df_test.shape[0]//2\n",
    "train = data[:n_train+n_val]\n",
    "test = data[n_train+n_val:]\n",
    "\n",
    "train_model_input = {name: train[name] for name in feature_names}\n",
    "test_model_input = {name: test[name] for name in feature_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "195/195 - 4s - loss: 0.3384 - label_income_loss: 0.1546 - label_marital_loss: 0.1833 - label_income_auc: 0.9027 - label_marital_auc_1: 0.9789 - val_loss: 0.2262 - val_label_income_loss: 0.1299 - val_label_marital_loss: 0.0957 - val_label_income_auc: 0.9370 - val_label_marital_auc_1: 0.9940\n",
      "Epoch 2/100\n",
      "195/195 - 2s - loss: 0.2226 - label_income_loss: 0.1266 - label_marital_loss: 0.0954 - label_income_auc: 0.9391 - label_marital_auc_1: 0.9939 - val_loss: 0.2219 - val_label_income_loss: 0.1267 - val_label_marital_loss: 0.0945 - val_label_income_auc: 0.9444 - val_label_marital_auc_1: 0.9942\n",
      "Epoch 3/100\n",
      "195/195 - 2s - loss: 0.2191 - label_income_loss: 0.1246 - label_marital_loss: 0.0938 - label_income_auc: 0.9407 - label_marital_auc_1: 0.9942 - val_loss: 0.2172 - val_label_income_loss: 0.1229 - val_label_marital_loss: 0.0936 - val_label_income_auc: 0.9446 - val_label_marital_auc_1: 0.9943\n",
      "Epoch 4/100\n",
      "195/195 - 2s - loss: 0.2151 - label_income_loss: 0.1227 - label_marital_loss: 0.0917 - label_income_auc: 0.9430 - label_marital_auc_1: 0.9944 - val_loss: 0.2169 - val_label_income_loss: 0.1219 - val_label_marital_loss: 0.0942 - val_label_income_auc: 0.9458 - val_label_marital_auc_1: 0.9944\n",
      "Epoch 5/100\n",
      "195/195 - 1s - loss: 0.2139 - label_income_loss: 0.1223 - label_marital_loss: 0.0908 - label_income_auc: 0.9436 - label_marital_auc_1: 0.9945 - val_loss: 0.2170 - val_label_income_loss: 0.1221 - val_label_marital_loss: 0.0941 - val_label_income_auc: 0.9455 - val_label_marital_auc_1: 0.9944\n",
      "Epoch 6/100\n",
      "195/195 - 1s - loss: 0.2133 - label_income_loss: 0.1220 - label_marital_loss: 0.0903 - label_income_auc: 0.9434 - label_marital_auc_1: 0.9945 - val_loss: 0.2158 - val_label_income_loss: 0.1217 - val_label_marital_loss: 0.0932 - val_label_income_auc: 0.9461 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 7/100\n",
      "195/195 - 1s - loss: 0.2119 - label_income_loss: 0.1213 - label_marital_loss: 0.0896 - label_income_auc: 0.9443 - label_marital_auc_1: 0.9946 - val_loss: 0.2144 - val_label_income_loss: 0.1229 - val_label_marital_loss: 0.0905 - val_label_income_auc: 0.9443 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 8/100\n",
      "195/195 - 1s - loss: 0.2103 - label_income_loss: 0.1207 - label_marital_loss: 0.0886 - label_income_auc: 0.9452 - label_marital_auc_1: 0.9947 - val_loss: 0.2136 - val_label_income_loss: 0.1220 - val_label_marital_loss: 0.0906 - val_label_income_auc: 0.9452 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 9/100\n",
      "195/195 - 1s - loss: 0.2101 - label_income_loss: 0.1209 - label_marital_loss: 0.0882 - label_income_auc: 0.9452 - label_marital_auc_1: 0.9948 - val_loss: 0.2140 - val_label_income_loss: 0.1225 - val_label_marital_loss: 0.0905 - val_label_income_auc: 0.9458 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 10/100\n",
      "195/195 - 1s - loss: 0.2094 - label_income_loss: 0.1203 - label_marital_loss: 0.0881 - label_income_auc: 0.9457 - label_marital_auc_1: 0.9948 - val_loss: 0.2123 - val_label_income_loss: 0.1210 - val_label_marital_loss: 0.0902 - val_label_income_auc: 0.9461 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 11/100\n",
      "195/195 - 1s - loss: 0.2097 - label_income_loss: 0.1207 - label_marital_loss: 0.0879 - label_income_auc: 0.9452 - label_marital_auc_1: 0.9948 - val_loss: 0.2122 - val_label_income_loss: 0.1209 - val_label_marital_loss: 0.0902 - val_label_income_auc: 0.9463 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 12/100\n",
      "195/195 - 1s - loss: 0.2088 - label_income_loss: 0.1197 - label_marital_loss: 0.0880 - label_income_auc: 0.9462 - label_marital_auc_1: 0.9948 - val_loss: 0.2147 - val_label_income_loss: 0.1215 - val_label_marital_loss: 0.0921 - val_label_income_auc: 0.9458 - val_label_marital_auc_1: 0.9944\n",
      "Epoch 13/100\n",
      "195/195 - 1s - loss: 0.2084 - label_income_loss: 0.1198 - label_marital_loss: 0.0875 - label_income_auc: 0.9460 - label_marital_auc_1: 0.9949 - val_loss: 0.2117 - val_label_income_loss: 0.1210 - val_label_marital_loss: 0.0896 - val_label_income_auc: 0.9461 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 14/100\n",
      "195/195 - 2s - loss: 0.2081 - label_income_loss: 0.1195 - label_marital_loss: 0.0874 - label_income_auc: 0.9463 - label_marital_auc_1: 0.9949 - val_loss: 0.2117 - val_label_income_loss: 0.1206 - val_label_marital_loss: 0.0900 - val_label_income_auc: 0.9474 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 15/100\n",
      "195/195 - 1s - loss: 0.2082 - label_income_loss: 0.1194 - label_marital_loss: 0.0877 - label_income_auc: 0.9465 - label_marital_auc_1: 0.9948 - val_loss: 0.2135 - val_label_income_loss: 0.1200 - val_label_marital_loss: 0.0922 - val_label_income_auc: 0.9460 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 16/100\n",
      "195/195 - 1s - loss: 0.2077 - label_income_loss: 0.1188 - label_marital_loss: 0.0876 - label_income_auc: 0.9470 - label_marital_auc_1: 0.9948 - val_loss: 0.2117 - val_label_income_loss: 0.1202 - val_label_marital_loss: 0.0903 - val_label_income_auc: 0.9465 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 17/100\n",
      "195/195 - 2s - loss: 0.2071 - label_income_loss: 0.1185 - label_marital_loss: 0.0873 - label_income_auc: 0.9471 - label_marital_auc_1: 0.9949 - val_loss: 0.2115 - val_label_income_loss: 0.1201 - val_label_marital_loss: 0.0902 - val_label_income_auc: 0.9464 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 18/100\n",
      "195/195 - 1s - loss: 0.2073 - label_income_loss: 0.1191 - label_marital_loss: 0.0869 - label_income_auc: 0.9465 - label_marital_auc_1: 0.9949 - val_loss: 0.2106 - val_label_income_loss: 0.1196 - val_label_marital_loss: 0.0898 - val_label_income_auc: 0.9472 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 19/100\n",
      "195/195 - 1s - loss: 0.2060 - label_income_loss: 0.1180 - label_marital_loss: 0.0867 - label_income_auc: 0.9475 - label_marital_auc_1: 0.9950 - val_loss: 0.2118 - val_label_income_loss: 0.1198 - val_label_marital_loss: 0.0907 - val_label_income_auc: 0.9459 - val_label_marital_auc_1: 0.9944\n",
      "Epoch 20/100\n",
      "195/195 - 1s - loss: 0.2061 - label_income_loss: 0.1178 - label_marital_loss: 0.0870 - label_income_auc: 0.9475 - label_marital_auc_1: 0.9949 - val_loss: 0.2116 - val_label_income_loss: 0.1193 - val_label_marital_loss: 0.0911 - val_label_income_auc: 0.9474 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 21/100\n",
      "195/195 - 1s - loss: 0.2056 - label_income_loss: 0.1179 - label_marital_loss: 0.0864 - label_income_auc: 0.9481 - label_marital_auc_1: 0.9950 - val_loss: 0.2108 - val_label_income_loss: 0.1194 - val_label_marital_loss: 0.0900 - val_label_income_auc: 0.9464 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 22/100\n",
      "195/195 - 1s - loss: 0.2054 - label_income_loss: 0.1178 - label_marital_loss: 0.0863 - label_income_auc: 0.9476 - label_marital_auc_1: 0.9950 - val_loss: 0.2116 - val_label_income_loss: 0.1202 - val_label_marital_loss: 0.0901 - val_label_income_auc: 0.9475 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 23/100\n",
      "195/195 - 1s - loss: 0.2052 - label_income_loss: 0.1175 - label_marital_loss: 0.0864 - label_income_auc: 0.9481 - label_marital_auc_1: 0.9950 - val_loss: 0.2125 - val_label_income_loss: 0.1192 - val_label_marital_loss: 0.0919 - val_label_income_auc: 0.9468 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 24/100\n",
      "195/195 - 1s - loss: 0.2049 - label_income_loss: 0.1173 - label_marital_loss: 0.0863 - label_income_auc: 0.9484 - label_marital_auc_1: 0.9950 - val_loss: 0.2099 - val_label_income_loss: 0.1192 - val_label_marital_loss: 0.0895 - val_label_income_auc: 0.9468 - val_label_marital_auc_1: 0.9947\n",
      "Epoch 25/100\n",
      "195/195 - 1s - loss: 0.2044 - label_income_loss: 0.1171 - label_marital_loss: 0.0859 - label_income_auc: 0.9483 - label_marital_auc_1: 0.9950 - val_loss: 0.2115 - val_label_income_loss: 0.1200 - val_label_marital_loss: 0.0902 - val_label_income_auc: 0.9460 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 26/100\n",
      "195/195 - 1s - loss: 0.2048 - label_income_loss: 0.1175 - label_marital_loss: 0.0858 - label_income_auc: 0.9479 - label_marital_auc_1: 0.9950 - val_loss: 0.2105 - val_label_income_loss: 0.1195 - val_label_marital_loss: 0.0897 - val_label_income_auc: 0.9464 - val_label_marital_auc_1: 0.9947\n",
      "Epoch 27/100\n",
      "195/195 - 1s - loss: 0.2041 - label_income_loss: 0.1170 - label_marital_loss: 0.0857 - label_income_auc: 0.9482 - label_marital_auc_1: 0.9951 - val_loss: 0.2120 - val_label_income_loss: 0.1198 - val_label_marital_loss: 0.0907 - val_label_income_auc: 0.9460 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 28/100\n",
      "195/195 - 2s - loss: 0.2040 - label_income_loss: 0.1168 - label_marital_loss: 0.0858 - label_income_auc: 0.9489 - label_marital_auc_1: 0.9951 - val_loss: 0.2105 - val_label_income_loss: 0.1191 - val_label_marital_loss: 0.0900 - val_label_income_auc: 0.9462 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 29/100\n",
      "195/195 - 1s - loss: 0.2038 - label_income_loss: 0.1168 - label_marital_loss: 0.0855 - label_income_auc: 0.9485 - label_marital_auc_1: 0.9951 - val_loss: 0.2105 - val_label_income_loss: 0.1196 - val_label_marital_loss: 0.0896 - val_label_income_auc: 0.9461 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 30/100\n",
      "195/195 - 2s - loss: 0.2040 - label_income_loss: 0.1168 - label_marital_loss: 0.0857 - label_income_auc: 0.9486 - label_marital_auc_1: 0.9951 - val_loss: 0.2109 - val_label_income_loss: 0.1192 - val_label_marital_loss: 0.0903 - val_label_income_auc: 0.9469 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 31/100\n",
      "195/195 - 2s - loss: 0.2037 - label_income_loss: 0.1167 - label_marital_loss: 0.0856 - label_income_auc: 0.9487 - label_marital_auc_1: 0.9951 - val_loss: 0.2111 - val_label_income_loss: 0.1193 - val_label_marital_loss: 0.0903 - val_label_income_auc: 0.9471 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 32/100\n",
      "195/195 - 2s - loss: 0.2035 - label_income_loss: 0.1163 - label_marital_loss: 0.0858 - label_income_auc: 0.9491 - label_marital_auc_1: 0.9951 - val_loss: 0.2110 - val_label_income_loss: 0.1195 - val_label_marital_loss: 0.0900 - val_label_income_auc: 0.9471 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 33/100\n",
      "195/195 - 2s - loss: 0.2034 - label_income_loss: 0.1165 - label_marital_loss: 0.0853 - label_income_auc: 0.9489 - label_marital_auc_1: 0.9951 - val_loss: 0.2153 - val_label_income_loss: 0.1239 - val_label_marital_loss: 0.0899 - val_label_income_auc: 0.9453 - val_label_marital_auc_1: 0.9946\n",
      "Epoch 34/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "195/195 - 1s - loss: 0.2036 - label_income_loss: 0.1165 - label_marital_loss: 0.0856 - label_income_auc: 0.9488 - label_marital_auc_1: 0.9951 - val_loss: 0.2121 - val_label_income_loss: 0.1195 - val_label_marital_loss: 0.0911 - val_label_income_auc: 0.9470 - val_label_marital_auc_1: 0.9945\n",
      "Epoch 00034: early stopping\n",
      "test income AUC 0.9478\n",
      "test marital AUC 0.9947\n"
     ]
    }
   ],
   "source": [
    "#Test Shared_Bottom Model\n",
    "from shared_bottom import Shared_Bottom\n",
    "\n",
    "model = Shared_Bottom(dnn_feature_columns, num_tasks=2, task_types= ['binary', 'binary'], task_names=['label_income','label_marital'], bottom_dnn_units=[16], tower_dnn_units_lists=[[8],[8]])\n",
    "\n",
    "early_stopping_monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=0, patience=10, verbose=1,\n",
    "                                       mode='min',baseline=None,restore_best_weights=True)\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=[\"binary_crossentropy\", \"binary_crossentropy\"], metrics=['AUC'])\n",
    "history = model.fit(train_model_input, [train['label_income'].values, train['label_marital'].values],batch_size=1024, epochs=100, verbose=2,validation_split=0.2, callbacks=[early_stopping_monitor])\n",
    "pred_ans = model.predict(test_model_input, batch_size=1024)\n",
    "\n",
    "print(\"test income AUC\", round(roc_auc_score(test['label_income'], pred_ans[0]), 4))\n",
    "print(\"test marital AUC\", round(roc_auc_score(test['label_marital'], pred_ans[1]), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "195/195 - 2s - loss: 0.3644 - label_marital_loss: 0.1804 - label_income_loss: 0.1834 - label_marital_auc: 0.9814 - label_income_auc_1: 0.8624 - val_loss: 0.2874 - val_label_marital_loss: 0.1343 - val_label_income_loss: 0.1523 - val_label_marital_auc: 0.9886 - val_label_income_auc_1: 0.9238\n",
      "Epoch 2/100\n",
      "195/195 - 2s - loss: 0.2795 - label_marital_loss: 0.1298 - label_income_loss: 0.1488 - label_marital_auc: 0.9900 - label_income_auc_1: 0.9260 - val_loss: 0.2768 - val_label_marital_loss: 0.1243 - val_label_income_loss: 0.1516 - val_label_marital_auc: 0.9909 - val_label_income_auc_1: 0.9303\n",
      "Epoch 3/100\n",
      "195/195 - 2s - loss: 0.2712 - label_marital_loss: 0.1250 - label_income_loss: 0.1452 - label_marital_auc: 0.9906 - label_income_auc_1: 0.9307 - val_loss: 0.2712 - val_label_marital_loss: 0.1249 - val_label_income_loss: 0.1454 - val_label_marital_auc: 0.9910 - val_label_income_auc_1: 0.9352\n",
      "Epoch 4/100\n",
      "195/195 - 2s - loss: 0.2671 - label_marital_loss: 0.1230 - label_income_loss: 0.1432 - label_marital_auc: 0.9907 - label_income_auc_1: 0.9334 - val_loss: 0.2689 - val_label_marital_loss: 0.1252 - val_label_income_loss: 0.1428 - val_label_marital_auc: 0.9909 - val_label_income_auc_1: 0.9338\n",
      "Epoch 5/100\n",
      "195/195 - 2s - loss: 0.2646 - label_marital_loss: 0.1221 - label_income_loss: 0.1415 - label_marital_auc: 0.9907 - label_income_auc_1: 0.9358 - val_loss: 0.2673 - val_label_marital_loss: 0.1196 - val_label_income_loss: 0.1467 - val_label_marital_auc: 0.9912 - val_label_income_auc_1: 0.9341\n",
      "Epoch 6/100\n",
      "195/195 - 2s - loss: 0.2626 - label_marital_loss: 0.1216 - label_income_loss: 0.1401 - label_marital_auc: 0.9906 - label_income_auc_1: 0.9376 - val_loss: 0.2671 - val_label_marital_loss: 0.1158 - val_label_income_loss: 0.1503 - val_label_marital_auc: 0.9908 - val_label_income_auc_1: 0.9375\n",
      "Epoch 7/100\n",
      "195/195 - 1s - loss: 0.2618 - label_marital_loss: 0.1211 - label_income_loss: 0.1397 - label_marital_auc: 0.9906 - label_income_auc_1: 0.9382 - val_loss: 0.2683 - val_label_marital_loss: 0.1226 - val_label_income_loss: 0.1447 - val_label_marital_auc: 0.9890 - val_label_income_auc_1: 0.9351\n",
      "Epoch 8/100\n",
      "195/195 - 1s - loss: 0.2604 - label_marital_loss: 0.1204 - label_income_loss: 0.1390 - label_marital_auc: 0.9907 - label_income_auc_1: 0.9391 - val_loss: 0.2679 - val_label_marital_loss: 0.1289 - val_label_income_loss: 0.1380 - val_label_marital_auc: 0.9905 - val_label_income_auc_1: 0.9413\n",
      "Epoch 9/100\n",
      "195/195 - 1s - loss: 0.2596 - label_marital_loss: 0.1204 - label_income_loss: 0.1382 - label_marital_auc: 0.9907 - label_income_auc_1: 0.9401 - val_loss: 0.2629 - val_label_marital_loss: 0.1195 - val_label_income_loss: 0.1424 - val_label_marital_auc: 0.9904 - val_label_income_auc_1: 0.9355\n",
      "Epoch 10/100\n",
      "195/195 - 1s - loss: 0.2588 - label_marital_loss: 0.1199 - label_income_loss: 0.1379 - label_marital_auc: 0.9907 - label_income_auc_1: 0.9401 - val_loss: 0.2622 - val_label_marital_loss: 0.1211 - val_label_income_loss: 0.1401 - val_label_marital_auc: 0.9907 - val_label_income_auc_1: 0.9396\n",
      "Epoch 11/100\n",
      "195/195 - 1s - loss: 0.2580 - label_marital_loss: 0.1189 - label_income_loss: 0.1380 - label_marital_auc: 0.9909 - label_income_auc_1: 0.9398 - val_loss: 0.2610 - val_label_marital_loss: 0.1220 - val_label_income_loss: 0.1379 - val_label_marital_auc: 0.9900 - val_label_income_auc_1: 0.9397\n",
      "Epoch 12/100\n",
      "195/195 - 1s - loss: 0.2570 - label_marital_loss: 0.1189 - label_income_loss: 0.1370 - label_marital_auc: 0.9908 - label_income_auc_1: 0.9412 - val_loss: 0.2616 - val_label_marital_loss: 0.1184 - val_label_income_loss: 0.1421 - val_label_marital_auc: 0.9904 - val_label_income_auc_1: 0.9389\n",
      "Epoch 13/100\n",
      "195/195 - 1s - loss: 0.2566 - label_marital_loss: 0.1185 - label_income_loss: 0.1370 - label_marital_auc: 0.9908 - label_income_auc_1: 0.9411 - val_loss: 0.2611 - val_label_marital_loss: 0.1199 - val_label_income_loss: 0.1400 - val_label_marital_auc: 0.9908 - val_label_income_auc_1: 0.9396\n",
      "Epoch 14/100\n",
      "195/195 - 1s - loss: 0.2560 - label_marital_loss: 0.1180 - label_income_loss: 0.1368 - label_marital_auc: 0.9909 - label_income_auc_1: 0.9414 - val_loss: 0.2608 - val_label_marital_loss: 0.1247 - val_label_income_loss: 0.1350 - val_label_marital_auc: 0.9902 - val_label_income_auc_1: 0.9416\n",
      "Epoch 15/100\n",
      "195/195 - 1s - loss: 0.2557 - label_marital_loss: 0.1181 - label_income_loss: 0.1364 - label_marital_auc: 0.9908 - label_income_auc_1: 0.9417 - val_loss: 0.2604 - val_label_marital_loss: 0.1215 - val_label_income_loss: 0.1378 - val_label_marital_auc: 0.9911 - val_label_income_auc_1: 0.9411\n",
      "Epoch 16/100\n",
      "195/195 - 1s - loss: 0.2556 - label_marital_loss: 0.1179 - label_income_loss: 0.1364 - label_marital_auc: 0.9908 - label_income_auc_1: 0.9415 - val_loss: 0.2600 - val_label_marital_loss: 0.1165 - val_label_income_loss: 0.1423 - val_label_marital_auc: 0.9904 - val_label_income_auc_1: 0.9407\n",
      "Epoch 17/100\n",
      "195/195 - 1s - loss: 0.2551 - label_marital_loss: 0.1174 - label_income_loss: 0.1365 - label_marital_auc: 0.9910 - label_income_auc_1: 0.9416 - val_loss: 0.2592 - val_label_marital_loss: 0.1171 - val_label_income_loss: 0.1409 - val_label_marital_auc: 0.9909 - val_label_income_auc_1: 0.9405\n",
      "Epoch 18/100\n",
      "195/195 - 1s - loss: 0.2548 - label_marital_loss: 0.1173 - label_income_loss: 0.1363 - label_marital_auc: 0.9909 - label_income_auc_1: 0.9418 - val_loss: 0.2616 - val_label_marital_loss: 0.1145 - val_label_income_loss: 0.1459 - val_label_marital_auc: 0.9917 - val_label_income_auc_1: 0.9329\n",
      "Epoch 19/100\n",
      "195/195 - 1s - loss: 0.2547 - label_marital_loss: 0.1176 - label_income_loss: 0.1359 - label_marital_auc: 0.9910 - label_income_auc_1: 0.9421 - val_loss: 0.2622 - val_label_marital_loss: 0.1116 - val_label_income_loss: 0.1493 - val_label_marital_auc: 0.9918 - val_label_income_auc_1: 0.9388\n",
      "Epoch 20/100\n",
      "195/195 - 1s - loss: 0.2540 - label_marital_loss: 0.1168 - label_income_loss: 0.1359 - label_marital_auc: 0.9911 - label_income_auc_1: 0.9422 - val_loss: 0.2589 - val_label_marital_loss: 0.1180 - val_label_income_loss: 0.1397 - val_label_marital_auc: 0.9911 - val_label_income_auc_1: 0.9415\n",
      "Epoch 21/100\n",
      "195/195 - 1s - loss: 0.2542 - label_marital_loss: 0.1170 - label_income_loss: 0.1359 - label_marital_auc: 0.9910 - label_income_auc_1: 0.9419 - val_loss: 0.2614 - val_label_marital_loss: 0.1210 - val_label_income_loss: 0.1391 - val_label_marital_auc: 0.9914 - val_label_income_auc_1: 0.9417\n",
      "Epoch 22/100\n",
      "195/195 - 1s - loss: 0.2538 - label_marital_loss: 0.1169 - label_income_loss: 0.1356 - label_marital_auc: 0.9911 - label_income_auc_1: 0.9428 - val_loss: 0.2605 - val_label_marital_loss: 0.1224 - val_label_income_loss: 0.1368 - val_label_marital_auc: 0.9902 - val_label_income_auc_1: 0.9423\n",
      "Epoch 23/100\n",
      "195/195 - 1s - loss: 0.2535 - label_marital_loss: 0.1165 - label_income_loss: 0.1356 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9425 - val_loss: 0.2597 - val_label_marital_loss: 0.1172 - val_label_income_loss: 0.1411 - val_label_marital_auc: 0.9910 - val_label_income_auc_1: 0.9416\n",
      "Epoch 24/100\n",
      "195/195 - 1s - loss: 0.2540 - label_marital_loss: 0.1168 - label_income_loss: 0.1358 - label_marital_auc: 0.9911 - label_income_auc_1: 0.9423 - val_loss: 0.2596 - val_label_marital_loss: 0.1176 - val_label_income_loss: 0.1406 - val_label_marital_auc: 0.9905 - val_label_income_auc_1: 0.9416\n",
      "Epoch 25/100\n",
      "195/195 - 1s - loss: 0.2531 - label_marital_loss: 0.1164 - label_income_loss: 0.1353 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9432 - val_loss: 0.2597 - val_label_marital_loss: 0.1223 - val_label_income_loss: 0.1360 - val_label_marital_auc: 0.9896 - val_label_income_auc_1: 0.9403\n",
      "Epoch 26/100\n",
      "195/195 - 1s - loss: 0.2528 - label_marital_loss: 0.1161 - label_income_loss: 0.1353 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9429 - val_loss: 0.2602 - val_label_marital_loss: 0.1172 - val_label_income_loss: 0.1417 - val_label_marital_auc: 0.9903 - val_label_income_auc_1: 0.9416\n",
      "Epoch 27/100\n",
      "195/195 - 1s - loss: 0.2535 - label_marital_loss: 0.1165 - label_income_loss: 0.1356 - label_marital_auc: 0.9911 - label_income_auc_1: 0.9426 - val_loss: 0.2618 - val_label_marital_loss: 0.1196 - val_label_income_loss: 0.1409 - val_label_marital_auc: 0.9919 - val_label_income_auc_1: 0.9400\n",
      "Epoch 28/100\n",
      "195/195 - 1s - loss: 0.2529 - label_marital_loss: 0.1164 - label_income_loss: 0.1352 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9432 - val_loss: 0.2587 - val_label_marital_loss: 0.1206 - val_label_income_loss: 0.1367 - val_label_marital_auc: 0.9904 - val_label_income_auc_1: 0.9432\n",
      "Epoch 29/100\n",
      "195/195 - 1s - loss: 0.2526 - label_marital_loss: 0.1163 - label_income_loss: 0.1349 - label_marital_auc: 0.9911 - label_income_auc_1: 0.9433 - val_loss: 0.2597 - val_label_marital_loss: 0.1183 - val_label_income_loss: 0.1400 - val_label_marital_auc: 0.9916 - val_label_income_auc_1: 0.9426\n",
      "Epoch 30/100\n",
      "195/195 - 1s - loss: 0.2523 - label_marital_loss: 0.1159 - label_income_loss: 0.1350 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9432 - val_loss: 0.2598 - val_label_marital_loss: 0.1242 - val_label_income_loss: 0.1342 - val_label_marital_auc: 0.9905 - val_label_income_auc_1: 0.9430\n",
      "Epoch 31/100\n",
      "195/195 - 1s - loss: 0.2524 - label_marital_loss: 0.1160 - label_income_loss: 0.1350 - label_marital_auc: 0.9913 - label_income_auc_1: 0.9428 - val_loss: 0.2593 - val_label_marital_loss: 0.1232 - val_label_income_loss: 0.1347 - val_label_marital_auc: 0.9902 - val_label_income_auc_1: 0.9428\n",
      "Epoch 32/100\n",
      "195/195 - 1s - loss: 0.2522 - label_marital_loss: 0.1162 - label_income_loss: 0.1346 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9441 - val_loss: 0.2602 - val_label_marital_loss: 0.1163 - val_label_income_loss: 0.1425 - val_label_marital_auc: 0.9907 - val_label_income_auc_1: 0.9394\n",
      "Epoch 33/100\n",
      "195/195 - 1s - loss: 0.2526 - label_marital_loss: 0.1163 - label_income_loss: 0.1348 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9434 - val_loss: 0.2591 - val_label_marital_loss: 0.1198 - val_label_income_loss: 0.1379 - val_label_marital_auc: 0.9910 - val_label_income_auc_1: 0.9423\n",
      "Epoch 34/100\n",
      "195/195 - 1s - loss: 0.2522 - label_marital_loss: 0.1160 - label_income_loss: 0.1347 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9435 - val_loss: 0.2595 - val_label_marital_loss: 0.1226 - val_label_income_loss: 0.1355 - val_label_marital_auc: 0.9906 - val_label_income_auc_1: 0.9423\n",
      "Epoch 35/100\n",
      "195/195 - 1s - loss: 0.2521 - label_marital_loss: 0.1160 - label_income_loss: 0.1347 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9434 - val_loss: 0.2593 - val_label_marital_loss: 0.1222 - val_label_income_loss: 0.1356 - val_label_marital_auc: 0.9910 - val_label_income_auc_1: 0.9438\n",
      "Epoch 36/100\n",
      "195/195 - 1s - loss: 0.2519 - label_marital_loss: 0.1157 - label_income_loss: 0.1348 - label_marital_auc: 0.9913 - label_income_auc_1: 0.9436 - val_loss: 0.2609 - val_label_marital_loss: 0.1147 - val_label_income_loss: 0.1448 - val_label_marital_auc: 0.9910 - val_label_income_auc_1: 0.9422\n",
      "Epoch 37/100\n",
      "195/195 - 1s - loss: 0.2522 - label_marital_loss: 0.1159 - label_income_loss: 0.1349 - label_marital_auc: 0.9913 - label_income_auc_1: 0.9436 - val_loss: 0.2615 - val_label_marital_loss: 0.1223 - val_label_income_loss: 0.1377 - val_label_marital_auc: 0.9916 - val_label_income_auc_1: 0.9397\n",
      "Epoch 38/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "195/195 - 1s - loss: 0.2522 - label_marital_loss: 0.1160 - label_income_loss: 0.1347 - label_marital_auc: 0.9912 - label_income_auc_1: 0.9435 - val_loss: 0.2588 - val_label_marital_loss: 0.1210 - val_label_income_loss: 0.1363 - val_label_marital_auc: 0.9902 - val_label_income_auc_1: 0.9430\n",
      "Epoch 00038: early stopping\n",
      "test marital AUC 0.9904\n",
      "test income AUC 0.9439\n"
     ]
    }
   ],
   "source": [
    "#Test ESSM Model\n",
    "from essm import ESSM\n",
    "#take marital as ctr task, take income as ctcvr task\n",
    "model = ESSM(dnn_feature_columns, task_type='binary', task_names=['label_marital', 'label_income'],\n",
    "        tower_dnn_units_lists=[[8],[8]])\n",
    "\n",
    "early_stopping_monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=0, patience=10, verbose=1,\n",
    "                                       mode='min',baseline=None,restore_best_weights=True)\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=[\"binary_crossentropy\", \"binary_crossentropy\"], metrics=['AUC'])\n",
    "history = model.fit(train_model_input, [train['label_marital'].values, train['label_income'].values],batch_size=1024, epochs=100, verbose=2, validation_split=0.2, callbacks=[early_stopping_monitor])\n",
    "\n",
    "pred_ans = model.predict(test_model_input, batch_size=1024)\n",
    "\n",
    "print(\"test marital AUC\", round(roc_auc_score(test['label_marital'], pred_ans[0]), 4))\n",
    "print(\"test income AUC\", round(roc_auc_score(test['label_income'], pred_ans[1]), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "195/195 - 4s - loss: 0.3040 - income_loss: 0.1597 - marital_loss: 0.1438 - income_auc: 0.8942 - marital_auc_1: 0.9867 - val_loss: 0.2277 - val_income_loss: 0.1288 - val_marital_loss: 0.0983 - val_income_auc: 0.9418 - val_marital_auc_1: 0.9942\n",
      "Epoch 2/100\n",
      "195/195 - 2s - loss: 0.2197 - income_loss: 0.1248 - marital_loss: 0.0942 - income_auc: 0.9413 - marital_auc_1: 0.9941 - val_loss: 0.2157 - val_income_loss: 0.1217 - val_marital_loss: 0.0933 - val_income_auc: 0.9454 - val_marital_auc_1: 0.9943\n",
      "Epoch 3/100\n",
      "195/195 - 3s - loss: 0.2162 - income_loss: 0.1232 - marital_loss: 0.0923 - income_auc: 0.9425 - marital_auc_1: 0.9943 - val_loss: 0.2147 - val_income_loss: 0.1214 - val_marital_loss: 0.0926 - val_income_auc: 0.9456 - val_marital_auc_1: 0.9944\n",
      "Epoch 4/100\n",
      "195/195 - 2s - loss: 0.2127 - income_loss: 0.1213 - marital_loss: 0.0906 - income_auc: 0.9444 - marital_auc_1: 0.9945 - val_loss: 0.2125 - val_income_loss: 0.1207 - val_marital_loss: 0.0910 - val_income_auc: 0.9466 - val_marital_auc_1: 0.9946\n",
      "Epoch 5/100\n",
      "195/195 - 2s - loss: 0.2104 - income_loss: 0.1202 - marital_loss: 0.0894 - income_auc: 0.9454 - marital_auc_1: 0.9947 - val_loss: 0.2124 - val_income_loss: 0.1211 - val_marital_loss: 0.0904 - val_income_auc: 0.9456 - val_marital_auc_1: 0.9947\n",
      "Epoch 6/100\n",
      "195/195 - 2s - loss: 0.2096 - income_loss: 0.1199 - marital_loss: 0.0888 - income_auc: 0.9458 - marital_auc_1: 0.9947 - val_loss: 0.2146 - val_income_loss: 0.1215 - val_marital_loss: 0.0923 - val_income_auc: 0.9456 - val_marital_auc_1: 0.9946\n",
      "Epoch 7/100\n",
      "195/195 - 3s - loss: 0.2086 - income_loss: 0.1193 - marital_loss: 0.0883 - income_auc: 0.9460 - marital_auc_1: 0.9948 - val_loss: 0.2116 - val_income_loss: 0.1216 - val_marital_loss: 0.0891 - val_income_auc: 0.9454 - val_marital_auc_1: 0.9947\n",
      "Epoch 8/100\n",
      "195/195 - 2s - loss: 0.2067 - income_loss: 0.1185 - marital_loss: 0.0873 - income_auc: 0.9470 - marital_auc_1: 0.9949 - val_loss: 0.2110 - val_income_loss: 0.1209 - val_marital_loss: 0.0892 - val_income_auc: 0.9463 - val_marital_auc_1: 0.9947\n",
      "Epoch 9/100\n",
      "195/195 - 2s - loss: 0.2059 - income_loss: 0.1178 - marital_loss: 0.0870 - income_auc: 0.9475 - marital_auc_1: 0.9949 - val_loss: 0.2111 - val_income_loss: 0.1205 - val_marital_loss: 0.0896 - val_income_auc: 0.9468 - val_marital_auc_1: 0.9948\n",
      "Epoch 10/100\n",
      "195/195 - 2s - loss: 0.2047 - income_loss: 0.1173 - marital_loss: 0.0863 - income_auc: 0.9480 - marital_auc_1: 0.9950 - val_loss: 0.2105 - val_income_loss: 0.1205 - val_marital_loss: 0.0889 - val_income_auc: 0.9461 - val_marital_auc_1: 0.9947\n",
      "Epoch 11/100\n",
      "195/195 - 2s - loss: 0.2038 - income_loss: 0.1168 - marital_loss: 0.0858 - income_auc: 0.9487 - marital_auc_1: 0.9951 - val_loss: 0.2132 - val_income_loss: 0.1208 - val_marital_loss: 0.0912 - val_income_auc: 0.9459 - val_marital_auc_1: 0.9946\n",
      "Epoch 12/100\n",
      "195/195 - 2s - loss: 0.2030 - income_loss: 0.1164 - marital_loss: 0.0853 - income_auc: 0.9490 - marital_auc_1: 0.9951 - val_loss: 0.2109 - val_income_loss: 0.1206 - val_marital_loss: 0.0890 - val_income_auc: 0.9463 - val_marital_auc_1: 0.9947\n",
      "Epoch 13/100\n",
      "195/195 - 2s - loss: 0.2015 - income_loss: 0.1152 - marital_loss: 0.0850 - income_auc: 0.9503 - marital_auc_1: 0.9952 - val_loss: 0.2134 - val_income_loss: 0.1221 - val_marital_loss: 0.0900 - val_income_auc: 0.9450 - val_marital_auc_1: 0.9947\n",
      "Epoch 14/100\n",
      "195/195 - 2s - loss: 0.2007 - income_loss: 0.1148 - marital_loss: 0.0845 - income_auc: 0.9503 - marital_auc_1: 0.9952 - val_loss: 0.2130 - val_income_loss: 0.1215 - val_marital_loss: 0.0902 - val_income_auc: 0.9448 - val_marital_auc_1: 0.9946\n",
      "Epoch 15/100\n",
      "195/195 - 2s - loss: 0.1993 - income_loss: 0.1135 - marital_loss: 0.0843 - income_auc: 0.9519 - marital_auc_1: 0.9952 - val_loss: 0.2136 - val_income_loss: 0.1215 - val_marital_loss: 0.0907 - val_income_auc: 0.9446 - val_marital_auc_1: 0.9946\n",
      "Epoch 16/100\n",
      "195/195 - 2s - loss: 0.1985 - income_loss: 0.1132 - marital_loss: 0.0837 - income_auc: 0.9519 - marital_auc_1: 0.9953 - val_loss: 0.2140 - val_income_loss: 0.1216 - val_marital_loss: 0.0908 - val_income_auc: 0.9450 - val_marital_auc_1: 0.9946\n",
      "Epoch 17/100\n",
      "195/195 - 2s - loss: 0.1970 - income_loss: 0.1124 - marital_loss: 0.0829 - income_auc: 0.9527 - marital_auc_1: 0.9954 - val_loss: 0.2144 - val_income_loss: 0.1223 - val_marital_loss: 0.0905 - val_income_auc: 0.9439 - val_marital_auc_1: 0.9946\n",
      "Epoch 18/100\n",
      "195/195 - 2s - loss: 0.1968 - income_loss: 0.1123 - marital_loss: 0.0828 - income_auc: 0.9527 - marital_auc_1: 0.9954 - val_loss: 0.2143 - val_income_loss: 0.1220 - val_marital_loss: 0.0906 - val_income_auc: 0.9442 - val_marital_auc_1: 0.9946\n",
      "Epoch 19/100\n",
      "195/195 - 2s - loss: 0.1957 - income_loss: 0.1111 - marital_loss: 0.0829 - income_auc: 0.9540 - marital_auc_1: 0.9954 - val_loss: 0.2169 - val_income_loss: 0.1227 - val_marital_loss: 0.0925 - val_income_auc: 0.9448 - val_marital_auc_1: 0.9943\n",
      "Epoch 20/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "195/195 - 2s - loss: 0.1943 - income_loss: 0.1102 - marital_loss: 0.0823 - income_auc: 0.9547 - marital_auc_1: 0.9955 - val_loss: 0.2153 - val_income_loss: 0.1219 - val_marital_loss: 0.0916 - val_income_auc: 0.9439 - val_marital_auc_1: 0.9944\n",
      "Epoch 00020: early stopping\n",
      "test income AUC 0.9463\n",
      "test marital AUC 0.9947\n"
     ]
    }
   ],
   "source": [
    "from mmoe import MMOE\n",
    "model = MMOE(dnn_feature_columns, num_tasks=2, task_types=['binary', 'binary'], task_names=['income','marital'], \n",
    "             num_experts=8, expert_dnn_units=[16], gate_dnn_units=None, tower_dnn_units_lists=[[8],[8]])\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=[\"binary_crossentropy\", \"binary_crossentropy\"],metrics=['AUC'])\n",
    "\n",
    "early_stopping_monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=0, patience=10, verbose=1,\n",
    "                                       mode='min',baseline=None,restore_best_weights=True)\n",
    "\n",
    "history = model.fit(train_model_input, [train['label_income'].values, train['label_marital'].values], \n",
    "                    batch_size=1024, epochs=100, verbose=2, validation_split=0.2, callbacks=[early_stopping_monitor])\n",
    "\n",
    "pred_ans = model.predict(test_model_input, batch_size=1024)\n",
    "print(\"test income AUC\", round(roc_auc_score(test['label_income'], pred_ans[0]), 4))\n",
    "print(\"test marital AUC\", round(roc_auc_score(test['label_marital'], pred_ans[1]), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "390/390 - 4s - loss: 0.2645 - income_loss: 0.1430 - marital_loss: 0.1210 - income_auc: 0.9185 - marital_auc_1: 0.9904 - val_loss: 0.2250 - val_income_loss: 0.1235 - val_marital_loss: 0.1010 - val_income_auc: 0.9436 - val_marital_auc_1: 0.9940\n",
      "Epoch 2/100\n",
      "390/390 - 3s - loss: 0.2177 - income_loss: 0.1241 - marital_loss: 0.0930 - income_auc: 0.9415 - marital_auc_1: 0.9942 - val_loss: 0.2138 - val_income_loss: 0.1212 - val_marital_loss: 0.0918 - val_income_auc: 0.9447 - val_marital_auc_1: 0.9944\n",
      "Epoch 3/100\n",
      "390/390 - 4s - loss: 0.2132 - income_loss: 0.1222 - marital_loss: 0.0902 - income_auc: 0.9431 - marital_auc_1: 0.9945 - val_loss: 0.2147 - val_income_loss: 0.1225 - val_marital_loss: 0.0913 - val_income_auc: 0.9446 - val_marital_auc_1: 0.9944\n",
      "Epoch 4/100\n",
      "390/390 - 4s - loss: 0.2118 - income_loss: 0.1214 - marital_loss: 0.0895 - income_auc: 0.9442 - marital_auc_1: 0.9946 - val_loss: 0.2124 - val_income_loss: 0.1208 - val_marital_loss: 0.0907 - val_income_auc: 0.9451 - val_marital_auc_1: 0.9945\n",
      "Epoch 5/100\n",
      "390/390 - 4s - loss: 0.2094 - income_loss: 0.1205 - marital_loss: 0.0880 - income_auc: 0.9446 - marital_auc_1: 0.9948 - val_loss: 0.2112 - val_income_loss: 0.1209 - val_marital_loss: 0.0892 - val_income_auc: 0.9473 - val_marital_auc_1: 0.9947\n",
      "Epoch 6/100\n",
      "390/390 - 4s - loss: 0.2086 - income_loss: 0.1200 - marital_loss: 0.0875 - income_auc: 0.9454 - marital_auc_1: 0.9949 - val_loss: 0.2127 - val_income_loss: 0.1204 - val_marital_loss: 0.0911 - val_income_auc: 0.9463 - val_marital_auc_1: 0.9946\n",
      "Epoch 7/100\n",
      "390/390 - 3s - loss: 0.2072 - income_loss: 0.1190 - marital_loss: 0.0870 - income_auc: 0.9467 - marital_auc_1: 0.9949 - val_loss: 0.2113 - val_income_loss: 0.1210 - val_marital_loss: 0.0891 - val_income_auc: 0.9460 - val_marital_auc_1: 0.9948\n",
      "Epoch 8/100\n",
      "390/390 - 3s - loss: 0.2063 - income_loss: 0.1185 - marital_loss: 0.0865 - income_auc: 0.9471 - marital_auc_1: 0.9950 - val_loss: 0.2109 - val_income_loss: 0.1201 - val_marital_loss: 0.0894 - val_income_auc: 0.9465 - val_marital_auc_1: 0.9947\n",
      "Epoch 9/100\n",
      "390/390 - 3s - loss: 0.2046 - income_loss: 0.1174 - marital_loss: 0.0858 - income_auc: 0.9483 - marital_auc_1: 0.9951 - val_loss: 0.2109 - val_income_loss: 0.1200 - val_marital_loss: 0.0895 - val_income_auc: 0.9469 - val_marital_auc_1: 0.9947\n",
      "Epoch 10/100\n",
      "390/390 - 3s - loss: 0.2033 - income_loss: 0.1165 - marital_loss: 0.0853 - income_auc: 0.9492 - marital_auc_1: 0.9951 - val_loss: 0.2175 - val_income_loss: 0.1258 - val_marital_loss: 0.0902 - val_income_auc: 0.9424 - val_marital_auc_1: 0.9946\n",
      "Epoch 11/100\n",
      "390/390 - 5s - loss: 0.2029 - income_loss: 0.1164 - marital_loss: 0.0849 - income_auc: 0.9497 - marital_auc_1: 0.9952 - val_loss: 0.2148 - val_income_loss: 0.1229 - val_marital_loss: 0.0902 - val_income_auc: 0.9447 - val_marital_auc_1: 0.9946\n",
      "Epoch 12/100\n",
      "390/390 - 5s - loss: 0.2009 - income_loss: 0.1153 - marital_loss: 0.0839 - income_auc: 0.9506 - marital_auc_1: 0.9953 - val_loss: 0.2163 - val_income_loss: 0.1232 - val_marital_loss: 0.0914 - val_income_auc: 0.9430 - val_marital_auc_1: 0.9946\n",
      "Epoch 13/100\n",
      "390/390 - 3s - loss: 0.2002 - income_loss: 0.1147 - marital_loss: 0.0836 - income_auc: 0.9511 - marital_auc_1: 0.9953 - val_loss: 0.2161 - val_income_loss: 0.1237 - val_marital_loss: 0.0906 - val_income_auc: 0.9427 - val_marital_auc_1: 0.9946\n",
      "Epoch 14/100\n",
      "390/390 - 3s - loss: 0.1984 - income_loss: 0.1136 - marital_loss: 0.0829 - income_auc: 0.9525 - marital_auc_1: 0.9954 - val_loss: 0.2178 - val_income_loss: 0.1224 - val_marital_loss: 0.0934 - val_income_auc: 0.9431 - val_marital_auc_1: 0.9944\n",
      "Epoch 15/100\n",
      "390/390 - 3s - loss: 0.1977 - income_loss: 0.1131 - marital_loss: 0.0826 - income_auc: 0.9530 - marital_auc_1: 0.9954 - val_loss: 0.2166 - val_income_loss: 0.1226 - val_marital_loss: 0.0920 - val_income_auc: 0.9432 - val_marital_auc_1: 0.9944\n",
      "Epoch 16/100\n",
      "390/390 - 3s - loss: 0.1962 - income_loss: 0.1124 - marital_loss: 0.0818 - income_auc: 0.9536 - marital_auc_1: 0.9955 - val_loss: 0.2171 - val_income_loss: 0.1229 - val_marital_loss: 0.0921 - val_income_auc: 0.9427 - val_marital_auc_1: 0.9944\n",
      "Epoch 17/100\n",
      "390/390 - 3s - loss: 0.1946 - income_loss: 0.1112 - marital_loss: 0.0812 - income_auc: 0.9546 - marital_auc_1: 0.9956 - val_loss: 0.2225 - val_income_loss: 0.1240 - val_marital_loss: 0.0963 - val_income_auc: 0.9421 - val_marital_auc_1: 0.9943\n",
      "Epoch 18/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "390/390 - 3s - loss: 0.1932 - income_loss: 0.1104 - marital_loss: 0.0806 - income_auc: 0.9562 - marital_auc_1: 0.9956 - val_loss: 0.2213 - val_income_loss: 0.1250 - val_marital_loss: 0.0940 - val_income_auc: 0.9404 - val_marital_auc_1: 0.9942\n",
      "Epoch 00018: early stopping\n",
      "test income AUC 0.9471\n",
      "test marital AUC 0.9947\n"
     ]
    }
   ],
   "source": [
    "from ple_cgc import PLE_CGC\n",
    "\n",
    "model = PLE_CGC(dnn_feature_columns, num_tasks=2, task_types=['binary', 'binary'], task_names=['income','marital'], \n",
    "                num_experts_specific=4, num_experts_shared=4, expert_dnn_units=[16], gate_dnn_units=None, tower_dnn_units_lists=[[8],[8]])\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=[\"binary_crossentropy\", \"binary_crossentropy\"], metrics=['AUC'])\n",
    "\n",
    "early_stopping_monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=0,patience=10, verbose=1,\n",
    "                                       mode='min',baseline=None,restore_best_weights=True)\n",
    "\n",
    "history = model.fit(train_model_input, [train['label_income'].values, train['label_marital'].values], \n",
    "                    batch_size=512, epochs=100, verbose=2, validation_split=0.2, callbacks=[early_stopping_monitor])\n",
    "\n",
    "pred_ans = model.predict(test_model_input, batch_size=512)\n",
    "print(\"test income AUC\", round(roc_auc_score(test['label_income'], pred_ans[0]), 4))\n",
    "print(\"test marital AUC\", round(roc_auc_score(test['label_marital'], pred_ans[1]), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "195/195 - 5s - loss: 0.3415 - income_loss: 0.1676 - marital_loss: 0.1734 - income_auc: 0.8845 - marital_auc_1: 0.9802 - val_loss: 0.2316 - val_income_loss: 0.1314 - val_marital_loss: 0.0996 - val_income_auc: 0.9406 - val_marital_auc_1: 0.9941\n",
      "Epoch 2/100\n",
      "195/195 - 3s - loss: 0.2241 - income_loss: 0.1294 - marital_loss: 0.0941 - income_auc: 0.9398 - marital_auc_1: 0.9941 - val_loss: 0.2211 - val_income_loss: 0.1278 - val_marital_loss: 0.0927 - val_income_auc: 0.9435 - val_marital_auc_1: 0.9943\n",
      "Epoch 3/100\n",
      "195/195 - 3s - loss: 0.2199 - income_loss: 0.1262 - marital_loss: 0.0930 - income_auc: 0.9423 - marital_auc_1: 0.9942 - val_loss: 0.2193 - val_income_loss: 0.1269 - val_marital_loss: 0.0916 - val_income_auc: 0.9439 - val_marital_auc_1: 0.9944\n",
      "Epoch 4/100\n",
      "195/195 - 3s - loss: 0.2155 - income_loss: 0.1239 - marital_loss: 0.0909 - income_auc: 0.9436 - marital_auc_1: 0.9945 - val_loss: 0.2139 - val_income_loss: 0.1223 - val_marital_loss: 0.0908 - val_income_auc: 0.9454 - val_marital_auc_1: 0.9945\n",
      "Epoch 5/100\n",
      "195/195 - 3s - loss: 0.2115 - income_loss: 0.1213 - marital_loss: 0.0893 - income_auc: 0.9445 - marital_auc_1: 0.9947 - val_loss: 0.2138 - val_income_loss: 0.1211 - val_marital_loss: 0.0918 - val_income_auc: 0.9473 - val_marital_auc_1: 0.9946\n",
      "Epoch 6/100\n",
      "195/195 - 3s - loss: 0.2093 - income_loss: 0.1200 - marital_loss: 0.0884 - income_auc: 0.9453 - marital_auc_1: 0.9948 - val_loss: 0.2132 - val_income_loss: 0.1224 - val_marital_loss: 0.0898 - val_income_auc: 0.9458 - val_marital_auc_1: 0.9947\n",
      "Epoch 7/100\n",
      "195/195 - 3s - loss: 0.2073 - income_loss: 0.1185 - marital_loss: 0.0878 - income_auc: 0.9470 - marital_auc_1: 0.9949 - val_loss: 0.2103 - val_income_loss: 0.1203 - val_marital_loss: 0.0890 - val_income_auc: 0.9469 - val_marital_auc_1: 0.9948\n",
      "Epoch 8/100\n",
      "195/195 - 3s - loss: 0.2059 - income_loss: 0.1179 - marital_loss: 0.0869 - income_auc: 0.9473 - marital_auc_1: 0.9949 - val_loss: 0.2112 - val_income_loss: 0.1203 - val_marital_loss: 0.0898 - val_income_auc: 0.9459 - val_marital_auc_1: 0.9947\n",
      "Epoch 9/100\n",
      "195/195 - 3s - loss: 0.2049 - income_loss: 0.1172 - marital_loss: 0.0865 - income_auc: 0.9481 - marital_auc_1: 0.9950 - val_loss: 0.2105 - val_income_loss: 0.1193 - val_marital_loss: 0.0901 - val_income_auc: 0.9476 - val_marital_auc_1: 0.9947\n",
      "Epoch 10/100\n",
      "195/195 - 3s - loss: 0.2035 - income_loss: 0.1164 - marital_loss: 0.0859 - income_auc: 0.9488 - marital_auc_1: 0.9951 - val_loss: 0.2103 - val_income_loss: 0.1194 - val_marital_loss: 0.0896 - val_income_auc: 0.9471 - val_marital_auc_1: 0.9947\n",
      "Epoch 11/100\n",
      "195/195 - 3s - loss: 0.2018 - income_loss: 0.1152 - marital_loss: 0.0853 - income_auc: 0.9500 - marital_auc_1: 0.9951 - val_loss: 0.2108 - val_income_loss: 0.1206 - val_marital_loss: 0.0889 - val_income_auc: 0.9457 - val_marital_auc_1: 0.9947\n",
      "Epoch 12/100\n",
      "195/195 - 3s - loss: 0.2004 - income_loss: 0.1143 - marital_loss: 0.0847 - income_auc: 0.9510 - marital_auc_1: 0.9952 - val_loss: 0.2110 - val_income_loss: 0.1200 - val_marital_loss: 0.0896 - val_income_auc: 0.9461 - val_marital_auc_1: 0.9947\n",
      "Epoch 13/100\n",
      "195/195 - 3s - loss: 0.1991 - income_loss: 0.1130 - marital_loss: 0.0845 - income_auc: 0.9523 - marital_auc_1: 0.9952 - val_loss: 0.2118 - val_income_loss: 0.1207 - val_marital_loss: 0.0896 - val_income_auc: 0.9447 - val_marital_auc_1: 0.9947\n",
      "Epoch 14/100\n",
      "195/195 - 3s - loss: 0.1973 - income_loss: 0.1120 - marital_loss: 0.0837 - income_auc: 0.9532 - marital_auc_1: 0.9953 - val_loss: 0.2132 - val_income_loss: 0.1205 - val_marital_loss: 0.0910 - val_income_auc: 0.9457 - val_marital_auc_1: 0.9944\n",
      "Epoch 15/100\n",
      "195/195 - 3s - loss: 0.1962 - income_loss: 0.1112 - marital_loss: 0.0833 - income_auc: 0.9538 - marital_auc_1: 0.9953 - val_loss: 0.2157 - val_income_loss: 0.1226 - val_marital_loss: 0.0914 - val_income_auc: 0.9441 - val_marital_auc_1: 0.9944\n",
      "Epoch 16/100\n",
      "195/195 - 3s - loss: 0.1940 - income_loss: 0.1097 - marital_loss: 0.0825 - income_auc: 0.9558 - marital_auc_1: 0.9954 - val_loss: 0.2150 - val_income_loss: 0.1218 - val_marital_loss: 0.0914 - val_income_auc: 0.9441 - val_marital_auc_1: 0.9944\n",
      "Epoch 17/100\n",
      "195/195 - 3s - loss: 0.1928 - income_loss: 0.1090 - marital_loss: 0.0819 - income_auc: 0.9561 - marital_auc_1: 0.9955 - val_loss: 0.2182 - val_income_loss: 0.1234 - val_marital_loss: 0.0929 - val_income_auc: 0.9424 - val_marital_auc_1: 0.9942\n",
      "Epoch 18/100\n",
      "195/195 - 4s - loss: 0.1906 - income_loss: 0.1078 - marital_loss: 0.0809 - income_auc: 0.9575 - marital_auc_1: 0.9956 - val_loss: 0.2190 - val_income_loss: 0.1245 - val_marital_loss: 0.0925 - val_income_auc: 0.9407 - val_marital_auc_1: 0.9944\n",
      "Epoch 19/100\n",
      "195/195 - 3s - loss: 0.1894 - income_loss: 0.1066 - marital_loss: 0.0808 - income_auc: 0.9589 - marital_auc_1: 0.9956 - val_loss: 0.2187 - val_income_loss: 0.1240 - val_marital_loss: 0.0926 - val_income_auc: 0.9423 - val_marital_auc_1: 0.9943\n",
      "Epoch 20/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "195/195 - 4s - loss: 0.1871 - income_loss: 0.1050 - marital_loss: 0.0799 - income_auc: 0.9598 - marital_auc_1: 0.9957 - val_loss: 0.2248 - val_income_loss: 0.1273 - val_marital_loss: 0.0953 - val_income_auc: 0.9384 - val_marital_auc_1: 0.9939\n",
      "Epoch 00020: early stopping\n",
      "test income AUC 0.948\n",
      "test marital AUC 0.9947\n"
     ]
    }
   ],
   "source": [
    "from ple import PLE\n",
    "\n",
    "model = PLE(dnn_feature_columns, num_tasks=2, task_types=['binary', 'binary'], task_names=['income','marital'], \n",
    "            num_levels=2, num_experts_specific=4, num_experts_shared=4, expert_dnn_units=[16],  gate_dnn_units=None,                                 tower_dnn_units_lists=[[8],[8]])\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=[\"binary_crossentropy\", \"binary_crossentropy\"], metrics=['AUC'])\n",
    "\n",
    "early_stopping_monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=0,patience=10, verbose=1,\n",
    "                                       mode='min',baseline=None,restore_best_weights=True)\n",
    "\n",
    "history = model.fit(train_model_input, [train['label_income'].values, train['label_marital'].values], \n",
    "                    batch_size=1024, epochs=100, verbose=2, validation_split=0.2, callbacks=[early_stopping_monitor])\n",
    "\n",
    "pred_ans = model.predict(test_model_input, batch_size=1024)\n",
    "print(\"test income AUC\", round(roc_auc_score(test['label_income'], pred_ans[0]), 4))\n",
    "print(\"test marital AUC\", round(roc_auc_score(test['label_marital'], pred_ans[1]), 4))"
   ]
  }
 ]
}