3.MMoE-example.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3.MMoE-example.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "37Lzwxf-fDWT"
      },
      "source": [
        "## MMoE多任务多目标建模"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s6srBeg382ee"
      },
      "source": [
        "## 加载数据"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kloudYEG6Cws"
      },
      "source": [
        "import joblib\n",
        "train = joblib.load('./data_and_feature/train.txt')\n",
        "val = joblib.load('./data_and_feature/val.txt')\n",
        "test = joblib.load('./data_and_feature/test.txt')\n",
        "encoder = joblib.load('./data_and_feature/encoder.txt')\n",
        "\n",
        "train_num = len(train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TfVav1G3NM_Q"
      },
      "source": [
        "## 导入工具库"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atU3210yKot0"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "import gc\n",
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras.backend as K\n",
        "\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.python.keras.layers import Layer\n",
        "from tensorflow.keras import regularizers\n",
        "\n",
        "from tensorflow.keras.models import Model,load_model\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint,ReduceLROnPlateau,EarlyStopping\n",
        "\n",
        "from tensorflow.keras import optimizers,initializers\n",
        "from tensorflow.python.keras.initializers import glorot_normal"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XlJgKlOlNTcV"
      },
      "source": [
        "## 搭建MMoE模型"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FunCHvVMzKq"
      },
      "source": [
        "class MeanPoolLayer(Layer):\n",
        "    def __init__(self, axis, **kwargs):\n",
        "        super(MeanPoolLayer, self).__init__(**kwargs)\n",
        "        self.axis = axis\n",
        "\n",
        "    def call(self, x, mask):\n",
        "        mask = tf.expand_dims(tf.cast(mask,tf.float32),axis = -1)\n",
        "        x = x * mask\n",
        "        return K.sum(x, axis=self.axis) / (K.sum(mask, axis=self.axis) + 1e-9)\n",
        "\n",
        "class MmoeLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self,expert_dim,n_expert,n_task):\n",
        "        super(MmoeLayer, self).__init__()\n",
        "        self.n_task = n_task\n",
        "        self.expert_layer = [Dense(expert_dim,activation = 'relu') for i in range(n_expert)]\n",
        "        self.gate_layers = [Dense(n_expert,activation = 'softmax') for i in range(n_task)]\n",
        "    \n",
        "    def call(self,x):\n",
        "        #多个专家网络\n",
        "        E_net = [expert(x) for expert in self.expert_layer]\n",
        "        E_net = Concatenate(axis = 1)([e[:,tf.newaxis,:] for e in E_net]) #(bs,n_expert,n_dims)\n",
        "        #多个门网络\n",
        "        gate_net = [gate(x) for gate in self.gate_layers]     #n_task个(bs,n_expert)\n",
        "        \n",
        "        #每个towers等于，对应的门网络乘上所有的专家网络。\n",
        "        towers = []\n",
        "        for i in range(self.n_task):\n",
        "            g = tf.expand_dims(gate_net[i],axis = -1)  #(bs,n_expert,1)\n",
        "            _tower = tf.matmul(E_net, g,transpose_a=True)\n",
        "            towers.append(Flatten()(_tower))           #(bs,expert_dim)\n",
        "            \n",
        "        return towers\n",
        "\n",
        "def build_mmoe(sparse_cols,dense_cols,sparse_max_len,embed_dim,expert_dim,\n",
        "              varlens_cols,varlens_max_len,n_expert,n_task,target = [],\n",
        "              dnn_hidden_units = (64,),dnn_reg_l2 = 1e-5,drop_rate = 0.1,\n",
        "                embedding_reg_l2 = 1e-6):\n",
        "    \n",
        "    \n",
        "    #输入部分，分为sparse,varlens,dense部分。\n",
        "    sparse_inputs = {f:Input([1],name = f) for f in sparse_cols}\n",
        "    dense_inputs = {f:Input([1],name = f) for f in dense_cols}\n",
        "    varlens_inputs = {f:Input([None,1],name = f) for f in varlens_cols}\n",
        "        \n",
        "    input_embed = {}\n",
        "    #离散特征，embedding到k维\n",
        "    for f in sparse_cols:\n",
        "        _input = sparse_inputs[f]\n",
        "        embedding = Embedding(sparse_max_len[f], embed_dim, \n",
        "            embeddings_regularizer=tf.keras.regularizers.l2(embedding_reg_l2)) \n",
        "        input_embed[f] =Flatten()(embedding(_input)) #(bs,k)\n",
        "        \n",
        "    #多标签离散变量\n",
        "    for f in varlens_inputs:\n",
        "        _input = varlens_inputs[f]\n",
        "        mask = Masking(mask_value = 0).compute_mask(_input)\n",
        "        embedding = Embedding(varlens_max_len[f], embed_dim,\n",
        "            embeddings_regularizer=tf.keras.regularizers.l2(1e-6))\n",
        "        _embed =Reshape([-1,embed_dim])(embedding(_input))\n",
        "        out_embed = MeanPoolLayer(axis=1)(_embed,mask)\n",
        "        input_embed[f] = out_embed\n",
        "        \n",
        "    input_embed.update(dense_inputs) #加入连续变量\n",
        "    input_embed = Concatenate(axis = -1)([input_embed[f] for f in input_embed])\n",
        "    for num in dnn_hidden_units:\n",
        "        input_embed = Dropout(drop_rate)(Dense(num,activation = 'relu',\n",
        "                    kernel_regularizer=regularizers.l2(dnn_reg_l2))(input_embed))\n",
        "    \n",
        "    #mmoe网络层\n",
        "    towers = MmoeLayer(expert_dim,n_expert,n_task)(input_embed)\n",
        "    outputs = [Dense(1,activation = 'sigmoid', kernel_regularizer=regularizers.l2(dnn_reg_l2),\n",
        "                     name = f,use_bias = True)(_t) for _t,f in zip(towers,target)]\n",
        "    inputs = [sparse_inputs[f] for f in sparse_inputs]+[varlens_inputs[f] for f in varlens_inputs]\\\n",
        "                +[dense_inputs[f] for f in dense_inputs]\n",
        "    model = Model(inputs,outputs) \n",
        "    return model"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hSbZkTbuOzYp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b830a16-bde5-4647-e96d-00db9e46d9e1"
      },
      "source": [
        "target = [\"read_comment\", \"like\", \"click_avatar\", \"forward\"]\n",
        "sparse_features = ['userid', 'feedid', 'authorid', 'bgm_song_id', 'bgm_singer_id']\n",
        "varlen_features = ['manual_tag_list','manual_keyword_list']\n",
        "dense_features = ['videoplayseconds']\n",
        "\n",
        "# 生成输入特征设置\n",
        "sparse_max_len = {f:len(encoder[f]) + 1 for f in sparse_features}\n",
        "varlens_max_len = {f:len(encoder[f]) + 1 for f in varlen_features}\n",
        "feature_names = sparse_features+varlen_features+dense_features\n",
        "\n",
        "# 构建输入数据\n",
        "train_model_input = {name: train[name] if name not in varlen_features else np.stack(train[name]) for name in feature_names } #训练模型的输入，字典类型。名称和具体值\n",
        "val_model_input = {name: val[name] if name not in varlen_features else np.stack(val[name]) for name in feature_names }\n",
        "test_model_input = {name: test[name] if name not in varlen_features else np.stack(test[name]) for name in feature_names}\n",
        "\n",
        "train_labels = [train[y].values for y in target]\n",
        "val_labels = [val[y].values for y in target]\n",
        "\n",
        "# 多余的特征删除，释放内存\n",
        "del train,val\n",
        "gc.collect()\n",
        "\n",
        "# 构建模型，训练和评估\n",
        "model = build_mmoe(sparse_features,dense_features,sparse_max_len,embed_dim = 16,expert_dim = 32,\n",
        "          n_task = 4,n_expert = 4,varlens_cols = varlen_features,varlens_max_len = varlens_max_len,\n",
        "          dnn_hidden_units = (64,64),target = target,dnn_reg_l2 = 1e-5,drop_rate = 0.1)\n",
        "\n",
        "adam = optimizers.Adam(learning_rate=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
        "model.compile(adam, loss = 'binary_crossentropy' ,metrics = [tf.keras.metrics.AUC()],)\n",
        "\n",
        "history = model.fit(train_model_input, train_labels,validation_data = (val_model_input,val_labels),\n",
        "                    batch_size=10240, epochs=4, verbose=1)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4\n",
            "656/656 [==============================] - 37s 46ms/step - loss: 0.2885 - read_comment_loss: 0.1024 - like_loss: 0.1022 - click_avatar_loss: 0.0443 - forward_loss: 0.0263 - read_comment_auc: 0.9066 - like_auc: 0.8086 - click_avatar_auc: 0.7547 - forward_auc: 0.7429 - val_loss: 0.2560 - val_read_comment_loss: 0.0938 - val_like_loss: 0.0907 - val_click_avatar_loss: 0.0369 - val_forward_loss: 0.0190 - val_read_comment_auc: 0.9195 - val_like_auc: 0.8290 - val_click_avatar_auc: 0.8163 - val_forward_auc: 0.7755\n",
            "Epoch 2/4\n",
            "656/656 [==============================] - 29s 44ms/step - loss: 0.2531 - read_comment_loss: 0.0908 - like_loss: 0.0901 - click_avatar_loss: 0.0358 - forward_loss: 0.0199 - read_comment_auc: 0.9337 - like_auc: 0.8581 - click_avatar_auc: 0.8376 - forward_auc: 0.8379 - val_loss: 0.2535 - val_read_comment_loss: 0.0931 - val_like_loss: 0.0894 - val_click_avatar_loss: 0.0364 - val_forward_loss: 0.0186 - val_read_comment_auc: 0.9245 - val_like_auc: 0.8338 - val_click_avatar_auc: 0.8166 - val_forward_auc: 0.7871\n",
            "Epoch 3/4\n",
            "656/656 [==============================] - 29s 44ms/step - loss: 0.2513 - read_comment_loss: 0.0898 - like_loss: 0.0892 - click_avatar_loss: 0.0351 - forward_loss: 0.0195 - read_comment_auc: 0.9363 - like_auc: 0.8632 - click_avatar_auc: 0.8490 - forward_auc: 0.8507 - val_loss: 0.2545 - val_read_comment_loss: 0.0933 - val_like_loss: 0.0894 - val_click_avatar_loss: 0.0364 - val_forward_loss: 0.0184 - val_read_comment_auc: 0.9241 - val_like_auc: 0.8369 - val_click_avatar_auc: 0.8237 - val_forward_auc: 0.8138\n",
            "Epoch 4/4\n",
            "656/656 [==============================] - 29s 44ms/step - loss: 0.2504 - read_comment_loss: 0.0892 - like_loss: 0.0886 - click_avatar_loss: 0.0347 - forward_loss: 0.0192 - read_comment_auc: 0.9376 - like_auc: 0.8661 - click_avatar_auc: 0.8543 - forward_auc: 0.8574 - val_loss: 0.2545 - val_read_comment_loss: 0.0928 - val_like_loss: 0.0894 - val_click_avatar_loss: 0.0363 - val_forward_loss: 0.0184 - val_read_comment_auc: 0.9241 - val_like_auc: 0.8370 - val_click_avatar_auc: 0.8223 - val_forward_auc: 0.8267\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wfDOMfjEevAW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}