inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "46da2cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import lightgbm as lgbm\n",
    "import catboost as cat\n",
    "import xgboost as xgb\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5db50a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb92a5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def score(res,label,pre):\n",
    "    res['diff'] = abs((res[pre]-res[label])/7)\n",
    "    s = 1-sum(res['diff'])/len(res)\n",
    "    return s\n",
    "\n",
    "def get_label(row):\n",
    "    max_date = row['date_max']\n",
    "    min_date = row['date_min']\n",
    "    date = row['date']\n",
    "    date_list = row['date_list']\n",
    "    \n",
    "    if date+7 <= max_date:\n",
    "        return sum([1 for x in set(date_list) if date < x < date+8])\n",
    "    else:\n",
    "        if date>153:\n",
    "            if row['user_id'] not in user_enddate:\n",
    "                return -999\n",
    "            else:\n",
    "                if user_enddate[row['user_id']] < date+7:\n",
    "                    return -999\n",
    "                else:\n",
    "                    return sum([1 for x in set(date_list) if date < x < date+8])\n",
    "        elif date<130:\n",
    "            return -999\n",
    "        else:\n",
    "            return sum([1 for x in set(date_list) if date < x < date+8])\n",
    "        \n",
    "def extend_list(row):\n",
    "    date_min = row['date_min']\n",
    "    date_max = row['date_max']\n",
    "    \n",
    "    if row['user_id'] in user_enddate:\n",
    "        end_date = user_enddate[row['user_id']]\n",
    "        return list(set([x for x in range(date_min-1, end_date-6)]+interval))\n",
    "    else:\n",
    "        return list(set([x for x in range(date_min-1, date_max-6)]+interval))\n",
    "    \n",
    "    \n",
    "def get_date_list(row):\n",
    "    ls = row['date_list']\n",
    "    end_date = row['date']\n",
    "    \n",
    "    ls_new = [x for x in ls if x<=end_date]\n",
    "    return ls_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f6e63a9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "interval = [x for x in range(131,154)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19715266",
   "metadata": {},
   "source": [
    "## 处理b榜数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cdcfc0d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/app_launch_logs.csv')\n",
    "test_a = pd.read_csv('data/test_B数据集.csv')\n",
    "\n",
    "test = pd.read_csv('data/test_B数据集.csv')\n",
    "test.columns = ['user_id','date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8398371b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['user_id','date']).reset_index(drop=True)\n",
    "df = df[['user_id','date']].drop_duplicates().reset_index(drop=True)\n",
    "df = df[df['user_id'].isin(test_a['user_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "08e5d8b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_group = df.groupby('user_id').agg(list).reset_index()\n",
    "df_group['date_max'] = df_group['date'].apply(lambda x: max(x))\n",
    "df_group['date_min'] = df_group['date'].apply(lambda x: min(x))\n",
    "user_enddate = dict(zip(test_a['user_id'],test_a['end_date']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "284120a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_group['date_all'] = df_group.apply(extend_list, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "383c715d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ = df_group.explode('date_all')\n",
    "df_.rename(columns = {'date':'date_list','date_all':'date'},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6b533820",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_['label'] = df_.apply(get_label, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "58952de1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df_[df_['label']!=-999]\n",
    "train = train.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b55bafaa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "37728522",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pb = pd.read_csv('features/online_train_pb.csv')\n",
    "train = pd.merge(train, train_pb, how='left')\n",
    "\n",
    "test_pb = pd.read_csv('features/online_test_pb.csv')\n",
    "test = pd.merge(test, test_pb, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "71ea60d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_trait = pd.read_csv('features/user_trait_feature.csv')\n",
    "\n",
    "train = pd.merge(train, user_trait, how='left')\n",
    "test = pd.merge(test, user_trait, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "db31fd02",
   "metadata": {},
   "outputs": [],
   "source": [
    "launch_train = pd.read_csv('features/launch_online_train.csv')\n",
    "train = pd.merge(train, launch_train, how='left')\n",
    "\n",
    "launch_test = pd.read_csv('features/launch_online_test.csv')\n",
    "test = pd.merge(test, launch_test, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "379e531c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['week'] = train['date'].apply(lambda x: (x-130)%7+1)\n",
    "test['week'] = test['date'].apply(lambda x: (x-130)%7+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "925b1ff0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c490005e",
   "metadata": {},
   "outputs": [],
   "source": [
    "playback = pd.read_csv('data/user_playback_data.csv')\n",
    "playback = playback[playback['user_id'].isin(test['user_id'])]\n",
    "user_list =set(playback['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f15f7487",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5bfe937e",
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = ['playtime_last0', 'video_count_last0', 'playtime_last1', 'video_count_last1', 'playtime_last2', 'video_count_last2',\n",
    "         'playtime_last3', 'video_count_last3', 'playtime_last4', 'video_count_last4', 'playtime_last5', 'video_count_last5',\n",
    "         'playtime_last6', 'video_count_last6', 'playtime_last7', 'video_count_last7', 'device_type', 'sex','age', 'education',\n",
    "         'occupation_status','device_ram_new','device_rom_new','diff_near','is_launch','launch_type_new',\n",
    "         'launchNum','NumLastWeek','preds_median_30',\n",
    "         'preds_mean_4','preds_mean_4_weighted','weighted_median','week']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "cc41737d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f82b12c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_list = ['device_type','sex','age','education','occupation_status','week','is_launch','launch_type_new']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65d2a581",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "acb1c899",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pos = train[train['label']>0]\n",
    "train_neg = train[train['label']==0]\n",
    "train_neg_new = train_neg.sample(frac=0.7,random_state=2).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f9b97fef",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train_pos, train_neg_new])\n",
    "train = train.sample(frac=1,random_state=2).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c3d3c22",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b4f4e1df",
   "metadata": {},
   "outputs": [],
   "source": [
    "for each_feat in cat_list:\n",
    "    train[each_feat] = train[each_feat].fillna(0)\n",
    "    test[each_feat] = test[each_feat].fillna(0)\n",
    "    \n",
    "    train[each_feat] = train[each_feat].astype(int)\n",
    "    test[each_feat] = test[each_feat].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0e1b7e7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "370d04d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for each_feat in cat_list:\n",
    "    df_tmp = train.groupby(each_feat,as_index=False)['label'].median()\n",
    "    dict_tmp = dict(zip(df_tmp[each_feat],df_tmp['label']))\n",
    "    \n",
    "    train[each_feat+'_label'] = train[each_feat].apply(lambda x: dict_tmp[x])\n",
    "    test[each_feat+'_label'] = test[each_feat].apply(lambda x: dict_tmp[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d09dcac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_list_new = [x+'_label' for x in cat_list]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5067e628",
   "metadata": {},
   "source": [
    "## lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4370bdc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = lgbm.LGBMRegressor( objective='regression',max_depth=5, num_leaves=32, learning_rate=0.01, n_estimators=2500\n",
    "                         , reg_alpha=0.1,reg_lambda=0.1, random_state=2021, subsample = 0.8, min_child_samples=500)\n",
    "clf.fit(train[feats],train['label'],categorical_feature=cat_list)\n",
    "test['pre_lgb'] = clf.predict(test[feats])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c96dc2f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "test['pre_lgb'] = test['pre_lgb'].apply(lambda x: 0 if x<0 else x)\n",
    "test['pre_lgb'] = test['pre_lgb'].apply(lambda x: 7 if x>7 else x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "485f8405",
   "metadata": {},
   "source": [
    "## catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c1921b62",
   "metadata": {},
   "outputs": [],
   "source": [
    "cbt = cat.CatBoostRegressor(\n",
    "        iterations=500, learning_rate=0.1,\n",
    "        depth=6, l2_leaf_reg=3,\n",
    "        verbose=False,\n",
    "        random_seed=2021)\n",
    "\n",
    "cbt.fit(train[feats],train['label'],cat_features=cat_list)\n",
    "test['pre_cat'] = cbt.predict(test[feats])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "baaf5aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "test['pre_cat'] = test['pre_cat'].apply(lambda x: 0 if x<0 else x)\n",
    "test['pre_cat'] = test['pre_cat'].apply(lambda x: 7 if x>7 else x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff3fe85f",
   "metadata": {},
   "source": [
    "## xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "4a6d0f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_feats = [x for x in feats+cat_list_new if x not in cat_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ef67c7f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_xgb = xgb.XGBRegressor(max_depth=5,n_estimators=200,learning_rate=0.15,subsample=0.8,\n",
    "                          reg_alpha=0.1,reg_lambda=0.2,base_score=0, min_child_weight=5,\n",
    "                          )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b5544906",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_xgb.fit(train[new_feats],train['label'])\n",
    "test['pre_xgb'] = clf_xgb.predict(test[new_feats])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6ced773c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test['pre_xgb'] = test['pre_xgb'].apply(lambda x: 0 if x<0 else x)\n",
    "test['pre_xgb'] = test['pre_xgb'].apply(lambda x: 7 if x>7 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e426d795",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f42133ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.merge(test, df_group[['user_id','date_max']],how='left')\n",
    "res['diff_date'] = res['date'] - res['date_max']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e000748",
   "metadata": {},
   "source": [
    "## tensorflow decision forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "30b7eba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfdf = pd.read_csv('res/tfdf_gbt_online.csv')\n",
    "tfdf.columns = ['user_id','pre_gbdt']\n",
    "res = pd.merge(res, tfdf, how='left')\n",
    "res['pre_gbdt'] = res['pre_gbdt'].apply(lambda x: 0 if x<0 else x)\n",
    "res['pre_gbdt'] = res['pre_gbdt'].apply(lambda x: 7 if x>7 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a47e46c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3577acd3",
   "metadata": {},
   "source": [
    "## 后处理（大于30天未登录设为0，预测值小于0.5且没看过视频设为0）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0f50f338",
   "metadata": {},
   "outputs": [],
   "source": [
    "def revise_pre(row):\n",
    "    if row['user_id'] not in user_list and row['pre_lgb']<0.5:\n",
    "        return 0\n",
    "    else:\n",
    "        return row['pre_lgb']\n",
    "\n",
    "res.loc[res['diff_date']>=30,'pre_lgb'] = 0\n",
    "\n",
    "res['pre_lgb'] = res.apply(revise_pre, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "26de3d46",
   "metadata": {},
   "outputs": [],
   "source": [
    "def revise_pre(row):\n",
    "    if row['user_id'] not in user_list and row['pre_cat']<0.5:\n",
    "        return 0\n",
    "    else:\n",
    "        return row['pre_cat']\n",
    "\n",
    "res.loc[res['diff_date']>=30,'pre_cat'] = 0\n",
    "\n",
    "res['pre_cat'] = res.apply(revise_pre, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c7a53255",
   "metadata": {},
   "outputs": [],
   "source": [
    "def revise_pre(row):\n",
    "    if row['user_id'] not in user_list and row['pre_xgb']<0.5:\n",
    "        return 0\n",
    "    else:\n",
    "        return row['pre_xgb']\n",
    "\n",
    "res.loc[res['diff_date']>=30,'pre_xgb'] = 0\n",
    "\n",
    "res['pre_xgb'] = res.apply(revise_pre, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "8b487881",
   "metadata": {},
   "outputs": [],
   "source": [
    "def revise_pre(row):\n",
    "    if row['user_id'] not in user_list and row['pre_gbdt']<0.5:\n",
    "        return 0\n",
    "    else:\n",
    "        return row['pre_gbdt']\n",
    "\n",
    "res.loc[res['diff_date']>=30,'pre_gbdt'] = 0\n",
    "\n",
    "res['pre_gbdt'] = res.apply(revise_pre, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "149d8dac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "b33fba57",
   "metadata": {},
   "outputs": [],
   "source": [
    "res['pre_avg'] = 4/( 1/(res['pre_xgb']+0.00001)  + 1/(res['pre_lgb']+0.00001)  + 1/(res['pre_cat']+0.00001) + 1/(res['pre_gbdt']+0.00001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "be3da635",
   "metadata": {},
   "outputs": [],
   "source": [
    "res['pre_avg'] = res['pre_avg'].apply(lambda x: 7 if x>6.5 else x)\n",
    "res['pre_avg'] = res['pre_avg'].apply(lambda x: 0 if x<0.4 else x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3259a12",
   "metadata": {},
   "source": [
    "## 用CNN分类结果对标签0进行修正"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b21ef031",
   "metadata": {},
   "outputs": [],
   "source": [
    "zero = pd.read_csv('res/binary_zero.csv')\n",
    "zero.columns = ['user_id','cnn_zero']\n",
    "test = pd.merge(test, zero, how='left')\n",
    "test['rank0'] = test['cnn_zero'].rank(ascending=False)\n",
    "res_0 = test[test['rank0']<=4500]\n",
    "userid_0 = res_0['user_id']\n",
    "res.loc[res['user_id'].isin(userid_0),'pre_avg'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93b892be",
   "metadata": {},
   "source": [
    "## 用CNN分类结果对标签7进行修正"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "7b4f0d69",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn = pd.read_csv('res/binary_seven.csv')\n",
    "cnn.columns = ['user_id','cnn_binary']\n",
    "test = pd.merge(test, cnn, how='left')\n",
    "test['rank'] = test['cnn_binary'].rank(ascending=False)\n",
    "res_7 = test.loc[test['rank']<=1200]\n",
    "userid_7 = res_7['user_id']\n",
    "res.loc[res['user_id'].isin(userid_7),'pre_avg'] = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4a6f3fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "res['pre_avg'] = res['pre_avg'].apply(lambda x: round(x,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "80b5d904",
   "metadata": {},
   "outputs": [],
   "source": [
    "res.loc[res['pre_avg']<0.5,'pre_avg'] = 0\n",
    "res.loc[(res['pre_avg']>0.6)&(res['pre_avg']<1.4),'pre_avg'] = 1\n",
    "res.loc[(res['pre_avg']>1.55)&(res['pre_avg']<2.4),'pre_avg'] = 2\n",
    "res.loc[(res['pre_avg']>2.55)&(res['pre_avg']<3.4),'pre_avg'] = 3\n",
    "res.loc[(res['pre_avg']>3.55)&(res['pre_avg']<4.4),'pre_avg'] = 4\n",
    "res.loc[(res['pre_avg']>4.55)&(res['pre_avg']<5.2),'pre_avg'] = 5\n",
    "res.loc[(res['pre_avg']>5.55)&(res['pre_avg']<6.2),'pre_avg'] = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "78633820",
   "metadata": {},
   "outputs": [],
   "source": [
    "res[['user_id','pre_avg']].to_csv('res/submission.csv',index=False, header=False, float_format=\"%.2f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf688385",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cabde10",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}