feature_engineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn import preprocessing\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import Isomap\n",
    "from sklearn.manifold import LocallyLinearEmbedding\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.decomposition import NMF\n",
    "from multiprocessing import Pool\n",
    "import time\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/LAB/yanhao/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"data/sample_train.txt\", delimiter=\"\\t\")\n",
    "valid = pd.read_csv(\"data/valid_id.txt\", delimiter=\"\\t\")\n",
    "test = pd.read_csv(\"data/test_id.txt\", delimiter=\"\\t\")\n",
    "df = pd.concat([train, test, valid], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_graph():\n",
    "    edge = pd.read_csv(\"data/dat_edge.txt\", delimiter=\"\\t\")\n",
    "    from_id = []\n",
    "    to_id = []\n",
    "    dates = []\n",
    "    nums = []\n",
    "    weights = []\n",
    "    for i, row in edge.iterrows():\n",
    "        for t in row.info.split(\",\"):\n",
    "            from_id.append(row.from_id)\n",
    "            to_id.append(row.to_id)\n",
    "\n",
    "            date, nums_weight = t.split(\":\")\n",
    "            num, weight = nums_weight.split(\"_\")\n",
    "\n",
    "            dates.append(date)\n",
    "            nums.append(num)\n",
    "            weights.append(weight)\n",
    "    graph = pd.DataFrame({\"from_id\": from_id, \"to_id\": to_id, \"date\": dates, \"num\":nums, \"weight\":weights})\n",
    "    graph.to_csv(\"data/graph\", index=False)\n",
    "    graph.weight = graph.weight + 1\n",
    "    a = graph.groupby([\"from_id\", \"to_id\"]).weight.max()\n",
    "    a = a.reset_index()\n",
    "\n",
    "    with open(\"data/graph_for_emb.txt\", \"w\") as f:\n",
    "        for i, row in a.iterrows(): \n",
    "            f.write(\"%d %d %d\\n\" % (row.from_id, row.to_id, row.weight))\n",
    "\n",
    "    graph_filter = graph[graph.from_id.isin(df.id) | graph.to_id.isin(df.id)]\n",
    "    graph_filter.to_csv(\"data/graph_filter.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "build_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 链路分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def link_analysis():\n",
    "    graph = pd.read_csv(\"data/graph\")\n",
    "\n",
    "    a = graph.groupby([\"from_id\", \"to_id\"]).weight.sum()\n",
    "    a = a.reset_index()\n",
    "\n",
    "    with open(\"data/graph_for_pagerank.txt\", \"w\") as f:\n",
    "        for i, row in a.iterrows(): \n",
    "            f.write(\"%d %d %d\\n\" % (row.from_id, row.to_id, row.weight))\n",
    "\n",
    "    G = nx.DiGraph()\n",
    "    with open(\"data/graph_for_pagerank.txt\", \"r\") as f:\n",
    "        for line in f:\n",
    "            from_id, to_id, weight = line.strip().split()\n",
    "            G.add_edge(int(from_id), int(to_id), weight=int(weight))\n",
    "\n",
    "    pr = nx.pagerank(G)\n",
    "\n",
    "    with open(\"data/edge/pagerank.plk\", \"wb\") as f:\n",
    "        pickle.dump(pr, f)\n",
    "\n",
    "    graph_filter = pd.read_csv(\"data/graph_filter.csv\")\n",
    "    graph_filter_ids = set(graph_filter.to_id.tolist()) | set(graph_filter.from_id.tolist())\n",
    "\n",
    "    h,a=nx.hits(G)\n",
    "\n",
    "    with open(\"data/edge/h.plk\", \"wb\") as f:\n",
    "        pickle.dump(h, f)\n",
    "\n",
    "    with open(\"data/edge/a.plk\", \"wb\") as f:\n",
    "        pickle.dump(a, f)\n",
    "        \n",
    "    dc = degree_centrality(G)\n",
    "    with open(\"data/edge/degree_centrality.plk\", \"wb\") as f:\n",
    "        pickle.dump(dc, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "link_analysis()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 筛选embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = pd.read_csv(\"data/graph\")\n",
    "graph_filter = graph[graph.from_id.isin(df.id) | graph.to_id.isin(df.id)]\n",
    "graph_filter.to_csv(\"data/graph_filter.csv\", index=False)\n",
    "\n",
    "graph_emb = pd.read_csv(\"data/deepwalk_192.emb\", delimiter=\" \", names=[\"id\"] + [\"dp_%d\" % i for i in range(192)], skiprows=1)\n",
    "graph_emb[graph_emb.id.isin(df.id)].to_csv(\"features/graph/deepwalk_192_filtered.emb\", index=False)\n",
    "\n",
    "graph_emb = pd.read_csv(\"data/deepwalk_128.emb\", delimiter=\" \", names=[\"id\"] + [\"dp_%d\" % i for i in range(128)], skiprows=1)\n",
    "graph_emb[graph_emb.id.isin(df.id)].to_csv(\"features/graph/deepwalk_128_filtered.emb\", index=False)\n",
    "\n",
    "graph_emb = pd.read_csv(\"data/deepwalk_256.emb\", delimiter=\" \", names=[\"id\"] + [\"dp_%d\" % i for i in range(256)], skiprows=1)\n",
    "graph_emb[graph_emb.id.isin(df.id)].to_csv(\"features/graph/deepwalk_256_filtered.emb\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基本特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def risk_feature():\n",
    "    risk = pd.read_csv(\"data/dat_risk.txt\", delimiter=\"\\t\")\n",
    "    risk['total'] = risk[[\"a_cnt\", \"b_cnt\", \"c_cnt\", \"d_cnt\", \"e_cnt\"]].sum(axis=1)\n",
    "    for c in [\"a_cnt\", \"b_cnt\", \"c_cnt\", \"d_cnt\", \"e_cnt\"]:\n",
    "        risk[c + \"_ratio\"] = risk[c] / risk.total\n",
    "    risk.to_csv(\"features/risk/risk.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def symbol_feature():\n",
    "    dat_symbol = pd.read_csv(\"data/dat_symbol.txt\", delimiter=\"\\t\")\n",
    "    dat_symbol['cat_count'] = dat_symbol.symbol.apply(lambda x: len(x.split(\",\")))\n",
    "    dat_symbol['symbol'] = dat_symbol.symbol.apply(lambda x:\" \".join([i for i in x.split(\",\")]))\n",
    "    dat_symbol['symbol_1'] = dat_symbol.symbol.apply(lambda x:\" \".join([i.split(\"_\")[0] for i in x.split(\" \")]))\n",
    "    \n",
    "    vectorizer = CountVectorizer()\n",
    "    a = vectorizer.fit_transform(dat_symbol.symbol)\n",
    "    lev2 = pd.DataFrame(a.toarray(), columns=[\"lev_2_\" + str(i) for i in range(44)])\n",
    "    b = vectorizer.fit_transform(dat_symbol.symbol_1)\n",
    "    lev1 = pd.DataFrame(b.toarray(), columns=[\"lev_1_\" + str(i) for i in range(24)])\n",
    "\n",
    "    dat_symbol = dat_symbol.join(lev1)\n",
    "    dat_symbol = dat_symbol.join(lev2)\n",
    "\n",
    "    dat_symbol.drop([\"symbol\", \"symbol_1\"], axis=1, inplace=True)\n",
    "    dat_symbol.to_csv(\"features/symbol/symbol.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def graph_feature(df, graph_filter):\n",
    "    graph_filter.date = pd.to_datetime(graph_filter.date)\n",
    "\n",
    "    out_degree = graph_filter.groupby(\"from_id\").to_id.count().reset_index().rename(columns={\"from_id\":\"id\", \"to_id\": \"out_degree\"})\n",
    "    in_degree = graph_filter.groupby(\"to_id\").from_id.count().reset_index().rename(columns={\"to_id\": \"id\", \"from_id\":\"in_degree\"})\n",
    "\n",
    "    out_num = graph_filter.groupby(\"from_id\").num.sum().reset_index().rename(columns={\"from_id\":\"id\", \"num\": \"out_sum\"})\n",
    "    in_num = graph_filter.groupby(\"to_id\").num.sum().reset_index().rename(columns={\"to_id\": \"id\", \"num\":\"in_sum\"})\n",
    "\n",
    "    in_weight = graph_filter.groupby(\"to_id\").weight.sum().reset_index().rename(columns={\"to_id\": \"id\", \"weight\":\"in_weight\"})\n",
    "    out_weight = graph_filter.groupby(\"from_id\").weight.sum().reset_index().rename(columns={\"from_id\":\"id\", \"weight\": \"out_weight\"})\n",
    "\n",
    "#     graph_filter = graph_filter.sort_values(\"date\")\n",
    "\n",
    "#     in_span = (graph_filter.groupby(\"to_id\").date.last().dt.year - graph_filter.groupby(\"to_id\").date.first().dt.year) * 12 + (graph_filter.groupby(\"to_id\").date.last().dt.month - graph_filter.groupby(\"to_id\").date.first().dt.month)\n",
    "#     out_span = (graph_filter.groupby(\"from_id\").date.last().dt.year - graph_filter.groupby(\"from_id\").date.first().dt.year) * 12 + (graph_filter.groupby(\"from_id\").date.last().dt.month - graph_filter.groupby(\"from_id\").date.first().dt.month)\n",
    "#     in_span = in_span.reset_index().rename(columns={\"to_id\":\"id\", \"date\": \"in_span\"})\n",
    "#     out_span = out_span.reset_index().rename(columns={\"from_id\":\"id\", \"date\": \"out_span\"})\n",
    "\n",
    "    in_unique = graph_filter.groupby(\"to_id\").from_id.nunique().reset_index().rename(columns={\"to_id\":\"id\", \"from_id\": \"in_nunique\"})\n",
    "    out_unique = graph_filter.groupby(\"from_id\").to_id.nunique().reset_index().rename(columns={\"from_id\":\"id\", \"to_id\": \"out_nunique\"})\n",
    "\n",
    "    graph_info = df[['id']]\n",
    "    graph_info = graph_info.merge(out_degree, on=\"id\")\n",
    "    graph_info = graph_info.merge(in_degree, on=\"id\")\n",
    "\n",
    "    graph_info = graph_info.merge(out_num, on=\"id\")\n",
    "    graph_info = graph_info.merge(in_num, on=\"id\")\n",
    "\n",
    "    graph_info = graph_info.merge(out_weight, on=\"id\")\n",
    "    graph_info = graph_info.merge(in_weight, on=\"id\")\n",
    "\n",
    "    graph_info = graph_info.merge(out_unique, on=\"id\")\n",
    "    graph_info = graph_info.merge(in_unique, on=\"id\")\n",
    "    \n",
    "    common_id = set(graph_filter.from_id.tolist()) & set(graph_filter.to_id.tolist())\n",
    "\n",
    "    from_dict = {}\n",
    "    for i in df.id:\n",
    "        from_dict[i] = set(graph_filter[graph_filter.from_id == i].to_id.values)\n",
    "\n",
    "    to_dict = {}\n",
    "    for i in df.id:\n",
    "        to_dict[i] = set(graph_filter[graph_filter.to_id == i].from_id.values)\n",
    "\n",
    "    common_id = {}\n",
    "    for i in df.id:\n",
    "        common_id[i] = from_dict[i] & to_dict[i]\n",
    "\n",
    "    graph_info['common_num'] = graph_info.id.apply(lambda x: len(common_id[x]))\n",
    "\n",
    "    graph_info.to_csv(\"features/graph/graph_info.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def app_feature(graph_filter):\n",
    "    app = pd.read_csv(\"data/dat_app.txt\", delimiter=\"\\t\", header=None, names=[\"id\", \"app_list\"])\n",
    "    app = app[app.id.isin(graph_filter.from_id) | app.id.isin(graph_filter.to_id)]\n",
    "    # app = app[app.id.isin(graph2_id & app_id)]\n",
    "    # app = app[app.id.isin(df.id)]\n",
    "\n",
    "    app['apps'] = app.app_list.apply(lambda x: \" \".join(x.split(\",\")))\n",
    "\n",
    "    d = {}\n",
    "    def count(x):\n",
    "        for i in x.split(\" \"):\n",
    "            d[i] = d.get(i, 0) + 1\n",
    "\n",
    "    _ = app.apps.apply(count)\n",
    "\n",
    "    app['app_num'] = app.apps.apply(lambda x: len(x.split(\" \")))\n",
    "    app[\"app_freq_sum\"] = app.apps.apply(lambda x: sum([d[i] for i in x.split(\" \")]))\n",
    "    app['app_num_mean'] = app.app_freq_sum / app.app_num\n",
    "\n",
    "    app['app_freq_max'] = app.apps.apply(lambda x: max([d[i] for i in x.split(\" \")]))\n",
    "    app['app_freq_min'] = app.apps.apply(lambda x: min([d[i] for i in x.split(\" \")]))\n",
    "    app['app_freq_median'] = app.apps.apply(lambda x: np.median([d[i] for i in x.split(\" \")]))\n",
    "    app['app_freq_var'] = app.apps.apply(lambda x: np.var([d[i] for i in x.split(\" \")]))\n",
    "\n",
    "    app_info = app[app.id.isin(df.id)]\n",
    "    app_info[[\"id\", \"app_num\", \"app_freq_sum\", \"app_num_mean\", \"app_freq_median\", \"app_freq_var\"]].to_csv(\"features/app/app_info.csv\", index=False)\n",
    "\n",
    "    a = pd.DataFrame({\"app\":list(d.keys()), \"count\": list(d.values())})\n",
    "    vocab = a.sort_values(\"count\", ascending=False).head(4000).app.tolist()\n",
    "    vectorizer = CountVectorizer(vocabulary=vocab)\n",
    "    vector = vectorizer.fit_transform(app.apps)\n",
    "\n",
    "    dim = 16\n",
    "    pca = PCA(n_components=dim)\n",
    "    pca_res = pca.fit_transform(vector.toarray())\n",
    "    app_pca = pd.DataFrame(pca_res, columns=[\"pca_%d\" % i for i in range(dim)])\n",
    "    app_pca[\"id\"] = app.id.values\n",
    "    app_pca.to_csv(\"features/app/app_pca_%d.csv\" % dim, index=False)\n",
    "\n",
    "    dim = 16\n",
    "    lda = LatentDirichletAllocation(n_components=dim, n_jobs=32)\n",
    "    lda_res = lda.fit_transform(vector.toarray())\n",
    "    app_lda = pd.DataFrame(lda_res, columns=[\"lda_%d\" % i for i in range(dim)])\n",
    "    app_lda[\"id\"] = app.id.values\n",
    "    app_lda.to_csv(\"features/app/app_lda_%d.csv\" % dim, index=False)\n",
    "\n",
    "    dim = 16\n",
    "    nmf = NMF(n_components=dim, init='random', random_state=0)\n",
    "    nmf_res = nmf.fit_transform(vector.toarray())\n",
    "    app_nmf = pd.DataFrame(nmf_res, columns=[\"nmf_%d\" % i for i in range(dim)])\n",
    "    app_nmf[\"id\"] = app.id.values\n",
    "    app_nmf.to_csv(\"features/app/app_nmf_%d.csv\" % dim, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_filter = pd.read_csv(\"data/graph_filter.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/LAB/yanhao/anaconda3/lib/python3.6/site-packages/sklearn/decomposition/online_lda.py:536: DeprecationWarning: The default value for 'learning_method' will be changed from 'online' to 'batch' in the release 0.20. This warning was introduced in 0.18.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "graph_feature(df, graph_filter)\n",
    "app_feature(graph_filter)\n",
    "symbol_feature()\n",
    "risk_feature()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 联系特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_with_graph(graph_filter, other_df, feature_cols, to_dir, new_col_name, func, weight_type, ids=df.id):\n",
    "    task_name = \"%s%s_%s\" % (new_col_name, weight_type, func)\n",
    "    start = time.time()\n",
    "    to_df = graph_filter.rename(columns={\"to_id\": \"id\"}).merge(other_df, on=\"id\", how=\"left\").drop(\"id\", axis=1).rename(columns={\"from_id\":\"id\"})\n",
    "    from_df = graph_filter.rename(columns={\"from_id\": \"id\"}).merge(other_df, on=\"id\", how=\"left\").drop(\"id\", axis=1).rename(columns={\"to_id\":\"id\"})\n",
    "    \n",
    "    to_df = to_df.merge(to_df.groupby(\"id\")[\"num\", \"weight\"].sum().reset_index().rename(columns={\"num\":\"num_sum_total\", \"weight\":\"weight_sum_total\"}), on=\"id\", how=\"left\")\n",
    "    from_df = from_df.merge(from_df.groupby(\"id\")[\"num\", \"weight\"].sum().reset_index().rename(columns={\"num\":\"num_sum_total\", \"weight\":\"weight_sum_total\"}), on=\"id\", how=\"left\")\n",
    "    \n",
    "    if weight_type == \"_num\":\n",
    "        for f in feature_cols:\n",
    "            to_df[f] = to_df[f] * to_df[\"num\"]\n",
    "            from_df[f] = from_df[f] * from_df[\"num\"]\n",
    "    elif weight_type == \"_weight\":\n",
    "        for f in feature_cols:\n",
    "            to_df[f] = to_df[f] * to_df[\"weight\"]\n",
    "            from_df[f] = from_df[f] * from_df[\"weight\"]\n",
    "\n",
    "    if weight_type in [\"_num\", \"_weight\"] and func == \"mean\":\n",
    "        to_df[f] /= to_df[weight_type[1:] + \"_sum_total\"]\n",
    "        from_df[f] /= from_df[weight_type[1:] + \"_sum_total\"]\n",
    "        a = to_df.groupby(\"id\")[feature_cols].agg(\"sum\").reset_index()\n",
    "        b = from_df.groupby(\"id\")[feature_cols].agg(\"sum\").reset_index()\n",
    "    else:\n",
    "        a = to_df.groupby(\"id\")[feature_cols].agg(func).reset_index()\n",
    "        b = from_df.groupby(\"id\")[feature_cols].agg(func).reset_index()\n",
    "    \n",
    "    if new_col_name == \"symbol\":\n",
    "        a['to_%s%s_count' % (new_col_name, weight_type)] = a[[c for c in a.columns if c != \"id\"]].sum(axis=1)\n",
    "        b['from_%s%s_count' % (new_col_name, weight_type)] = b[[c for c in b.columns if c != \"id\"]].sum(axis=1)\n",
    "\n",
    "    a.columns = [\"id\"] + [\"to_%s%s_%s_%d\" % (new_col_name, weight_type, func, i) for i in range(1, len(a.columns))]\n",
    "    b.columns = [\"id\"] + [\"from_%s%s_%s_%d\" % (new_col_name, weight_type, func, i) for i in range(1, len(a.columns))]\n",
    "\n",
    "    a[a.id.isin(ids)].to_csv(\"features/%s/to_%s%s_%s.csv\" % (to_dir, new_col_name, weight_type, func), index=False)\n",
    "    b[b.id.isin(ids)].to_csv(\"features/%s/from_%s%s_%s.csv\" % (to_dir, new_col_name, weight_type, func), index=False)\n",
    "    \n",
    "    end = time.time()\n",
    "    print('Task %s runs %0.2f seconds.' % (task_name, (end - start)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一度联系人"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "symbol_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat_symbol = pd.read_csv(\"features/symbol/symbol.csv\")\n",
    "lev_f = []\n",
    "for f in dat_symbol.columns:\n",
    "    if f[:5] == \"lev_1\":\n",
    "        lev_f.append(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "symbol_args_list = [\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"sum\", \"\"),\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"sum\", \"_num\"),\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"mean\", \"\"),\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"mean\", \"_num\"),\n",
    "    (graph_filter, dat_symbol, lev_f, \"symbol_graph\", \"symbol\", \"mean\", \"_weight\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "app_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_pca = pd.read_csv(\"features/app/app_pca_16.csv\")\n",
    "app_lda = pd.read_csv(\"features/app/app_lda_16.csv\")\n",
    "app_nmf = pd.read_csv(\"features/app/app_nmf_16.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_args_list = [\n",
    "    (graph_filter, app_pca, [\"pca_%d\" % i for i in range(16)], \"app_graph\", \"app_pca\", \"mean\", \"\"),\n",
    "    (graph_filter, app_lda, [\"lda_%d\" % i for i in range(16)], \"app_graph\", \"app_lda\", \"mean\", \"\"),\n",
    "    (graph_filter, app_nmf, [\"nmf_%d\" % i for i in range(16)], \"app_graph\", \"app_nmf\", \"mean\", \"\"),\n",
    "    (graph_filter, app_pca, [\"pca_%d\" % i for i in range(16)], \"app_graph\", \"app_pca\", \"mean\", \"_num\"),\n",
    "    (graph_filter, app_lda, [\"lda_%d\" % i for i in range(16)], \"app_graph\", \"app_lda\", \"mean\", \"_num\"),\n",
    "    (graph_filter, app_nmf, [\"nmf_%d\" % i for i in range(16)], \"app_graph\", \"app_nmf\", \"mean\", \"_num\"),\n",
    "    (graph_filter, app_pca, [\"pca_%d\" % i for i in range(16)], \"app_graph\", \"app_pca\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, app_lda, [\"lda_%d\" % i for i in range(16)], \"app_graph\", \"app_lda\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, app_nmf, [\"nmf_%d\" % i for i in range(16)], \"app_graph\", \"app_nmf\", \"mean\", \"_weight\")\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "risk_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pd.read_csv(\"features/risk/risk.csv\")\n",
    "risk_f = [\"a_cnt\", \"b_cnt\", \"c_cnt\", \"d_cnt\", \"e_cnt\", \"total\"]\n",
    "risk_ratio_f = [\"a_cnt_ratio\", \"b_cnt_ratio\", \"c_cnt_ratio\", \"d_cnt_ratio\", \"e_cnt_ratio\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk_args_list = [\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"mean\", \"\"),\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"mean\", \"_num\"),\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"sum\", \"\"),\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"sum\", \"_num\"),\n",
    "    (graph_filter, risk, risk_f, \"risk_graph\", \"risk\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, risk, risk_ratio_f, \"risk_graph\", \"risk_ratio\", \"mean\", \"\"),\n",
    "    (graph_filter, risk, risk_ratio_f, \"risk_graph\", \"risk_ratio\", \"mean\", \"_num\"),\n",
    "    (graph_filter, risk, risk_ratio_f, \"risk_graph\", \"risk_ratio\", \"mean\", \"_weight\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task symbol_sum runs 63.57 seconds.\n",
      "Task app_pca_mean runs 32.21 seconds.\n",
      "Task app_lda_mean runs 31.75 seconds.\n",
      "Task app_nmf_mean runs 33.76 seconds.\n",
      "Task app_pca_num_mean runs 36.23 seconds.\n",
      "Task symbol_weight_sum runs 66.53 seconds.\n",
      "Task symbol_num_sum runs 75.36 seconds.\n",
      "Task app_lda_num_mean runs 30.99 seconds.\n",
      "Task app_nmf_num_mean runs 30.29 seconds.\n",
      "Task app_pca_weight_mean runs 31.69 seconds.\n",
      "Task symbol_mean runs 78.49 seconds.\n",
      "Task app_lda_weight_mean runs 31.54 seconds.\n",
      "Task symbol_num_mean runs 76.20 seconds.\n",
      "Task app_nmf_weight_mean runs 32.91 seconds.\n",
      "Task symbol_weight_mean runs 77.83 seconds.\n",
      "Task risk_mean runs 31.79 seconds.\n",
      "Task risk_num_mean runs 36.27 seconds.\n",
      "Task risk_weight_mean runs 33.93 seconds.\n",
      "Task risk_sum runs 33.67 seconds.\n",
      "Task risk_weight_sum runs 28.49 seconds.\n",
      "Task risk_num_sum runs 33.47 seconds.\n",
      "Task risk_ratio_mean runs 34.42 seconds.\n",
      "Task risk_ratio_num_mean runs 35.89 seconds.\n",
      "Task risk_ratio_weight_mean runs 32.52 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(10)\n",
    "for args in symbol_args_list + app_args_list + risk_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = pd.read_csv(\"data/graph\")\n",
    "\n",
    "graph_filter_ids = set(graph_filter.to_id.tolist()) | set(graph_filter.from_id.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from_filtered = graph[graph.from_id.isin(graph_filter_ids)]\n",
    "to_filtered = graph[graph.to_id.isin(graph_filter_ids)]\n",
    "\n",
    "d1_to = from_filtered.groupby(\"from_id\").to_id.count()\n",
    "d1_from = to_filtered.groupby(\"to_id\").from_id.count()\n",
    "\n",
    "d1_to_sum = from_filtered.groupby(\"from_id\")[\"num\", \"weight\"].sum()\n",
    "d1_from_sum = to_filtered.groupby(\"to_id\")[\"num\", \"weight\"].sum()\n",
    "\n",
    "d1_to = d1_to.reset_index().merge(d1_to_sum.reset_index(), on=\"from_id\", how=\"left\").rename(columns={\"to_id\":\"count\"})\n",
    "d1_from = d1_from.reset_index().merge(d1_from_sum.reset_index(), on=\"to_id\", how=\"left\").rename(columns={\"from_id\":\"count\"})\n",
    "\n",
    "d1_to = d1_to.rename(columns={\"from_id\":\"id\", \"num\": \"num_sum\", \"weight\":\"weight_sum\"})\n",
    "d1_from = d1_from.rename(columns={\"to_id\":\"id\", \"num\": \"num_sum\", \"weight\":\"weight_sum\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1_to[\"num_mean\"] = d1_to.num_sum / d1_to[\"count\"]\n",
    "d1_to[\"weight_mean\"] = d1_to.weight_sum / d1_to[\"count\"]\n",
    "\n",
    "d1_from[\"num_mean\"] = d1_from.num_sum / d1_from[\"count\"]\n",
    "d1_from[\"weight_mean\"] = d1_from.weight_sum / d1_from[\"count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# d2_f = [\"count\", \"num_mean\", \"weight_mean\"]\n",
    "d2_f = [\"count\", \"num_mean\", \"weight_mean\"]\n",
    "d2_args_list = [\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"sum\", \"\"),\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"sum\", \"_num\"),\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"mean\", \"\"),\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, d1_to, d2_f, \"graph\", \"d2_to\", \"mean\", \"_num\"),\n",
    "    \n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"sum\", \"\"),\n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"sum\", \"_num\"),\n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"mean\", \"\"),\n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"mean\", \"_num\"),\n",
    "    (graph_filter, d1_from, d2_f, \"graph\", \"d2_from\", \"mean\", \"_weight\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task d2_to_sum runs 24.58 seconds.\n",
      "Task d2_to_weight_sum runs 27.43 seconds.\n",
      "Task d2_to_mean runs 26.62 seconds.\n",
      "Task d2_to_num_sum runs 29.36 seconds.\n",
      "Task d2_to_weight_mean runs 28.06 seconds.\n",
      "Task d2_to_num_mean runs 28.30 seconds.\n",
      "Task d2_from_sum runs 29.14 seconds.\n",
      "Task d2_from_weight_sum runs 28.73 seconds.\n",
      "Task d2_from_num_sum runs 28.76 seconds.\n",
      "Task d2_from_mean runs 27.15 seconds.\n",
      "Task d2_from_num_mean runs 19.73 seconds.\n",
      "Task d2_from_weight_mean runs 22.14 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(10)\n",
    "for args in d2_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PageRank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/edge/pagerank.plk\", \"rb\") as f:\n",
    "    pr = pickle.load(f)\n",
    "\n",
    "pr_df = pd.DataFrame({\"id\": list(pr.keys()), \"pr\":list(pr.values())})\n",
    "pr_df[pr_df.id.isin(df.id)].to_csv(\"features/graph/pagerank.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr_f = ['pr']\n",
    "pr_args_list = [\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"sum\", \"\", ),\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"sum\", \"_weight\", graph_filter_ids),\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"sum\", \"_num\", graph_filter_ids),\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"mean\", \"\", graph_filter_ids),\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"mean\", \"_weight\", graph_filter_ids),\n",
    "    (graph_filter, pr_df, pr_f, \"graph\", \"pagerank\", \"mean\", \"_num\", graph_filter_ids),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task pagerank_sum runs 65.19 seconds.\n",
      "Task pagerank_weight_sum runs 77.54 seconds.\n",
      "Task pagerank_num_sum runs 76.52 seconds.\n",
      "Task pagerank_mean runs 75.12 seconds.\n",
      "Task pagerank_weight_mean runs 75.42 seconds.\n",
      "Task pagerank_num_mean runs 77.08 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(6)\n",
    "for args in pr_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PageRank 二度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "gf = graph[graph.from_id.isin(graph_filter_ids) | graph.to_id.isin(graph_filter_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task pagerank_sum runs 1045.24 seconds.\n",
      "Task pagerank_weight_sum runs 1064.47 seconds.\n",
      "Task pagerank_num_sum runs 1053.86 seconds.\n",
      "Task pagerank_mean runs 1052.96 seconds.\n",
      "Task pagerank_weight_mean runs 1063.16 seconds.\n",
      "Task pagerank_num_mean runs 1066.43 seconds.\n"
     ]
    }
   ],
   "source": [
    "pr_f = ['pr']\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"sum\", \"\", graph_filter_ids)\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"sum\", \"_weight\", graph_filter_ids)\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"sum\", \"_num\", graph_filter_ids)\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"mean\", \"\", graph_filter_ids)\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"mean\", \"_weight\",graph_filter_ids)\n",
    "feature_with_graph(gf, pr_df, pr_f, \"temp\", \"pagerank\", \"mean\", \"_num\", graph_filter_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_num_mean = pd.read_csv(\"features/temp/to_pagerank_num_mean.csv\")\n",
    "from_num_mean = pd.read_csv(\"features/temp/from_pagerank_num_mean.csv\")\n",
    "\n",
    "to_weight_mean = pd.read_csv(\"features/temp/to_pagerank_weight_mean.csv\")\n",
    "from_weight_mean = pd.read_csv(\"features/temp/from_pagerank_weight_mean.csv\")\n",
    "\n",
    "to_num_sum = pd.read_csv(\"features/temp/to_pagerank_num_sum.csv\")\n",
    "from_num_sum = pd.read_csv(\"features/temp/from_pagerank_num_sum.csv\")\n",
    "\n",
    "to_weight_sum = pd.read_csv(\"features/temp/to_pagerank_weight_sum.csv\")\n",
    "from_weight_sum = pd.read_csv(\"features/temp/from_pagerank_weight_sum.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>from_pagerank_num_mean_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22</td>\n",
       "      <td>2.940365e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>8.918203e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>80</td>\n",
       "      <td>9.372466e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>91</td>\n",
       "      <td>1.578345e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>117</td>\n",
       "      <td>9.312466e-06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id  from_pagerank_num_mean_1\n",
       "0   22              2.940365e-08\n",
       "1   24              8.918203e-07\n",
       "2   80              9.372466e-07\n",
       "3   91              1.578345e-08\n",
       "4  117              9.312466e-06"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from_num_mean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr2_args_list = [\n",
    "    (graph_filter, to_num_mean, ['to_pagerank_num_mean_1'], \"graph\", \"pg_to_num_mean\", \"mean\", \"_num\"),\n",
    "    (graph_filter, from_num_mean, [\"from_pagerank_num_mean_1\"], \"graph\", \"pg_from_num_mean\", \"mean\", \"_num\"),\n",
    "    (graph_filter, to_weight_mean, ['to_pagerank_weight_mean_1'], \"graph\", \"pg_to_weight_mean\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, from_weight_mean, [\"from_pagerank_weight_mean_1\"], \"graph\", \"pg_from_weight_mean\", \"mean\", \"_weight\"),\n",
    "    \n",
    "    (graph_filter, to_num_sum, ['to_pagerank_num_sum_1'], \"graph\", \"pg_to_num_sum\", \"sum\", \"_num\"),\n",
    "    (graph_filter, from_num_sum, [\"from_pagerank_num_sum_1\"], \"graph\", \"pg_from_num_sum\", \"sum\", \"_num\"),\n",
    "    (graph_filter, to_weight_sum, ['to_pagerank_weight_sum_1'], \"graph\", \"pg_to_weight_sum\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, from_weight_sum, [\"from_pagerank_weight_sum_1\"], \"graph\", \"pg_from_weight_sum\", \"sum\", \"_weight\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task pg_to_num_mean_num_mean runs 19.40 seconds.\n",
      "Task pg_from_num_mean_num_mean runs 18.77 seconds.\n",
      "Task pg_to_weight_mean_weight_mean runs 19.28 seconds.\n",
      "Task pg_from_weight_mean_weight_mean runs 20.31 seconds.\n",
      "Task pg_from_num_sum_num_sum runs 16.08 seconds.\n",
      "Task pg_to_num_sum_num_sum runs 16.96 seconds.\n",
      "Task pg_to_weight_sum_weight_sum runs 16.75 seconds.\n",
      "Task pg_from_weight_sum_weight_sum runs 16.41 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(4)\n",
    "for args in pr2_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/edge/a.plk\", \"rb\") as f:\n",
    "    a = pickle.load(f)\n",
    "\n",
    "a_df = pd.DataFrame({\"id\": list(a.keys()), \"a\":list(a.values())})\n",
    "\n",
    "with open(\"data/edge/h.plk\", \"rb\") as f:\n",
    "    h = pickle.load(f)\n",
    "\n",
    "h_df = pd.DataFrame({\"id\": list(a.keys()), \"h\":list(h.values())})\n",
    "\n",
    "\n",
    "hits = a_df.merge(h_df, on=\"id\")\n",
    "hits[a_df.id.isin(df.id)].to_csv(\"features/graph/hits.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>a</th>\n",
       "      <th>h</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.297083e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16872051</td>\n",
       "      <td>4.546588e-17</td>\n",
       "      <td>5.140984e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.163187e-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6907348</td>\n",
       "      <td>4.136847e-21</td>\n",
       "      <td>2.879042e-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7911933</td>\n",
       "      <td>7.092638e-23</td>\n",
       "      <td>8.419130e-18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id             a             h\n",
       "0         2  0.000000e+00  2.297083e-21\n",
       "1  16872051  4.546588e-17  5.140984e-21\n",
       "2         3  0.000000e+00  1.163187e-24\n",
       "3   6907348  4.136847e-21  2.879042e-16\n",
       "4   7911933  7.092638e-23  8.419130e-18"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "hits_f = ['a', \"h\"]\n",
    "hits_args_list = [\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"sum\", \"\"),\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"sum\", \"_num\"),\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"mean\", \"\"),\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, hits, hits_f, \"graph\", \"hits\", \"mean\", \"_num\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task hits_sum runs 65.34 seconds.\n",
      "Task hits_weight_sum runs 66.21 seconds.\n",
      "Task hits_num_sum runs 68.90 seconds.\n",
      "Task hits_mean runs 64.11 seconds.\n",
      "Task hits_weight_mean runs 63.83 seconds.\n",
      "Task hits_num_mean runs 62.85 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(10)\n",
    "for args in hits_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "hits 二度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task hits_sum runs 1122.12 seconds.\n",
      "Task hits_weight_sum runs 1138.46 seconds.\n",
      "Task hits_num_sum runs 1144.98 seconds.\n",
      "Task hits_mean runs 1152.57 seconds.\n",
      "Task hits_weight_mean runs 1169.23 seconds.\n",
      "Task hits_num_mean runs 1146.93 seconds.\n"
     ]
    }
   ],
   "source": [
    "hits_f = ['a', \"h\"]\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"sum\", \"\", graph_filter_ids)\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"sum\", \"_weight\", graph_filter_ids)\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"sum\", \"_num\", graph_filter_ids)\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"mean\", \"\", graph_filter_ids)\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"mean\", \"_weight\",graph_filter_ids)\n",
    "feature_with_graph(gf, hits, hits_f, \"temp\", \"hits\", \"mean\", \"_num\", graph_filter_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_num_mean = pd.read_csv(\"features/temp/to_hits_num_mean.csv\")\n",
    "from_num_mean = pd.read_csv(\"features/temp/from_hits_num_mean.csv\")\n",
    "\n",
    "to_weight_mean = pd.read_csv(\"features/temp/to_hits_weight_mean.csv\")\n",
    "from_weight_mean = pd.read_csv(\"features/temp/from_hits_weight_mean.csv\")\n",
    "\n",
    "to_num_sum = pd.read_csv(\"features/temp/to_hits_num_sum.csv\")\n",
    "from_num_sum = pd.read_csv(\"features/temp/from_hits_num_sum.csv\")\n",
    "\n",
    "to_weight_sum = pd.read_csv(\"features/temp/to_hits_weight_sum.csv\")\n",
    "from_weight_sum = pd.read_csv(\"features/temp/from_hits_weight_sum.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "hits2_args_list = [\n",
    "    (graph_filter, to_num_mean, ['to_hits_num_mean_1', 'to_hits_num_mean_2'], \"graph\", \"hits_to_num_mean\", \"mean\", \"_num\"),\n",
    "    (graph_filter, from_num_mean, [\"from_hits_num_mean_1\", \"from_hits_num_mean_2\"], \"graph\", \"hits_from_num_mean\", \"mean\", \"_num\"),\n",
    "    (graph_filter, to_weight_mean, ['to_hits_weight_mean_1', 'to_hits_weight_mean_2'], \"graph\", \"hits_to_weight_mean\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, from_weight_mean, [\"from_hits_weight_mean_1\", \"from_hits_weight_mean_2\"], \"graph\", \"hits_from_weight_mean\", \"mean\", \"_weight\"),\n",
    "    \n",
    "    (graph_filter, to_num_sum, ['to_hits_num_sum_1', 'to_hits_num_sum_2'], \"graph\", \"hits_to_num_sum\", \"sum\", \"_num\"),\n",
    "    (graph_filter, from_num_sum, [\"from_hits_num_sum_1\", \"from_hits_num_sum_2\"], \"graph\", \"hits_from_num_sum\", \"sum\", \"_num\"),\n",
    "    (graph_filter, to_weight_sum, ['to_hits_weight_sum_1', 'to_hits_weight_sum_2'], \"graph\", \"hits_to_weight_sum\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, from_weight_sum, [\"from_hits_weight_sum_1\", \"from_hits_weight_sum_2\"], \"graph\", \"hits_from_weight_sum\", \"sum\", \"_weight\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task hits_to_num_mean_num_mean runs 21.48 seconds.\n",
      "Task hits_from_num_mean_num_mean runs 23.92 seconds.\n",
      "Task hits_from_weight_mean_weight_mean runs 23.51 seconds.\n",
      "Task hits_to_weight_mean_weight_mean runs 25.11 seconds.\n",
      "Task hits_to_num_sum_num_sum runs 23.70 seconds.\n",
      "Task hits_from_num_sum_num_sum runs 23.28 seconds.\n",
      "Task hits_to_weight_sum_weight_sum runs 22.55 seconds.\n",
      "Task hits_from_weight_sum_weight_sum runs 22.57 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "p = Pool(10)\n",
    "for args in hits2_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/edge/degree_centrality.plk\", \"rb\") as f:\n",
    "    a = pickle.load(f)\n",
    "\n",
    "a_df = pd.DataFrame({\"id\": list(a.keys()), \"a\":list(a.values())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_df[a_df.id.isin(df.id)].to_csv(\"features/graph/dc.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task dc_sum runs 62.91 seconds.\n",
      "Task dc_weight_sum runs 64.62 seconds.\n",
      "Task dc_num_sum runs 62.13 seconds.\n",
      "Task dc_mean runs 61.85 seconds.\n",
      "Task dc_weight_mean runs 61.26 seconds.\n",
      "Task dc_num_mean runs 62.25 seconds.\n",
      "All subprocesses done.\n"
     ]
    }
   ],
   "source": [
    "f = ['a']\n",
    "a_df_args_list = [\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"sum\", \"\"),\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"sum\", \"_weight\"),\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"sum\", \"_num\"),\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"mean\", \"\"),\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"mean\", \"_weight\"),\n",
    "    (graph_filter, a_df, f, \"graph\", \"dc\", \"mean\", \"_num\"),\n",
    "]\n",
    "\n",
    "p = Pool(10)\n",
    "for args in a_df_args_list:\n",
    "    p.apply_async(feature_with_graph, args)\n",
    "p.close()\n",
    "p.join()\n",
    "print('All subprocesses done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}