process-3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MAXP 2021初赛数据探索和处理-3\n",
    "\n",
    "使用步骤1里处理好的节点的ID，来构建DGL的graph所需要的边列表。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import dgl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path\n",
    "base_path = './dataset'\n",
    "publish_path = ''\n",
    "\n",
    "link_p1_path = os.path.join(base_path, publish_path, 'link_phase1.csv')\n",
    "nodes_path = os.path.join(base_path, publish_path, 'IDandLabels.csv')\n",
    "# phase2\n",
    "link_p2_path = os.path.join(base_path, publish_path, 'link_phase2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取节点列表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5346177, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node_idx</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>Label</th>\n",
       "      <th>Split_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5346173</th>\n",
       "      <td>5346173</td>\n",
       "      <td>caed47d55d1e193ecb1fa97a415c13dd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5346174</th>\n",
       "      <td>5346174</td>\n",
       "      <td>c82eb6be79a245392fb626b9a7e1f246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5346175</th>\n",
       "      <td>5346175</td>\n",
       "      <td>926a31f6b378575204aae30b5dfa6dd3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5346176</th>\n",
       "      <td>5346176</td>\n",
       "      <td>bbace2419c3f827158ea4602f3eb35fa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         node_idx                          paper_id Label  Split_ID\n",
       "5346173   5346173  caed47d55d1e193ecb1fa97a415c13dd   NaN         1\n",
       "5346174   5346174  c82eb6be79a245392fb626b9a7e1f246   NaN         1\n",
       "5346175   5346175  926a31f6b378575204aae30b5dfa6dd3   NaN         1\n",
       "5346176   5346176  bbace2419c3f827158ea4602f3eb35fa   NaN         1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_df = pd.read_csv(nodes_path, dtype={'Label':str})\n",
    "print(nodes_df.shape)\n",
    "nodes_df.tail(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node_idx</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>Label</th>\n",
       "      <th>Split_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>bfdee5ab86ef5e68da974d48a138c28e</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>78f43b8b62f040347fec0be44e5f08bd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>a971601a0286d2701aa5cde46e63a9fd</td>\n",
       "      <td>G</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>ac4b88a72146bae66cedfd1c13e1552d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>a48c92cc8f67a8327adac7ff62d24a53</td>\n",
       "      <td>W</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   node_idx                          paper_id Label  Split_ID\n",
       "0         0  bfdee5ab86ef5e68da974d48a138c28e     S         0\n",
       "1         1  78f43b8b62f040347fec0be44e5f08bd   NaN         0\n",
       "2         2  a971601a0286d2701aa5cde46e63a9fd     G         0\n",
       "3         3  ac4b88a72146bae66cedfd1c13e1552d   NaN         0\n",
       "4         4  a48c92cc8f67a8327adac7ff62d24a53     W         0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取边列表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48159841, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_id</th>\n",
       "      <th>reference_paper_id</th>\n",
       "      <th>phase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65679d473736e72a8f12984d4c42e955</td>\n",
       "      <td>05f085e5ba961ce715912b6cb4836e96</td>\n",
       "      <td>phase2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c90b0164d72bd4f00b4c5efb7c216dc5</td>\n",
       "      <td>33c1a5ee63412996aed098f0a8f01c53</td>\n",
       "      <td>phase2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d5945e460d2d63118c6b601c7330e649</td>\n",
       "      <td>50603e852362bde38fa27ea2c9b7039b</td>\n",
       "      <td>phase2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f10da75ad1eaf16eb2ffe0d85b76b332</td>\n",
       "      <td>711ef25bdb2c2421c0131af77b3ede1d</td>\n",
       "      <td>phase1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9ac5a4327bd4f3dcb424c93ca9b84087</td>\n",
       "      <td>2d91c73304c5e8a94a0e5b4956093f71</td>\n",
       "      <td>phase1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           paper_id                reference_paper_id   phase\n",
       "0  65679d473736e72a8f12984d4c42e955  05f085e5ba961ce715912b6cb4836e96  phase2\n",
       "1  c90b0164d72bd4f00b4c5efb7c216dc5  33c1a5ee63412996aed098f0a8f01c53  phase2\n",
       "2  d5945e460d2d63118c6b601c7330e649  50603e852362bde38fa27ea2c9b7039b  phase2\n",
       "3  f10da75ad1eaf16eb2ffe0d85b76b332  711ef25bdb2c2421c0131af77b3ede1d  phase1\n",
       "4  9ac5a4327bd4f3dcb424c93ca9b84087  2d91c73304c5e8a94a0e5b4956093f71  phase1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edges_df = pd.read_csv(link_p2_path)\n",
    "print(edges_df.shape)\n",
    "edges_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# phase2\n",
    "# edges_df2 = pd.read_csv(link_p2_path)\n",
    "# print(edges_df2.shape)\n",
    "# edges_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concat phase1 and phase2\n",
    "# edges_df = pd.concat([edges_df, edges_df2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join点列表和边列表以生成从0开始的边列表\n",
    "\n",
    "DGL默认节点是从0开始，并以最大的ID为容量构建Graph，因此这里我们先构建从0开始的边列表。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48159841, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_id_x</th>\n",
       "      <th>reference_paper_id</th>\n",
       "      <th>phase</th>\n",
       "      <th>node_idx_x</th>\n",
       "      <th>Label_x</th>\n",
       "      <th>Split_ID_x</th>\n",
       "      <th>node_idx_y</th>\n",
       "      <th>paper_id_y</th>\n",
       "      <th>Label_y</th>\n",
       "      <th>Split_ID_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65679d473736e72a8f12984d4c42e955</td>\n",
       "      <td>05f085e5ba961ce715912b6cb4836e96</td>\n",
       "      <td>phase2</td>\n",
       "      <td>3926192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>985918</td>\n",
       "      <td>05f085e5ba961ce715912b6cb4836e96</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c90b0164d72bd4f00b4c5efb7c216dc5</td>\n",
       "      <td>33c1a5ee63412996aed098f0a8f01c53</td>\n",
       "      <td>phase2</td>\n",
       "      <td>3790465</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1188633</td>\n",
       "      <td>33c1a5ee63412996aed098f0a8f01c53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d5945e460d2d63118c6b601c7330e649</td>\n",
       "      <td>50603e852362bde38fa27ea2c9b7039b</td>\n",
       "      <td>phase2</td>\n",
       "      <td>4780332</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>3503589</td>\n",
       "      <td>50603e852362bde38fa27ea2c9b7039b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f10da75ad1eaf16eb2ffe0d85b76b332</td>\n",
       "      <td>711ef25bdb2c2421c0131af77b3ede1d</td>\n",
       "      <td>phase1</td>\n",
       "      <td>529879</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2364950</td>\n",
       "      <td>711ef25bdb2c2421c0131af77b3ede1d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         paper_id_x                reference_paper_id   phase  \\\n",
       "0  65679d473736e72a8f12984d4c42e955  05f085e5ba961ce715912b6cb4836e96  phase2   \n",
       "1  c90b0164d72bd4f00b4c5efb7c216dc5  33c1a5ee63412996aed098f0a8f01c53  phase2   \n",
       "2  d5945e460d2d63118c6b601c7330e649  50603e852362bde38fa27ea2c9b7039b  phase2   \n",
       "3  f10da75ad1eaf16eb2ffe0d85b76b332  711ef25bdb2c2421c0131af77b3ede1d  phase1   \n",
       "\n",
       "   node_idx_x Label_x  Split_ID_x  node_idx_y  \\\n",
       "0     3926192     NaN           1      985918   \n",
       "1     3790465     NaN           1     1188633   \n",
       "2     4780332     NaN           1     3503589   \n",
       "3      529879     NaN           0     2364950   \n",
       "\n",
       "                         paper_id_y Label_y  Split_ID_y  \n",
       "0  05f085e5ba961ce715912b6cb4836e96     NaN           0  \n",
       "1  33c1a5ee63412996aed098f0a8f01c53     NaN           0  \n",
       "2  50603e852362bde38fa27ea2c9b7039b     NaN           1  \n",
       "3  711ef25bdb2c2421c0131af77b3ede1d     NaN           0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge paper_id列\n",
    "edges = edges_df.merge(nodes_df, on='paper_id', how='left')\n",
    "# Merge reference_paper_id列\n",
    "edges = edges.merge(nodes_df, left_on='reference_paper_id', right_on='paper_id', how='left')\n",
    "print(edges.shape)\n",
    "edges.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 修改node_idx_* 列的名称作为新的node id，并只保留需要的列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src_nid</th>\n",
       "      <th>dst_nid</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>reference_paper_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3926192</td>\n",
       "      <td>985918</td>\n",
       "      <td>65679d473736e72a8f12984d4c42e955</td>\n",
       "      <td>05f085e5ba961ce715912b6cb4836e96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3790465</td>\n",
       "      <td>1188633</td>\n",
       "      <td>c90b0164d72bd4f00b4c5efb7c216dc5</td>\n",
       "      <td>33c1a5ee63412996aed098f0a8f01c53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4780332</td>\n",
       "      <td>3503589</td>\n",
       "      <td>d5945e460d2d63118c6b601c7330e649</td>\n",
       "      <td>50603e852362bde38fa27ea2c9b7039b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>529879</td>\n",
       "      <td>2364950</td>\n",
       "      <td>f10da75ad1eaf16eb2ffe0d85b76b332</td>\n",
       "      <td>711ef25bdb2c2421c0131af77b3ede1d</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   src_nid  dst_nid                          paper_id  \\\n",
       "0  3926192   985918  65679d473736e72a8f12984d4c42e955   \n",
       "1  3790465  1188633  c90b0164d72bd4f00b4c5efb7c216dc5   \n",
       "2  4780332  3503589  d5945e460d2d63118c6b601c7330e649   \n",
       "3   529879  2364950  f10da75ad1eaf16eb2ffe0d85b76b332   \n",
       "\n",
       "                 reference_paper_id  \n",
       "0  05f085e5ba961ce715912b6cb4836e96  \n",
       "1  33c1a5ee63412996aed098f0a8f01c53  \n",
       "2  50603e852362bde38fa27ea2c9b7039b  \n",
       "3  711ef25bdb2c2421c0131af77b3ede1d  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edges.rename(columns={'paper_id_x': 'paper_id', 'node_idx_x':'src_nid', 'node_idx_y':'dst_nid'}, inplace=True)\n",
    "edges = edges[['src_nid', 'dst_nid', 'paper_id', 'reference_paper_id']]\n",
    "edges.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建DGL的Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 讲源节点和目标节点转换成Numpy的NDArray\n",
    "src_nid = edges.src_nid.to_numpy()\n",
    "dst_nid = edges.dst_nid.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Graph(num_nodes=5346177, num_edges=48159841,\n",
      "      ndata_schemes={}\n",
      "      edata_schemes={})\n"
     ]
    }
   ],
   "source": [
    "# 构建一个DGL的graph\n",
    "graph = dgl.graph((src_nid, dst_nid))\n",
    "print(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存Graph为二进制格式方便后面建模时的快速读取\n",
    "graph_path = os.path.join(base_path, publish_path, 'graph.bin')\n",
    "dgl.data.utils.save_graphs(graph_path, [graph])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7d6e9f8b297c76fb07a0d121ca94fd696bee4f63bd4011f44f51b92fd8c25d8a"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 64-bit ('env_pyg2': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}