Basis Datasets Generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "geological-behavior",
   "metadata": {},
   "source": [
    "In this notebook, we generate the basis datasets using which we retrofit our embeddings. Each of these basis datasets has pairs of nodes and the `similarity_value` indicating the extent of retrofitting needed between them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "individual-diamond",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import scipy.stats as stats\n",
    "import json\n",
    "from tqdm.notebook import tqdm\n",
    "import gzip\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "necessary-shopper",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filepaths\n",
    "\n",
    "# DWD Files\n",
    "DERIVED_IS_COUNTS_FILE = '../source_dataset_files/wikidata-20210215-dwd-v2/dwd_isa_class_count.compact.tsv.gz'\n",
    "LABELS_FILE = \"../source_dataset_files/wikidata-20210215-dwd-v2/labels.en.tsv.gz\"\n",
    "DESCRIPTIONS_FILE = \"../source_dataset_files/wikidata-20210215-dwd-v2/descriptions.en.tsv.gz\"\n",
    "P279_EDGES_FILE = \"../source_dataset_files/wikidata-20210215-dwd-v2/derived.P279.tsv.gz\"\n",
    "P279STAR_EDGES_FILE = \"../source_dataset_files/wikidata-20210215-dwd-v2/derived.P279star.tsv.gz\"\n",
    "CLASS_COUNTS_FILE = \"../data/supplementary_files/class-counts.tsv\"\n",
    "CLASS_TRANSITIVE_P279_COUNTS_FILE = \"../data/supplementary_files/class-counts-P279star.tsv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "chubby-vehicle",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279_CHILD_PAR_INTERM_FILE = \"../data/basis/intermediate_files/P279_ChildPar.labDesc.csv\"\n",
    "P279_CHILD_PAR_DISTILBERT_COSSIM_FILE = \"../data/basis/P279_ChildPar.all-distilroberta-v1.csv\"\n",
    "P279_CHILD_PAR_CLASSSIM_FILE = \"../data/basis/P279_ChildPar.classSim.csv\"\n",
    "\n",
    "P279_SIBLINGS_INTERM_FILE = \"../data/basis/intermediate_files/P279_Siblings.all_combinations.csv\"\n",
    "P279_SIBLINGS_DISTILBERT_COSSIM_FILE = \"../data/basis/P279_Siblings.all-distilroberta-v1.csv\"\n",
    "P279_SIBLINGS_CLASSSIM_FILE = \"../data/basis/P279_Siblings.classSim.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "parental-liverpool",
   "metadata": {},
   "outputs": [],
   "source": [
    "WORDSIM_DF = '../data/evaluation/wordsim353_with_r3.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "subsequent-phrase",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBASE_SOURCE_DIR = \"../source_dataset_files/probase/\"\n",
    "PROBASE_INTERM_FILE = \"../data/basis/intermediate_files/probase_WQnodes.csv\"\n",
    "PROBASE_FINAL_FILE = '../data/basis/intermediate_files/probase_WQnodes_subset_and_sim.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "joint-wrestling",
   "metadata": {},
   "source": [
    "# Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "respiratory-average",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71f276d6b5b642609801caf1fa94bb41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/41845781 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def fetch_labels_to_qnodes_dict():\n",
    "    labels_to_qnodes_dict = defaultdict(str)\n",
    "    with gzip.open(LABELS_FILE) as lab_f:\n",
    "        first_line = True\n",
    "        for line in tqdm(lab_f, total=41845781):\n",
    "            if first_line:\n",
    "                first_line = False\n",
    "                continue\n",
    "            line = line.decode('utf-8').strip().split()\n",
    "            str1 = line[3][1:-4]\n",
    "            qnode = line[1]\n",
    "            if str1 in labels_to_qnodes_dict:\n",
    "                if qnode < labels_to_qnodes_dict[str1]:\n",
    "                    labels_to_qnodes_dict[str1] = qnode\n",
    "            else:\n",
    "                labels_to_qnodes_dict[str1] = qnode\n",
    "    return labels_to_qnodes_dict\n",
    "labels_to_qnodes_dict = fetch_labels_to_qnodes_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "damaged-value",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_worsim_labels_to_qnodes_dict():\n",
    "    wordsim_DF = pd.read_csv(WORDSIM_DF)\n",
    "    labels_to_qnodes_dict = {row['Word 1']: row['word1_kg_id'] for _, row in wordsim_DF.iterrows()}\n",
    "    temp2 = {row['Word 2']: row['word2_kg_id'] for _, row in wordsim_DF.iterrows()}\n",
    "    labels_to_qnodes_dict.update(temp2)\n",
    "    return labels_to_qnodes_dict\n",
    "\n",
    "wordsim_labels_to_qnodes_dict = fetch_worsim_labels_to_qnodes_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "removable-number",
   "metadata": {},
   "source": [
    "# Class Counts File Construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "genuine-memorabilia",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Took almost 1.5 hours to execute\n",
    "# !zcat ../source_dataset_files/wikidata-20210215-dwd-v2/dwd_isa_class_count.compact.tsv.gz | cut -f 3 | tail -n +2 | sed -e 's/|/\\n/g' -e 's/:/\\t/g' | sort | uniq > ../data/supplementary_files/class-counts.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pressed-influence",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_counts = defaultdict(int)\n",
    "with gzip.open(P279STAR_EDGES_FILE, 'r') as f:\n",
    "    first_line = True\n",
    "    for line in tqdm(f):\n",
    "        if first_line:\n",
    "            first_line = False\n",
    "            continue\n",
    "        line = line.decode('utf-8').strip().split('\\t')\n",
    "        class_counts[line[2]] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "centered-school",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_counts_df = pd.DataFrame.from_dict(class_counts, orient='index',columns=['class_count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "catholic-criminal",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_counts_df.reset_index().rename(columns={'index': '0'}).to_csv(CLASS_TRANSITIVE_P279_COUNTS_FILE, sep='\\t', header=None, index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thorough-murray",
   "metadata": {},
   "source": [
    "# P279 Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "threatened-pacific",
   "metadata": {},
   "source": [
    "## P279 ChildPar Dataset\n",
    "\n",
    "Use descriptions wherever available, if not use labels, if not skip rows\n",
    "\n",
    "**Old Version Details:**\n",
    "\n",
    "[Wikidata OS File (Wikidata 2021-02-15 DWD version)](https://drive.google.com/drive/folders/168j3OfdVGXMTKcs6VyH8rq_p0n6w0GGj?usp=sharing)\n",
    "\n",
    "There are 721983 rows in P279 dataset, 606996 unique number of nodes in this dataset, 511841 labels and 299844 descriptions\n",
    "\n",
    "There are 373463 rows in P279 dataset s.t. both nodes have either a label or a description.     There are 373463 rows where both nodes have labels.     There are 373463 rows where both nodes have descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "polyphonic-county",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !kgtk filter -i ../data/wikidataos.for.text-embedding.tsv.gz -p \";P279;\" -o ../data/P279_dataset/wikidata-P279.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "silent-liver",
   "metadata": {},
   "outputs": [],
   "source": [
    "p279DF = pd.read_csv(P279_EDGES_FILE,sep='\\t')\n",
    "p279DFNodesSet = set(p279DF.node1.to_list() + p279DF.node2.to_list())\n",
    "labelsDF = pd.read_csv(LABELS_FILE, sep='\\t')\n",
    "descriptionsDF = pd.read_csv(DESCRIPTIONS_FILE, compression='gzip', sep='\\t')\n",
    "labelsDF = labelsDF[labelsDF.node1.apply(lambda p: p in p279DFNodesSet)]\n",
    "descriptionsDF = descriptionsDF[descriptionsDF.node1.apply(lambda p: p in p279DFNodesSet)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cubic-taxation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 3077831 rows in P279 dataset, 2503943 unique number of nodes in this dataset, 2406580 labels and 1369514 descriptions\n"
     ]
    }
   ],
   "source": [
    "print(f\"There are {len(p279DF)} rows in P279 dataset, {len(p279DFNodesSet)} unique number of nodes in this dataset, {len(labelsDF)} labels and {len(descriptionsDF)} descriptions\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "damaged-driving",
   "metadata": {},
   "outputs": [],
   "source": [
    "p279DFNew = p279DF\n",
    "temp1 = p279DFNew.set_index('node1').join(labelsDF.set_index('node1'), rsuffix='_label')\n",
    "temp1 = temp1[['id','label','node2','node2_label']]\n",
    "temp2 = temp1.reset_index().set_index('node2').join(labelsDF.set_index('node1'), rsuffix='_label2')\n",
    "temp3 = temp2[['id','node1','label','node2', 'node2_label']].rename(columns={'node2_label': 'node1_label', 'node2':'node2_label'}).reset_index().rename(columns={'index':'node2'})\n",
    "temp3 = temp3[['id','node1','node1_label','label','node2','node2_label']]\n",
    "temp3['node1_label'] = temp3['node1_label'].apply(lambda p: p[1:-4] if type(p) == str else None)\n",
    "temp3['node2_label'] = temp3['node2_label'].apply(lambda p: p[1:-4] if type(p) == str else None)\n",
    "hierDF = temp3.copy()\n",
    "\n",
    "p279DFNew = hierDF\n",
    "temp1 = p279DFNew.set_index('node1').join(descriptionsDF.set_index('node1'), rsuffix='_desc')\n",
    "temp1 = temp1[['id','label','node2','node1_label','node2_label', 'node2_desc']]\n",
    "temp2 = temp1.reset_index().set_index('node2').join(descriptionsDF.set_index('node1'), rsuffix='_desc2')\n",
    "temp3 = temp2[['id','node1','label','node2', 'node1_label', 'node2_label', 'node2_desc']].rename(columns={'node2_desc': 'node1_desc', 'node2':'node2_desc'}).reset_index().rename(columns={'index':'node2'})\n",
    "temp3['node1_desc'] = temp3['node1_desc'].apply(lambda p: p[1:-4] if type(p) == str else None)\n",
    "temp3['node2_desc'] = temp3['node2_desc'].apply(lambda p: p[1:-4] if type(p) == str else None)\n",
    "hierDF = temp3.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dress-yellow",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combineLabDesc(row, nodeNum):\n",
    "    if type(row['node'+str(nodeNum)+'_desc']) == str and type(row['node'+str(nodeNum)+'_label']) == str :\n",
    "        return row['node'+str(nodeNum)+'_label'] + ' ' + row['node'+str(nodeNum)+'_desc']\n",
    "    else:\n",
    "        return None\n",
    "hierDF['node1_labDesc'] = hierDF.apply(combineLabDesc, axis=1, args=(1,))\n",
    "hierDF['node2_labDesc'] = hierDF.apply(combineLabDesc, axis=1, args=(2,))\n",
    "hierDF = hierDF[~hierDF['node1_labDesc'].isna()]\n",
    "hierDF = hierDF[~hierDF['node2_labDesc'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "adaptive-measure",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1898589 rows in P279 dataset s.t. both nodes have either a label or a description.     There are 1898589 rows where both nodes have labels.     There are 1898589 rows where both nodes have descriptions.\n"
     ]
    }
   ],
   "source": [
    "print(f\"There are {len(hierDF)} rows in P279 dataset s.t. both nodes have either a label or a description. \\\n",
    "    There are {len(hierDF[(~hierDF.node1_label.isna()) & (~hierDF.node2_label.isna())])} rows where both nodes have labels. \\\n",
    "    There are {len(hierDF[(~hierDF.node1_desc.isna()) & (~hierDF.node2_desc.isna())])} rows where both nodes have descriptions.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "disturbed-adaptation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node2</th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>node2_desc</th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>node1_desc</th>\n",
       "      <th>node1_labDesc</th>\n",
       "      <th>node2_labDesc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>YF-16</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial prototype of the F-16 fighter aircraft</td>\n",
       "      <td>YF-16 initial prototype of the F-16 fighter ai...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16A/B Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16A/B Fighting Falcon initial series of the ...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16C/D Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>multirole series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16C/D Fighting Falcon multirole series of th...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16E/F Desert Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>export strike fighter series of the F-16 fight...</td>\n",
       "      <td>F-16E/F Desert Falcon export strike fighter se...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16 VISTA</td>\n",
       "      <td>F-16</td>\n",
       "      <td>experimental aircraft</td>\n",
       "      <td>F-16 VISTA experimental aircraft</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     node2                                 id      node1 label  \\\n",
       "1  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "2  Q100026  Q17372377-P279-Q100026-fd42bd71-0  Q17372377  P279   \n",
       "3  Q100026  Q17372444-P279-Q100026-ca0fc4bd-0  Q17372444  P279   \n",
       "4  Q100026  Q17372455-P279-Q100026-c2b1bf36-0  Q17372455  P279   \n",
       "5  Q100026   Q2029940-P279-Q100026-ceba4380-0   Q2029940  P279   \n",
       "\n",
       "                                          node2_desc              node1_label  \\\n",
       "1  1944 fighter aircraft family by Oberst R. Lehmann                    YF-16   \n",
       "2  1944 fighter aircraft family by Oberst R. Lehmann  F-16A/B Fighting Falcon   \n",
       "3  1944 fighter aircraft family by Oberst R. Lehmann  F-16C/D Fighting Falcon   \n",
       "4  1944 fighter aircraft family by Oberst R. Lehmann    F-16E/F Desert Falcon   \n",
       "5  1944 fighter aircraft family by Oberst R. Lehmann               F-16 VISTA   \n",
       "\n",
       "  node2_label                                         node1_desc  \\\n",
       "1        F-16     initial prototype of the F-16 fighter aircraft   \n",
       "2        F-16        initial series of the F-16 fighter aircraft   \n",
       "3        F-16      multirole series of the F-16 fighter aircraft   \n",
       "4        F-16  export strike fighter series of the F-16 fight...   \n",
       "5        F-16                              experimental aircraft   \n",
       "\n",
       "                                       node1_labDesc  \\\n",
       "1  YF-16 initial prototype of the F-16 fighter ai...   \n",
       "2  F-16A/B Fighting Falcon initial series of the ...   \n",
       "3  F-16C/D Fighting Falcon multirole series of th...   \n",
       "4  F-16E/F Desert Falcon export strike fighter se...   \n",
       "5                   F-16 VISTA experimental aircraft   \n",
       "\n",
       "                                       node2_labDesc  \n",
       "1  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "2  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "3  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "4  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "5  F-16 1944 fighter aircraft family by Oberst R....  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "plain-jurisdiction",
   "metadata": {},
   "source": [
    "### Filter based on transitive P279 counts\n",
    "\n",
    "6769\n",
    "(310053, 303284)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "broken-intermediate",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hierDF = pd.read_csv('../data/Master_P279_dataset/P279ChildPar_filtered.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "trained-parent",
   "metadata": {},
   "outputs": [],
   "source": [
    "transitive_P279Counts = pd.read_csv(CLASS_COUNTS_FILE, sep='\\t', header=None)\n",
    "transitive_P279Counts = transitive_P279Counts.set_index(0)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "great-engine",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_pairs_with_top_k_class_counts(transitive_P279Counts, k, hierDF):\n",
    "    transitive_P279Counts = transitive_P279Counts[transitive_P279Counts >= k]\n",
    "    transitive_P279CountsNeededSet = set(transitive_P279Counts.index.to_list())\n",
    "    hierDF_transP279_filtered = hierDF[hierDF.node2.apply(lambda p: p in transitive_P279CountsNeededSet)]\n",
    "#     print(\"No. of pairs with identical desc:\", (hierDF_transP279_filtered.node1_desc == hierDF_transP279_filtered.node2_desc).sum())\n",
    "    hierDF_transP279_filtered1 = hierDF_transP279_filtered[hierDF_transP279_filtered.node1_desc != hierDF_transP279_filtered.node2_desc]\n",
    "#     print(len(hierDF_transP279_filtered), len(hierDF_transP279_filtered1))\n",
    "    hierDF_transP279_filtered1 = hierDF_transP279_filtered1.groupby(\"node2\").apply(lambda g: g if len(g) <= 500 else g.sample(500))\n",
    "    return hierDF_transP279_filtered1, k, len(hierDF_transP279_filtered1), hierDF_transP279_filtered1.node2.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "worldwide-designation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 304654, 28511)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered1, k, cnt, n_pars = extract_pairs_with_top_k_class_counts(transitive_P279Counts, 10, hierDF)\n",
    "k, cnt, n_pars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "medical-hierarchy",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tempRes = []\n",
    "# for k in range(10000,100000,10000):\n",
    "#     _, k, cnt, n_pars = extract_pairs_with_top_k_class_counts(transitive_P279Counts, k, hierDF)\n",
    "#     tempRes.append((k, cnt, n_pars))\n",
    "# pd.DataFrame(tempRes, columns=['Thresh', 'No. of rows', 'No. of unique parents'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "peaceful-protocol",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    28511.000000\n",
       "mean        10.685490\n",
       "std         28.508339\n",
       "min          1.000000\n",
       "25%          2.000000\n",
       "50%          4.000000\n",
       "75%         10.000000\n",
       "max        500.000000\n",
       "Name: node2, dtype: float64"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered1.node2.value_counts().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "spiritual-ballot",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered1.to_csv(P279_CHILD_PAR_INTERM_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "english-alfred",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF = pd.read_csv(P279_CHILD_PAR_INTERM_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "interpreted-institute",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node2</th>\n",
       "      <th>Unnamed: 1</th>\n",
       "      <th>node2.1</th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>node2_desc</th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>node1_desc</th>\n",
       "      <th>node1_labDesc</th>\n",
       "      <th>node2_labDesc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>1</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>YF-16</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial prototype of the F-16 fighter aircraft</td>\n",
       "      <td>YF-16 initial prototype of the F-16 fighter ai...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>2</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16A/B Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16A/B Fighting Falcon initial series of the ...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>3</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16C/D Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>multirole series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16C/D Fighting Falcon multirole series of th...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>4</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16E/F Desert Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>export strike fighter series of the F-16 fight...</td>\n",
       "      <td>F-16E/F Desert Falcon export strike fighter se...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>5</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16 VISTA</td>\n",
       "      <td>F-16</td>\n",
       "      <td>experimental aircraft</td>\n",
       "      <td>F-16 VISTA experimental aircraft</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     node2  Unnamed: 1  node2.1                                 id      node1  \\\n",
       "0  Q100026           1  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279   \n",
       "1  Q100026           2  Q100026  Q17372377-P279-Q100026-fd42bd71-0  Q17372377   \n",
       "2  Q100026           3  Q100026  Q17372444-P279-Q100026-ca0fc4bd-0  Q17372444   \n",
       "3  Q100026           4  Q100026  Q17372455-P279-Q100026-c2b1bf36-0  Q17372455   \n",
       "4  Q100026           5  Q100026   Q2029940-P279-Q100026-ceba4380-0   Q2029940   \n",
       "\n",
       "  label                                         node2_desc  \\\n",
       "0  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "1  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "2  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "3  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "4  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "\n",
       "               node1_label node2_label  \\\n",
       "0                    YF-16        F-16   \n",
       "1  F-16A/B Fighting Falcon        F-16   \n",
       "2  F-16C/D Fighting Falcon        F-16   \n",
       "3    F-16E/F Desert Falcon        F-16   \n",
       "4               F-16 VISTA        F-16   \n",
       "\n",
       "                                          node1_desc  \\\n",
       "0     initial prototype of the F-16 fighter aircraft   \n",
       "1        initial series of the F-16 fighter aircraft   \n",
       "2      multirole series of the F-16 fighter aircraft   \n",
       "3  export strike fighter series of the F-16 fight...   \n",
       "4                              experimental aircraft   \n",
       "\n",
       "                                       node1_labDesc  \\\n",
       "0  YF-16 initial prototype of the F-16 fighter ai...   \n",
       "1  F-16A/B Fighting Falcon initial series of the ...   \n",
       "2  F-16C/D Fighting Falcon multirole series of th...   \n",
       "3  F-16E/F Desert Falcon export strike fighter se...   \n",
       "4                   F-16 VISTA experimental aircraft   \n",
       "\n",
       "                                       node2_labDesc  \n",
       "0  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "1  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "2  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "3  F-16 1944 fighter aircraft family by Oberst R....  \n",
       "4  F-16 1944 fighter aircraft family by Oberst R....  "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "committed-diamond",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from time import time\n",
    "import pandas as pd\n",
    "\n",
    "def getSentEmbeddings(valSeries, modelName):\n",
    "    model = SentenceTransformer(modelName, device='cuda:0')\n",
    "    start = time()\n",
    "    encodings = model.encode(valSeries.to_list(), show_progress_bar=True, batch_size=500)\n",
    "    print(time()-start,'s')\n",
    "    return encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "alpine-virus",
   "metadata": {},
   "outputs": [],
   "source": [
    "modelName = 'sentence-transformers/all-distilroberta-v1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "funny-asian",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF = hierDF.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "stunning-consolidation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF.node1_labDesc.isna().sum(), hierDF.node2_labDesc.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "warming-dinner",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip. Response 404\n",
      "SentenceTransformer-Model https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip not found. Try to create it from scratch\n",
      "Try to create Transformer Model sentence-transformers/all-distilroberta-v1 with mean pooling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149.29957628250122 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip. Response 404\n",
      "SentenceTransformer-Model https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip not found. Try to create it from scratch\n",
      "Try to create Transformer Model sentence-transformers/all-distilroberta-v1 with mean pooling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103.25607371330261 s\n"
     ]
    }
   ],
   "source": [
    "hierDF['node1_emb'] = pd.Series(list(getSentEmbeddings(hierDF.node1_labDesc, modelName)))\n",
    "hierDF['node2_emb'] = pd.Series(list(getSentEmbeddings(hierDF.node2_labDesc, modelName)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "noticed-electric",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>node2</th>\n",
       "      <th>Unnamed: 1</th>\n",
       "      <th>node2.1</th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>node2_desc</th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>node1_desc</th>\n",
       "      <th>node1_labDesc</th>\n",
       "      <th>node2_labDesc</th>\n",
       "      <th>node1_emb</th>\n",
       "      <th>node2_emb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>1</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>YF-16</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial prototype of the F-16 fighter aircraft</td>\n",
       "      <td>YF-16 initial prototype of the F-16 fighter ai...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "      <td>[-0.0003189822, -0.31257084, 0.4071587, 0.4445...</td>\n",
       "      <td>[-0.074766316, -0.20337783, 0.23880175, 0.4511...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>2</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16A/B Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>initial series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16A/B Fighting Falcon initial series of the ...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "      <td>[-0.066473946, -0.17350475, 0.24638157, 0.4571...</td>\n",
       "      <td>[-0.074766316, -0.20337783, 0.23880175, 0.4511...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>3</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16C/D Fighting Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>multirole series of the F-16 fighter aircraft</td>\n",
       "      <td>F-16C/D Fighting Falcon multirole series of th...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "      <td>[-0.110435985, -0.09184393, 0.24768817, 0.2641...</td>\n",
       "      <td>[-0.074766316, -0.20337783, 0.23880175, 0.4511...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>4</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16E/F Desert Falcon</td>\n",
       "      <td>F-16</td>\n",
       "      <td>export strike fighter series of the F-16 fight...</td>\n",
       "      <td>F-16E/F Desert Falcon export strike fighter se...</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "      <td>[-0.40679273, -0.026886985, 0.032887716, 0.641...</td>\n",
       "      <td>[-0.074766316, -0.20337783, 0.23880175, 0.4511...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>5</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>P279</td>\n",
       "      <td>1944 fighter aircraft family by Oberst R. Lehmann</td>\n",
       "      <td>F-16 VISTA</td>\n",
       "      <td>F-16</td>\n",
       "      <td>experimental aircraft</td>\n",
       "      <td>F-16 VISTA experimental aircraft</td>\n",
       "      <td>F-16 1944 fighter aircraft family by Oberst R....</td>\n",
       "      <td>[-0.18904628, 0.10738095, 0.3032565, 0.6507382...</td>\n",
       "      <td>[-0.074766316, -0.20337783, 0.23880175, 0.4511...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index    node2  Unnamed: 1  node2.1                                 id  \\\n",
       "0      0  Q100026           1  Q100026  Q17372279-P279-Q100026-beba8cd1-0   \n",
       "1      1  Q100026           2  Q100026  Q17372377-P279-Q100026-fd42bd71-0   \n",
       "2      2  Q100026           3  Q100026  Q17372444-P279-Q100026-ca0fc4bd-0   \n",
       "3      3  Q100026           4  Q100026  Q17372455-P279-Q100026-c2b1bf36-0   \n",
       "4      4  Q100026           5  Q100026   Q2029940-P279-Q100026-ceba4380-0   \n",
       "\n",
       "       node1 label                                         node2_desc  \\\n",
       "0  Q17372279  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "1  Q17372377  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "2  Q17372444  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "3  Q17372455  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "4   Q2029940  P279  1944 fighter aircraft family by Oberst R. Lehmann   \n",
       "\n",
       "               node1_label node2_label  \\\n",
       "0                    YF-16        F-16   \n",
       "1  F-16A/B Fighting Falcon        F-16   \n",
       "2  F-16C/D Fighting Falcon        F-16   \n",
       "3    F-16E/F Desert Falcon        F-16   \n",
       "4               F-16 VISTA        F-16   \n",
       "\n",
       "                                          node1_desc  \\\n",
       "0     initial prototype of the F-16 fighter aircraft   \n",
       "1        initial series of the F-16 fighter aircraft   \n",
       "2      multirole series of the F-16 fighter aircraft   \n",
       "3  export strike fighter series of the F-16 fight...   \n",
       "4                              experimental aircraft   \n",
       "\n",
       "                                       node1_labDesc  \\\n",
       "0  YF-16 initial prototype of the F-16 fighter ai...   \n",
       "1  F-16A/B Fighting Falcon initial series of the ...   \n",
       "2  F-16C/D Fighting Falcon multirole series of th...   \n",
       "3  F-16E/F Desert Falcon export strike fighter se...   \n",
       "4                   F-16 VISTA experimental aircraft   \n",
       "\n",
       "                                       node2_labDesc  \\\n",
       "0  F-16 1944 fighter aircraft family by Oberst R....   \n",
       "1  F-16 1944 fighter aircraft family by Oberst R....   \n",
       "2  F-16 1944 fighter aircraft family by Oberst R....   \n",
       "3  F-16 1944 fighter aircraft family by Oberst R....   \n",
       "4  F-16 1944 fighter aircraft family by Oberst R....   \n",
       "\n",
       "                                           node1_emb  \\\n",
       "0  [-0.0003189822, -0.31257084, 0.4071587, 0.4445...   \n",
       "1  [-0.066473946, -0.17350475, 0.24638157, 0.4571...   \n",
       "2  [-0.110435985, -0.09184393, 0.24768817, 0.2641...   \n",
       "3  [-0.40679273, -0.026886985, 0.032887716, 0.641...   \n",
       "4  [-0.18904628, 0.10738095, 0.3032565, 0.6507382...   \n",
       "\n",
       "                                           node2_emb  \n",
       "0  [-0.074766316, -0.20337783, 0.23880175, 0.4511...  \n",
       "1  [-0.074766316, -0.20337783, 0.23880175, 0.4511...  \n",
       "2  [-0.074766316, -0.20337783, 0.23880175, 0.4511...  \n",
       "3  [-0.074766316, -0.20337783, 0.23880175, 0.4511...  \n",
       "4  [-0.074766316, -0.20337783, 0.23880175, 0.4511...  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "surrounded-thinking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF.node1_emb.isna().sum(), hierDF.node2_emb.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "frozen-console",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF['similarity_value'] = hierDF.apply(lambda p: cosine_similarity(p.node1_emb.reshape(1,-1), p.node2_emb.reshape(1,-1))[0][0], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "optimum-sewing",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF[['id', 'node1', 'label', 'node2', 'similarity_value']].to_csv(P279_CHILD_PAR_DISTILBERT_COSSIM_FILE, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "selective-nashville",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    304654.000000\n",
       "mean          0.499092\n",
       "std           0.186887\n",
       "min          -0.130950\n",
       "25%           0.371610\n",
       "50%           0.499928\n",
       "75%           0.626436\n",
       "max           0.989182\n",
       "Name: similarity_value, dtype: float64"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF['similarity_value'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "interior-treat",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba6879c669da4812919bee809747ece9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/747 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # Alt scoring\n",
    "# from sentence_transformers.cross_encoder import CrossEncoder\n",
    "# crossEncModel = 'cross-encoder/stsb-roberta-large'\n",
    "# model = CrossEncoder(crossEncModel, device='cuda:3')\n",
    "# crossEncSimVals = model.predict(list(zip(hierDF.node1_labDesc.to_list(), hierDF.node2_labDesc.to_list())), show_progress_bar=True, batch_size=500)\n",
    "# hierDF['similarity_value'] = crossEncSimVals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "consolidated-librarian",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "secret-buddy",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "confirmed-canberra",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "controlled-perfume",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "through-penalty",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "broke-tolerance",
   "metadata": {},
   "source": [
    "## P279 Siblings Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "going-custom",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>node2</th>\n",
       "      <th>similarity_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>0.592530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>P279</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>0.660948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>P279</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>0.634662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>P279</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>0.552555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>P279</td>\n",
       "      <td>Q100026</td>\n",
       "      <td>0.504576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  id      node1 label    node2  \\\n",
       "0  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279  Q100026   \n",
       "1  Q17372377-P279-Q100026-fd42bd71-0  Q17372377  P279  Q100026   \n",
       "2  Q17372444-P279-Q100026-ca0fc4bd-0  Q17372444  P279  Q100026   \n",
       "3  Q17372455-P279-Q100026-c2b1bf36-0  Q17372455  P279  Q100026   \n",
       "4   Q2029940-P279-Q100026-ceba4380-0   Q2029940  P279  Q100026   \n",
       "\n",
       "   similarity_value  \n",
       "0          0.592530  \n",
       "1          0.660948  \n",
       "2          0.634662  \n",
       "3          0.552555  \n",
       "4          0.504576  "
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered = pd.read_csv(P279_CHILD_PAR_DISTILBERT_COSSIM_FILE)\n",
    "hierDF_transP279_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "normal-playback",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_left = hierDF_transP279_filtered.set_index('node2')\n",
    "hierDF_transP279_filtered_right = hierDF_transP279_filtered.copy().set_index('node2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "interpreted-elimination",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.04 s, sys: 684 ms, total: 2.72 s\n",
      "Wall time: 2.72 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hierDF_transP279_filtered_sibs = hierDF_transP279_filtered_left.join(hierDF_transP279_filtered_right, rsuffix='_right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "consolidated-hampshire",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26426178"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hierDF_transP279_filtered_sibs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "blessed-microwave",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>similarity_value</th>\n",
       "      <th>id_right</th>\n",
       "      <th>node1_right</th>\n",
       "      <th>label_right</th>\n",
       "      <th>similarity_value_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>node2</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q100026</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.592530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q100026</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.660948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q100026</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.634662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q100026</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.552555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q100026</th>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.504576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        id      node1 label  similarity_value  \\\n",
       "node2                                                                           \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279           0.59253   \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279           0.59253   \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279           0.59253   \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279           0.59253   \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279           0.59253   \n",
       "\n",
       "                                  id_right node1_right label_right  \\\n",
       "node2                                                                \n",
       "Q100026  Q17372279-P279-Q100026-beba8cd1-0   Q17372279        P279   \n",
       "Q100026  Q17372377-P279-Q100026-fd42bd71-0   Q17372377        P279   \n",
       "Q100026  Q17372444-P279-Q100026-ca0fc4bd-0   Q17372444        P279   \n",
       "Q100026  Q17372455-P279-Q100026-c2b1bf36-0   Q17372455        P279   \n",
       "Q100026   Q2029940-P279-Q100026-ceba4380-0    Q2029940        P279   \n",
       "\n",
       "         similarity_value_right  \n",
       "node2                            \n",
       "Q100026                0.592530  \n",
       "Q100026                0.660948  \n",
       "Q100026                0.634662  \n",
       "Q100026                0.552555  \n",
       "Q100026                0.504576  "
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered_sibs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "compliant-measure",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_sibs1 = hierDF_transP279_filtered_sibs.drop(columns=['label_right']).reset_index().rename(columns={'node2': 'par_node', 'node1_right': 'node2'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "anticipated-occasion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_node</th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>similarity_value</th>\n",
       "      <th>id_right</th>\n",
       "      <th>node2</th>\n",
       "      <th>similarity_value_right</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>0.592530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372377-P279-Q100026-fd42bd71-0</td>\n",
       "      <td>Q17372377</td>\n",
       "      <td>0.660948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372444-P279-Q100026-ca0fc4bd-0</td>\n",
       "      <td>Q17372444</td>\n",
       "      <td>0.634662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q17372455-P279-Q100026-c2b1bf36-0</td>\n",
       "      <td>Q17372455</td>\n",
       "      <td>0.552555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q100026</td>\n",
       "      <td>Q17372279-P279-Q100026-beba8cd1-0</td>\n",
       "      <td>Q17372279</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.59253</td>\n",
       "      <td>Q2029940-P279-Q100026-ceba4380-0</td>\n",
       "      <td>Q2029940</td>\n",
       "      <td>0.504576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  par_node                                 id      node1 label  \\\n",
       "0  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "1  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "2  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "3  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "4  Q100026  Q17372279-P279-Q100026-beba8cd1-0  Q17372279  P279   \n",
       "\n",
       "   similarity_value                           id_right      node2  \\\n",
       "0           0.59253  Q17372279-P279-Q100026-beba8cd1-0  Q17372279   \n",
       "1           0.59253  Q17372377-P279-Q100026-fd42bd71-0  Q17372377   \n",
       "2           0.59253  Q17372444-P279-Q100026-ca0fc4bd-0  Q17372444   \n",
       "3           0.59253  Q17372455-P279-Q100026-c2b1bf36-0  Q17372455   \n",
       "4           0.59253   Q2029940-P279-Q100026-ceba4380-0   Q2029940   \n",
       "\n",
       "   similarity_value_right  \n",
       "0                0.592530  \n",
       "1                0.660948  \n",
       "2                0.634662  \n",
       "3                0.552555  \n",
       "4                0.504576  "
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered_sibs1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "steady-jacket",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26426178"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hierDF_transP279_filtered_sibs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "italian-slovak",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_sibs1 = hierDF_transP279_filtered_sibs1[hierDF_transP279_filtered_sibs1.node1 != hierDF_transP279_filtered_sibs1.node2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "digital-oxygen",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26121338"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hierDF_transP279_filtered_sibs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "fewer-rebel",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_sibs1.to_csv(P279_SIBLINGS_INTERM_FILE, index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "failing-accent",
   "metadata": {},
   "source": [
    "### Add labels + descs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "controlled-width",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils.random import sample_without_replacement\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "prompt-grade",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000000\n"
     ]
    }
   ],
   "source": [
    "hierDF_transP279_filtered_sibs = pd.read_csv(P279_SIBLINGS_INTERM_FILE)\n",
    "hierDF_transP279_filtered_sibs1 = hierDF_transP279_filtered_sibs.iloc[sample_without_replacement(len(hierDF_transP279_filtered_sibs), 1000000, random_state=13)]\n",
    "hierDF_transP279_filtered_sibs1 = hierDF_transP279_filtered_sibs1.reset_index()\n",
    "print(len(hierDF_transP279_filtered_sibs1))\n",
    "labelsDF = pd.read_csv(LABELS_FILE, compression='gzip', sep='\\t')\n",
    "hierDF_transP279_filtered_set = set(hierDF_transP279_filtered_sibs1.node1.to_list() + hierDF_transP279_filtered_sibs1.node2.to_list() + hierDF_transP279_filtered_sibs1.par_node.to_list())\n",
    "labelsDF = labelsDF[labelsDF.node1.apply(lambda p: p in hierDF_transP279_filtered_set)]\n",
    "labelsDict = {row['node1']: row['node2'] for _, row in labelsDF.iterrows() if row['node1'] in hierDF_transP279_filtered_set}\n",
    "descriptionsDF = pd.read_csv(DESCRIPTIONS_FILE, compression='gzip', sep='\\t')\n",
    "descriptionsDF = descriptionsDF[descriptionsDF.node1.apply(lambda p: p in hierDF_transP279_filtered_set)]\n",
    "descDict = {row['node1']: row['node2'] for _, row in descriptionsDF.iterrows() if row['node1'] in hierDF_transP279_filtered_set}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "american-porter",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_sibs1['node1_label'] = hierDF_transP279_filtered_sibs1.node1.apply(lambda p: labelsDict[p][1:-4] if p in labelsDict else None)\n",
    "hierDF_transP279_filtered_sibs1['node2_label'] = hierDF_transP279_filtered_sibs1.node2.apply(lambda p: labelsDict[p][1:-4] if p in labelsDict else None)\n",
    "hierDF_transP279_filtered_sibs1['par_label'] = hierDF_transP279_filtered_sibs1.par_node.apply(lambda p: labelsDict[p][1:-4] if p in labelsDict else None)\n",
    "hierDF_transP279_filtered_sibs1['node1_desc'] = hierDF_transP279_filtered_sibs1.node1.apply(lambda p: descDict[p][1:-4] if p in descDict else None)\n",
    "hierDF_transP279_filtered_sibs1['node2_desc'] = hierDF_transP279_filtered_sibs1.node2.apply(lambda p: descDict[p][1:-4] if p in descDict else None)\n",
    "hierDF_transP279_filtered_sibs1['node1_sent'] = hierDF_transP279_filtered_sibs1.apply(lambda p: p['node1_label'] + ' ' + p['node1_desc'] + ' is ' + p['par_label'],axis=1)\n",
    "hierDF_transP279_filtered_sibs1['node2_sent'] = hierDF_transP279_filtered_sibs1.apply(lambda p: p['node2_label'] + ' ' + p['node2_desc'] + ' is ' + p['par_label'],axis=1)\n",
    "hierDF_transP279_filtered_sibs2 = hierDF_transP279_filtered_sibs1[hierDF_transP279_filtered_sibs1.node1_desc != hierDF_transP279_filtered_sibs1.node2_desc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "official-separation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset reduced from 1000000 to 785418 by eliminating rows where node1_desc == node2_desc\n"
     ]
    }
   ],
   "source": [
    "print(f\"Dataset reduced from {len(hierDF_transP279_filtered_sibs1)} to {len(hierDF_transP279_filtered_sibs2)} by eliminating rows where node1_desc == node2_desc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "weighted-mirror",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>par_node</th>\n",
       "      <th>id</th>\n",
       "      <th>node1</th>\n",
       "      <th>label</th>\n",
       "      <th>similarity_value</th>\n",
       "      <th>id_right</th>\n",
       "      <th>node2</th>\n",
       "      <th>similarity_value_right</th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>par_label</th>\n",
       "      <th>node1_desc</th>\n",
       "      <th>node2_desc</th>\n",
       "      <th>node1_sent</th>\n",
       "      <th>node2_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24998287</td>\n",
       "      <td>Q89</td>\n",
       "      <td>Q96627253-P279-Q89-51483924-0</td>\n",
       "      <td>Q96627253</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.460166</td>\n",
       "      <td>Q595003-P279-Q89-ddc59387-0</td>\n",
       "      <td>Q595003</td>\n",
       "      <td>0.553182</td>\n",
       "      <td>Sevillana (manzana)</td>\n",
       "      <td>Calville Blanc d\\'hiver</td>\n",
       "      <td>apple</td>\n",
       "      <td>apple cultivar</td>\n",
       "      <td>apple cultivar</td>\n",
       "      <td>Sevillana (manzana) apple cultivar is apple</td>\n",
       "      <td>Calville Blanc d\\'hiver apple cultivar is apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6348351</td>\n",
       "      <td>Q17589470</td>\n",
       "      <td>Q56942-P279-Q17589470-d0932570-0</td>\n",
       "      <td>Q56942</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.527086</td>\n",
       "      <td>Q69974281-P279-Q17589470-ed5eb339-0</td>\n",
       "      <td>Q69974281</td>\n",
       "      <td>0.466616</td>\n",
       "      <td>Wii U</td>\n",
       "      <td>Gamatic 7600</td>\n",
       "      <td>home video game console</td>\n",
       "      <td>eight-generation home video game console by Ni...</td>\n",
       "      <td>Pong console. The same of Otron Gamatic 7600</td>\n",
       "      <td>Wii U eight-generation home video game console...</td>\n",
       "      <td>Gamatic 7600 Pong console. The same of Otron G...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3244723</td>\n",
       "      <td>Q13357858</td>\n",
       "      <td>Q63845311-P279-Q13357858-c40a679c-0</td>\n",
       "      <td>Q63845311</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.695520</td>\n",
       "      <td>Q79132633-P279-Q13357858-4ccf52ce-0</td>\n",
       "      <td>Q79132633</td>\n",
       "      <td>0.679767</td>\n",
       "      <td>2013 World Senior Badminton Championships O35</td>\n",
       "      <td>Slovak Badminton Championships U17</td>\n",
       "      <td>badminton tournament</td>\n",
       "      <td>badminton championships</td>\n",
       "      <td>badminton championships</td>\n",
       "      <td>2013 World Senior Badminton Championships O35 ...</td>\n",
       "      <td>Slovak Badminton Championships U17 badminton c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5923963</td>\n",
       "      <td>Q174736</td>\n",
       "      <td>Q583164-P279-Q174736-262a2fe0-0</td>\n",
       "      <td>Q583164</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.459580</td>\n",
       "      <td>Q2692256-P279-Q174736-e1333ac9-0</td>\n",
       "      <td>Q2692256</td>\n",
       "      <td>0.592024</td>\n",
       "      <td>Project 30bis</td>\n",
       "      <td>Bagley-class destroyer</td>\n",
       "      <td>destroyer</td>\n",
       "      <td>1949 Soviet destroyer class</td>\n",
       "      <td>class of U.S. destroyers</td>\n",
       "      <td>Project 30bis 1949 Soviet destroyer class is d...</td>\n",
       "      <td>Bagley-class destroyer class of U.S. destroyer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10867070</td>\n",
       "      <td>Q215627</td>\n",
       "      <td>Q19840821-P279-Q215627-eb7ec03a-0</td>\n",
       "      <td>Q19840821</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.201813</td>\n",
       "      <td>Q40554819-P279-Q215627-95d720c4-0</td>\n",
       "      <td>Q40554819</td>\n",
       "      <td>0.104633</td>\n",
       "      <td>Singaporeans</td>\n",
       "      <td>Sicilians</td>\n",
       "      <td>person</td>\n",
       "      <td>citizens or residents of Singapore</td>\n",
       "      <td>ethnic group indigenous to the island of Sicil...</td>\n",
       "      <td>Singaporeans citizens or residents of Singapor...</td>\n",
       "      <td>Sicilians ethnic group indigenous to the islan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>15478058</td>\n",
       "      <td>Q3382119</td>\n",
       "      <td>Q75086913-P279-Q3382119-a9ff163a-0</td>\n",
       "      <td>Q75086913</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.769255</td>\n",
       "      <td>Q75828417-P279-Q3382119-59b8950c-0</td>\n",
       "      <td>Q75828417</td>\n",
       "      <td>0.621310</td>\n",
       "      <td>Sub-prefect of Nogent-le-Rotrou</td>\n",
       "      <td>Sub-prefect of Pont-Audemer</td>\n",
       "      <td>sub-prefect</td>\n",
       "      <td>French official position</td>\n",
       "      <td>former French official position (1800-1926)</td>\n",
       "      <td>Sub-prefect of Nogent-le-Rotrou French officia...</td>\n",
       "      <td>Sub-prefect of Pont-Audemer former French offi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999996</th>\n",
       "      <td>23394433</td>\n",
       "      <td>Q785745</td>\n",
       "      <td>Q23022056-P279-Q785745-f48ca89b-0</td>\n",
       "      <td>Q23022056</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.405016</td>\n",
       "      <td>Q3511974-P279-Q785745-42da354d-0</td>\n",
       "      <td>Q3511974</td>\n",
       "      <td>0.306730</td>\n",
       "      <td>Tasmanian Government Railways J class</td>\n",
       "      <td>Alsace-Lorraine E 6</td>\n",
       "      <td>tank locomotive</td>\n",
       "      <td>class of 1 Australian 2-6-4-0T locomotive</td>\n",
       "      <td>class of 13 German (later French) metre-gauge ...</td>\n",
       "      <td>Tasmanian Government Railways J class class of...</td>\n",
       "      <td>Alsace-Lorraine E 6 class of 13 German (later ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999997</th>\n",
       "      <td>19035228</td>\n",
       "      <td>Q483373</td>\n",
       "      <td>Q682971-P279-Q483373-6ff4adf5-0</td>\n",
       "      <td>Q682971</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.314112</td>\n",
       "      <td>Q4598091-P279-Q483373-fd15b986-0</td>\n",
       "      <td>Q4598091</td>\n",
       "      <td>0.213652</td>\n",
       "      <td>Rhaetian Railway ABe 4/16</td>\n",
       "      <td>CTA 2000 series</td>\n",
       "      <td>electric multiple unit</td>\n",
       "      <td>multiple unit</td>\n",
       "      <td>class of Chicago Transit Authority cars</td>\n",
       "      <td>Rhaetian Railway ABe 4/16 multiple unit is ele...</td>\n",
       "      <td>CTA 2000 series class of Chicago Transit Autho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999998</th>\n",
       "      <td>20618370</td>\n",
       "      <td>Q625151</td>\n",
       "      <td>Q841350-P279-Q625151-bd2c0728-0</td>\n",
       "      <td>Q841350</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.459560</td>\n",
       "      <td>Q463577-P279-Q625151-4cfe245b-0</td>\n",
       "      <td>Q463577</td>\n",
       "      <td>0.466203</td>\n",
       "      <td>OSE class 120</td>\n",
       "      <td>BLS Re 4/4</td>\n",
       "      <td>electric locomotive</td>\n",
       "      <td>class of Greek electric locomotives</td>\n",
       "      <td>class of 35 Swiss electric locomoties</td>\n",
       "      <td>OSE class 120 class of Greek electric locomoti...</td>\n",
       "      <td>BLS Re 4/4 class of 35 Swiss electric locomoti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999999</th>\n",
       "      <td>2023217</td>\n",
       "      <td>Q11513337</td>\n",
       "      <td>Q13382122-P279-Q11513337-2d44cd4f-0</td>\n",
       "      <td>Q13382122</td>\n",
       "      <td>P279</td>\n",
       "      <td>0.559249</td>\n",
       "      <td>Q13854733-P279-Q11513337-8c2aca91-0</td>\n",
       "      <td>Q13854733</td>\n",
       "      <td>0.626328</td>\n",
       "      <td>high jumper</td>\n",
       "      <td>steeplechase runner</td>\n",
       "      <td>athletics competitor</td>\n",
       "      <td>sportsperson taking part in high jump competit...</td>\n",
       "      <td>sportsperson taking part in steeplechasing com...</td>\n",
       "      <td>high jumper sportsperson taking part in high j...</td>\n",
       "      <td>steeplechase runner sportsperson taking part i...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           index   par_node                                   id      node1  \\\n",
       "0       24998287        Q89        Q96627253-P279-Q89-51483924-0  Q96627253   \n",
       "1        6348351  Q17589470     Q56942-P279-Q17589470-d0932570-0     Q56942   \n",
       "2        3244723  Q13357858  Q63845311-P279-Q13357858-c40a679c-0  Q63845311   \n",
       "3        5923963    Q174736      Q583164-P279-Q174736-262a2fe0-0    Q583164   \n",
       "4       10867070    Q215627    Q19840821-P279-Q215627-eb7ec03a-0  Q19840821   \n",
       "...          ...        ...                                  ...        ...   \n",
       "999995  15478058   Q3382119   Q75086913-P279-Q3382119-a9ff163a-0  Q75086913   \n",
       "999996  23394433    Q785745    Q23022056-P279-Q785745-f48ca89b-0  Q23022056   \n",
       "999997  19035228    Q483373      Q682971-P279-Q483373-6ff4adf5-0    Q682971   \n",
       "999998  20618370    Q625151      Q841350-P279-Q625151-bd2c0728-0    Q841350   \n",
       "999999   2023217  Q11513337  Q13382122-P279-Q11513337-2d44cd4f-0  Q13382122   \n",
       "\n",
       "       label  similarity_value                             id_right  \\\n",
       "0       P279          0.460166          Q595003-P279-Q89-ddc59387-0   \n",
       "1       P279          0.527086  Q69974281-P279-Q17589470-ed5eb339-0   \n",
       "2       P279          0.695520  Q79132633-P279-Q13357858-4ccf52ce-0   \n",
       "3       P279          0.459580     Q2692256-P279-Q174736-e1333ac9-0   \n",
       "4       P279          0.201813    Q40554819-P279-Q215627-95d720c4-0   \n",
       "...      ...               ...                                  ...   \n",
       "999995  P279          0.769255   Q75828417-P279-Q3382119-59b8950c-0   \n",
       "999996  P279          0.405016     Q3511974-P279-Q785745-42da354d-0   \n",
       "999997  P279          0.314112     Q4598091-P279-Q483373-fd15b986-0   \n",
       "999998  P279          0.459560      Q463577-P279-Q625151-4cfe245b-0   \n",
       "999999  P279          0.559249  Q13854733-P279-Q11513337-8c2aca91-0   \n",
       "\n",
       "            node2  similarity_value_right  \\\n",
       "0         Q595003                0.553182   \n",
       "1       Q69974281                0.466616   \n",
       "2       Q79132633                0.679767   \n",
       "3        Q2692256                0.592024   \n",
       "4       Q40554819                0.104633   \n",
       "...           ...                     ...   \n",
       "999995  Q75828417                0.621310   \n",
       "999996   Q3511974                0.306730   \n",
       "999997   Q4598091                0.213652   \n",
       "999998    Q463577                0.466203   \n",
       "999999  Q13854733                0.626328   \n",
       "\n",
       "                                          node1_label  \\\n",
       "0                                 Sevillana (manzana)   \n",
       "1                                               Wii U   \n",
       "2       2013 World Senior Badminton Championships O35   \n",
       "3                                       Project 30bis   \n",
       "4                                        Singaporeans   \n",
       "...                                               ...   \n",
       "999995                Sub-prefect of Nogent-le-Rotrou   \n",
       "999996          Tasmanian Government Railways J class   \n",
       "999997                      Rhaetian Railway ABe 4/16   \n",
       "999998                                  OSE class 120   \n",
       "999999                                    high jumper   \n",
       "\n",
       "                               node2_label                par_label  \\\n",
       "0                  Calville Blanc d\\'hiver                    apple   \n",
       "1                             Gamatic 7600  home video game console   \n",
       "2       Slovak Badminton Championships U17     badminton tournament   \n",
       "3                   Bagley-class destroyer                destroyer   \n",
       "4                                Sicilians                   person   \n",
       "...                                    ...                      ...   \n",
       "999995         Sub-prefect of Pont-Audemer              sub-prefect   \n",
       "999996                 Alsace-Lorraine E 6          tank locomotive   \n",
       "999997                     CTA 2000 series   electric multiple unit   \n",
       "999998                          BLS Re 4/4      electric locomotive   \n",
       "999999                 steeplechase runner     athletics competitor   \n",
       "\n",
       "                                               node1_desc  \\\n",
       "0                                          apple cultivar   \n",
       "1       eight-generation home video game console by Ni...   \n",
       "2                                 badminton championships   \n",
       "3                             1949 Soviet destroyer class   \n",
       "4                      citizens or residents of Singapore   \n",
       "...                                                   ...   \n",
       "999995                           French official position   \n",
       "999996          class of 1 Australian 2-6-4-0T locomotive   \n",
       "999997                                      multiple unit   \n",
       "999998                class of Greek electric locomotives   \n",
       "999999  sportsperson taking part in high jump competit...   \n",
       "\n",
       "                                               node2_desc  \\\n",
       "0                                          apple cultivar   \n",
       "1            Pong console. The same of Otron Gamatic 7600   \n",
       "2                                 badminton championships   \n",
       "3                                class of U.S. destroyers   \n",
       "4       ethnic group indigenous to the island of Sicil...   \n",
       "...                                                   ...   \n",
       "999995        former French official position (1800-1926)   \n",
       "999996  class of 13 German (later French) metre-gauge ...   \n",
       "999997            class of Chicago Transit Authority cars   \n",
       "999998              class of 35 Swiss electric locomoties   \n",
       "999999  sportsperson taking part in steeplechasing com...   \n",
       "\n",
       "                                               node1_sent  \\\n",
       "0             Sevillana (manzana) apple cultivar is apple   \n",
       "1       Wii U eight-generation home video game console...   \n",
       "2       2013 World Senior Badminton Championships O35 ...   \n",
       "3       Project 30bis 1949 Soviet destroyer class is d...   \n",
       "4       Singaporeans citizens or residents of Singapor...   \n",
       "...                                                   ...   \n",
       "999995  Sub-prefect of Nogent-le-Rotrou French officia...   \n",
       "999996  Tasmanian Government Railways J class class of...   \n",
       "999997  Rhaetian Railway ABe 4/16 multiple unit is ele...   \n",
       "999998  OSE class 120 class of Greek electric locomoti...   \n",
       "999999  high jumper sportsperson taking part in high j...   \n",
       "\n",
       "                                               node2_sent  \n",
       "0         Calville Blanc d\\'hiver apple cultivar is apple  \n",
       "1       Gamatic 7600 Pong console. The same of Otron G...  \n",
       "2       Slovak Badminton Championships U17 badminton c...  \n",
       "3       Bagley-class destroyer class of U.S. destroyer...  \n",
       "4       Sicilians ethnic group indigenous to the islan...  \n",
       "...                                                   ...  \n",
       "999995  Sub-prefect of Pont-Audemer former French offi...  \n",
       "999996  Alsace-Lorraine E 6 class of 13 German (later ...  \n",
       "999997  CTA 2000 series class of Chicago Transit Autho...  \n",
       "999998  BLS Re 4/4 class of 35 Swiss electric locomoti...  \n",
       "999999  steeplechase runner sportsperson taking part i...  \n",
       "\n",
       "[1000000 rows x 16 columns]"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered_sibs1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "third-canon",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip. Response 404\n",
      "SentenceTransformer-Model https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip not found. Try to create it from scratch\n",
      "Try to create Transformer Model sentence-transformers/all-distilroberta-v1 with mean pooling\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff153c8a9ba8458a8c4edf70cee2648d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/786 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "693.1311120986938 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip. Response 404\n",
      "SentenceTransformer-Model https://sbert.net/models/sentence-transformers/all-distilroberta-v1.zip not found. Try to create it from scratch\n",
      "Try to create Transformer Model sentence-transformers/all-distilroberta-v1 with mean pooling\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4beaeda275a64f0b867ab8f25b138435",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/786 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "688.8171112537384 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count    785418.000000\n",
       "mean          0.512485\n",
       "std           0.160710\n",
       "min          -0.145408\n",
       "25%           0.399946\n",
       "50%           0.515756\n",
       "75%           0.632409\n",
       "max           0.997993\n",
       "Name: similarity_value, dtype: float64"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from time import time\n",
    "import pandas as pd\n",
    "\n",
    "def getSentEmbeddings(valSeries, modelName):\n",
    "    model = SentenceTransformer(modelName, device='cuda:0')\n",
    "    start = time()\n",
    "    encodings = model.encode(valSeries.to_list(), show_progress_bar=True, batch_size=1000)\n",
    "    print(time()-start,'s')\n",
    "    return encodings\n",
    "modelName = 'sentence-transformers/all-distilroberta-v1'\n",
    "hierDF_transP279_filtered_sibs2 = hierDF_transP279_filtered_sibs2.reset_index()\n",
    "hierDF_transP279_filtered_sibs2['node1_emb'] = pd.Series(list(getSentEmbeddings(hierDF_transP279_filtered_sibs2.node1_sent, modelName)))\n",
    "hierDF_transP279_filtered_sibs2['node2_emb'] = pd.Series(list(getSentEmbeddings(hierDF_transP279_filtered_sibs2.node2_sent, modelName)))\n",
    "hierDF_transP279_filtered_sibs2['similarity_value'] = hierDF_transP279_filtered_sibs2.apply(lambda p: cosine_similarity(p.node1_emb.reshape(1,-1), p.node2_emb.reshape(1,-1))[0][0], axis=1)\n",
    "hierDF_transP279_filtered_sibs2['similarity_value'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "invalid-yahoo",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered_sibs2[['id', 'node1', 'label', 'node2', 'par_node', 'similarity_value']].to_csv(P279_SIBLINGS_DISTILBERT_COSSIM_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "infrared-success",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a35cdeb3c28e445dbfb84b318876a709",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/629 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "432a9ea31712417b8b6910e4b16c2008",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.42G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fea59ce42344dfaa116390c720ade24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4f09ee455fd4edcb3d7cde46fa6b141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ed068be329844b696662aaa852d938c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/772 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84eafe207915471b804c37f346d69474",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/139 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56aa42aa7cbb4980992bbe5feb3078db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/3708 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # Alt scoring\n",
    "# from sentence_transformers.cross_encoder import CrossEncoder\n",
    "# crossEncModel = 'cross-encoder/stsb-roberta-large'\n",
    "# model = CrossEncoder(crossEncModel, device='cuda:3')\n",
    "# crossEncSimVals = model.predict(list(zip(hierDF_transP279_filtered_sibs2.node1_sent.to_list(), hierDF_transP279_filtered_sibs2.node2_sent.to_list())), show_progress_bar=True, batch_size=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "olive-exercise",
   "metadata": {},
   "source": [
    "# Probase Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "frozen-warrant",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>no_of_relations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>factor</td>\n",
       "      <td>age</td>\n",
       "      <td>35167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>free rich company datum</td>\n",
       "      <td>size</td>\n",
       "      <td>33222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>free rich company datum</td>\n",
       "      <td>revenue</td>\n",
       "      <td>33185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>state</td>\n",
       "      <td>california</td>\n",
       "      <td>18062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>supplement</td>\n",
       "      <td>msm glucosamine sulfate</td>\n",
       "      <td>15942</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               node1_label              node2_label  no_of_relations\n",
       "0                   factor                      age            35167\n",
       "1  free rich company datum                     size            33222\n",
       "2  free rich company datum                  revenue            33185\n",
       "3                    state               california            18062\n",
       "4               supplement  msm glucosamine sulfate            15942"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probDF = pd.read_csv(PROBASE_SOURCE_DIR + 'data-concept/data-concept-instance-relations.txt',header=None, sep='\\t')\n",
    "probDF.columns=['node1_label', 'node2_label', 'no_of_relations']\n",
    "probDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "multiple-official",
   "metadata": {},
   "outputs": [],
   "source": [
    "probDF['node1'] = probDF.node1_label.apply(lambda p: labels_to_qnodes_dict[p] if p in labels_to_qnodes_dict else None)\n",
    "probDF['node2'] = probDF.node2_label.apply(lambda p: labels_to_qnodes_dict[p] if p in labels_to_qnodes_dict else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "front-bryan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acc4b8e178af4e5ebb4358c8d92fb7b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/33377320 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def fix_qnodes_to_wsim(row):\n",
    "    if row.node1_label in wordsim_labels_to_qnodes_dict:\n",
    "        row['node1'] = wordsim_labels_to_qnodes_dict[row.node1_label]\n",
    "    if row.node2_label in wordsim_labels_to_qnodes_dict:\n",
    "        row['node2'] = wordsim_labels_to_qnodes_dict[row.node2_label]\n",
    "    return row\n",
    "tqdm.pandas()\n",
    "probDF1 = probDF.progress_apply(fix_qnodes_to_wsim, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "associate-count",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>node1_label</th>\n",
       "      <th>node2_label</th>\n",
       "      <th>no_of_relations</th>\n",
       "      <th>node1</th>\n",
       "      <th>node2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>factor</td>\n",
       "      <td>age</td>\n",
       "      <td>35167</td>\n",
       "      <td>Q103858669</td>\n",
       "      <td>Q100343219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>free rich company datum</td>\n",
       "      <td>size</td>\n",
       "      <td>33222</td>\n",
       "      <td>None</td>\n",
       "      <td>Q322481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>free rich company datum</td>\n",
       "      <td>revenue</td>\n",
       "      <td>33185</td>\n",
       "      <td>None</td>\n",
       "      <td>Q850210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>state</td>\n",
       "      <td>california</td>\n",
       "      <td>18062</td>\n",
       "      <td>P1310</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>supplement</td>\n",
       "      <td>msm glucosamine sulfate</td>\n",
       "      <td>15942</td>\n",
       "      <td>Q2915731</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33377315</th>\n",
       "      <td>popular legacy datum structure</td>\n",
       "      <td>binary search tree</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33377316</th>\n",
       "      <td>norwegian food</td>\n",
       "      <td>lefse</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33377317</th>\n",
       "      <td>freeze skill</td>\n",
       "      <td>new fot</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33377318</th>\n",
       "      <td>enhanced enforcement initiative</td>\n",
       "      <td>monthly impact inspections of problem mine</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33377319</th>\n",
       "      <td>enhanced enforcement initiative</td>\n",
       "      <td>pattern of violation notification</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33377320 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              node1_label  \\\n",
       "0                                  factor   \n",
       "1                 free rich company datum   \n",
       "2                 free rich company datum   \n",
       "3                                   state   \n",
       "4                              supplement   \n",
       "...                                   ...   \n",
       "33377315   popular legacy datum structure   \n",
       "33377316                   norwegian food   \n",
       "33377317                     freeze skill   \n",
       "33377318  enhanced enforcement initiative   \n",
       "33377319  enhanced enforcement initiative   \n",
       "\n",
       "                                         node2_label  no_of_relations  \\\n",
       "0                                                age            35167   \n",
       "1                                               size            33222   \n",
       "2                                            revenue            33185   \n",
       "3                                         california            18062   \n",
       "4                            msm glucosamine sulfate            15942   \n",
       "...                                              ...              ...   \n",
       "33377315                          binary search tree                1   \n",
       "33377316                                       lefse                1   \n",
       "33377317                                     new fot                1   \n",
       "33377318  monthly impact inspections of problem mine                1   \n",
       "33377319           pattern of violation notification                1   \n",
       "\n",
       "               node1       node2  \n",
       "0         Q103858669  Q100343219  \n",
       "1               None     Q322481  \n",
       "2               None     Q850210  \n",
       "3              P1310        None  \n",
       "4           Q2915731        None  \n",
       "...              ...         ...  \n",
       "33377315        None        None  \n",
       "33377316        None        None  \n",
       "33377317        None        None  \n",
       "33377318        None        None  \n",
       "33377319        None        None  \n",
       "\n",
       "[33377320 rows x 5 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probDF1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "tribal-teens",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33377320"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(probDF1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "capable-charm",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "955210"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((~probDF1.node1.isna()) & (~probDF1.node2.isna())).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "southwest-fortune",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "factor                         364111\n",
       "feature                        203549\n",
       "issue                          201986\n",
       "product                        172106\n",
       "item                           158829\n",
       "                                ...  \n",
       "non opec supplier                   1\n",
       "proactive industry group            1\n",
       "car s important information         1\n",
       "mesofronts                          1\n",
       "bucket related offer                1\n",
       "Name: node1_label, Length: 5376524, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probDF1.node1_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "aggressive-richardson",
   "metadata": {},
   "outputs": [],
   "source": [
    "probDF1.to_csv(PROBASE_INTERM_FILE,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "received-unemployment",
   "metadata": {},
   "outputs": [],
   "source": [
    "probDF_Qnodes_DF_WQnodes = pd.read_csv(PROBASE_INTERM_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "amino-sarah",
   "metadata": {},
   "outputs": [],
   "source": [
    "probDF_Qnodes_DF_WQnodes1_subset = probDF_Qnodes_DF_WQnodes[(~probDF_Qnodes_DF_WQnodes.node1.isna()) & (~probDF_Qnodes_DF_WQnodes.node2.isna())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "hindu-roberts",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    955210.000000\n",
       "mean         10.028577\n",
       "std          85.410631\n",
       "min           1.000000\n",
       "25%           1.000000\n",
       "50%           2.000000\n",
       "75%           5.000000\n",
       "max       35167.000000\n",
       "Name: no_of_relations, dtype: float64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probDF_Qnodes_DF_WQnodes1_subset['no_of_relations'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "contained-tamil",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-36-38e5ab73861c>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  probDF_Qnodes_DF_WQnodes1_subset['similarity_value'] = probDF_Qnodes_DF_WQnodes1_subset['no_of_relations'].apply(np.log10) / np.log10(maxNR)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count    955210.000000\n",
       "mean          0.091717\n",
       "std           0.112667\n",
       "min           0.000000\n",
       "25%           0.000000\n",
       "50%           0.066217\n",
       "75%           0.153750\n",
       "max           1.000000\n",
       "Name: similarity_value, dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxNR = probDF_Qnodes_DF_WQnodes1_subset['no_of_relations'].max()\n",
    "probDF_Qnodes_DF_WQnodes1_subset['similarity_value'] = probDF_Qnodes_DF_WQnodes1_subset['no_of_relations'].apply(np.log10) / np.log10(maxNR)\n",
    "probDF_Qnodes_DF_WQnodes1_subset.similarity_value.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "coastal-composition",
   "metadata": {},
   "outputs": [],
   "source": [
    "probDF_Qnodes_DF_WQnodes1_subset.to_csv(PROBASE_FINAL_FILE, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "polyphonic-louisiana",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "neither-consumer",
   "metadata": {},
   "source": [
    "# Class Similarity Datasets\n",
    "\n",
    "`zcat derived.dwd_isa_class_count.compact.tsv.gz | cut -f 3 | tail -n +2 | sed -e 's/|/\\n/g' -e 's/:/\\t/g' | sort | uniq > class-counts.tsv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "short-heritage",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "import sys\n",
    "\n",
    "hierDF_transP279_filtered2_class_arr = []\n",
    "\n",
    "def fetchClassSim(row, similarity_type):\n",
    "    resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=\"+similarity_type)\n",
    "    try:\n",
    "        row['similarity_value'] = float(resp.json()['similarity']) if resp else -1\n",
    "    except Exception as exc:\n",
    "        print(exc)\n",
    "        print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "        row['similarity_value'] = None\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "interested-boxing",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279childPar_class = pd.read_csv(P279_CHILD_PAR_DISTILBERT_COSSIM_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "alternate-trance",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5667a15f08d470ca3907d5200ca13e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/304654 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-205-cc4159d2906b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mP279childPar_class_new\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mP279childPar_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetchClassSim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'class'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(df, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    808\u001b[0m                 \u001b[0;31m# on the df using our wrapper (which provides bar updating)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 810\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    811\u001b[0m                 \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    812\u001b[0m                     \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, axis, raw, result_type, args, **kwds)\u001b[0m\n\u001b[1;32m   7763\u001b[0m             \u001b[0mkwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7764\u001b[0m         )\n\u001b[0;32m-> 7765\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   7766\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7767\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapplymap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mna_action\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mget_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    183\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_raw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_empty_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    275\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 276\u001b[0;31m         \u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_series_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    278\u001b[0m         \u001b[0;31m# wrap results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_series_generator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    288\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseries_gen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m                 \u001b[0;31m# ignore SettingWithCopy here in case the user mutates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m                 \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    291\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m                     \u001b[0;31m# If we have a view on v, we need to make a copy because\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mf\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m             \u001b[0;32mdef\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    803\u001b[0m                     \u001b[0;31m# take a fast or slow code path; so stop when t.total==t.n\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    804\u001b[0m                     \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 805\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    806\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    807\u001b[0m                 \u001b[0;31m# Apply the provided function (in **kwargs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-201-4df33f71c8eb>\u001b[0m in \u001b[0;36mfetchClassSim\u001b[0;34m(row, similarity_type)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfetchClassSim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimilarity_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"https://kgtk.isi.edu/similarity_api?q1=\"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'node1'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"&q2=\"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'node2'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"&embedding_type=\"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0msimilarity_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'similarity_value'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'similarity'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresp\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    540\u001b[0m         }\n\u001b[1;32m    541\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    543\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    653\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    654\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 655\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    656\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    657\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    438\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mchunked\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 439\u001b[0;31m                 resp = conn.urlopen(\n\u001b[0m\u001b[1;32m    440\u001b[0m                     \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    441\u001b[0m                     \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    697\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m             \u001b[0;31m# Make the request on the httplib connection object.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 699\u001b[0;31m             httplib_response = self._make_request(\n\u001b[0m\u001b[1;32m    700\u001b[0m                 \u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m                 \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    380\u001b[0m         \u001b[0;31m# Trigger any extra validation we need to do.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSocketTimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBaseSSLError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m             \u001b[0;31m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   1008\u001b[0m         \u001b[0;31m# Force connect early to allow us to validate the connection.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1009\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"sock\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# AppEngine might not have  `.sock`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1010\u001b[0;31m             \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1011\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1012\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_verified\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/urllib3/connection.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    409\u001b[0m             \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_default_certs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m         self.sock = ssl_wrap_socket(\n\u001b[0m\u001b[1;32m    412\u001b[0m             \u001b[0msock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    413\u001b[0m             \u001b[0mkeyfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/urllib3/util/ssl_.py\u001b[0m in \u001b[0;36mssl_wrap_socket\u001b[0;34m(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)\u001b[0m\n\u001b[1;32m    379\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mca_certs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mca_cert_dir\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mca_cert_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m             \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_verify_locations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mca_certs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mca_cert_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mca_cert_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    382\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mIOError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mSSLError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tqdm.pandas()\n",
    "P279childPar_class_new = P279childPar_class.progress_apply(fetchClassSim, axis=1, args=('class',))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adolescent-nurse",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279childPar_class_new.to_csv(P279_CHILD_PAR_CLASSSIM_FILE, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "warming-vertical",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "imported-mounting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0f3c043918e4634a2dc5bdc7f767c7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/304654 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-202-93d379bb189a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mP279childPar_class_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mParallel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelayed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetchClassSim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'class'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP279childPar_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP279childPar_class\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1052\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1054\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1055\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1056\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    931\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    932\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 933\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    934\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    935\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    540\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[1;32m    541\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 542\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    543\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mCfTimeoutError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    432\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 434\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kgtkEnv/lib/python3.8/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    300\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "P279childPar_class_arr = Parallel(n_jobs=5)(delayed(fetchClassSim)(row, 'class') for _, row in tqdm(P279childPar_class.iterrows(), total=len(P279childPar_class)))\n",
    "                                                                                                                                                                                                                                                                                  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "spread-snapshot",
   "metadata": {},
   "source": [
    "## ChildPar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "supposed-newport",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2 = pd.read_csv('../data/Master_P279_dataset/P279ChildPar_transP279_filtered_min_cols.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "reliable-publisher",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2[['node1', 'node2']].rename(columns={'node1':'q1', 'node2':'q2'}).to_csv('../data/Master_P279_dataset/P279ChildPar_transP279_filtered_min_cols_FOR_CLASS.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "fifty-garden",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21f6e6799b7d425fa5c2d3d79d6ae2dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "215bbc5968f94456a3a50e8d182f06d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class_dict = {}\n",
    "hierDF_transP279_filtered = pd.read_csv(P279_CHILD_PAR_DISTILBERT_COSSIM_FILE)\n",
    "p279QnodesList = set(hierDF_transP279_filtered.node1.to_list() + hierDF_transP279_filtered.node2.to_list())\n",
    "with gzip.open(DERIVED_IS_COUNTS_FILE) as class_file:\n",
    "    for line in tqdm(class_file):\n",
    "        line = line.decode('utf-8').strip().split('\\t')\n",
    "        if line[0] in p279QnodesList:\n",
    "            class_dict[line[0]] = line[2].strip()\n",
    "all_class_count_dict = {}\n",
    "with open(CLASS_COUNTS_FILE) as class_file:\n",
    "    for line in tqdm(class_file):\n",
    "        line = line.split('\\t')\n",
    "        if line[0] in p279QnodesList:\n",
    "            all_class_count_dict[line[0]] = line[1].strip()\n",
    "\n",
    "\n",
    "class SemanticSimilarity(object):\n",
    "    def __init__(self):\n",
    "#         self.config = config\n",
    "#         self.embeddings_type = ['class']\n",
    "        self.N = float(52546967)\n",
    "        self.all_class_count_dict = all_class_count_dict\n",
    "        \n",
    "    def get_qnode_details(self, qnodes: list):\n",
    "        global class_dict\n",
    "        qnodes_dict = {}\n",
    "        qnodes_dict[qnodes[0]] = class_dict.get(qnodes[0],'')\n",
    "        qnodes_dict[qnodes[1]] = class_dict.get(qnodes[1],'')\n",
    "        return qnodes_dict\n",
    "        \n",
    "    def compute_class_similarity(self, q1, q2):\n",
    "        qnodes_dict = self.get_qnode_details([q1, q2])\n",
    "#         print(qnodes_dict)\n",
    "        feature_dict, feature_count_dict = self.build_qnode_feature_dict(qnodes_dict)\n",
    "#         print(feature_dict, feature_count_dict)\n",
    "        normalized_classes_idf = self.normalize_idf_classes(feature_dict, feature_count_dict)\n",
    "        if q1 in feature_dict and q2 in feature_dict:\n",
    "            q1_cl = set(feature_dict[q1])\n",
    "            q2_cl = set(feature_dict[q2])\n",
    "            q1_q2_intersection = q1_cl.intersection(q2_cl)\n",
    "\n",
    "            _similarity = sum([normalized_classes_idf.get(c) for c in q1_q2_intersection])\n",
    "            return {\n",
    "                'similarity': _similarity\n",
    "            }\n",
    "        return {\n",
    "            'similarity': None\n",
    "        }\n",
    "\n",
    "    def build_qnode_feature_dict(self, qnodes_dict: dict) -> (dict, dict):\n",
    "        feature_dict = {}\n",
    "        feature_count_dict = {}\n",
    "\n",
    "        for qnode in qnodes_dict:\n",
    "            feature_val = []\n",
    "            \n",
    "            if '|' in qnodes_dict[qnode]:\n",
    "                cl = qnodes_dict[qnode].split(\"|\")\n",
    "\n",
    "                for c in cl:\n",
    "                    vals = c.split(\":\")\n",
    "                    feature_val.append(vals[0])\n",
    "                    feature_count_dict[vals[0]] = float(vals[1])\n",
    "            if qnode not in feature_val:\n",
    "                feature_val.append(qnode)\n",
    "            if qnode not in feature_count_dict:\n",
    "                feature_count_dict[qnode] = float(self.all_class_count_dict.get(qnode, 1.0))\n",
    "            feature_dict[qnode] = feature_val\n",
    "\n",
    "        return feature_dict, feature_count_dict\n",
    "\n",
    "    def normalize_idf_classes(self, feature_dict, feature_count_dict):\n",
    "\n",
    "        classes_count = {}\n",
    "        for qnode in feature_dict:\n",
    "\n",
    "            classes = feature_dict[qnode]\n",
    "            for c in classes:\n",
    "                if c not in classes_count:\n",
    "                    classes_count[c] = 0\n",
    "                classes_count[c] += 1\n",
    "\n",
    "        classes_idf = self.calculate_idf_features(feature_count_dict)\n",
    "\n",
    "        # multiply class count with idf\n",
    "        for c in classes_idf:\n",
    "            classes_idf[c] = classes_count[c] * classes_idf[c]\n",
    "\n",
    "        # normalize the idf scores so that they sum to 1\n",
    "        classes_idf_sum = sum([classes_idf[x] for x in classes_idf])\n",
    "        for c in classes_idf:\n",
    "            classes_idf[c] = classes_idf[c] / classes_idf_sum\n",
    "\n",
    "        return classes_idf\n",
    "\n",
    "    def calculate_idf_features(self, feature_count_dict):\n",
    "        _ = {}\n",
    "        for c in feature_count_dict:\n",
    "            _[c] = math.log(self.N / feature_count_dict[c])\n",
    "        return _"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "streaming-match",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "flying-perception",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import requests\n",
    "# from tqdm.notebook import tqdm\n",
    "# import json\n",
    "# from joblib import Parallel, delayed\n",
    "# import sys\n",
    "\n",
    "# hierDF_transP279_filtered2_class_arr = []\n",
    "\n",
    "# def fetchClassSim(row):\n",
    "#     resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\")\n",
    "#     try:\n",
    "#         row['classSim'] = float(resp.json()['similarity']) if resp else -1\n",
    "#     except Exception as exc:\n",
    "#         print(exc)\n",
    "#         print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "#         row['classSim'] = None\n",
    "#     row['Resp_code'] = resp\n",
    "#     return row\n",
    "\n",
    "# hierDF_transP279_filtered2_class_arr = Parallel(n_jobs=5)(delayed(fetchClassSim)(row) for _, row in tqdm(hierDF_transP279_filtered2.iloc[:100].iterrows(), total=len(hierDF_transP279_filtered2)))\n",
    "                                                                                                                                          \n",
    "                                                                                                                                          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "empirical-suspect",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2_class_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "helpful-medline",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2_class = pd.DataFrame(hierDF_transP279_filtered2_class_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "activated-excuse",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered2_class.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "baking-input",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "302108"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered2_class.classSim.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cognitive-decade",
   "metadata": {},
   "source": [
    "## Siblings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "pacific-warehouse",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279_3M_data = pd.read_csv('../data/Master_P279_dataset/P279Siblings_transP279_filtered_min_cols_with_desc_dups_removed.csv')\n",
    "P279_10M_data = pd.read_csv('../data/Master_P279_dataset/P279Siblings_transP279_filtered_min_cols.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "outdoor-sense",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import requests\n",
    "# from tqdm.notebook import tqdm\n",
    "# import json\n",
    "# from joblib import Parallel, delayed\n",
    "\n",
    "# P279_3M_data_class_arr = []\n",
    "\n",
    "# def fetchClassSim(row):\n",
    "#     try:\n",
    "#         resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\").json()['similarity']\n",
    "#         row['classSim'] = float(resp) if resp else -1\n",
    "#     except:\n",
    "#         print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "#         row['classSim'] = None\n",
    "#     return row\n",
    "\n",
    "# P279_3M_data_class_arr = Parallel(n_jobs=20)(delayed(fetchClassSim)(row) for _, row in tqdm(P279_3M_data.iterrows(), total=len(P279_3M_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "handy-coral",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import requests\n",
    "# from tqdm.notebook import tqdm\n",
    "# import json\n",
    "# from joblib import Parallel, delayed\n",
    "\n",
    "# P279_10M_data_class_arr = []\n",
    "\n",
    "# def fetchClassSim(row):\n",
    "#     try:\n",
    "#         resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\").json()['similarity']\n",
    "#         row['classSim'] = float(resp) if resp else -1\n",
    "#     except:\n",
    "#         print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "#         row['classSim'] = None\n",
    "#     return row\n",
    "\n",
    "# P279_10M_data_class_arr = Parallel(n_jobs=20)(delayed(fetchClassSim)(row) for _, row in tqdm(P279_10M_data.iterrows(), total=len(P279_10M_data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wired-treatment",
   "metadata": {},
   "source": [
    "# JC Similarity Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "textile-productivity",
   "metadata": {},
   "source": [
    "## ChildPar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "aggregate-tiger",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2 = pd.read_csv('../data/Master_P279_dataset/P279ChildPar_filtered.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "cardiac-fence",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "978c7892341342029d881b0bf31a6f66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/303284 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "could not convert string to float: ''\n",
      "Resp not found for Q102108504, Q100171002\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "import sys\n",
    "\n",
    "hierDF_transP279_filtered2_class_arr = []\n",
    "\n",
    "def fetchJCSim(row):\n",
    "    resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\")\n",
    "    try:\n",
    "        row['similarity_value'] = float(resp.json()['similarity']) if resp else -1\n",
    "    except Exception as exc:\n",
    "        print(exc)\n",
    "        print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "        row['similarity_value'] = None\n",
    "    row['Resp_code'] = resp\n",
    "    return row\n",
    "\n",
    "hierDF_transP279_filtered2_class_arr = Parallel(n_jobs=5)(delayed(fetchClassSim)(row) for _, row in tqdm(hierDF_transP279_filtered2.iloc[:100].iterrows(), total=len(hierDF_transP279_filtered2)))\n",
    "                                                                                                                                          \n",
    "                                                                                                                                          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "attached-novelty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Unnamed: 0                                      0\n",
       " id              Q17372279-P279-Q100026-beba8cd1-0\n",
       " node1                                   Q17372279\n",
       " label                                        P279\n",
       " node2                                     Q100026\n",
       " bert2SentSim                             0.666525\n",
       " classSim                                 0.835782\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 0, dtype: object,\n",
       " Unnamed: 0                                      1\n",
       " id              Q17372377-P279-Q100026-fd42bd71-0\n",
       " node1                                   Q17372377\n",
       " label                                        P279\n",
       " node2                                     Q100026\n",
       " bert2SentSim                             0.700515\n",
       " classSim                                  0.94674\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 1, dtype: object,\n",
       " Unnamed: 0                                      2\n",
       " id              Q17372444-P279-Q100026-ca0fc4bd-0\n",
       " node1                                   Q17372444\n",
       " label                                        P279\n",
       " node2                                     Q100026\n",
       " bert2SentSim                             0.683245\n",
       " classSim                                 0.946647\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 2, dtype: object,\n",
       " Unnamed: 0                                      3\n",
       " id              Q17372455-P279-Q100026-c2b1bf36-0\n",
       " node1                                   Q17372455\n",
       " label                                        P279\n",
       " node2                                     Q100026\n",
       " bert2SentSim                             0.689516\n",
       " classSim                                 0.943256\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 3, dtype: object,\n",
       " Unnamed: 0                                     4\n",
       " id              Q2029940-P279-Q100026-ceba4380-0\n",
       " node1                                   Q2029940\n",
       " label                                       P279\n",
       " node2                                    Q100026\n",
       " bert2SentSim                            0.603161\n",
       " classSim                                0.809487\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 4, dtype: object,\n",
       " Unnamed: 0                                    5\n",
       " id              Q209480-P279-Q100026-a218e96b-0\n",
       " node1                                   Q209480\n",
       " label                                      P279\n",
       " node2                                   Q100026\n",
       " bert2SentSim                            0.61352\n",
       " classSim                               0.975716\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 5, dtype: object,\n",
       " Unnamed: 0                                      6\n",
       " id              Q77929250-P279-Q100026-964ec021-0\n",
       " node1                                   Q77929250\n",
       " label                                        P279\n",
       " node2                                     Q100026\n",
       " bert2SentSim                              0.76767\n",
       " classSim                                 0.961308\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 6, dtype: object,\n",
       " Unnamed: 0                                      7\n",
       " id              Q1087043-P279-Q1000371-945e7797-0\n",
       " node1                                    Q1087043\n",
       " label                                        P279\n",
       " node2                                    Q1000371\n",
       " bert2SentSim                             0.699843\n",
       " classSim                                 0.671266\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 7, dtype: object,\n",
       " Unnamed: 0                                    8\n",
       " id              Q384656-P279-Q100047-6147b9a8-0\n",
       " node1                                   Q384656\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.681549\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 8, dtype: object,\n",
       " Unnamed: 0                                    9\n",
       " id              Q471089-P279-Q100047-473b504b-0\n",
       " node1                                   Q471089\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.679593\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 9, dtype: object,\n",
       " Unnamed: 0                                   10\n",
       " id              Q471094-P279-Q100047-14b1b1df-0\n",
       " node1                                   Q471094\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.681549\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 10, dtype: object,\n",
       " Unnamed: 0                                   11\n",
       " id              Q471102-P279-Q100047-886a847c-0\n",
       " node1                                   Q471102\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.681549\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 11, dtype: object,\n",
       " Unnamed: 0                                   12\n",
       " id              Q471127-P279-Q100047-6738fc46-0\n",
       " node1                                   Q471127\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.681549\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 12, dtype: object,\n",
       " Unnamed: 0                                   13\n",
       " id              Q471144-P279-Q100047-766e1fa3-0\n",
       " node1                                   Q471144\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.811462\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 13, dtype: object,\n",
       " Unnamed: 0                                   14\n",
       " id              Q471158-P279-Q100047-7955aa40-0\n",
       " node1                                   Q471158\n",
       " label                                      P279\n",
       " node2                                   Q100047\n",
       " bert2SentSim                           0.681549\n",
       " classSim                               0.943953\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 14, dtype: object,\n",
       " Unnamed: 0                                         15\n",
       " id              Q100052562-P279-Q100052008-1a4fa432-0\n",
       " node1                                      Q100052562\n",
       " label                                            P279\n",
       " node2                                      Q100052008\n",
       " bert2SentSim                                 0.515993\n",
       " classSim                                     0.913344\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 15, dtype: object,\n",
       " Unnamed: 0                                         16\n",
       " id              Q100052791-P279-Q100052008-1d15b1ae-0\n",
       " node1                                      Q100052791\n",
       " label                                            P279\n",
       " node2                                      Q100052008\n",
       " bert2SentSim                                 0.478483\n",
       " classSim                                     0.907455\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 16, dtype: object,\n",
       " Unnamed: 0                                         17\n",
       " id              Q100119502-P279-Q100052008-d2c330a4-0\n",
       " node1                                      Q100119502\n",
       " label                                            P279\n",
       " node2                                      Q100052008\n",
       " bert2SentSim                                  0.51221\n",
       " classSim                                     0.913229\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 17, dtype: object,\n",
       " Unnamed: 0                                        18\n",
       " id              Q88594442-P279-Q100052008-badba21b-0\n",
       " node1                                      Q88594442\n",
       " label                                           P279\n",
       " node2                                     Q100052008\n",
       " bert2SentSim                                0.424672\n",
       " classSim                                     0.91916\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 18, dtype: object,\n",
       " Unnamed: 0                                     19\n",
       " id              Q1133024-P279-Q1000660-6d8a1a17-0\n",
       " node1                                    Q1133024\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.556017\n",
       " classSim                                  0.96031\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 19, dtype: object,\n",
       " Unnamed: 0                                     20\n",
       " id              Q1231309-P279-Q1000660-656bad41-0\n",
       " node1                                    Q1231309\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.473025\n",
       " classSim                                 0.987153\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 20, dtype: object,\n",
       " Unnamed: 0                                     21\n",
       " id              Q1892554-P279-Q1000660-b60addd4-0\n",
       " node1                                    Q1892554\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.710084\n",
       " classSim                                 0.986632\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 21, dtype: object,\n",
       " Unnamed: 0                                     22\n",
       " id              Q2835946-P279-Q1000660-ff0a5cf9-0\n",
       " node1                                    Q2835946\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                              0.69158\n",
       " classSim                                 0.986632\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 22, dtype: object,\n",
       " Unnamed: 0                                     23\n",
       " id              Q3413373-P279-Q1000660-9d565f5e-0\n",
       " node1                                    Q3413373\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.676028\n",
       " classSim                                 0.986632\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 23, dtype: object,\n",
       " Unnamed: 0                                     24\n",
       " id              Q5611316-P279-Q1000660-6b051008-0\n",
       " node1                                    Q5611316\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.652025\n",
       " classSim                                 0.986632\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 24, dtype: object,\n",
       " Unnamed: 0                                    25\n",
       " id              Q649977-P279-Q1000660-cc4475c9-0\n",
       " node1                                    Q649977\n",
       " label                                       P279\n",
       " node2                                   Q1000660\n",
       " bert2SentSim                            0.548393\n",
       " classSim                                0.986632\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 25, dtype: object,\n",
       " Unnamed: 0                                     26\n",
       " id              Q7269432-P279-Q1000660-8976e0ff-0\n",
       " node1                                    Q7269432\n",
       " label                                        P279\n",
       " node2                                    Q1000660\n",
       " bert2SentSim                             0.439381\n",
       " classSim                                 0.989483\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 26, dtype: object,\n",
       " Unnamed: 0                                    27\n",
       " id              Q844980-P279-Q1000660-5f020e98-0\n",
       " node1                                    Q844980\n",
       " label                                       P279\n",
       " node2                                   Q1000660\n",
       " bert2SentSim                            0.678084\n",
       " classSim                                0.986632\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 27, dtype: object,\n",
       " Unnamed: 0                                     28\n",
       " id              Q1051606-P279-Q1000809-995e5221-0\n",
       " node1                                    Q1051606\n",
       " label                                        P279\n",
       " node2                                    Q1000809\n",
       " bert2SentSim                             0.740599\n",
       " classSim                                 0.912943\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 28, dtype: object,\n",
       " Unnamed: 0                                     29\n",
       " id              Q4984171-P279-Q1000809-95749f41-0\n",
       " node1                                    Q4984171\n",
       " label                                        P279\n",
       " node2                                    Q1000809\n",
       " bert2SentSim                             0.808338\n",
       " classSim                                 0.957835\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 29, dtype: object,\n",
       " Unnamed: 0                                     30\n",
       " id              Q2047562-P279-Q1001041-45feee81-0\n",
       " node1                                    Q2047562\n",
       " label                                        P279\n",
       " node2                                    Q1001041\n",
       " bert2SentSim                             0.359032\n",
       " classSim                                 0.707243\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 30, dtype: object,\n",
       " Unnamed: 0                                     31\n",
       " id              Q8069073-P279-Q1001041-3f13f5bb-0\n",
       " node1                                    Q8069073\n",
       " label                                        P279\n",
       " node2                                    Q1001041\n",
       " bert2SentSim                             0.769952\n",
       " classSim                                 0.849692\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 31, dtype: object,\n",
       " Unnamed: 0                                     32\n",
       " id              Q1070955-P279-Q1001051-5df03dac-0\n",
       " node1                                    Q1070955\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.800508\n",
       " classSim                                  0.86936\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 32, dtype: object,\n",
       " Unnamed: 0                                     33\n",
       " id              Q1153367-P279-Q1001051-34783d55-0\n",
       " node1                                    Q1153367\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.837581\n",
       " classSim                                 0.876525\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 33, dtype: object,\n",
       " Unnamed: 0                                     34\n",
       " id              Q1958065-P279-Q1001051-d6eb5968-0\n",
       " node1                                    Q1958065\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.848892\n",
       " classSim                                 0.707989\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 34, dtype: object,\n",
       " Unnamed: 0                                    35\n",
       " id              Q209680-P279-Q1001051-072eeb1b-0\n",
       " node1                                    Q209680\n",
       " label                                       P279\n",
       " node2                                   Q1001051\n",
       " bert2SentSim                            0.656872\n",
       " classSim                                0.724838\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 35, dtype: object,\n",
       " Unnamed: 0                                     36\n",
       " id              Q2986204-P279-Q1001051-7b21695a-0\n",
       " node1                                    Q2986204\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.816212\n",
       " classSim                                 0.796208\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 36, dtype: object,\n",
       " Unnamed: 0                                    37\n",
       " id              Q385530-P279-Q1001051-17db3704-0\n",
       " node1                                    Q385530\n",
       " label                                       P279\n",
       " node2                                   Q1001051\n",
       " bert2SentSim                             0.75061\n",
       " classSim                                0.780191\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 37, dtype: object,\n",
       " Unnamed: 0                                      38\n",
       " id              Q41795401-P279-Q1001051-5747dcc4-0\n",
       " node1                                    Q41795401\n",
       " label                                         P279\n",
       " node2                                     Q1001051\n",
       " bert2SentSim                              0.787191\n",
       " classSim                                   0.88381\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 38, dtype: object,\n",
       " Unnamed: 0                                    39\n",
       " id              Q612405-P279-Q1001051-bb6fa17c-0\n",
       " node1                                    Q612405\n",
       " label                                       P279\n",
       " node2                                   Q1001051\n",
       " bert2SentSim                             0.72366\n",
       " classSim                                 0.86936\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 39, dtype: object,\n",
       " Unnamed: 0                                    40\n",
       " id              Q668439-P279-Q1001051-bbba9ea3-0\n",
       " node1                                    Q668439\n",
       " label                                       P279\n",
       " node2                                   Q1001051\n",
       " bert2SentSim                            0.698926\n",
       " classSim                                 0.86936\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 40, dtype: object,\n",
       " Unnamed: 0                                     41\n",
       " id              Q7133779-P279-Q1001051-4fc71627-0\n",
       " node1                                    Q7133779\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.576617\n",
       " classSim                                  0.86936\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 41, dtype: object,\n",
       " Unnamed: 0                                     42\n",
       " id              Q7630946-P279-Q1001051-2e78f5b5-0\n",
       " node1                                    Q7630946\n",
       " label                                        P279\n",
       " node2                                    Q1001051\n",
       " bert2SentSim                             0.632298\n",
       " classSim                                 0.796208\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 42, dtype: object,\n",
       " Unnamed: 0                                     43\n",
       " id              Q1056352-P279-Q1001059-c1976180-0\n",
       " node1                                    Q1056352\n",
       " label                                        P279\n",
       " node2                                    Q1001059\n",
       " bert2SentSim                             0.717131\n",
       " classSim                                 0.886053\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 43, dtype: object,\n",
       " Unnamed: 0                                     44\n",
       " id              Q2123051-P279-Q1001059-18b4564b-0\n",
       " node1                                    Q2123051\n",
       " label                                        P279\n",
       " node2                                    Q1001059\n",
       " bert2SentSim                             0.606046\n",
       " classSim                                 0.465094\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 44, dtype: object,\n",
       " Unnamed: 0                                    45\n",
       " id              Q444278-P279-Q1001059-0c74af28-0\n",
       " node1                                    Q444278\n",
       " label                                       P279\n",
       " node2                                   Q1001059\n",
       " bert2SentSim                            0.481163\n",
       " classSim                                0.881922\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 45, dtype: object,\n",
       " Unnamed: 0                                     46\n",
       " id              Q7240696-P279-Q1001059-31dfa177-0\n",
       " node1                                    Q7240696\n",
       " label                                        P279\n",
       " node2                                    Q1001059\n",
       " bert2SentSim                             0.752948\n",
       " classSim                                 0.881922\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 46, dtype: object,\n",
       " Unnamed: 0                                      47\n",
       " id              Q85776190-P279-Q1001059-b7a4aad6-0\n",
       " node1                                    Q85776190\n",
       " label                                         P279\n",
       " node2                                     Q1001059\n",
       " bert2SentSim                              0.747289\n",
       " classSim                                   0.53963\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 47, dtype: object,\n",
       " Unnamed: 0                                         48\n",
       " id              Q100052008-P279-Q100116222-d1597eca-0\n",
       " node1                                      Q100052008\n",
       " label                                            P279\n",
       " node2                                      Q100116222\n",
       " bert2SentSim                                 0.725062\n",
       " classSim                                     0.909364\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 48, dtype: object,\n",
       " Unnamed: 0                                        49\n",
       " id              Q27303706-P279-Q100116222-526530c6-0\n",
       " node1                                      Q27303706\n",
       " label                                           P279\n",
       " node2                                     Q100116222\n",
       " bert2SentSim                                0.537653\n",
       " classSim                                    0.917887\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 49, dtype: object,\n",
       " Unnamed: 0                                        50\n",
       " id              Q99565932-P279-Q100116222-139f958c-0\n",
       " node1                                      Q99565932\n",
       " label                                           P279\n",
       " node2                                     Q100116222\n",
       " bert2SentSim                                0.691907\n",
       " classSim                                    0.892978\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 50, dtype: object,\n",
       " Unnamed: 0                                     51\n",
       " id              Q1416523-P279-Q1001329-2b6d0c1e-0\n",
       " node1                                    Q1416523\n",
       " label                                        P279\n",
       " node2                                    Q1001329\n",
       " bert2SentSim                             0.160931\n",
       " classSim                                 0.871646\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 51, dtype: object,\n",
       " Unnamed: 0                                     52\n",
       " id              Q2963918-P279-Q1001329-c85a352f-0\n",
       " node1                                    Q2963918\n",
       " label                                        P279\n",
       " node2                                    Q1001329\n",
       " bert2SentSim                             0.469497\n",
       " classSim                                 0.878703\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 52, dtype: object,\n",
       " Unnamed: 0                                      53\n",
       " id              Q56756850-P279-Q1001329-6cb334b1-0\n",
       " node1                                    Q56756850\n",
       " label                                         P279\n",
       " node2                                     Q1001329\n",
       " bert2SentSim                              0.412103\n",
       " classSim                                  0.363595\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 53, dtype: object,\n",
       " Unnamed: 0                                      54\n",
       " id              Q60782200-P279-Q1001329-0b4da903-0\n",
       " node1                                    Q60782200\n",
       " label                                         P279\n",
       " node2                                     Q1001329\n",
       " bert2SentSim                              0.239958\n",
       " classSim                                  0.871646\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 54, dtype: object,\n",
       " Unnamed: 0                                      55\n",
       " id              Q78841106-P279-Q1001329-d474c822-0\n",
       " node1                                    Q78841106\n",
       " label                                         P279\n",
       " node2                                     Q1001329\n",
       " bert2SentSim                              0.496099\n",
       " classSim                                  0.871646\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 55, dtype: object,\n",
       " Unnamed: 0                                    56\n",
       " id              Q798863-P279-Q1001329-c284c2ec-0\n",
       " node1                                    Q798863\n",
       " label                                       P279\n",
       " node2                                   Q1001329\n",
       " bert2SentSim                            0.597623\n",
       " classSim                                0.871646\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 56, dtype: object,\n",
       " Unnamed: 0                                      57\n",
       " id              Q96337569-P279-Q1001329-b8b30fb9-0\n",
       " node1                                    Q96337569\n",
       " label                                         P279\n",
       " node2                                     Q1001329\n",
       " bert2SentSim                              0.210435\n",
       " classSim                                  0.871646\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 57, dtype: object,\n",
       " Unnamed: 0                                       58\n",
       " id              Q3403683-P279-Q100135295-fd0a0a92-0\n",
       " node1                                      Q3403683\n",
       " label                                          P279\n",
       " node2                                    Q100135295\n",
       " bert2SentSim                               0.371629\n",
       " classSim                                   0.811291\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 58, dtype: object,\n",
       " Unnamed: 0                                         59\n",
       " id              Q100095237-P279-Q100136951-a7f6f753-0\n",
       " node1                                      Q100095237\n",
       " label                                            P279\n",
       " node2                                      Q100136951\n",
       " bert2SentSim                                 0.883361\n",
       " classSim                                     0.909817\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 59, dtype: object,\n",
       " Unnamed: 0                                         60\n",
       " id              Q100136066-P279-Q100136951-1e2368cd-0\n",
       " node1                                      Q100136066\n",
       " label                                            P279\n",
       " node2                                      Q100136951\n",
       " bert2SentSim                                 0.883361\n",
       " classSim                                     0.843155\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 60, dtype: object,\n",
       " Unnamed: 0                                      61\n",
       " id              Q235041-P279-Q100136951-07d30679-0\n",
       " node1                                      Q235041\n",
       " label                                         P279\n",
       " node2                                   Q100136951\n",
       " bert2SentSim                              0.607752\n",
       " classSim                                  0.719618\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 61, dtype: object,\n",
       " Unnamed: 0                                      62\n",
       " id              Q22006653-P279-Q1001378-98eddb62-0\n",
       " node1                                    Q22006653\n",
       " label                                         P279\n",
       " node2                                     Q1001378\n",
       " bert2SentSim                              0.366579\n",
       " classSim                                    0.5263\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 62, dtype: object,\n",
       " Unnamed: 0                                     63\n",
       " id              Q2895476-P279-Q1001378-9d15228e-0\n",
       " node1                                    Q2895476\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                             0.682378\n",
       " classSim                                  0.75066\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 63, dtype: object,\n",
       " Unnamed: 0                                     64\n",
       " id              Q2918907-P279-Q1001378-39c09697-0\n",
       " node1                                    Q2918907\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                             0.813292\n",
       " classSim                                 0.775328\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 64, dtype: object,\n",
       " Unnamed: 0                                     65\n",
       " id              Q3516833-P279-Q1001378-30384a7c-0\n",
       " node1                                    Q3516833\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                             0.684669\n",
       " classSim                                 0.916831\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 65, dtype: object,\n",
       " Unnamed: 0                                     66\n",
       " id              Q4129855-P279-Q1001378-3ad464f7-0\n",
       " node1                                    Q4129855\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                             0.740775\n",
       " classSim                                 0.905545\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 66, dtype: object,\n",
       " Unnamed: 0                                     67\n",
       " id              Q5197004-P279-Q1001378-513f9939-0\n",
       " node1                                    Q5197004\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                             0.642596\n",
       " classSim                                 0.892815\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 67, dtype: object,\n",
       " Unnamed: 0                                     68\n",
       " id              Q7237336-P279-Q1001378-55e73e1a-0\n",
       " node1                                    Q7237336\n",
       " label                                        P279\n",
       " node2                                    Q1001378\n",
       " bert2SentSim                              0.58928\n",
       " classSim                                 0.892815\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 68, dtype: object,\n",
       " Unnamed: 0                                    69\n",
       " id              Q738160-P279-Q1001378-b69af0cb-0\n",
       " node1                                    Q738160\n",
       " label                                       P279\n",
       " node2                                   Q1001378\n",
       " bert2SentSim                            0.813292\n",
       " classSim                                0.892815\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 69, dtype: object,\n",
       " Unnamed: 0                                  70\n",
       " id              Q8317-P279-Q1001378-a56c669f-0\n",
       " node1                                    Q8317\n",
       " label                                     P279\n",
       " node2                                 Q1001378\n",
       " bert2SentSim                          0.636886\n",
       " classSim                              0.541961\n",
       " Resp_code                     <Response [200]>\n",
       " Name: 70, dtype: object,\n",
       " Unnamed: 0                                      71\n",
       " id              Q86908314-P279-Q1001378-7a81e91b-0\n",
       " node1                                    Q86908314\n",
       " label                                         P279\n",
       " node2                                     Q1001378\n",
       " bert2SentSim                              0.421366\n",
       " classSim                                  0.892815\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 71, dtype: object,\n",
       " Unnamed: 0                                      72\n",
       " id              Q86910224-P279-Q1001378-b427331c-0\n",
       " node1                                    Q86910224\n",
       " label                                         P279\n",
       " node2                                     Q1001378\n",
       " bert2SentSim                              0.584772\n",
       " classSim                                  0.892815\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 72, dtype: object,\n",
       " Unnamed: 0                                         73\n",
       " id              Q100148346-P279-Q100148352-003654ac-0\n",
       " node1                                      Q100148346\n",
       " label                                            P279\n",
       " node2                                      Q100148352\n",
       " bert2SentSim                                 0.872339\n",
       " classSim                                     0.951283\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 73, dtype: object,\n",
       " Unnamed: 0                                        74\n",
       " id              Q27147769-P279-Q100148352-8254b7c4-0\n",
       " node1                                      Q27147769\n",
       " label                                           P279\n",
       " node2                                     Q100148352\n",
       " bert2SentSim                                0.886007\n",
       " classSim                                    0.924902\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 74, dtype: object,\n",
       " Unnamed: 0                                     75\n",
       " id              Q14747611-P279-Q100159-567e45a7-0\n",
       " node1                                   Q14747611\n",
       " label                                        P279\n",
       " node2                                     Q100159\n",
       " bert2SentSim                              0.50114\n",
       " classSim                                 0.866312\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 75, dtype: object,\n",
       " Unnamed: 0                                     76\n",
       " id              Q17089549-P279-Q100159-3247e0d4-0\n",
       " node1                                   Q17089549\n",
       " label                                        P279\n",
       " node2                                     Q100159\n",
       " bert2SentSim                             0.705335\n",
       " classSim                                 0.866312\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 76, dtype: object,\n",
       " Unnamed: 0                                     77\n",
       " id              Q18923049-P279-Q100159-04db261f-0\n",
       " node1                                   Q18923049\n",
       " label                                        P279\n",
       " node2                                     Q100159\n",
       " bert2SentSim                             0.861815\n",
       " classSim                                 0.719858\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 77, dtype: object,\n",
       " Unnamed: 0                                    78\n",
       " id              Q2056084-P279-Q100159-94d1b7b8-0\n",
       " node1                                   Q2056084\n",
       " label                                       P279\n",
       " node2                                    Q100159\n",
       " bert2SentSim                            0.782052\n",
       " classSim                                0.866312\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 78, dtype: object,\n",
       " Unnamed: 0                                    79\n",
       " id              Q5097845-P279-Q100159-c2bd0b24-0\n",
       " node1                                   Q5097845\n",
       " label                                       P279\n",
       " node2                                    Q100159\n",
       " bert2SentSim                            0.551256\n",
       " classSim                                0.866312\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 79, dtype: object,\n",
       " Unnamed: 0                                    80\n",
       " id              Q6061403-P279-Q100159-d5101564-0\n",
       " node1                                   Q6061403\n",
       " label                                       P279\n",
       " node2                                    Q100159\n",
       " bert2SentSim                            0.565718\n",
       " classSim                                0.866312\n",
       " Resp_code                       <Response [200]>\n",
       " Name: 80, dtype: object,\n",
       " Unnamed: 0                                   81\n",
       " id              Q652653-P279-Q100159-bd6c5d3a-0\n",
       " node1                                   Q652653\n",
       " label                                      P279\n",
       " node2                                   Q100159\n",
       " bert2SentSim                           0.501037\n",
       " classSim                               0.837687\n",
       " Resp_code                      <Response [200]>\n",
       " Name: 81, dtype: object,\n",
       " Unnamed: 0                                     82\n",
       " id              Q66293226-P279-Q100159-ad4b8016-0\n",
       " node1                                   Q66293226\n",
       " label                                        P279\n",
       " node2                                     Q100159\n",
       " bert2SentSim                               0.8089\n",
       " classSim                                 0.866312\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 82, dtype: object,\n",
       " Unnamed: 0                                         83\n",
       " id              Q100166380-P279-Q100166391-4684ea65-0\n",
       " node1                                      Q100166380\n",
       " label                                            P279\n",
       " node2                                      Q100166391\n",
       " bert2SentSim                                 0.752609\n",
       " classSim                                     0.799499\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 83, dtype: object,\n",
       " Unnamed: 0                                      84\n",
       " id              Q244326-P279-Q100166391-35359099-0\n",
       " node1                                      Q244326\n",
       " label                                         P279\n",
       " node2                                   Q100166391\n",
       " bert2SentSim                              0.522545\n",
       " classSim                                  0.470975\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 84, dtype: object,\n",
       " Unnamed: 0                                     85\n",
       " id              Q40551-P279-Q100166391-7242baf9-0\n",
       " node1                                      Q40551\n",
       " label                                        P279\n",
       " node2                                  Q100166391\n",
       " bert2SentSim                              0.74986\n",
       " classSim                                 0.781795\n",
       " Resp_code                        <Response [200]>\n",
       " Name: 85, dtype: object,\n",
       " Unnamed: 0                                         86\n",
       " id              Q102108504-P279-Q100171002-7ccf3934-0\n",
       " node1                                      Q102108504\n",
       " label                                            P279\n",
       " node2                                      Q100171002\n",
       " bert2SentSim                                 0.680867\n",
       " classSim                                         None\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 86, dtype: object,\n",
       " Unnamed: 0                                       87\n",
       " id              Q1150647-P279-Q100171002-fb1cad26-0\n",
       " node1                                      Q1150647\n",
       " label                                          P279\n",
       " node2                                    Q100171002\n",
       " bert2SentSim                               0.664551\n",
       " classSim                                   0.795416\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 87, dtype: object,\n",
       " Unnamed: 0                                       88\n",
       " id              Q1503443-P279-Q100171002-8282ae74-0\n",
       " node1                                      Q1503443\n",
       " label                                          P279\n",
       " node2                                    Q100171002\n",
       " bert2SentSim                               0.748802\n",
       " classSim                                   0.822611\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 88, dtype: object,\n",
       " Unnamed: 0                                       89\n",
       " id              Q2616305-P279-Q100171002-49dc8346-0\n",
       " node1                                      Q2616305\n",
       " label                                          P279\n",
       " node2                                    Q100171002\n",
       " bert2SentSim                               0.646817\n",
       " classSim                                   0.887266\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 89, dtype: object,\n",
       " Unnamed: 0                                        90\n",
       " id              Q24050179-P279-Q100195948-90966900-0\n",
       " node1                                      Q24050179\n",
       " label                                           P279\n",
       " node2                                     Q100195948\n",
       " bert2SentSim                                0.652214\n",
       " classSim                                    0.698738\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 90, dtype: object,\n",
       " Unnamed: 0                                        91\n",
       " id              Q29479539-P279-Q100195948-e222789a-0\n",
       " node1                                      Q29479539\n",
       " label                                           P279\n",
       " node2                                     Q100195948\n",
       " bert2SentSim                                0.689148\n",
       " classSim                                    0.681145\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 91, dtype: object,\n",
       " Unnamed: 0                                      92\n",
       " id              Q308859-P279-Q100195948-a5841742-0\n",
       " node1                                      Q308859\n",
       " label                                         P279\n",
       " node2                                   Q100195948\n",
       " bert2SentSim                              0.750328\n",
       " classSim                                  0.279686\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 92, dtype: object,\n",
       " Unnamed: 0                                         93\n",
       " id              Q100251305-P279-Q100251275-02ddd08e-0\n",
       " node1                                      Q100251305\n",
       " label                                            P279\n",
       " node2                                      Q100251275\n",
       " bert2SentSim                                 0.486286\n",
       " classSim                                      0.94603\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 93, dtype: object,\n",
       " Unnamed: 0                                         94\n",
       " id              Q100251705-P279-Q100251275-fa0c89c7-0\n",
       " node1                                      Q100251705\n",
       " label                                            P279\n",
       " node2                                      Q100251275\n",
       " bert2SentSim                                 0.370606\n",
       " classSim                                      0.94603\n",
       " Resp_code                            <Response [200]>\n",
       " Name: 94, dtype: object,\n",
       " Unnamed: 0                                        95\n",
       " id              Q16007968-P279-Q100251275-318d5598-0\n",
       " node1                                      Q16007968\n",
       " label                                           P279\n",
       " node2                                     Q100251275\n",
       " bert2SentSim                                0.478886\n",
       " classSim                                      0.9303\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 95, dtype: object,\n",
       " Unnamed: 0                                       96\n",
       " id              Q3310017-P279-Q100251275-772909ea-0\n",
       " node1                                      Q3310017\n",
       " label                                          P279\n",
       " node2                                    Q100251275\n",
       " bert2SentSim                                0.57202\n",
       " classSim                                   0.929488\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 96, dtype: object,\n",
       " Unnamed: 0                                       97\n",
       " id              Q4855428-P279-Q100251275-fbf2100a-0\n",
       " node1                                      Q4855428\n",
       " label                                          P279\n",
       " node2                                    Q100251275\n",
       " bert2SentSim                               0.545835\n",
       " classSim                                   0.931676\n",
       " Resp_code                          <Response [200]>\n",
       " Name: 97, dtype: object,\n",
       " Unnamed: 0                                      98\n",
       " id              Q862086-P279-Q100251275-faf57342-0\n",
       " node1                                      Q862086\n",
       " label                                         P279\n",
       " node2                                   Q100251275\n",
       " bert2SentSim                              0.630551\n",
       " classSim                                  0.948207\n",
       " Resp_code                         <Response [200]>\n",
       " Name: 98, dtype: object,\n",
       " Unnamed: 0                                        99\n",
       " id              Q15209696-P279-Q100257358-c4b60f40-0\n",
       " node1                                      Q15209696\n",
       " label                                           P279\n",
       " node2                                     Q100257358\n",
       " bert2SentSim                                0.484455\n",
       " classSim                                    0.610338\n",
       " Resp_code                           <Response [200]>\n",
       " Name: 99, dtype: object]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered2_class_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "worth-croatia",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierDF_transP279_filtered2_class = pd.DataFrame(hierDF_transP279_filtered2_class_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "changing-burning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered2_class.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "portuguese-practitioner",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "302108"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hierDF_transP279_filtered2_class.classSim.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "breathing-bicycle",
   "metadata": {},
   "source": [
    "## Siblings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "historic-maria",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279_3M_data = pd.read_csv('../data/Master_P279_dataset/P279Siblings_transP279_filtered_min_cols_with_desc_dups_removed.csv')\n",
    "P279_10M_data = pd.read_csv('../data/Master_P279_dataset/P279Siblings_transP279_filtered_min_cols.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spanish-killer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "P279_3M_data_class_arr = []\n",
    "\n",
    "def fetchClassSim(row):\n",
    "    try:\n",
    "        resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\").json()['similarity']\n",
    "        row['classSim'] = float(resp) if resp else -1\n",
    "    except:\n",
    "        print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "        row['classSim'] = None\n",
    "    return row\n",
    "\n",
    "P279_3M_data_class_arr = Parallel(n_jobs=20)(delayed(fetchClassSim)(row) for _, row in tqdm(P279_3M_data.iterrows(), total=len(P279_3M_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "collectible-modern",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "P279_10M_data_class_arr = []\n",
    "\n",
    "def fetchClassSim(row):\n",
    "    try:\n",
    "        resp = requests.get(\"https://kgtk.isi.edu/similarity_api?q1=\"+row['node1']+\"&q2=\"+row['node2']+\"&embedding_type=class\").json()['similarity']\n",
    "        row['classSim'] = float(resp) if resp else -1\n",
    "    except:\n",
    "        print(f\"Resp not found for {row['node1']}, {row['node2']}\")\n",
    "        row['classSim'] = None\n",
    "    return row\n",
    "\n",
    "P279_10M_data_class_arr = Parallel(n_jobs=20)(delayed(fetchClassSim)(row) for _, row in tqdm(P279_10M_data.iterrows(), total=len(P279_10M_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "removable-watershed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fossil-psychiatry",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rapid-drink",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "liquid-symphony",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "intellectual-rainbow",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "integrated-intention",
   "metadata": {},
   "source": [
    "# Wiki-CS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "functioning-scholarship",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unique-weekend",
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiDF = pd.read_csv('../data/wikidata-cs-20200504.tsv',sep='\\t')\n",
    "mapper1 = {'/r/DistinctFrom': 'distinctness',\n",
    "          '/r/Antonym': 'antonymy',\n",
    "          '/r/Synonym': 'synonymy',\n",
    "          '/r/SimilarTo': 'similarity',\n",
    "          '/r/DerivedFrom': 'derivation',\n",
    "          '/r/IsA': 'inheritance',\n",
    "          '/r/PartOf': 'meronymy',\n",
    "          '/r/MadeOf': 'material',\n",
    "          '/r/CreatedBy': 'attribution',\n",
    "          '/r/UsedFor': 'utility',\n",
    "          '/r/HasProperty': 'properties',\n",
    "          '/r/Causes': 'causation',\n",
    "          '/r/HasPrerequisite': 'ordering',\n",
    "          '/r/HasContext': 'context',\n",
    "          '/r/RelatedTo': 'other'}\n",
    "\n",
    "wikiDF['dim'] = wikiDF['relation'].apply(lambda p: mapper1[p])\n",
    "wikiDF['category'] = wikiDF['dim'].apply(lambda p: 'I' if p == 'synonymy' or p == 'similarity' else 'M')\n",
    "wikiDF['dim'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "located-swaziland",
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiDF['dim'].apply(lambda p: 'I' if p == 'synonymy' else 'M').value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sharing-scientist",
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiDF1 = wikiDF[wikiDF.apply(lambda p: p['node1'].startswith('Q') and p['node2'].startswith('Q'), axis=1)]\n",
    "wikiDF1.category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fewer-place",
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiDF1 = wikiDF1.rename(columns={'node1':'word1_kg_id', 'node2':'word2_kg_id'})\n",
    "wikiDF1[['word1_kg_id', 'word2_kg_id', 'category', 'node1;label', 'node2;label']].to_csv('../data/wikidata-cs_categorized.csv',index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "better-planet",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "enhanced-insured",
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiDF = pd.read_csv('../data/wikidata-cs_categorized.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "complex-character",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "M    100110\n",
       "I      1415\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikiDF.category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "exciting-anderson",
   "metadata": {},
   "outputs": [],
   "source": [
    "I_set_size = len(wikiDF[wikiDF.category == 'I'])\n",
    "M_sampled_set = wikiDF[wikiDF.category == 'M'].sample(I_set_size, random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "thrown-graham",
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import combinations\n",
    "\n",
    "def find_relation_mapping(mainset, subset, needed_size):\n",
    "    nodes = set([row['word1_kg_id'] for _,row in subset.iterrows()] \n",
    "            + [row['word2_kg_id'] for _,row in subset.iterrows()])\n",
    "    node_label_mappings = {row['word1_kg_id']: row['node1;label'] for _, row in subset.iterrows()}.update(\n",
    "                                {row['word2_kg_id']: row['node2;label'] for _, row in subset.iterrows()}\n",
    "                        )\n",
    "    mainset_pairs = set([(row['word1_kg_id'], row['word2_kg_id']) for _,row in mainset.iterrows()])\n",
    "\n",
    "    all_pairs = set([pair for pair in list(combinations(list(nodes), 2))[10*needed_size]])\n",
    "    \n",
    "    removed_pairs = set()\n",
    "    \n",
    "    first_line = True\n",
    "    with gzip.open('../data/wikidata-20210215-dwd/claims.tsv.gz', 'r') as claims_file:\n",
    "        for line in tqdm(claims_file,total=98482585):\n",
    "            if first_line:\n",
    "                first_line = False\n",
    "                continue\n",
    "            line = line.decode(\"utf-8\").strip().split('\\t')\n",
    "            curr_pair = (line[1], line[3])\n",
    "            if curr_pair in all_pairs:\n",
    "                removed_pairs.add(curr_pair)\n",
    "    newDF = []\n",
    "    for pair in all_pairs - removed_pairs - mainset_pairs:\n",
    "        try:\n",
    "            if pair[0] is None or pair[1] is None:\n",
    "                continue\n",
    "            newDF.append(pair[0], pair[1], 'U', node_label_mappings[pair[0]], node_label_mappings[pair[1]])\n",
    "        except:\n",
    "            print(pair)\n",
    "    return pd.DataFrame(newDF, columns=subset.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "immune-jesus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3df524d9d4fd47e6864fe57299e1f8e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/98482585 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q1070669\n",
      "Q20755059\n",
      "CPU times: user 13min 20s, sys: 1min 24s, total: 14min 44s\n",
      "Wall time: 14min 40s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "U_sampled_set = find_relation_mapping(wikiDF, wikiDF[wikiDF.category == 'M'], I_set_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "transparent-weather",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(U_sampled_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "graduate-guarantee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amended-support",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "assigned-knowing",
   "metadata": {},
   "source": [
    "# Check Diff in versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "under-simon",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "275927\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "def get_all_nodes():\n",
    "    p279ChildPar = pd.read_csv('../data/Master_P279_dataset/P279ChildPar_transP279_filtered_baremetal.csv')\n",
    "    wordsim_df = pd.read_csv('../data/wordsim353_with_r3.csv')\n",
    "    wiki_cs_df = pd.read_csv('../data/wikidata-cs_categorized.csv')\n",
    "    concept_net_df = pd.read_csv('../data/kgtk_conceptnet_evaluation.csv')\n",
    "    p279QnodesList = set(p279ChildPar.node1.to_list() \n",
    "                        + p279ChildPar.node2.to_list()\n",
    "                        + wordsim_df['word1_kg_id'].to_list() \n",
    "                        + wordsim_df['word2_kg_id'].to_list()\n",
    "                        + wiki_cs_df['word1_kg_id'].to_list() \n",
    "                        + wiki_cs_df['word2_kg_id'].to_list()\n",
    "                        + concept_net_df['word1_kg_id'].to_list()\n",
    "                        + concept_net_df['word2_kg_id'].to_list())\n",
    "    print(len(p279QnodesList))\n",
    "    return p279QnodesList\n",
    "allNodes = get_all_nodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "excess-killer",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "files_set_1 = ['../data/wikidata-20210215-dwd/descriptions.en.tsv.gz',\n",
    "              '../data/wikidata-20210215-dwd/sitelinks.en.tsv.gz']\n",
    "\n",
    "files_set_2 = [DESCRIPTIONS_FILE,\n",
    "              '../../wd-correctness/gdrive-kgtk-dump-2020-12-07/sitelinks.en.tsv.gz']\n",
    "\n",
    "\n",
    "all_nodes_set = set(allNodes)\n",
    "for file1, file2 in zip(files_set_1, files_set_2):\n",
    "    file1_dict, file2_dict = {}, {}\n",
    "    with gzip.open(file1) as file1:\n",
    "        for line in tqdm(file1):\n",
    "            line = line.decode(\"utf-8\").strip().split('\\t')\n",
    "            if line[1] in all_nodes_set:\n",
    "                file1_dict[line[1]] = line[3][1:-4]\n",
    "    with gzip.open(file2) as file2:\n",
    "        for line in tqdm(file2):\n",
    "            line = line.decode(\"utf-8\").strip().split('\\t')\n",
    "            if line[1] in all_nodes_set:\n",
    "                file2_dict[line[1]] = line[3][1:-4]\n",
    "    for elem in file1_dict.keys():\n",
    "        try:\n",
    "            if file1_dict[elem] != file2_dict[elem]:\n",
    "                print(elem, file1_dict[elem], file2_dict[elem])\n",
    "        except:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "integrated-publisher",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "romantic-rebecca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "needed-convergence",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "communist-thumbnail",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kgtkEnv",
   "language": "python",
   "name": "kgtkenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "278px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}