1b-Generate-2000-queries-datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import gensim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "ROOT = os.path.dirname(os.getcwd())\n",
    "#path_data = os.path.join(ROOT, 'data')\n",
    "path_data = 'C:\\\\Users\\\\joris\\\\Documents\\\\eScience_data\\\\data'\n",
    "sys.path.insert(0, ROOT)\n",
    "sys.path.insert(0, \"C:\\\\Users\\\\joris\\\\Documents\\\\eScience_data\\\\spec2vec_gnps_data_analysis\\\\custom_functions\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import pre-processed dataset \"AllPositive\" and post process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\joris\\Documents\\eScience_data\\data\\gnps_positive_ionmode_cleaned_by_matchms_and_lookups.pickle\n",
      "loading took 17.44 s\n",
      "number of spectra: 112956\n"
     ]
    }
   ],
   "source": [
    "#loading data as pickled object goes a lot quicker\n",
    "import time\n",
    "import pickle\n",
    "outfile = os.path.join(path_data, 'gnps_positive_ionmode_cleaned_by_matchms_and_lookups.pickle')\n",
    "print(outfile)\n",
    "start = time.time()\n",
    "with open(outfile, 'rb') as inf:\n",
    "        spectrums = pickle.load(inf)\n",
    "end = time.time()\n",
    "print('loading took {:.2f} s'.format(end-start))\n",
    "print(\"number of spectra:\", len(spectrums))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95320 remaining spectra.\n"
     ]
    }
   ],
   "source": [
    "from matchms.filtering import normalize_intensities\n",
    "from matchms.filtering import require_minimum_number_of_peaks\n",
    "from matchms.filtering import select_by_mz\n",
    "from matchms.filtering import select_by_relative_intensity\n",
    "from matchms.filtering import reduce_to_number_of_peaks\n",
    "from matchms.filtering import add_losses\n",
    "\n",
    "def post_process_s2v(s):\n",
    "    s = normalize_intensities(s)\n",
    "    s = select_by_mz(s, mz_from=0, mz_to=1000)\n",
    "    s = require_minimum_number_of_peaks(s, n_required=10)\n",
    "    s = reduce_to_number_of_peaks(s, n_required=10, ratio_desired=0.5)\n",
    "    if s is None:\n",
    "        return None\n",
    "    s_remove_low_peaks = select_by_relative_intensity(s, intensity_from=0.001)\n",
    "    if len(s_remove_low_peaks.peaks) >= 10:\n",
    "        s = s_remove_low_peaks\n",
    "        \n",
    "    s = add_losses(s, loss_mz_from=5.0, loss_mz_to=200.0)\n",
    "    return s\n",
    "\n",
    "# apply post processing steps to the data\n",
    "spectrums_s2v = [post_process_s2v(s) for s in spectrums]\n",
    "\n",
    "# omit spectrums that didn't qualify for analysis\n",
    "spectrums_s2v = [s for s in spectrums_s2v if s is not None]\n",
    "\n",
    "print(\"{} remaining spectra.\".format(len(spectrums_s2v)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspect annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Inchikeys = []\n",
    "for spec in spectrums_s2v:\n",
    "    Inchikeys.append(spec.get(\"inchikey\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NEGQHKSYEYVFTD    432\n",
       "SULIDBRAXVDKBU    426\n",
       "IQGPMZRCLCCXAG    308\n",
       "WTJKGGKOPKCXLL    295\n",
       "IIZPXYDJLKNOIY    235\n",
       "RWKUXQNLWDTSLO    234\n",
       "QIBZFHLFHCIUOT    225\n",
       "ZAYXPDDGEIJGGW    212\n",
       "QJWDAOSZZYVBJZ    210\n",
       "KILNVBDSWZSGLL    205\n",
       "RFVFQQWKPSOBED    202\n",
       "IESVDEZGAHUQJU    197\n",
       "LFUDDCMNKWEORN    195\n",
       "JLPULHDHAOZNQI    182\n",
       "LSOWKZULVQWMLY    180\n",
       "XGGMHQYOVYWRLV    178\n",
       "LLHISNQVRRYJGL    175\n",
       "JFISYPWOVQNHLS    174\n",
       "SRIGHEHXEGELQJ    166\n",
       "ACTIUHUUMQJHFO    163\n",
       "PZNPLUBHRSSFHT    162\n",
       "YLWSJLLZUHSIEA    158\n",
       "GPWHCUUIQMGELX    150\n",
       "BLZVZPYMHLXLHG    148\n",
       "QEDPUVGSSDPBMD    146\n",
       "IGZPHNNYPPAPLA    142\n",
       "CITHEXJVPOWHKC    141\n",
       "YEJYLHKQOBOSCP    131\n",
       "SXNXGNVZTLZDHE    131\n",
       "AXZGUCXCTZMPTR    128\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inchikeys_pd = pd.Series([x for x in Inchikeys if x])\n",
    "inchikeys_pd.str[:14].value_counts()[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inchikey14</th>\n",
       "      <th>occurences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NEGQHKSYEYVFTD</td>\n",
       "      <td>432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SULIDBRAXVDKBU</td>\n",
       "      <td>426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IQGPMZRCLCCXAG</td>\n",
       "      <td>308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WTJKGGKOPKCXLL</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IIZPXYDJLKNOIY</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RWKUXQNLWDTSLO</td>\n",
       "      <td>234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>QIBZFHLFHCIUOT</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ZAYXPDDGEIJGGW</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>QJWDAOSZZYVBJZ</td>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>KILNVBDSWZSGLL</td>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>RFVFQQWKPSOBED</td>\n",
       "      <td>202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>IESVDEZGAHUQJU</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LFUDDCMNKWEORN</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>JLPULHDHAOZNQI</td>\n",
       "      <td>182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LSOWKZULVQWMLY</td>\n",
       "      <td>180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        inchikey14  occurences\n",
       "0   NEGQHKSYEYVFTD         432\n",
       "1   SULIDBRAXVDKBU         426\n",
       "2   IQGPMZRCLCCXAG         308\n",
       "3   WTJKGGKOPKCXLL         295\n",
       "4   IIZPXYDJLKNOIY         235\n",
       "5   RWKUXQNLWDTSLO         234\n",
       "6   QIBZFHLFHCIUOT         225\n",
       "7   ZAYXPDDGEIJGGW         212\n",
       "8   QJWDAOSZZYVBJZ         210\n",
       "9   KILNVBDSWZSGLL         205\n",
       "10  RFVFQQWKPSOBED         202\n",
       "11  IESVDEZGAHUQJU         197\n",
       "12  LFUDDCMNKWEORN         195\n",
       "13  JLPULHDHAOZNQI         182\n",
       "14  LSOWKZULVQWMLY         180"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suitable_inchikeys = pd.DataFrame(inchikeys_pd.str[:14].value_counts()[inchikeys_pd.str[:14].value_counts().values >= 5])\n",
    "suitable_inchikeys.reset_index(level=suitable_inchikeys.index.names, inplace=True)\n",
    "suitable_inchikeys.columns = (['inchikey14', 'occurences'])\n",
    "\n",
    "# Important: sort values to make it reproducible (same occurences have random order otherwise!)\n",
    "suitable_inchikeys = suitable_inchikeys.sort_values(['occurences', 'inchikey14'], ascending=False)\n",
    "suitable_inchikeys.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of spectra with inchikeys that exist >= 5 times: 4079\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of spectra with inchikeys that exist >= 5 times:\", suitable_inchikeys.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomly select 1000 inchikeys that exist >=5  times in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  33 3233 1556 2547  457 3598  857 1878  705  803 1642  495 2866 1034\n",
      " 1517  564 2826  568 2575 3956  670 3317 1498 2971   96]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['SRRQPVVYXBTRQK', 'JUZYLCPPVHEVSV', 'UZXMLGUMBQQVME',\n",
       "       'PDAKXMIQFUHWQC', 'JFVKWCYZKMUTLH', 'HQPCSDADVLFHHO',\n",
       "       'HUPGTAGQEXENPN', 'YPKUMLKVFXFYOT', 'KWIUHFFTVRNATP',\n",
       "       'QJVXKWHHAMZTBY', 'PGERTGWKXFAEFR', 'MIHLTJZHRJACQQ',\n",
       "       'YFAGHNZHGGCZAX', 'LKWWJGGLULNRBP', 'XWTYSIMOBUGWOL',\n",
       "       'BPEXJHGGARTCIR', 'FHHVIBPVBBRLOR', 'YFPYXTNSQOUHPS',\n",
       "       'LTLYEAJONXGNFG', 'INOGLHRUEYDAHX', 'VGOJYSUPEJWUNN',\n",
       "       'BPICBUSOMSTKRF', 'ZQHJXKYYELWEOK', 'MXHRCPNRJAMMIM',\n",
       "       'QXMHHXQBBKDSSL'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_spectra = 1000\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "selection = np.random.choice(suitable_inchikeys.shape[0], num_spectra, replace=False)\n",
    "print(selection[:25])\n",
    "selected_inchikeys = suitable_inchikeys['inchikey14'].values[selection]\n",
    "selected_inchikeys[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3079\n",
      "Number of old selected inchikeys in new list: 0\n",
      "[  63 2775 2016  331 2739  594 2244  346 2312 1472 2785  194 2370 2032\n",
      " 2538  246  256  486  978 1961  144  801  744  457  416] 1000\n",
      "Overlap between new and old list: 0\n"
     ]
    }
   ],
   "source": [
    "#Randomly select another 1000 inchikeys that exist >=5 times in the dataset and don't occur in the first list selected_inchikeys\n",
    "new_suitable_inchikeys = suitable_inchikeys.copy()\n",
    "drop_selection = suitable_inchikeys.index[selection] #get index names of first selection\n",
    "new_suitable_inchikeys.drop(drop_selection, inplace = True) #remove first selection\n",
    "print(new_suitable_inchikeys.shape[0])\n",
    "print('Number of old selected inchikeys in new list:',len(set(new_suitable_inchikeys['inchikey14']) & set(selected_inchikeys)))\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "new_selection = np.random.choice(new_suitable_inchikeys.shape[0], num_spectra, replace=False)\n",
    "print(new_selection[:25], len(new_selection))\n",
    "new_selected_inchikeys = new_suitable_inchikeys['inchikey14'].values[new_selection]\n",
    "print('Overlap between new and old list:', len(set(new_selected_inchikeys) & set(selected_inchikeys)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomly pick one spectra for each of the chosen inchikeys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_spectra = []\n",
    "inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "for inchikey in selected_inchikeys:\n",
    "    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values\n",
    "    selected_spectra.append(int(np.random.choice(matches,1)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pick one spectrum for each of the new chosen inchikeys\n",
    "new_selected_spectra = []\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "for inchikey in new_selected_inchikeys:\n",
    "    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values\n",
    "    new_selected_spectra.append(int(np.random.choice(matches,1)[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomly pick 1000 inchikeys (spectra) that are unqiue in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inchikey14</th>\n",
       "      <th>occurences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2854</th>\n",
       "      <td>ZZYHCCDMBJTROG</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4383</th>\n",
       "      <td>ZZTCNNZHOWDRPS</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907</th>\n",
       "      <td>ZZQHNBGRWRQWFI</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327</th>\n",
       "      <td>ZZPAWQYZQVUVHX</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2651</th>\n",
       "      <td>ZZNVCZGRNCQHCQ</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>ZZHLYYDVIOPZBE</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3181</th>\n",
       "      <td>ZZAJQOPSWWVMBI</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934</th>\n",
       "      <td>ZYYNEJWFGGVJQZ</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4729</th>\n",
       "      <td>ZYVYPNZFOCZLEM</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3347</th>\n",
       "      <td>ZYUDDSCUGFXAKK</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1775</th>\n",
       "      <td>ZYTHYMZQQBWDDS</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980</th>\n",
       "      <td>ZYGYNVJVDLQWDG</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2969</th>\n",
       "      <td>ZYGSNVAGWKWKKI</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ZYEMGPIYFIJGTP</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2323</th>\n",
       "      <td>ZXZPFGGUUZDXND</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          inchikey14  occurences\n",
       "2854  ZZYHCCDMBJTROG           1\n",
       "4383  ZZTCNNZHOWDRPS           1\n",
       "2907  ZZQHNBGRWRQWFI           1\n",
       "1327  ZZPAWQYZQVUVHX           1\n",
       "2651  ZZNVCZGRNCQHCQ           1\n",
       "3826  ZZHLYYDVIOPZBE           1\n",
       "3181  ZZAJQOPSWWVMBI           1\n",
       "4934  ZYYNEJWFGGVJQZ           1\n",
       "4729  ZYVYPNZFOCZLEM           1\n",
       "3347  ZYUDDSCUGFXAKK           1\n",
       "1775  ZYTHYMZQQBWDDS           1\n",
       "1980  ZYGYNVJVDLQWDG           1\n",
       "2969  ZYGSNVAGWKWKKI           1\n",
       "28    ZYEMGPIYFIJGTP           1\n",
       "2323  ZXZPFGGUUZDXND           1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_inchikeys = pd.DataFrame(inchikeys_pd.str[:14].value_counts()[inchikeys_pd.str[:14].value_counts().values == 1])\n",
    "\n",
    "unique_inchikeys.reset_index(level=unique_inchikeys.index.names, inplace=True)\n",
    "unique_inchikeys.columns = (['inchikey14', 'occurences'])\n",
    "\n",
    "# Important: sort values to make it reproducible (same occurences have random order otherwise!)\n",
    "unique_inchikeys = unique_inchikeys.sort_values(['occurences', 'inchikey14'], ascending=False)\n",
    "unique_inchikeys.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5352\n"
     ]
    }
   ],
   "source": [
    "print(len(unique_inchikeys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4114 1650  401 4221 1684   33  290 3265 5324 3235 4213 3403 5004 3699\n",
      " 1075  925 4074 4446 2642 2972  810 3204 3872 2707 2651]\n",
      "['GQIJYUMTOUBHSH' 'RVWQXFWAUMMLKE' 'YGULWPYYGQCFMP' 'GDPHPXYFLPDZGH'\n",
      " 'RRNSPXUFTKJIEZ' 'ZWNKGUOHTAKRBN' 'YTNXARXLPJGCHV' 'KMRZMOYMAAVHNA'\n",
      " 'ADDCNOCQPWDJSR' 'KQFUXLQBMQGNRT' 'GEJNVNUFFYATKR' 'JVRVKOOOXHGJKI'\n",
      " 'BOVRCQYBOHNUIF' 'INOKSQLHQGQUNF' 'VGGCLGHLLZKCSW' 'VYGQXRZAHIZHQV'\n",
      " 'GUOQUXNJZHGPQF' 'FAZKBTFQSATIIN' 'NISZLIJBKRGIAU' 'LVUQCTGSDJLWCE'\n",
      " 'WMAITHDYVBQITD' 'KSZJPQHIPKEEMF' 'HSLDNQNDHKZKNZ' 'NBLBCGUCPBXKOV'\n",
      " 'NHNODHRSCRALBF']\n",
      "1000\n"
     ]
    }
   ],
   "source": [
    "num_spectra = 1000\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "unique_selection = np.random.choice(unique_inchikeys.shape[0], num_spectra, replace=False)\n",
    "print(unique_selection[:25])\n",
    "selected_unique_inchikeys = unique_inchikeys['inchikey14'].values[unique_selection]\n",
    "print(selected_unique_inchikeys[:25])\n",
    "\n",
    "#get the spectrum that belongs with the unique chosen inchikeys\n",
    "selected_unique_spectra = []\n",
    "inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "for inchikey in selected_unique_inchikeys:\n",
    "    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values\n",
    "    selected_unique_spectra.append(int(matches[0]))\n",
    "print(len(selected_unique_spectra))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4352\n",
      "Number of old selected inchikeys in new list: 0\n",
      "[ 505 1611 3107  511 1467  410 3678 2677 3370 4235 3853 3217  149 4332\n",
      " 1295 2457 2589 1261  287 3038   17  755  305 3276 4084] 1000\n",
      "Overlap between new and old list: 0\n",
      "1000\n"
     ]
    }
   ],
   "source": [
    "#Randomly select another 1000 inchikeys that are unique in the dataset and don't occur in the first list selected_unique_inchikeys\n",
    "new_unique_inchikeys = unique_inchikeys.copy()\n",
    "drop_selection = unique_inchikeys.index[unique_selection] #get index names of first selection\n",
    "new_unique_inchikeys.drop(drop_selection, inplace = True) #remove first selection\n",
    "print(new_unique_inchikeys.shape[0])\n",
    "print('Number of old selected inchikeys in new list:',len(set(new_unique_inchikeys['inchikey14']) & set(selected_unique_inchikeys)))\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "new_unique_selection = np.random.choice(new_unique_inchikeys.shape[0], num_spectra, replace=False)\n",
    "print(new_unique_selection[:25], len(new_unique_selection))\n",
    "new_selected_unique_inchikeys = new_unique_inchikeys['inchikey14'].values[new_unique_selection]\n",
    "print('Overlap between new and old list:', len(set(new_selected_unique_inchikeys) & set(selected_unique_inchikeys)))\n",
    "\n",
    "#get the spectrum that belongs with the unique chosen inchikeys\n",
    "new_selected_unique_spectra = []\n",
    "inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!\n",
    "\n",
    "np.random.seed(42) # to make it reproducible\n",
    "for inchikey in new_selected_unique_inchikeys:\n",
    "    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values\n",
    "    new_selected_unique_spectra.append(int(matches[0]))\n",
    "print(len(new_selected_unique_spectra))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the two datasets together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#add unique dataset together with old test set of 1000 spectra >5 occurrence\n",
    "old_and_unique_test_set = selected_spectra + selected_unique_spectra\n",
    "len(old_and_unique_test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#add second unique test set together with new test set of 1000 spectra >5 occurrence\n",
    "new_and_unique2_test_set = new_selected_spectra + new_selected_unique_spectra\n",
    "len(new_and_unique2_test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split spectra into library and query set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_and_unique_documents_library_s2v = [SpectrumDocument(s, n_decimals=2) for ind, s in enumerate(spectrums_s2v) if ind not in old_and_unique_test_set]\n",
    "new_and_unique2_documents_library_s2v = [SpectrumDocument(s, n_decimals=2) for ind, s in enumerate(spectrums_s2v) if ind not in new_and_unique2_test_set]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_and_unique_documents_query_s2v = [SpectrumDocument(spectrums_s2v[i], n_decimals=2) for i in old_and_unique_test_set]\n",
    "new_and_unique2_documents_query_s2v = [SpectrumDocument(spectrums_s2v[i], n_decimals=2) for i in new_and_unique2_test_set]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}