t-SNE and UMAP comparison-cleared.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary of work"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook draws preliminary comparisons between the UMAP embeddings, a currently generated t-SNE embedding, and an earlier t-SNE embedding by Joris Louwen. \n",
    "\n",
    "Because the interactive plots created with plotly inflated the notebook beyond the data limit of Github, I've created HTML versions of the plots and uploaded them via the Bioinformatics server. You can view them via the links provided below the code for the plots."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries, data and functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import gensim\n",
    "import spec2vec\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import umap\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import IncrementalPCA\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "from processing import get_ids_for_unique_inchikeys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def colour_dict(var_column, blackout_list=[]):\n",
    "    # Creates a colour dictionary with 20 distinct colours. Blackout_list offers the possibility to directly assign the colour black to groups.  \n",
    "    # If the number of groups in the variable column exceeds 20, the only largest groups will be given the distinct colours.\n",
    "    # If the number of groups is smaller than 20, the first n colours will be used.\n",
    "    # Colours by Sasha Trubetskoy at https://sashamaps.net/docs/resources/20-colors/\n",
    "    \n",
    "    colours = ['#E6194B', '#3CB44B', '#FFE119', '#4363D8', '#F58231',\n",
    "                '#911EB4', '#46F0F0', '#F032E6', '#BCF60C', '#FABEBE',\n",
    "                '#008080', '#E6BEFF', '#9A6324', '#0000CD', '#800000', # replaced FFFAC8 (beige) with #0000CD (medium blue) for better visibility against light background\n",
    "                '#AAFFC3', '#808000', '#FFD8B1', '#000075', '#808080']\n",
    "    \n",
    "    groups = list(var_column.value_counts(ascending=False).index)\n",
    "    for elem in blackout_list:\n",
    "        groups.remove(elem)\n",
    "    while (len(groups) > len(colours)):\n",
    "           colours.append(\"#000000\")\n",
    "    colour_dict = dict(zip(groups, colours))\n",
    "    for elem in blackout_list:\n",
    "        colour_dict[elem] = \"#000000\"\n",
    "    return colour_dict\n",
    "\n",
    "def collapse_classes(df, var, top_n = 20):\n",
    "    # Collapses all the minority classes of a variable into one super group, to increase legibility of plots. \n",
    "    # It returns a dataframe with the var column altered to reflect the directed change.\n",
    "    top_classes = list(df[var].value_counts(ascending = False).index[:top_n])\n",
    "    df.loc[~df[var].isin(top_classes), var] = \"Other\"\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"C:/Users/Artur/Documents/Werk/Spec2Vec/Data/\"\n",
    "model_dir = \"C:/Users/Artur/Documents/Werk/Spec2Vec/Model/\"\n",
    "embedding_dir = \"C:/Users/Artur/Documents/Werk/Spec2Vec/Embeddings/\"\n",
    "\n",
    "# For now, we are only using the spectra that were obtained in positive ion-mode\n",
    "pretrained_model = gensim.models.Word2Vec.load(model_dir+\"ALL_GNPS_210409_positive_cleaned_spec2vec_embedding_iter_15.model\")\n",
    "spectra = pd.read_pickle(data_dir+\"ALL_GNPS_210409_positive_cleaned_peaks_processed_s2v.pickle\")\n",
    "\n",
    "# Load the class predictions for each inchikey and shorten the inchikey to the first 14 characters. In case of duplicates, we keep the first occurence and drop the others.\n",
    "# We only use the first 14 characters of the inchikey (the so-called planar inchikey) because MS spectra cannot be used to meaningfully distinguish compounds beyond these features.\n",
    "inchikey_classifications = pd.read_csv(data_dir+\"ALL_GNPS_210409_positive_processed_annotated_CF_NPC_classes.txt\", sep = \"\\t\")\n",
    "inchikey_classifications.rename(columns = {\"inchi_key\": \"inchikey\"}, inplace = True) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What metadata do we have?\n",
    "spectra[0].metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187152 out of 199780 spectra have an Inchikey\n"
     ]
    }
   ],
   "source": [
    "# We retrieve the inchikey and source instrument for all spectra, and look up the predicted classes for the inchikey.\n",
    "spectrum_id = []\n",
    "inchikeys = []\n",
    "instruments = []\n",
    "\n",
    "for spec in spectra:\n",
    "    #short_inchikey = spec.get(\"inchikey\")[:14]\n",
    "    inchikeys.append(spec.get(\"inchikey\"))\n",
    "    instruments.append(spec.get(\"instrument\"))\n",
    "    spectrum_id.append(spec.get(\"spectrum_id\"))\n",
    "spectrum_metadata = pd.DataFrame({\"ID\": spectrum_id, \"inchikey\":inchikeys, \"instrument\": instruments})\n",
    "\n",
   "# We drop all spectral records without inchikey and match the remaining records with npclassifier and classyfire compound class predictions\n",
    "spectrum_metadata['spectrum_index'] = spectrum_metadata.index # Merging dataframes changes the row order, so we're saving the index now as a column to restore it after merging\n",
    "spectrum_metadata = spectrum_metadata[spectrum_metadata[\"inchikey\"] != \"\"]\n",
    "spectrum_metadata = spectrum_metadata.merge(inchikey_classifications, how='left', on='inchikey', sort=False).sort_values(by=\"spectrum_index\")\n",
    "\n",
    "spectrum_metadata[\"planar_inchi\"] = [key[:14] for key in spectrum_metadata[\"inchikey\"]] # Add shortened planar inchikey to metadata\n",
    "inchi_spectra = [spectra[i] for i in spectrum_metadata[\"spectrum_index\"]] # Subset the spectra to only the spectra with inchikeys\n",
    "print(len(inchi_spectra), \"out of\", len(spectra), \"spectra have an Inchikey\")\n",
    "spectrum_metadata.reset_index(drop = True, inplace = True) # Match indices to inchi_spectra indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieve SpectrumDocuments and convert to vectors\n",
    "spectrum_documents = [spec2vec.SpectrumDocument(s, n_decimals=2) for i, s in enumerate(inchi_spectra)]\n",
    "spectrum_vectors = pd.DataFrame([spec2vec.calc_vector(pretrained_model, document, intensity_weighting_power=0.5) for i, document in enumerate(spectrum_documents)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16358 out of 187152 annotated spectra have a unique Inchikey\n"
     ]
    }
   ],
   "source": [
    "# We retrieve the IDs of spectra with unique planar Inchikeys\n",
    "unique_inchi = get_ids_for_unique_inchikeys(inchi_spectra)\n",
    "print(f'{len(unique_inchi)} out of {len(inchi_spectra)} annotated spectra have a unique Inchikey')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We save the spectrum_vectors so we can don't have to calculate those again later\n",
    "spectrum_vectors.to_csv(data_dir+\"ALL_GNPS_210409_positive_spectrumvectors_weighted_0.5.csv\", header = False, index = False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "spectrum_vectors = pd.read_csv(data_dir+\"ALL_GNPS_210409_positive_spectrumvectors_weighted_0.5.csv\", sep = \"\\t\", header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is what the metadata we've collected looks like for the first three spectra\n",
    "# The class predictions are split up between two different classifiers, ClassyFire (cf), and NPClassifier (npc)\n",
    "spectrum_metadata.iloc[0:3,:].T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reduce dimensions with UMAP and t-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduce dimensions with umap\n",
    "umap_df = pd.DataFrame(umap.UMAP(n_components=2, a=1, b=1, n_neighbors=15).fit_transform(spectrum_vectors), columns = [\"x\", \"y\"]); umap_df.index = spectrum_metadata.index\n",
    "umap_df = pd.concat([umap_df, spectrum_metadata], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduce dimensions with t-SNE\n",
    "tsne_df = pd.DataFrame(TSNE(n_components=2, random_state=42).fit_transform(spectrum_vectors), columns = [\"x\", \"y\"]); tsne_df.index = spectrum_metadata.index\n",
    "tsne_df = pd.concat([tsne_df, spectrum_metadata], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combination PCA + t-SNE\n",
    "vector_pca = IncrementalPCA(n_components = 50, batch_size = 7500).fit_transform(spectrum_vectors)\n",
    "pca_tsne = pd.DataFrame(TSNE(n_components=2, random_state=42).fit_transform(vector_pca), columns = [\"x\", \"y\"]); pca_tsne.index = spectrum_metadata.index\n",
    "pca_tsne = pd.concat([pca_tsne, spectrum_metadata], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We save the UMAP and t-SNE dataframes we made, so we don't have to calculate those again later\n",
    "umap_df.to_csv(data_dir+\"Annotated_weighted0.5_GNPS_210409_positive_UMAP_a1b1_neighbours15.csv\", index = True, sep=\"\\t\")\n",
    "tsne_df.to_csv(data_dir+\"Annotated_weighted0.5_GNPS_210409_positive_t-SNE.csv\", index = False, sep=\"\\t\")\n",
    "pca_tsne.to_csv(data_dir+\"Annotated_weighted0.5_GNPS_210409_positive_PCA_t-SNE.csv\", index = False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read UMAP and t-SNE dataframes\n",
    "umap_df = pd.read_csv(embedding_dir+\"Annotated_weighted0.5_GNPS_210409_positive_UMAP_a1b1_neighbours15.csv\", sep=\"\\t\", index_col = 0)\n",
    "tsne_df = pd.read_csv(embedding_dir+\"Python395_sklearn0242_weighted_t-SNE_state42.csv\", sep=\",\", index_col = 0)\n",
    "tsne_df = pd.concat([tsne_df, spectrum_metadata], axis = 1)\n",
    "\n",
    "pca_tsne = pd.read_csv(embedding_dir+\"Annotated_weighted0.5_GNPS_210409_positive_PCA_t-SNE.csv\", sep=\"\\t\")\n",
    "\n",
    "tsne_louwen = pd.read_csv(embedding_dir+\"ALL_GNPS_210409_positive_cleaned_peaks_processed_s2v_only_annotated_tsne2D.csv\", sep=\",\", index_col = 0, names = [\"x\", \"y\"])\n",
    "tsne_louwen = tsne_louwen.merge(spectrum_metadata, how = \"inner\", left_index = True, right_on = \"ID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot UMAP and 21/7 t-SNE\n",
    "hue_var = \"npc_superclass_results\"\n",
    "plot_df = collapse_classes(df = umap_df, var = hue_var, top_n = 20)\n",
    "colourdict = colour_dict(plot_df[hue_var], blackout_list=[\"Other\"])\n",
    "\n",
    "small_marker = 2; big_marker = 3.5\n",
    "\n",
    "fig = make_subplots(rows=2, cols=2, horizontal_spacing = 0.1, vertical_spacing = 0.1, \n",
    "                    subplot_titles=(f'UMAP: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'21/7 t-SNE Joris: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'UMAP: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey',\n",
    "                                    f'21/7 t-SNE Joris: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey'))\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.25, legendgroup=\"all\"), row=1, col=1)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x),\n",
    "    row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.25, legendgroup=\"all\",  showlegend=False), row=2, col=1)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=1)\n",
    "    \n",
    "plot_df = collapse_classes(df = tsne_louwen, var = hue_var, top_n = 20)\n",
    "colourdict = colour_dict(plot_df[hue_var], blackout_list=[\"Other\"])\n",
    "    \n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.5, legendgroup=\"all\", showlegend=False), row=1, col=2)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x, showlegend=False),\n",
    "    row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.5, legendgroup=\"all\",  showlegend=False), row=2, col=2)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=2)\n",
    "\n",
    "fig.update_layout(height=900, width=1600, title_text=f\"UMAP versus t-SNE with data colored based on {hue_var}\", title_x=0.5, titlefont=dict(size=24), template='ggplot2', legend={'itemsizing': 'constant'})\n",
    "fig.show()\n",
    "\n",
    "fig.write_html(data_dir+\"UMAP_vs_21_7_tSNE_Joris.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View plot at:\n",
    "https://www.bioinformatics.nl/~vanb001/UMAP_vs_21_7_tSNE_Joris.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot UMAP and 21/7 t-SNE\n",
    "hue_var = \"npc_superclass_results\"\n",
    "plot_df = collapse_classes(df = tsne_df, var = hue_var, top_n = 20)\n",
    "colourdict = colour_dict(plot_df[hue_var], blackout_list=[\"Other\"])\n",
    "\n",
    "small_marker = 2; big_marker = 3.5\n",
    "\n",
    "fig = make_subplots(rows=2, cols=2, vertical_spacing = 0.1, horizontal_spacing = 0.1,\n",
    "                    subplot_titles=(f'3/8 Artur: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'21/7 Joris: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'3/8 Artur: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey',\n",
    "                                    f'21/7 Joris: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey'))\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.1, legendgroup=\"all\"), row=1, col=1)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x),\n",
    "    row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.5, legendgroup=\"all\",  showlegend=False), row=2, col=1)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=1)\n",
    "    \n",
    "plot_df = collapse_classes(df = tsne_louwen, var = hue_var, top_n = 20)\n",
    "colourdict = colour_dict(plot_df[hue_var], blackout_list=[\"Other\"])\n",
    "    \n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.5, legendgroup=\"all\", showlegend=False), row=1, col=2)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x, showlegend=False),\n",
    "    row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.5, legendgroup=\"all\",  showlegend=False), row=2, col=2)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=2)\n",
    "\n",
    "fig.update_layout(height=900, width=1600, title_text=f\"21/7 t-SNE Joris versus 3/8 t-SNE Artur with data colored based on {hue_var}\", title_x=0.5, titlefont=dict(size=24), template='ggplot2', legend={'itemsizing': 'constant'})\n",
    "fig.update_yaxes(range=[-45, 45]); fig.update_xaxes(range=[-50, 50])\n",
    "fig.show()\n",
    "\n",
    "fig.write_html(data_dir+\"tSNE_Artur_vs_21_7_tSNE_Joris.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View plot at:\n",
    "https://www.bioinformatics.nl/~vanb001/tSNE_Artur_vs_21_7_tSNE_Joris.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Plot t-SNE\n",
    "hue_var = \"npc_superclass_results\"\n",
    "plot_df = collapse_classes(df = tsne_df, var = hue_var, top_n = 20)\n",
    "colourdict = colour_dict(plot_df[hue_var], blackout_list=[\"Other\"])\n",
    "\n",
    "small_marker = 2; big_marker = 3.5\n",
    "\n",
    "fig = make_subplots(rows=2, cols=2, vertical_spacing = 0.1, horizontal_spacing = 0.1,\n",
    "                    subplot_titles=(f'Only t-SNE: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'PCA + t-SNE: all {len(plot_df)} spectra with an Inchikey', \n",
    "                                    f'Only t-SNE: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey',\n",
    "                                    f'PCA + t-SNE: all {len(plot_df.iloc[unique_inchi, :])} spectra with a unique planar Inchikey'))\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.1, legendgroup=\"all\"), row=1, col=1)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x),\n",
    "    row=1, col=1)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.5, legendgroup=\"all\",  showlegend=False), row=2, col=1)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=1)\n",
    "\n",
    "    \n",
    "plot_df = collapse_classes(df = pca_tsne, var = hue_var, top_n = 20)\n",
    "    \n",
    "fig.add_trace(go.Scattergl(x=plot_df[\"x\"], y=plot_df[\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = small_marker, marker_line=dict(color='grey', width=0.01), opacity = 0.1, legendgroup=\"all\", showlegend=False), row=1, col=2)\n",
    "for x, group in plot_df.groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = x, mode='markers', marker_color = colourdict[x], marker_size = small_marker, legendgroup=x, showlegend=False),\n",
    "    row=1, col=2)\n",
    "\n",
    "fig.add_trace(go.Scattergl(x=plot_df.iloc[unique_inchi][\"x\"], y=plot_df.iloc[unique_inchi][\"y\"], name = \"All spectra\", mode='markers', marker_color = \"white\", marker_size = big_marker, opacity = 0.5, legendgroup=\"all\",  showlegend=False), row=2, col=2)\n",
    "for x, group in plot_df.iloc[unique_inchi,:].groupby(hue_var):\n",
    "    fig.add_trace(go.Scattergl(x=group.x, y=group.y, name = \"\", mode='markers',  marker_color = colourdict[x], marker_size = big_marker, legendgroup=x, showlegend=False),\n",
    "    row=2, col=2)\n",
    "\n",
    "fig.update_layout(height=900, width=1800, title_text=f\"t-SNE of spec2vec vectors colored based on {hue_var}\", title_x=0.5, titlefont=dict(size=24), template='ggplot2', legend={'itemsizing': 'constant'})\n",
    "fig.update_yaxes(range=[-45, 45]); fig.update_xaxes(range=[-50, 50])\n",
    "fig.show()\n",
    "\n",
    "fig.write_html(data_dir+\"tSNE_Artur_vs_PCA_tSNE.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View plot at:\n",
    "https://www.bioinformatics.nl/~vanb001/tSNE_Artur_vs_PCA_tSNE.html"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc-autonumbering": true,
  "toc-showmarkdowntxt": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}