vader_spark (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "91e1c9f3",
   "metadata": {},
   "source": [
    "# Sentiment analysis (Vader) and Word2Vec for Reddit comment data (r/TheRedPill, 2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c36323c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e91d0502d18411fa22bced3bc8a70d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>0</td><td>application_1622917956355_0001</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-39-114.ec2.internal:20888/proxy/application_1622917956355_0001/\" class=\"emr-proxy-link\" emr-resource=\"j-321QCRQH1104N\n",
       "\" application-id=\"application_1622917956355_0001\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-44-84.ec2.internal:8042/node/containerlogs/container_1622917956355_0001_01_000001/livy\" >Link</a></td><td>✔</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "An error was encountered:\n",
      "Package already installed for current Spark context!\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/spark/python/lib/pyspark.zip/pyspark/context.py\", line 1110, in install_pypi_package\n",
      "    raise ValueError(\"Package already installed for current Spark context!\")\n",
      "ValueError: Package already installed for current Spark context!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sc.install_pypi_package(\"seaborn\")\n",
    "sc.install_pypi_package(\"pandas\")\n",
    "sc.install_pypi_package(\"nltk\")\n",
    "sc.install_pypi_package(\"pandas\")\n",
    "sc.install_pypi_package(\"pickle\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "92117a1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f876f52adedc467aa6fd31d09060ee6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting vader\n",
      "  Downloading https://files.pythonhosted.org/packages/be/0d/df60a0ae9ffb63c409849d2909883963855d10e2ee9a5a71c97be41da300/vader-0.0.2-py3-none-any.whl (45kB)\n",
      "Collecting scikit-learn (from vader)\n",
      "  Downloading https://files.pythonhosted.org/packages/6f/6b/10881b09340d69d4a941e5624bfbf1ba853be8cdf2141077e66dda0b088e/scikit_learn-0.24.2-cp37-cp37m-manylinux1_x86_64.whl (20.0MB)\n",
      "Collecting sonopy (from vader)\n",
      "  Downloading https://files.pythonhosted.org/packages/2b/4d/862855fb391bc30351f90d6c50ea913df9d18b0ae3b6b8ef3c7aa3ac976f/sonopy-0.1.2.tar.gz\n",
      "Requirement already satisfied: numpy in /usr/local/lib64/python3.7/site-packages (from vader)\n",
      "Requirement already satisfied: scipy in /mnt/tmp/1622918341249-0/lib/python3.7/site-packages (from vader)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib64/python3.7/site-packages (from scikit-learn->vader)\n",
      "Collecting threadpoolctl>=2.0.0 (from scikit-learn->vader)\n",
      "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
      "Building wheels for collected packages: sonopy\n",
      "  Running setup.py bdist_wheel for sonopy: started\n",
      "  Running setup.py bdist_wheel for sonopy: finished with status 'done'\n",
      "  Stored in directory: /var/lib/livy/.cache/pip/wheels/b6/39/ba/b2f21d4fbcb362658c73f83c9502782300b0399aef3693b506\n",
      "Successfully built sonopy\n",
      "Installing collected packages: threadpoolctl, scikit-learn, sonopy, vader\n",
      "Successfully installed scikit-learn-0.24.2 sonopy-0.1.2 threadpoolctl-2.1.0 vader-0.0.2"
     ]
    }
   ],
   "source": [
    "sc.install_pypi_package(\"vader\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48cdf159",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "568a89593dd9426581b94ca5e4f2f1e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gensim\n",
      "  Downloading https://files.pythonhosted.org/packages/44/52/f1417772965652d4ca6f901515debcd9d6c5430969e8c02ee7737e6de61c/gensim-4.0.1-cp37-cp37m-manylinux1_x86_64.whl (23.9MB)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /mnt/tmp/1622918341249-0/lib/python3.7/site-packages (from gensim)\n",
      "Collecting smart-open>=1.8.1 (from gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/e9/90/6ca525991e281ecdf204c5c1de854da6334068e44121c384b68c6a838e14/smart_open-5.1.0-py3-none-any.whl (57kB)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib64/python3.7/site-packages (from gensim)\n",
      "Installing collected packages: smart-open, gensim\n",
      "Successfully installed gensim-4.0.1 smart-open-5.1.0"
     ]
    }
   ],
   "source": [
    "sc.install_pypi_package(\"gensim\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a3641c8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16de55c95b8f4868b11340ff4987e07f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                    Version  \n",
      "-------------------------- ---------\n",
      "beautifulsoup4             4.9.3    \n",
      "boto                       2.49.0   \n",
      "click                      7.1.2    \n",
      "cycler                     0.10.0   \n",
      "gensim                     4.0.1    \n",
      "jmespath                   0.10.0   \n",
      "joblib                     1.0.1    \n",
      "kiwisolver                 1.3.1    \n",
      "lxml                       4.6.2    \n",
      "matplotlib                 3.4.2    \n",
      "mysqlclient                1.4.2    \n",
      "nltk                       3.5      \n",
      "nose                       1.3.4    \n",
      "numpy                      1.16.5   \n",
      "pandas                     1.2.4    \n",
      "Pillow                     8.2.0    \n",
      "pip                        9.0.1    \n",
      "py-dateutil                2.2      \n",
      "pyparsing                  2.4.7    \n",
      "python-dateutil            2.8.1    \n",
      "python37-sagemaker-pyspark 1.4.1    \n",
      "pytz                       2021.1   \n",
      "PyYAML                     5.4.1    \n",
      "regex                      2021.3.17\n",
      "scikit-learn               0.24.2   \n",
      "scipy                      1.6.3    \n",
      "seaborn                    0.11.1   \n",
      "setuptools                 28.8.0   \n",
      "six                        1.13.0   \n",
      "smart-open                 5.1.0    \n",
      "sonopy                     0.1.2    \n",
      "threadpoolctl              2.1.0    \n",
      "tqdm                       4.59.0   \n",
      "vader                      0.0.2    \n",
      "wheel                      0.29.0   \n",
      "windmill                   1.6"
     ]
    }
   ],
   "source": [
    "sc.list_packages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "baacf00f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2db56abcf3f34150b75dd7cce0935faf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- subreddit: string (nullable = true)\n",
      " |-- comment: string (nullable = true)\n",
      " |-- time created: string (nullable = true)\n",
      " |-- cleaned_text: string (nullable = true)\n",
      " |-- lemmas: string (nullable = true)\n",
      " |-- lemmas_str: string (nullable = true)\n",
      " |-- neg: string (nullable = true)\n",
      " |-- compound: string (nullable = true)\n",
      " |-- pos: string (nullable = true)"
     ]
    }
   ],
   "source": [
    "df = spark.read.format(\"csv\").option(\"header\", \"true\").option(\"delimiter\", \"\\t\").option(\"mode\", \"DROPMALFORMED\").csv(\"s3://aws-emr-resources-729741394700-us-east-1/reddit_comments_with_sentiment.tsv\")\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "79f6c8ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bdbef560b7d491d9c7672c7fe4dd98e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+\n",
      "| subreddit|             comment|        time created|        cleaned_text|              lemmas|          lemmas_str|  neg|compound|  pos|\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+\n",
      "|TheRedPill|Why the fuck woul...|2016-01-01 00:00:...|fuck would friend...|['fuck', 'would',...|fuck would friend ex|0.402| -0.0772|0.368|\n",
      "|TheRedPill|Great story full ...|2016-01-01 00:01:...|great story full ...|['great', 'story'...|great story full ...|  0.0|  0.8519|0.482|\n",
      "|TheRedPill|Brazilian here to...|2016-01-01 00:03:...|brazilian man don...|['brazilian', 'ma...|brazilian man don...|0.042| -0.0772|  0.0|\n",
      "|TheRedPill|Absolutely the la...|2016-01-01 00:06:...|absolutely last t...|['absolutely', 'l...|absolutely last t...|0.112| -0.4404|  0.0|\n",
      "|TheRedPill|How would you cou...|2016-01-01 00:06:...|would counter shi...|['would', 'counte...|would counter shi...|0.343| -0.1779|0.276|\n",
      "|TheRedPill|Hmm could you ela...|2016-01-01 00:06:...|hmm could elabora...|['hmm', 'could', ...|hmm could elabora...|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|           [removed]|2016-01-01 00:06:...|             removed|         ['removed']|             removed|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|u learn something...|2016-01-01 00:07:...|u learn something...|['u', 'learn', 's...|u learn something...|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|Reading these art...|2016-01-01 00:08:...|reading articles ...|['reading', 'arti...|reading article m...|0.481| -0.5719|  0.0|\n",
      "|TheRedPill|TRT, although inv...|2016-01-01 00:08:...|trt although invo...|['trt', 'although...|trt although invo...|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|Using Star Wars a...|2016-01-01 00:09:...|using star wars a...|['using', 'star',...|using star war an...| 0.16|  0.3612|0.263|\n",
      "|TheRedPill|The only woman I'...|2016-01-01 00:09:...|woman ive ever kn...|['woman', 'ive', ...|woman ive ever kn...|0.101|     0.0|0.101|\n",
      "|TheRedPill|I'll take a look ...|2016-01-01 00:11:...|ill take look mon...|['ill', 'take', '...|ill take look mon...|0.094|   0.743|0.338|\n",
      "|TheRedPill|Fame hunter. If s...|2016-01-01 00:12:...|fame hunter she s...|['fame', 'hunter'...|fame hunter she s...|  0.0|  0.7184| 0.25|\n",
      "|TheRedPill|wouldnt they want...|2016-01-01 00:14:...|wouldnt want brai...|['wouldnt', 'want...|wouldnt want brai...|0.288|  0.0459|0.221|\n",
      "|TheRedPill|           [removed]|2016-01-01 00:14:...|             removed|         ['removed']|             removed|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|Tell you what. Go...|2016-01-01 00:16:...|tell go get bank ...|['tell', 'go', 'g...|tell go get bank ...|  0.0|  0.2732|0.123|\n",
      "|TheRedPill|The sheer audacit...|2016-01-01 00:16:...|sheer audacity ab...|['sheer', 'audaci...|sheer audacity ab...|0.203|  0.2263|0.233|\n",
      "|TheRedPill|           [deleted]|2016-01-01 00:17:...|             deleted|         ['deleted']|             deleted|  0.0|     0.0|  0.0|\n",
      "|TheRedPill|Thanks for the ti...|2016-01-01 00:20:...|thanks tips actua...|['thanks', 'tip',...|thanks tip actual...|0.137|  0.7096|0.424|\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "fff467fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75ffd81651f7457e97da1894a3ff3aa3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_text = df.filter(df[\"lemmas_str\"].rlike(\"[a-zA-Z]+\")) # ensuring that only non-numeric data is processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f591f6dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98e0ca59ac1f4c0c99406ea9f377b5dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting vaderSentiment\n",
      "  Downloading https://files.pythonhosted.org/packages/76/fc/310e16254683c1ed35eeb97386986d6c00bc29df17ce280aed64d55537e9/vaderSentiment-3.3.2-py2.py3-none-any.whl (125kB)\n",
      "Collecting requests (from vaderSentiment)\n",
      "  Downloading https://files.pythonhosted.org/packages/29/c1/24814557f1d22c56d50280771a17307e6bf87b70727d975fd6b2ce6b014a/requests-2.25.1-py2.py3-none-any.whl (61kB)\n",
      "Collecting urllib3<1.27,>=1.21.1 (from requests->vaderSentiment)\n",
      "  Downloading https://files.pythonhosted.org/packages/0c/cd/1e2ec680ec7b09846dc6e605f5a7709dfb9d7128e51a026e7154e18a234e/urllib3-1.26.5-py2.py3-none-any.whl (138kB)\n",
      "Collecting chardet<5,>=3.0.2 (from requests->vaderSentiment)\n",
      "  Downloading https://files.pythonhosted.org/packages/19/c7/fa589626997dd07bd87d9269342ccb74b1720384a4d739a1872bd84fbe68/chardet-4.0.0-py2.py3-none-any.whl (178kB)\n",
      "Collecting idna<3,>=2.5 (from requests->vaderSentiment)\n",
      "  Downloading https://files.pythonhosted.org/packages/a2/38/928ddce2273eaa564f6f50de919327bf3a00f091b5baba8dfa9460f3a8a8/idna-2.10-py2.py3-none-any.whl (58kB)\n",
      "Collecting certifi>=2017.4.17 (from requests->vaderSentiment)\n",
      "  Downloading https://files.pythonhosted.org/packages/05/1b/0a0dece0e8aa492a6ec9e4ad2fe366b511558cdc73fd3abc82ba7348e875/certifi-2021.5.30-py2.py3-none-any.whl (145kB)\n",
      "Installing collected packages: urllib3, chardet, idna, certifi, requests, vaderSentiment\n",
      "Successfully installed certifi-2021.5.30 chardet-4.0.0 idna-2.10 requests-2.25.1 urllib3-1.26.5 vaderSentiment-3.3.2"
     ]
    }
   ],
   "source": [
    "sc.install_pypi_package(\"vaderSentiment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f3feae24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f600760bada84cf5aed3a178d19b9a0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "analyzer = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "db80f235",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e709df788ffb45a88df3be54201143d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql.functions import format_number as fmt\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "from pyspark.ml.feature import Tokenizer,StopWordsRemover,Word2Vec\n",
    "from pyspark.ml import PipelineModel\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "6083d1d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19c9c72db5f8461fa4b2558dd14feb7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def calc_sent_score(comment): \n",
    "    '''\n",
    "    Gets compound similarity score for a given comment\n",
    "    '''\n",
    "    analyzer = SentimentIntensityAnalyzer()\n",
    "    vs = analyzer.polarity_scores(comment)\n",
    "    return float(vs['compound'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "00371d75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e008b47c5f534c829465ee66440e872d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+---------------+\n",
      "| subreddit|             comment|        time created|        cleaned_text|              lemmas|          lemmas_str|  neg|compound|  pos|sentiment_score|\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+---------------+\n",
      "|TheRedPill|Why the fuck woul...|2016-01-01 00:00:...|fuck would friend...|['fuck', 'would',...|fuck would friend ex|0.402| -0.0772|0.368|        -0.0772|\n",
      "|TheRedPill|Great story full ...|2016-01-01 00:01:...|great story full ...|['great', 'story'...|great story full ...|  0.0|  0.8519|0.482|         0.8519|\n",
      "|TheRedPill|Brazilian here to...|2016-01-01 00:03:...|brazilian man don...|['brazilian', 'ma...|brazilian man don...|0.042| -0.0772|  0.0|        -0.0772|\n",
      "|TheRedPill|Absolutely the la...|2016-01-01 00:06:...|absolutely last t...|['absolutely', 'l...|absolutely last t...|0.112| -0.4404|  0.0|        -0.4404|\n",
      "|TheRedPill|How would you cou...|2016-01-01 00:06:...|would counter shi...|['would', 'counte...|would counter shi...|0.343| -0.1779|0.276|        -0.1779|\n",
      "|TheRedPill|Hmm could you ela...|2016-01-01 00:06:...|hmm could elabora...|['hmm', 'could', ...|hmm could elabora...|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|           [removed]|2016-01-01 00:06:...|             removed|         ['removed']|             removed|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|u learn something...|2016-01-01 00:07:...|u learn something...|['u', 'learn', 's...|u learn something...|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|Reading these art...|2016-01-01 00:08:...|reading articles ...|['reading', 'arti...|reading article m...|0.481| -0.5719|  0.0|        -0.5719|\n",
      "|TheRedPill|TRT, although inv...|2016-01-01 00:08:...|trt although invo...|['trt', 'although...|trt although invo...|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|Using Star Wars a...|2016-01-01 00:09:...|using star wars a...|['using', 'star',...|using star war an...| 0.16|  0.3612|0.263|         0.3612|\n",
      "|TheRedPill|The only woman I'...|2016-01-01 00:09:...|woman ive ever kn...|['woman', 'ive', ...|woman ive ever kn...|0.101|     0.0|0.101|            0.0|\n",
      "|TheRedPill|I'll take a look ...|2016-01-01 00:11:...|ill take look mon...|['ill', 'take', '...|ill take look mon...|0.094|   0.743|0.338|          0.743|\n",
      "|TheRedPill|Fame hunter. If s...|2016-01-01 00:12:...|fame hunter she s...|['fame', 'hunter'...|fame hunter she s...|  0.0|  0.7184| 0.25|         0.7184|\n",
      "|TheRedPill|wouldnt they want...|2016-01-01 00:14:...|wouldnt want brai...|['wouldnt', 'want...|wouldnt want brai...|0.288|  0.0459|0.221|         0.0459|\n",
      "|TheRedPill|           [removed]|2016-01-01 00:14:...|             removed|         ['removed']|             removed|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|Tell you what. Go...|2016-01-01 00:16:...|tell go get bank ...|['tell', 'go', 'g...|tell go get bank ...|  0.0|  0.2732|0.123|         0.2732|\n",
      "|TheRedPill|The sheer audacit...|2016-01-01 00:16:...|sheer audacity ab...|['sheer', 'audaci...|sheer audacity ab...|0.203|  0.2263|0.233|         0.2263|\n",
      "|TheRedPill|           [deleted]|2016-01-01 00:17:...|             deleted|         ['deleted']|             deleted|  0.0|     0.0|  0.0|            0.0|\n",
      "|TheRedPill|Thanks for the ti...|2016-01-01 00:20:...|thanks tips actua...|['thanks', 'tip',...|thanks tip actual...|0.137|  0.7096|0.424|         0.7096|\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+--------------------+-----+--------+-----+---------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "udf_sent_score = udf(calc_sent_score, FloatType())\n",
    "# Applying sentiment score function above to each comment (within 'lemmas_str', cleaned lemmatized comments)\n",
    "\n",
    "df_sent_score = df_text.withColumn('sentiment_score', udf_sent_score('lemmas_str'))\n",
    "df_sent_score.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "36e16c4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fb0687381f14ef9bc7aa2dcb1bbe18c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = Tokenizer(inputCol='lemmas_str', outputCol='words')\n",
    "df_tokenized = tokenizer.transform(df_sent_score) # tokenizing comments (in lemma_str) for word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "be64dfff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "615e2db148114d81b511515f15fc1add",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---------------+--------------------+\n",
      "|        cleaned_text|sentiment_score|               words|\n",
      "+--------------------+---------------+--------------------+\n",
      "|fuck would friend...|        -0.0772|[fuck, would, fri...|\n",
      "|great story full ...|         0.8519|[great, story, fu...|\n",
      "|brazilian man don...|        -0.0772|[brazilian, man, ...|\n",
      "|absolutely last t...|        -0.4404|[absolutely, last...|\n",
      "|would counter shi...|        -0.1779|[would, counter, ...|\n",
      "+--------------------+---------------+--------------------+\n",
      "only showing top 5 rows"
     ]
    }
   ],
   "source": [
    "df_tokenized.select('cleaned_text','sentiment_score','words').show(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f3c717de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2db20134450745c3b46e38c23e97eb12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|           w2_vector|\n",
      "+--------------------+\n",
      "|[-0.0012583285570...|\n",
      "|[-0.0590399055956...|\n",
      "|[0.00635056311805...|\n",
      "|[-0.0522531430469...|\n",
      "|[0.02936405253907...|\n",
      "+--------------------+\n",
      "only showing top 5 rows"
     ]
    }
   ],
   "source": [
    "word2Vec = Word2Vec(vectorSize=100, minCount=5, inputCol='words', outputCol='w2_vector')\n",
    "w2v_model = word2Vec.fit(df_tokenized)\n",
    "transform_w2v = w2v_model.transform(df_tokenized)\n",
    "transform_w2v.select('w2_vector').show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d43564e",
   "metadata": {},
   "source": [
    "### Most similar words to 'girl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "ca1a0bb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13a5e0842cdd48399b89c53c4b389897",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         word similarity\n",
      "0       chick    0.81825\n",
      "1         guy    0.63390\n",
      "2  girlfriend    0.56461\n",
      "3       woman    0.54702\n",
      "4         hb8    0.54700\n",
      "5         gal    0.54121\n",
      "6      blonde    0.53347\n",
      "7      hotter    0.52709\n",
      "8    sorority    0.51088\n",
      "9        cute    0.51087"
     ]
    }
   ],
   "source": [
    "top_n = 10\n",
    "\n",
    "sim_words_girl=w2v_model.findSynonyms('girl', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_girl.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98e4e9bd",
   "metadata": {},
   "source": [
    "### Most similar words to 'woman'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "1f998c1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b9a4899ea954f72a190a8014ee945ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               word similarity\n",
      "0               men    0.65025\n",
      "1       objectified    0.58079\n",
      "2              rele    0.56953\n",
      "3              girl    0.54702\n",
      "4          feminine    0.54008\n",
      "5           loyalty    0.51931\n",
      "6       promiscuous    0.51716\n",
      "7       pedastalize    0.51647\n",
      "8  husbandboyfriend    0.51427\n",
      "9         desirable    0.51371"
     ]
    }
   ],
   "source": [
    "sim_words_women=w2v_model.findSynonyms('woman', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_women.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30df19b1",
   "metadata": {},
   "source": [
    "### Most similar words to 'feminist' (since many incels on this subreddit talk negatively about feminists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "d7d87511",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac659ff2cd47439f92fc05bd885f6de3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         word similarity\n",
      "0         sjw    0.72513\n",
      "1    feminism    0.67483\n",
      "2        sjws    0.64657\n",
      "3     radical    0.61928\n",
      "4     liberal    0.61631\n",
      "5      openly    0.61582\n",
      "6        anti    0.61291\n",
      "7   thirdwave    0.60567\n",
      "8  misogynist    0.60542\n",
      "9   supremacy    0.60440"
     ]
    }
   ],
   "source": [
    "sim_words_feminist=w2v_model.findSynonyms('feminist', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_feminist.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d9d758e",
   "metadata": {},
   "source": [
    "### Most similar words to 'stacy' (word referring to 'perfect'/unobtainable woman)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "e744ee0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fba6e6e3fa1841f5adc319ba2308ec6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          word similarity\n",
      "0    obedience    0.72934\n",
      "1      whorish    0.71832\n",
      "2    protector    0.71243\n",
      "3       schlub    0.71231\n",
      "4  committment    0.70744\n",
      "5       maggot    0.70685\n",
      "6        putty    0.70662\n",
      "7      servile    0.70216\n",
      "8    masochist    0.70129\n",
      "9    misbehave    0.69837"
     ]
    }
   ],
   "source": [
    "sim_words_stacy=w2v_model.findSynonyms('stacy', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_stacy.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f91de0",
   "metadata": {},
   "source": [
    "### Most similar words to 'chad' (referring to perfect/alpha man)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4e7e3d77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "992801f95ed041d9966e68c5051d0fb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           word similarity\n",
      "0   thundercock    0.76120\n",
      "1         billy    0.66107\n",
      "2       widowed    0.64209\n",
      "3         alpha    0.64077\n",
      "4         chump    0.63740\n",
      "5            bb    0.62806\n",
      "6       betabux    0.62020\n",
      "7         widow    0.61716\n",
      "8  thundercocks    0.61616\n",
      "9        tyrone    0.61528"
     ]
    }
   ],
   "source": [
    "sim_words_chad=w2v_model.findSynonyms('chad', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_chad.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "795ff94f",
   "metadata": {},
   "source": [
    "### Most similar words to 'becky' (woman that's the opposite of a stacy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f7a0c9b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08e5bed4a22049a2b0f29d368f98e0da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          word similarity\n",
      "0     thanking    0.74624\n",
      "1          711    0.74476\n",
      "2       buster    0.74387\n",
      "3  coordinator    0.74126\n",
      "4      peacock    0.73053\n",
      "5       shrill    0.72806\n",
      "6     etcetera    0.72385\n",
      "7          cab    0.72318\n",
      "8        spree    0.72243\n",
      "9         prod    0.71904"
     ]
    }
   ],
   "source": [
    "sim_words_becky=w2v_model.findSynonyms('becky', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_becky.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "741c3aa6",
   "metadata": {},
   "source": [
    "### Most similar words to 'beta' (since many men on this subreddit see themselves as betas, not alphas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "90dbf21b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ac176ef24b64ebf86817343a30bb833",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        word similarity\n",
      "0      alpha    0.80729\n",
      "1        bux    0.76561\n",
      "2        fux    0.74707\n",
      "3      omega    0.70937\n",
      "4  fucksbeta    0.70718\n",
      "5       cuck    0.67084\n",
      "6      widow    0.67001\n",
      "7       buck    0.66542\n",
      "8    orbiter    0.66092\n",
      "9   provider    0.65838"
     ]
    }
   ],
   "source": [
    "sim_words_beta=w2v_model.findSynonyms('beta', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_beta.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc05de27",
   "metadata": {},
   "source": [
    "### Most similar words to 'love' (since many incels on this subreddit are seeking a girlfriend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "34e5724d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22c9b3bad8084c2b87a49426ed4cd6ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              word similarity\n",
      "0    unconditional    0.63432\n",
      "1  unconditionally    0.59699\n",
      "2            puppy    0.55327\n",
      "3        heartless    0.48756\n",
      "4         selflove    0.47339\n",
      "5        homemaker    0.46907\n",
      "6           loving    0.45918\n",
      "7        beautiful    0.45756\n",
      "8         kindness    0.45667\n",
      "9         motherly    0.44484"
     ]
    }
   ],
   "source": [
    "sim_words_love=w2v_model.findSynonyms('love', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_love.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f10dadd0",
   "metadata": {},
   "source": [
    "### Most similar words to 'wish' (since many incels on this subreddit feel inadequate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "f03b8db5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef6bb075c13744fd9e516f00d0cce16d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         word similarity\n",
      "0      upvote    0.49127\n",
      "1      relate    0.47049\n",
      "2        2001    0.43497\n",
      "3      brotha    0.42316\n",
      "4       loved    0.42003\n",
      "5      hahaha    0.41684\n",
      "6     glimmer    0.40989\n",
      "7   elaborate    0.40873\n",
      "8  motivating    0.40812\n",
      "9        hope    0.40192"
     ]
    }
   ],
   "source": [
    "sim_words_wish=w2v_model.findSynonyms('wish', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_wish.head(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde46f1c",
   "metadata": {},
   "source": [
    "### Most similar words to 'deserve' (since many incels on this subreddit feel that they are entitled to women)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "24861be2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f31820406ded440db07b8793d14c572e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         word similarity\n",
      "0   homemaker    0.59974\n",
      "1         owe    0.58529\n",
      "2       whine    0.55778\n",
      "3   heartless    0.53394\n",
      "4      coddle    0.53095\n",
      "5    bitching    0.51546\n",
      "6      fooled    0.50647\n",
      "7         shh    0.50047\n",
      "8        care    0.50014\n",
      "9  moralistic    0.49990"
     ]
    }
   ],
   "source": [
    "sim_words_deserve=w2v_model.findSynonyms('deserve', top_n).select('word', fmt('similarity', 5).alias('similarity')).toPandas()\n",
    "sim_words_deserve.head(top_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "280095b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}