lemmatizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from french_lefff_lemmatizer.french_lefff_lemmatizer import FrenchLefffLemmatizer\n",
    "from nltk import pos_tag, word_tokenize\n",
    "from nltk.tag import StanfordPOSTagger\n",
    "from nltk.corpus import wordnet\n",
    "import pandas as pd\n",
    "from langdetect import detect\n",
    "from stop_words import get_stop_words\n",
    "from collections import defaultdict\n",
    "import pprint\n",
    "from googletrans import Translator\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "#nltk.download('wordnet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = get_stop_words('fr')\n",
    "df = pd.read_csv('export_articles_EGC_2004_2018.csv', sep='\\t')\n",
    "translator = Translator()\n",
    "lemmatizer = FrenchLefffLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bonjour'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translator.translate('Hello', dest='fr').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.04 s, sys: 129 ms, total: 9.17 s\n",
      "Wall time: 9.16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def languagesDetectionDF(column):\n",
    "    languagesTitle = defaultdict(int)\n",
    "    for cell in df[column]:\n",
    "        if type(cell) is str:\n",
    "            languagesTitle[detect(cell)] += 1\n",
    "#         if lang != 'fr' and lang != 'en':\n",
    "#             print(title)\n",
    "    return languagesTitle\n",
    "\n",
    "languageTitle = languagesDetectionDF(\"title\")\n",
    "languageAbstract = languagesDetectionDF(\"abstract\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Langage des titres : defaultdict(<class 'int'>, {'fr': 1132, 'en': 124, 'tl': 1, 'it': 2, 'ca': 5, 'de': 2, 'fi': 1, 'da': 1, 'es': 1})\n",
      "Langage des abstract : defaultdict(<class 'int'>, {'fr': 991, 'en': 105})\n"
     ]
    }
   ],
   "source": [
    "print(\"Langage des titres :\", languageTitle)\n",
    "\n",
    "print(\"Langage des abstract :\", languageAbstract)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73.06666666666666\n",
      "13.884558054938314\n",
      "56\n",
      "98\n"
     ]
    }
   ],
   "source": [
    "invYr = df.groupby(['year']).count()\n",
    "\n",
    "print(invYr['abstract'].mean())\n",
    "print(invYr['abstract'].std())\n",
    "print(invYr['abstract'].min())\n",
    "print(invYr['abstract'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tagger fr\n",
    "jar = 'stanford-postagger-full-2018-10-16/stanford-postagger-3.9.2.jar'\n",
    "model = 'stanford-postagger-full-2018-10-16/models/french.tagger'\n",
    "import os\n",
    "java_path = \"/usr/bin/java\"\n",
    "os.environ['JAVAHOME'] = java_path\n",
    "\n",
    "#appel super long\n",
    "pos_tagger = StanfordPOSTagger(model, jar, encoding='utf8' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2 µs, sys: 1 µs, total: 3 µs\n",
      "Wall time: 4.77 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def wordNetTag(tag):\n",
    "    if tag.startswith('J') or tag.startswith('A'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "def lemma(tokenizedText):\n",
    "    taggedText = pos_tagger.tag(tokenizedText)\n",
    "    res = []\n",
    "    for word in taggedText:\n",
    "        tag = wordNetTag(word[1])\n",
    "        if tag:\n",
    "            res.append(lemmatizer.lemmatize(word[0], tag))\n",
    "\n",
    "        else:\n",
    "            res.append(lemmatizer.lemmatize(word[0]))\n",
    "    return(res)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14 µs, sys: 6 µs, total: 20 µs\n",
      "Wall time: 25.5 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "def preprocess(txt):\n",
    "    if(type(txt) == str):\n",
    "        if detect(txt) != 'fr':\n",
    "            txt = translator.translate(txt, dest='fr').text\n",
    "        tok = tokenizer.tokenize(txt)\n",
    "        tokens = [token for token in tok if (len(token) > 4) ] #Words that have fewer than 3 characters are removed.\n",
    "        tokens = [token.lower() for token in tokens if token.isalpha()] #stopwords are removed.\n",
    "        tokens = [w for w in tokens if not w in stop_words] #Lowercase the words and remove punctuation.\n",
    "        return lemma(tokens)\n",
    "    else:\n",
    "        return('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.23 s, sys: 8.78 s, total: 16 s\n",
      "Wall time: 9min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#C'est très long (10 minutes) parce que chaque utilisation du tagger prend environ une demie seconde et qu'on l'utilise pour chaque ligne\n",
    "\n",
    "titleLemma = df.title.apply(lambda x : preprocess(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.88 s, sys: 7.48 s, total: 15.4 s\n",
      "Wall time: 10min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#C'est très long (10 minutes) parce que chaque utilisation du tagger prend environ une demie seconde et qu'on l'utilise pour chaque ligne\n",
    "\n",
    "abstractLemma = df.abstract.apply(lambda x : preprocess(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titleLemma\"] = [\" \".join(cell) for cell in titleLemma]\n",
    "df[\"abstractLemma\"] = [\" \".join(cell) for cell in abstractLemma]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('dataLemmatized.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}