{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse des données pour le rapport\n",
    "\n",
    "## Lecture des fichiers de données et affichage d'un échantillon de données\n",
    "\n",
    "### Lecture des données des articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import wn_pos_from_ud_pos as wnud\n",
    "from nltk.corpus import wordnet as wn\n",
    "from french_lefff_lemmatizer.french_lefff_lemmatizer import FrenchLefffLemmatizer\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import emoji\n",
    "import ast\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.parse.corenlp import CoreNLPServer\n",
    "from nltk.parse.corenlp import CoreNLPParser\n",
    "parser = CoreNLPParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"pickle/textes_articles_df.pickle\",\"rb\")\n",
    "textes_articles_df = pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lecture des données des commentaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_comm = open(\"pickle/commentaires_df.pickle\",\"rb\")\n",
    "commentaires_df = pickle.load(f_comm)\n",
    "f_comm.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df = commentaires_df[commentaires_df[\"media\"]!='CNN']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description des corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nombre d'articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_medias = textes_articles_df.groupby(\"media\").count()[[\"post_id\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_medias.columns = [\"Nombre de publications\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_medias.to_latex(\"decompte_articles_medias.tex\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nombre de commentaires total par média"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_commentaires = commentaires_df.groupby(\"media\").count()[[\"comment_id\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_commentaires.columns = [\"Nombre de commentaires\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_commentaires.to_latex(\"decompte_comm_medias.tex\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proportion de commentaires contenant des emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb_comm = commentaires_df[\"emoji_dict\"].count()\n",
    "nb_comm_emoji = sum(commentaires_df[\"emoji_dict\"].apply(lambda x: len(x)) == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(nb_comm_emoji/nb_comm,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse des commentaires\n",
    "## Niveau de langage\n",
    "### Nombre de jetons dans WordNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = FrenchLefffLemmatizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatisation d'une liste de tokens en français"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lem_fr(tokens):\n",
    "    list_tokens = []\n",
    "    for token in tokens:\n",
    "        wn_pos = wnud.wn_pos_from_ud_pos(token[1]) # conversion de Universal Dependancies vers WordNet\n",
    "        if wn_pos is not None:\n",
    "            lem_token = lemmatizer.lemmatize(token[0],pos=wn_pos)\n",
    "        else:\n",
    "            lem_token = lemmatizer.lemmatize(token[0])\n",
    "        list_tokens.append((lem_token,token[1]))\n",
    "    return set(list_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"pos_dict_lem\"] = commentaires_df.apply(lambda x: lem_fr(x[\"pos_dict\"]), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Identification des synsets des tokens en français"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def synsets_fr(tokens):\n",
    "    list_synsets = []\n",
    "    for token in tokens:\n",
    "        wn_pos = wnud.wn_pos_from_ud_pos(token[1])\n",
    "        if wn_pos is not None:\n",
    "            synset = wn.synsets(token[0], lang='fra', pos=wn_pos)\n",
    "            list_synsets.append(synset)\n",
    "    return list_synsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"synsets\"] = commentaires_df.apply(lambda x: synsets_fr(x[\"pos_dict_lem\"]), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nombre de mots ayant au moins un synset associé"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"nb_synsets\"]=commentaires_df.apply(lambda y: sum([len(x)>0 for x in y[\"synsets\"]]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"nb_pos\"]=commentaires_df.apply(lambda y: len(y[\"pos_dict\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"prop_synsets_pos\"]=commentaires_df.apply(lambda y: y[\"nb_synsets\"]/max(y[\"nb_pos\"],1), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11704 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d000 = commentaires_df[[\"media\",\"nb_synsets\"]].copy()\n",
    "d000[\"nb_synsets_tronq\"] = d000.apply(lambda x: min(20,x[\"nb_synsets\"]), axis=1)\n",
    "g000 = sns.catplot(x=\"media\", y=\"nb_synsets_tronq\",\n",
    "            kind=\"boxen\",\n",
    "            data=d000);\n",
    "g000.set_axis_labels(\"Média\", \"Nombre de POS ayant un synset\\n(tronqué à 20)\")\n",
    "g000.despine(offset=10, trim=True)\n",
    "g000.savefig(\"g000.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9096 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d001 = commentaires_df[[\"media\",\"prop_synsets_pos\"]].copy()\n",
    "g001 = sns.catplot(x=\"media\", y=\"prop_synsets_pos\",\n",
    "            kind=\"boxen\",\n",
    "            data=d001);\n",
    "g001.set_axis_labels(\"Média\", \"Proportion de POS ayant un synset\")\n",
    "g001.despine(offset=10, trim=True)\n",
    "g001.savefig(\"g001.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nombre de types de POS dans le commentaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"nb_type_pos\"] = commentaires_df.apply(lambda y: len(set([x[1] for x in y[\"pos_dict\"]])), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7768 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d002 = commentaires_df[[\"media\",\"nb_type_pos\"]].copy()\n",
    "g002 = sns.catplot(x=\"media\", y=\"nb_type_pos\",\n",
    "            kind=\"boxen\",\n",
    "            data=d002);\n",
    "g002.set_axis_labels(\"Média\", \"Nombre de types de POS\")\n",
    "g002.despine(offset=10, trim=True)\n",
    "g002.savefig(\"g002.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Marqueurs d'emphase\n",
    "\n",
    "### Nombre d'emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict_emojis_to_list(emoji_dict):\n",
    "    emojis = []\n",
    "    for key in emoji_dict:\n",
    "        for value in emoji_dict[key]:\n",
    "            emojis.append(key)\n",
    "    return emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"emoji_list\"] = commentaires_df.apply(lambda x: dict_emojis_to_list(x[\"emoji_dict\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003 = commentaires_df[[\"media\",\"emoji_list\"]].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003df = pd.DataFrame({'media':np.repeat(d003.media.values, d003.emoji_list.str.len()),\n",
    "                       'emoji':np.concatenate(d003.emoji_list.values),\n",
    "                       'value':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cnt1 = d003df.groupby([\"media\",\"emoji\"]).agg(['sum']).reset_index(['media','emoji'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cnt1.columns = [\"media\",\"emoji\",\"media_count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cnt2 = d003df.groupby(\"emoji\").agg(['count']).reset_index(\"emoji\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cnt2 = d003cnt2[[\"emoji\",\"media\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cnt2.columns = [\"emoji\",\"total_count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "d003cntx = d003cnt1[d003cnt1.emoji != \":\"].merge(d003cnt2,on=\"emoji\").sort_values(\"total_count\",ascending=False).head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "del d003cntx[\"total_count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23508 characters]\n",
      "text/plain": [
       "<Figure size 415.75x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "g003 = sns.catplot(y=\"emoji\", x=\"media_count\", hue=\"media\", \n",
    "            kind=\"bar\",\n",
    "            palette=\"pastel\", edgecolor=\".6\",\n",
    "            data=d003cntx)\n",
    "g003.set_axis_labels(\"Fréquence\", \"Forme textuelle des emojis\")\n",
    "g003.despine(offset=10, trim=True)\n",
    "g003.savefig(\"g003.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mots en majuscules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df['nb_mots_majuscules'] = commentaires_df.apply(lambda x: min(10,len([word for word,_ in x[\"pos_dict\"] if word.isupper()])), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10944 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d004 = commentaires_df[['media','nb_mots_majuscules']].copy()\n",
    "g004 = sns.catplot(x=\"media\", y=\"nb_mots_majuscules\",\n",
    "            kind=\"boxen\",\n",
    "            data=d004);\n",
    "g004.set_axis_labels(\"Média\", \"Nombre de mots en majuscules\\n(limité à 10)\")\n",
    "g004.despine(offset=10, trim=True)\n",
    "g004.savefig(\"g004.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ponctuations successives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "punct_re = re.compile(r'([^\\w\\s\\:]{2,})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "demo = emoji.demojize(commentaires_df['comment_clean'][37765])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Ponctuations successives\n",
    "def nb_punct_succ(x):\n",
    "    seq_punct = re.findall(punct_re,x)\n",
    "    return sum([len(s) for s in seq_punct])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df['nb_punct_succ'] = commentaires_df.apply(lambda x: min(10,nb_punct_succ(x['comment_clean'])), axis=1)\n",
    "commentaires_df['units'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13660 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d005 = commentaires_df[['media','nb_punct_succ','units']].groupby([\"media\",'nb_punct_succ']).agg(\"sum\").reset_index([\"media\",'nb_punct_succ']).copy()\n",
    "g005 = sns.barplot(x='nb_punct_succ',y='units', hue='media',data=d005);\n",
    "g005.set_ylabel(\"Nombre de commentaires\")\n",
    "g005.set_xlabel(\"Ponctuations successives totales (limité à 10)\")\n",
    "g005.figure.savefig(\"g005.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Intertextualité\n",
    "\n",
    "### Mention de l'auteur d'un autre commentaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df['ind_auteurs_referes'] = commentaires_df.apply(lambda x: len(ast.literal_eval(x['auteurs_referes']))>0, axis=1) # j'avais sauvegardé une représentation littérale de la liste ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_auteur_refere = commentaires_df[['media','ind_auteurs_referes']].groupby('media').agg('mean').reset_index('media')\n",
    "decompte_auteur_refere.columns = ['Media','Proportion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_auteur_refere.to_latex(\"decompte_auteur_refere.tex\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pronoms et articles possessifs à la deuxième personne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = TweetTokenizer(preserve_case=False)\n",
    "mots_2epers = ['ton','ta','tes','toi','te','tu','t\\'','vous','votre','vos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df['ind_2e_pers'] = commentaires_df.apply(lambda x: len([token for token in tok.tokenize(x['comment_clean']) if token in mots_2epers])>0, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_2e_pers = commentaires_df[['media','ind_2e_pers']].groupby('media').agg('mean').reset_index('media')\n",
    "decompte_2e_pers.columns = ['Media','Proportion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "decompte_2e_pers.to_latex(\"decompte_2e_pers.tex\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse des liens entre les commentaires et les articles\n",
    "\n",
    "## Entités nommées"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "d006a = commentaires_df[commentaires_df['ner_dict'] != {}][['media','id','post_id','ner_dict']].copy()\n",
    "d006a['ner_list_comm'] = commentaires_df.apply(lambda x: list(x['ner_dict'].keys()), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "del d006a['ner_dict']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "d006b = textes_articles_df[textes_articles_df['ner_dict'] != {}][['media','post_id','ner_dict']].copy()\n",
    "d006b['ner_list_art'] = textes_articles_df.apply(lambda x: list(x['ner_dict'].keys()), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "del d006b['ner_dict']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "d006 = d006a.merge(d006b, on=['post_id','media'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection(lst1, lst2): \n",
    "    lst3 = [value for value in lst1 if value in lst2] \n",
    "    return lst3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "d006['ner_list_inter'] = d006.apply(lambda x: len(intersection(x['ner_list_comm'],x['ner_list_art'])), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "prop_ner_inter_post = d006[['media','post_id','ner_list_inter']].groupby(['media','post_id']).agg('mean').reset_index(['media','post_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "prop_ner_inter_post.columns = ['media','post_id','prop_ner_list_inter']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11792 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "g006 = sns.catplot(x=\"media\", y=\"prop_ner_list_inter\",\n",
    "            kind=\"boxen\",\n",
    "            data=prop_ner_inter_post);\n",
    "g006.set_axis_labels(\"Média\", \"Nombre moyen d'entités nommées communes, par article\")\n",
    "g006.despine(offset=10, trim=True)\n",
    "g006.savefig(\"g006.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Synsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "textes_articles_df[\"pos_dict_lem_art\"] = textes_articles_df.apply(lambda x: lem_fr(x[\"pos_dict\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "textes_articles_df[\"synsets_art\"] = textes_articles_df.apply(lambda x: synsets_fr(x[\"pos_dict_lem_art\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "textes_articles_df[\"synsets_art_merged\"] = textes_articles_df.apply(lambda x: list(set(sum(x[\"synsets_art\"],[]))), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"synsets_comm_merged\"] = commentaires_df.apply(lambda x: list(set(sum(x[\"synsets\"],[]))), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "commentaires_df[\"synsets_comm_merged_t\"] = commentaires_df.apply(lambda x: [str(i) for i in x[\"synsets_comm_merged\"]], axis=1)\n",
    "textes_articles_df[\"synsets_art_merged_t\"] = textes_articles_df.apply(lambda x: [str(i) for i in x[\"synsets_art_merged\"]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "d007a = commentaires_df[['media','id','post_id','synsets_comm_merged_t']].copy()\n",
    "d007a['nb_synsets'] = d007a.apply(lambda x: len(x['synsets_comm_merged_t']), axis=1)\n",
    "d007b = textes_articles_df[['media','post_id','synsets_art_merged_t']].copy()\n",
    "d007 = d007a.merge(d007b, on=['post_id','media'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "d007['synsets_inter'] = d007.apply(lambda x: len(intersection(x['synsets_comm_merged_t'],x['synsets_art_merged_t'])), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "d007['prop_synsets_inter'] = d007['synsets_inter']/d007['nb_synsets']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "prop_synsets_inter_post = d007[['media','post_id','prop_synsets_inter']].groupby(['media','post_id']).agg('mean').reset_index(['media','post_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "prop_synsets_inter_post.columns = ['media','post_id','prop_synsets_inter']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12144 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "g007 = sns.catplot(x=\"media\", y=\"prop_synsets_inter\",\n",
    "            kind=\"boxen\",\n",
    "            data=prop_synsets_inter_post);\n",
    "g007.set_axis_labels(\"Média\", \"Proportion moyenne de synsets communs, par article\")\n",
    "g007.despine(offset=10, trim=True)\n",
    "g007.savefig(\"g007.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
