3_binary_decision.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122467\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.2</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ID</th>\n",
       "      <th>ID_unit</th>\n",
       "      <th>Message_x</th>\n",
       "      <th>Message_y</th>\n",
       "      <th>Sentences</th>\n",
       "      <th>Page Name</th>\n",
       "      <th>Post Created Date</th>\n",
       "      <th>sents</th>\n",
       "      <th>object</th>\n",
       "      <th>group</th>\n",
       "      <th>preds_representation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FBVL95605</td>\n",
       "      <td>PARL0000</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>['Dit zien en even beseffen hoe geweldig mooi ...</td>\n",
       "      <td>Maurits Vande Reyde</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Dit zien en even beseffen hoe geweldig mooi ...</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0001</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0002</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Met dankbaarheid om wat het mocht zijn, kijk i...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0003</td>\n",
       "      <td>Met dankbaarheid om wat het mocht zijn, kijk i...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Met dankbaarheid om wat het mocht zijn, kijk...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>FBVL95607</td>\n",
       "      <td>PARL0004</td>\n",
       "      <td>Jonge artiesten steunen! 📸 Eindwerk van Hannah...</td>\n",
       "      <td>Jonge artiesten steunen! 📸 Eindwerk van Hannah...</td>\n",
       "      <td>['Jonge artiesten steunen!', '📸 Eindwerk van H...</td>\n",
       "      <td>Stephanie D'Hose</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Jonge artiesten steunen!', '📸 Eindwerk van H...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.2  Unnamed: 0.1  Unnamed: 0         ID   ID_unit  \\\n",
       "0             0             0           0  FBVL95605  PARL0000   \n",
       "1             1             1           1  FBVL95606  PARL0001   \n",
       "2             2             2           2  FBVL95606  PARL0002   \n",
       "3             3             3           3  FBVL95606  PARL0003   \n",
       "4             4             4           4  FBVL95607  PARL0004   \n",
       "\n",
       "                                           Message_x  \\\n",
       "0  Dit zien en even beseffen hoe geweldig mooi de...   \n",
       "1  Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "2  Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "3  Met dankbaarheid om wat het mocht zijn, kijk i...   \n",
       "4  Jonge artiesten steunen! 📸 Eindwerk van Hannah...   \n",
       "\n",
       "                                           Message_y  \\\n",
       "0  Dit zien en even beseffen hoe geweldig mooi de...   \n",
       "1  Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "2  Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "3  Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "4  Jonge artiesten steunen! 📸 Eindwerk van Hannah...   \n",
       "\n",
       "                                           Sentences            Page Name  \\\n",
       "0  ['Dit zien en even beseffen hoe geweldig mooi ...  Maurits Vande Reyde   \n",
       "1  ['Prachtige kaart gevonden in mijn brievenbus,...     Ann De Martelaer   \n",
       "2  ['Prachtige kaart gevonden in mijn brievenbus,...     Ann De Martelaer   \n",
       "3  ['Prachtige kaart gevonden in mijn brievenbus,...     Ann De Martelaer   \n",
       "4  ['Jonge artiesten steunen!', '📸 Eindwerk van H...     Stephanie D'Hose   \n",
       "\n",
       "  Post Created Date                                              sents  \\\n",
       "0        2022-06-23  ['Dit zien en even beseffen hoe geweldig mooi ...   \n",
       "1        2022-06-23  ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "2        2022-06-23  ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "3        2022-06-23  ['Met dankbaarheid om wat het mocht zijn, kijk...   \n",
       "4        2022-06-23  ['Jonge artiesten steunen!', '📸 Eindwerk van H...   \n",
       "\n",
       "                                              object group  \\\n",
       "0  Dit zien en even beseffen hoe geweldig mooi de...    []   \n",
       "1                                                NaN    []   \n",
       "2  Met dankbaarheid om wat het mocht zijn, kijk i...    []   \n",
       "3                                                NaN    []   \n",
       "4                                                NaN    []   \n",
       "\n",
       "   preds_representation  \n",
       "0                     0  \n",
       "1                     0  \n",
       "2                     0  \n",
       "3                     0  \n",
       "4                     1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load in data\n",
    "data = pd.read_csv(\"data/final_data_object_rep.csv\")\n",
    "print(len(data))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.2</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ID</th>\n",
       "      <th>ID_unit</th>\n",
       "      <th>Message_x</th>\n",
       "      <th>Message_y</th>\n",
       "      <th>Sentences</th>\n",
       "      <th>Page Name</th>\n",
       "      <th>Post Created Date</th>\n",
       "      <th>sents</th>\n",
       "      <th>object</th>\n",
       "      <th>group</th>\n",
       "      <th>preds_representation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>FBVL95611</td>\n",
       "      <td>PARL0007</td>\n",
       "      <td>We hebben een nieuwe Letterzetter! Astrid Haer...</td>\n",
       "      <td>We hebben een nieuwe Letterzetter! Astrid Haer...</td>\n",
       "      <td>['We hebben een nieuwe Letterzetter!', 'Astrid...</td>\n",
       "      <td>Axel Ronse</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['We hebben een nieuwe Letterzetter!', 'Astrid...</td>\n",
       "      <td>Astrid Haerens zal de komende jaren Kortrijk n...</td>\n",
       "      <td>['jonge schrijvers']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>FBVL95612</td>\n",
       "      <td>PARL0010</td>\n",
       "      <td>Waar een wil is, is altijd een weg! 💪 Bij ART2...</td>\n",
       "      <td>Waar een wil is, is altijd een weg! 💪 Bij ART2...</td>\n",
       "      <td>['Waar een wil is, is altijd een weg!', '💪 Bij...</td>\n",
       "      <td>Sven Gatz</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Waar een wil is, is altijd een weg!', '💪 Bij...</td>\n",
       "      <td>💪 Bij ART2WORK geeft men jonge Brusselaars een...</td>\n",
       "      <td>['jonge Brusselaars']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>FBVL95612</td>\n",
       "      <td>PARL0011</td>\n",
       "      <td>💪 Bij ART2WORK geeft men jonge Brusselaars een...</td>\n",
       "      <td>Waar een wil is, is altijd een weg! 💪 Bij ART2...</td>\n",
       "      <td>['Waar een wil is, is altijd een weg!', '💪 Bij...</td>\n",
       "      <td>Sven Gatz</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['💪 Bij ART2WORK geeft men jonge Brusselaars e...</td>\n",
       "      <td>Met trajecten op maat &amp; verschillende workshop...</td>\n",
       "      <td>['werkzoekende jongeren']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>FBVL95612</td>\n",
       "      <td>PARL0011</td>\n",
       "      <td>💪 Bij ART2WORK geeft men jonge Brusselaars een...</td>\n",
       "      <td>Waar een wil is, is altijd een weg! 💪 Bij ART2...</td>\n",
       "      <td>['Waar een wil is, is altijd een weg!', '💪 Bij...</td>\n",
       "      <td>Sven Gatz</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['💪 Bij ART2WORK geeft men jonge Brusselaars e...</td>\n",
       "      <td>Met trajecten op maat &amp; verschillende workshop...</td>\n",
       "      <td>['personen zonder diploma']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>28</td>\n",
       "      <td>FBVL95615</td>\n",
       "      <td>PARL0028</td>\n",
       "      <td>Er zijn er die zouden zeggen dat ze blij zijn ...</td>\n",
       "      <td>Beste allen 𝗜𝗸 𝗵𝗲𝗯 𝗻𝗶𝗲𝘂𝘄𝘀 𝘃𝗼𝗼𝗿 𝗷𝘂𝗹𝗹𝗶𝗲: na lang...</td>\n",
       "      <td>['Beste allen 𝗜𝗸 𝗵𝗲𝗯 𝗻𝗶𝗲𝘂𝘄𝘀 𝘃𝗼𝗼𝗿 𝗷𝘂𝗹𝗹𝗶𝗲: na la...</td>\n",
       "      <td>Kurt Vanryckeghem</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Er zijn er die zouden zeggen dat ze blij zij...</td>\n",
       "      <td>Maar ook in het Vlaams Parlement kon ik deze l...</td>\n",
       "      <td>['onze lokale ondernemingen']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122437</th>\n",
       "      <td>122437</td>\n",
       "      <td>122437</td>\n",
       "      <td>114808</td>\n",
       "      <td>FBVL125549</td>\n",
       "      <td>PARL114808</td>\n",
       "      <td>Gelukkig startte minister Ben Weyts project Ta...</td>\n",
       "      <td>Nog niet alle scholen weten goed hoe ze met ho...</td>\n",
       "      <td>['Nog niet alle scholen weten goed hoe ze met ...</td>\n",
       "      <td>Kathleen Krekels</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Gelukkig startte minister Ben Weyts project ...</td>\n",
       "      <td>👨‍🏫 Daarnaast namen we grendels weg zodat deze...</td>\n",
       "      <td>['deze leerlingen']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122443</th>\n",
       "      <td>122443</td>\n",
       "      <td>122443</td>\n",
       "      <td>114814</td>\n",
       "      <td>FBVL125553</td>\n",
       "      <td>PARL114814</td>\n",
       "      <td>Dat is wanneer de meest wijze beslissingen wor...</td>\n",
       "      <td>“Volg altijd je gevoel. Dat is wanneer de mees...</td>\n",
       "      <td>['“Volg altijd je gevoel.', 'Dat is wanneer de...</td>\n",
       "      <td>Steve Vandenberghe</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Dat is wanneer de meest wijze beslissingen w...</td>\n",
       "      <td>(Oprah Winfrey) Heel veel dankbaarheid en resp...</td>\n",
       "      <td>['alle schooldirecteurs die zich dag en nacht ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122446</th>\n",
       "      <td>122446</td>\n",
       "      <td>122446</td>\n",
       "      <td>114817</td>\n",
       "      <td>FBVL125554</td>\n",
       "      <td>PARL114817</td>\n",
       "      <td>Het stikstofakkoord van de regering-Jambon is ...</td>\n",
       "      <td>Het stikstofakkoord van de regering-Jambon is ...</td>\n",
       "      <td>['Het stikstofakkoord van de regering-Jambon i...</td>\n",
       "      <td>Immanuel De Reuse</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Het stikstofakkoord van de regering-Jambon i...</td>\n",
       "      <td>Ook West-Vlaamse landbouwbedrijven investeren ...</td>\n",
       "      <td>['West-Vlaamse landbouwbedrijven']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122447</th>\n",
       "      <td>122447</td>\n",
       "      <td>122447</td>\n",
       "      <td>114818</td>\n",
       "      <td>FBVL125554</td>\n",
       "      <td>PARL114818</td>\n",
       "      <td>Ook West-Vlaamse landbouwbedrijven investeren ...</td>\n",
       "      <td>Het stikstofakkoord van de regering-Jambon is ...</td>\n",
       "      <td>['Het stikstofakkoord van de regering-Jambon i...</td>\n",
       "      <td>Immanuel De Reuse</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Ook West-Vlaamse landbouwbedrijven investere...</td>\n",
       "      <td>Voor het Vlaams Belang is het duidelijk: onze ...</td>\n",
       "      <td>['onze landbouw']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122462</th>\n",
       "      <td>122462</td>\n",
       "      <td>122462</td>\n",
       "      <td>114833</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114833</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>On-begrijpelijk en on-aanvaardbaar, want maar ...</td>\n",
       "      <td>['1 op 5 Brusselse scholieren']</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26202 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0.2  Unnamed: 0.1  Unnamed: 0          ID     ID_unit  \\\n",
       "7                  7             7           7   FBVL95611    PARL0007   \n",
       "10                10            10          10   FBVL95612    PARL0010   \n",
       "11                11            11          11   FBVL95612    PARL0011   \n",
       "12                12            12          11   FBVL95612    PARL0011   \n",
       "29                29            29          28   FBVL95615    PARL0028   \n",
       "...              ...           ...         ...         ...         ...   \n",
       "122437        122437        122437      114808  FBVL125549  PARL114808   \n",
       "122443        122443        122443      114814  FBVL125553  PARL114814   \n",
       "122446        122446        122446      114817  FBVL125554  PARL114817   \n",
       "122447        122447        122447      114818  FBVL125554  PARL114818   \n",
       "122462        122462        122462      114833  FBVL125558  PARL114833   \n",
       "\n",
       "                                                Message_x  \\\n",
       "7       We hebben een nieuwe Letterzetter! Astrid Haer...   \n",
       "10      Waar een wil is, is altijd een weg! 💪 Bij ART2...   \n",
       "11      💪 Bij ART2WORK geeft men jonge Brusselaars een...   \n",
       "12      💪 Bij ART2WORK geeft men jonge Brusselaars een...   \n",
       "29      Er zijn er die zouden zeggen dat ze blij zijn ...   \n",
       "...                                                   ...   \n",
       "122437  Gelukkig startte minister Ben Weyts project Ta...   \n",
       "122443  Dat is wanneer de meest wijze beslissingen wor...   \n",
       "122446  Het stikstofakkoord van de regering-Jambon is ...   \n",
       "122447  Ook West-Vlaamse landbouwbedrijven investeren ...   \n",
       "122462  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "\n",
       "                                                Message_y  \\\n",
       "7       We hebben een nieuwe Letterzetter! Astrid Haer...   \n",
       "10      Waar een wil is, is altijd een weg! 💪 Bij ART2...   \n",
       "11      Waar een wil is, is altijd een weg! 💪 Bij ART2...   \n",
       "12      Waar een wil is, is altijd een weg! 💪 Bij ART2...   \n",
       "29      Beste allen 𝗜𝗸 𝗵𝗲𝗯 𝗻𝗶𝗲𝘂𝘄𝘀 𝘃𝗼𝗼𝗿 𝗷𝘂𝗹𝗹𝗶𝗲: na lang...   \n",
       "...                                                   ...   \n",
       "122437  Nog niet alle scholen weten goed hoe ze met ho...   \n",
       "122443  “Volg altijd je gevoel. Dat is wanneer de mees...   \n",
       "122446  Het stikstofakkoord van de regering-Jambon is ...   \n",
       "122447  Het stikstofakkoord van de regering-Jambon is ...   \n",
       "122462  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "\n",
       "                                                Sentences           Page Name  \\\n",
       "7       ['We hebben een nieuwe Letterzetter!', 'Astrid...          Axel Ronse   \n",
       "10      ['Waar een wil is, is altijd een weg!', '💪 Bij...           Sven Gatz   \n",
       "11      ['Waar een wil is, is altijd een weg!', '💪 Bij...           Sven Gatz   \n",
       "12      ['Waar een wil is, is altijd een weg!', '💪 Bij...           Sven Gatz   \n",
       "29      ['Beste allen 𝗜𝗸 𝗵𝗲𝗯 𝗻𝗶𝗲𝘂𝘄𝘀 𝘃𝗼𝗼𝗿 𝗷𝘂𝗹𝗹𝗶𝗲: na la...   Kurt Vanryckeghem   \n",
       "...                                                   ...                 ...   \n",
       "122437  ['Nog niet alle scholen weten goed hoe ze met ...    Kathleen Krekels   \n",
       "122443  ['“Volg altijd je gevoel.', 'Dat is wanneer de...  Steve Vandenberghe   \n",
       "122446  ['Het stikstofakkoord van de regering-Jambon i...   Immanuel De Reuse   \n",
       "122447  ['Het stikstofakkoord van de regering-Jambon i...   Immanuel De Reuse   \n",
       "122462  ['Vlaams onderwijsminister Ben Weyts legt Scho...    Hannelore Goeman   \n",
       "\n",
       "       Post Created Date                                              sents  \\\n",
       "7             2022-06-23  ['We hebben een nieuwe Letterzetter!', 'Astrid...   \n",
       "10            2022-06-23  ['Waar een wil is, is altijd een weg!', '💪 Bij...   \n",
       "11            2022-06-23  ['💪 Bij ART2WORK geeft men jonge Brusselaars e...   \n",
       "12            2022-06-23  ['💪 Bij ART2WORK geeft men jonge Brusselaars e...   \n",
       "29            2022-06-23  ['Er zijn er die zouden zeggen dat ze blij zij...   \n",
       "...                  ...                                                ...   \n",
       "122437        2024-01-31  ['Gelukkig startte minister Ben Weyts project ...   \n",
       "122443        2024-01-31  ['Dat is wanneer de meest wijze beslissingen w...   \n",
       "122446        2024-01-31  ['Het stikstofakkoord van de regering-Jambon i...   \n",
       "122447        2024-01-31  ['Ook West-Vlaamse landbouwbedrijven investere...   \n",
       "122462        2024-01-31  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "\n",
       "                                                   object  \\\n",
       "7       Astrid Haerens zal de komende jaren Kortrijk n...   \n",
       "10      💪 Bij ART2WORK geeft men jonge Brusselaars een...   \n",
       "11      Met trajecten op maat & verschillende workshop...   \n",
       "12      Met trajecten op maat & verschillende workshop...   \n",
       "29      Maar ook in het Vlaams Parlement kon ik deze l...   \n",
       "...                                                   ...   \n",
       "122437  👨‍🏫 Daarnaast namen we grendels weg zodat deze...   \n",
       "122443  (Oprah Winfrey) Heel veel dankbaarheid en resp...   \n",
       "122446  Ook West-Vlaamse landbouwbedrijven investeren ...   \n",
       "122447  Voor het Vlaams Belang is het duidelijk: onze ...   \n",
       "122462  On-begrijpelijk en on-aanvaardbaar, want maar ...   \n",
       "\n",
       "                                                    group  \\\n",
       "7                                    ['jonge schrijvers']   \n",
       "10                                  ['jonge Brusselaars']   \n",
       "11                              ['werkzoekende jongeren']   \n",
       "12                            ['personen zonder diploma']   \n",
       "29                          ['onze lokale ondernemingen']   \n",
       "...                                                   ...   \n",
       "122437                                ['deze leerlingen']   \n",
       "122443  ['alle schooldirecteurs die zich dag en nacht ...   \n",
       "122446                 ['West-Vlaamse landbouwbedrijven']   \n",
       "122447                                  ['onze landbouw']   \n",
       "122462                    ['1 op 5 Brusselse scholieren']   \n",
       "\n",
       "        preds_representation  \n",
       "7                          1  \n",
       "10                         1  \n",
       "11                         1  \n",
       "12                         1  \n",
       "29                         1  \n",
       "...                      ...  \n",
       "122437                     1  \n",
       "122443                     1  \n",
       "122446                     1  \n",
       "122447                     1  \n",
       "122462                     1  \n",
       "\n",
       "[26202 rows x 14 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# According to our hypothesis, if there is an object present and representation, then these are representative claims\n",
    "\n",
    "data[(data['preds_representation'] == 1) & (data['group'] != '[]')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here, we keep the decision in a separate column\n",
    "data['preds_binary'] = ((data['preds_representation'] ==1) & (data['group'] !='[]')).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.2</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ID</th>\n",
       "      <th>ID_unit</th>\n",
       "      <th>Message_x</th>\n",
       "      <th>Message_y</th>\n",
       "      <th>Sentences</th>\n",
       "      <th>Page Name</th>\n",
       "      <th>Post Created Date</th>\n",
       "      <th>sents</th>\n",
       "      <th>object</th>\n",
       "      <th>group</th>\n",
       "      <th>preds_representation</th>\n",
       "      <th>preds_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FBVL95605</td>\n",
       "      <td>PARL0000</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>['Dit zien en even beseffen hoe geweldig mooi ...</td>\n",
       "      <td>Maurits Vande Reyde</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Dit zien en even beseffen hoe geweldig mooi ...</td>\n",
       "      <td>Dit zien en even beseffen hoe geweldig mooi de...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0001</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0002</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Met dankbaarheid om wat het mocht zijn, kijk i...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>FBVL95606</td>\n",
       "      <td>PARL0003</td>\n",
       "      <td>Met dankbaarheid om wat het mocht zijn, kijk i...</td>\n",
       "      <td>Prachtige kaart gevonden in mijn brievenbus, e...</td>\n",
       "      <td>['Prachtige kaart gevonden in mijn brievenbus,...</td>\n",
       "      <td>Ann De Martelaer</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Met dankbaarheid om wat het mocht zijn, kijk...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>FBVL95607</td>\n",
       "      <td>PARL0004</td>\n",
       "      <td>Jonge artiesten steunen! 📸 Eindwerk van Hannah...</td>\n",
       "      <td>Jonge artiesten steunen! 📸 Eindwerk van Hannah...</td>\n",
       "      <td>['Jonge artiesten steunen!', '📸 Eindwerk van H...</td>\n",
       "      <td>Stephanie D'Hose</td>\n",
       "      <td>2022-06-23</td>\n",
       "      <td>['Jonge artiesten steunen!', '📸 Eindwerk van H...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122462</th>\n",
       "      <td>122462</td>\n",
       "      <td>122462</td>\n",
       "      <td>114833</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114833</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>On-begrijpelijk en on-aanvaardbaar, want maar ...</td>\n",
       "      <td>['1 op 5 Brusselse scholieren']</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122463</th>\n",
       "      <td>122463</td>\n",
       "      <td>122463</td>\n",
       "      <td>114834</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114834</td>\n",
       "      <td>On-begrijpelijk en on-aanvaardbaar, want maar ...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['On-begrijpelijk en on-aanvaardbaar, want maa...</td>\n",
       "      <td>Zeker in onze diverse stad Brussel moeten derg...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122464</th>\n",
       "      <td>122464</td>\n",
       "      <td>122464</td>\n",
       "      <td>114835</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114835</td>\n",
       "      <td>Zeker in onze diverse stad Brussel moeten derg...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Zeker in onze diverse stad Brussel moeten de...</td>\n",
       "      <td>••• Le ministre flamand de l'Enseignement Ben ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122465</th>\n",
       "      <td>122465</td>\n",
       "      <td>122465</td>\n",
       "      <td>114836</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114836</td>\n",
       "      <td>••• Le ministre flamand de l'Enseignement Ben ...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['••• Le ministre flamand de l\\'Enseignement B...</td>\n",
       "      <td>Incompréhensible et inacceptable, car un écoli...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122466</th>\n",
       "      <td>122466</td>\n",
       "      <td>122466</td>\n",
       "      <td>114837</td>\n",
       "      <td>FBVL125558</td>\n",
       "      <td>PARL114837</td>\n",
       "      <td>Incompréhensible et inacceptable, car un écoli...</td>\n",
       "      <td>Vlaams onderwijsminister Ben Weyts legt School...</td>\n",
       "      <td>['Vlaams onderwijsminister Ben Weyts legt Scho...</td>\n",
       "      <td>Hannelore Goeman</td>\n",
       "      <td>2024-01-31</td>\n",
       "      <td>['Incompréhensible et inacceptable, car un éco...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>122467 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0.2  Unnamed: 0.1  Unnamed: 0          ID     ID_unit  \\\n",
       "0                  0             0           0   FBVL95605    PARL0000   \n",
       "1                  1             1           1   FBVL95606    PARL0001   \n",
       "2                  2             2           2   FBVL95606    PARL0002   \n",
       "3                  3             3           3   FBVL95606    PARL0003   \n",
       "4                  4             4           4   FBVL95607    PARL0004   \n",
       "...              ...           ...         ...         ...         ...   \n",
       "122462        122462        122462      114833  FBVL125558  PARL114833   \n",
       "122463        122463        122463      114834  FBVL125558  PARL114834   \n",
       "122464        122464        122464      114835  FBVL125558  PARL114835   \n",
       "122465        122465        122465      114836  FBVL125558  PARL114836   \n",
       "122466        122466        122466      114837  FBVL125558  PARL114837   \n",
       "\n",
       "                                                Message_x  \\\n",
       "0       Dit zien en even beseffen hoe geweldig mooi de...   \n",
       "1       Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "2       Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "3       Met dankbaarheid om wat het mocht zijn, kijk i...   \n",
       "4       Jonge artiesten steunen! 📸 Eindwerk van Hannah...   \n",
       "...                                                   ...   \n",
       "122462  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "122463  On-begrijpelijk en on-aanvaardbaar, want maar ...   \n",
       "122464  Zeker in onze diverse stad Brussel moeten derg...   \n",
       "122465  ••• Le ministre flamand de l'Enseignement Ben ...   \n",
       "122466  Incompréhensible et inacceptable, car un écoli...   \n",
       "\n",
       "                                                Message_y  \\\n",
       "0       Dit zien en even beseffen hoe geweldig mooi de...   \n",
       "1       Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "2       Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "3       Prachtige kaart gevonden in mijn brievenbus, e...   \n",
       "4       Jonge artiesten steunen! 📸 Eindwerk van Hannah...   \n",
       "...                                                   ...   \n",
       "122462  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "122463  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "122464  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "122465  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "122466  Vlaams onderwijsminister Ben Weyts legt School...   \n",
       "\n",
       "                                                Sentences  \\\n",
       "0       ['Dit zien en even beseffen hoe geweldig mooi ...   \n",
       "1       ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "2       ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "3       ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "4       ['Jonge artiesten steunen!', '📸 Eindwerk van H...   \n",
       "...                                                   ...   \n",
       "122462  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "122463  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "122464  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "122465  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "122466  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "\n",
       "                  Page Name Post Created Date  \\\n",
       "0       Maurits Vande Reyde        2022-06-23   \n",
       "1          Ann De Martelaer        2022-06-23   \n",
       "2          Ann De Martelaer        2022-06-23   \n",
       "3          Ann De Martelaer        2022-06-23   \n",
       "4          Stephanie D'Hose        2022-06-23   \n",
       "...                     ...               ...   \n",
       "122462     Hannelore Goeman        2024-01-31   \n",
       "122463     Hannelore Goeman        2024-01-31   \n",
       "122464     Hannelore Goeman        2024-01-31   \n",
       "122465     Hannelore Goeman        2024-01-31   \n",
       "122466     Hannelore Goeman        2024-01-31   \n",
       "\n",
       "                                                    sents  \\\n",
       "0       ['Dit zien en even beseffen hoe geweldig mooi ...   \n",
       "1       ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "2       ['Prachtige kaart gevonden in mijn brievenbus,...   \n",
       "3       ['Met dankbaarheid om wat het mocht zijn, kijk...   \n",
       "4       ['Jonge artiesten steunen!', '📸 Eindwerk van H...   \n",
       "...                                                   ...   \n",
       "122462  ['Vlaams onderwijsminister Ben Weyts legt Scho...   \n",
       "122463  ['On-begrijpelijk en on-aanvaardbaar, want maa...   \n",
       "122464  ['Zeker in onze diverse stad Brussel moeten de...   \n",
       "122465  ['••• Le ministre flamand de l\\'Enseignement B...   \n",
       "122466  ['Incompréhensible et inacceptable, car un éco...   \n",
       "\n",
       "                                                   object  \\\n",
       "0       Dit zien en even beseffen hoe geweldig mooi de...   \n",
       "1                                                     NaN   \n",
       "2       Met dankbaarheid om wat het mocht zijn, kijk i...   \n",
       "3                                                     NaN   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "122462  On-begrijpelijk en on-aanvaardbaar, want maar ...   \n",
       "122463  Zeker in onze diverse stad Brussel moeten derg...   \n",
       "122464  ••• Le ministre flamand de l'Enseignement Ben ...   \n",
       "122465  Incompréhensible et inacceptable, car un écoli...   \n",
       "122466                                                NaN   \n",
       "\n",
       "                                  group  preds_representation  preds_binary  \n",
       "0                                    []                     0             0  \n",
       "1                                    []                     0             0  \n",
       "2                                    []                     0             0  \n",
       "3                                    []                     0             0  \n",
       "4                                    []                     1             0  \n",
       "...                                 ...                   ...           ...  \n",
       "122462  ['1 op 5 Brusselse scholieren']                     1             1  \n",
       "122463                               []                     0             0  \n",
       "122464                               []                     0             0  \n",
       "122465                               []                     0             0  \n",
       "122466                               []                     0             0  \n",
       "\n",
       "[122467 rows x 15 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    96265\n",
       "1    26202\n",
       "Name: preds_binary, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['preds_binary'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0.2', 'Unnamed: 0.1', 'Unnamed: 0', 'ID', 'ID_unit',\n",
       "       'Message_x', 'Message_y', 'Sentences', 'Page Name', 'Post Created Date',\n",
       "       'sents', 'object', 'group', 'preds_representation', 'preds_binary'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save data\n",
    "data.to_csv('data/final_data_binary_decision.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Pol-Comp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}