4. transformer_reduced_bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:25:59.268000Z",
     "start_time": "2020-12-31T06:25:59.095747Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import metrics\n",
    "import transformers\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from transformers import BertTokenizer, BertModel, BertConfig\n",
    "from sklearn.metrics import recall_score, precision_score, f1_score, accuracy_score\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:24:02.502180Z",
     "start_time": "2020-12-31T06:24:02.498213Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T17:41:28.811383Z",
     "start_time": "2020-12-23T17:41:28.731961Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN Dataset: (5956, 6)\n",
      "TEST Dataset: (1489, 6)\n"
     ]
    }
   ],
   "source": [
    "train_dataset = pd.read_csv(\"./save_reduced_bert/train.csv\")\n",
    "test_dataset = pd.read_csv(\"./save_reduced_bert/test.csv\")\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(test_dataset.shape))\n",
    "\n",
    "train_dataset['list'] = train_dataset.iloc[:,1:].values.astype(int).tolist()\n",
    "test_dataset['list'] = test_dataset.iloc[:,1:].values.astype(int).tolist()\n",
    "\n",
    "train_dataset = train_dataset[['news', 'list']].rename(columns={\"news\": \"comment_text\"})\n",
    "test_dataset = test_dataset[['news', 'list']].rename(columns={\"news\": \"comment_text\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T17:41:33.601964Z",
     "start_time": "2020-12-23T17:41:32.996490Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUMklEQVR4nO3df/BddX3n8edLgiJIJZQ0G5NsA51Ul04tZiPiaLu2rvzciu52GegPU9dtOluYKdvObIPrFFuXGbqjWNxaatS04Co//J2FdGmgTlt3RiBBhABSUgmSNJAollB1VOx7/zifb7iNyTf3JN/7vfdLno+ZO/ec9znnnvf3e2/y+p4f95xUFZIkDet5425AkjS3GBySpF4MDklSLwaHJKkXg0OS1Mu8cTcwCieddFItW7Zs3G1I0pyyefPmr1XVgoPN95wMjmXLlrFp06ZxtyFJc0qSR4eZz11VkqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqRenpPfHJ+rlq25ZWzr3nbleWNbt6S5xS0OSVIvBockqReDQ5LUi8EhSerF4JAk9TKy4EiyNMnnkjyQ5P4kv9nq70yyI8k97XHuwDKXJdma5KEkZw3Uz261rUnWjKpnSdLBjfJ03GeA366qu5McD2xOsrFNe29VvXtw5iSnAhcCPwG8BLgtyY+3ye8H3gBsB+5Ksr6qHhhh75KkAxhZcFTVTmBnG346yYPA4mkWOR+4oaq+AzySZCtwepu2taq+ApDkhjavwSFJYzArxziSLANeAdzRSpckuTfJuiTzW20x8NjAYttb7UD1fdexOsmmJJt279490z+CJKkZeXAkeRHwSeDSqtoDXAP8GHAa3RbJe2ZiPVW1tqpWVtXKBQsOeq91SdIhGuklR5IcTRcaH62qTwFU1RMD0z8I3NxGdwBLBxZf0mpMU5ckzbJRnlUV4MPAg1V11UB90cBsbwa2tOH1wIVJXpDkZGA5cCdwF7A8yclJnk93AH39qPqWJE1vlFscrwF+BbgvyT2t9nbgoiSnAQVsA34doKruT3IT3UHvZ4CLq+r7AEkuAW4FjgLWVdX9I+xbkjSNUZ5V9Xkg+5m0YZplrgCu2E99w3TLSZJmj98clyT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqReDA5JUi8GhySpF4NDktTLvFG9cJKlwHXAQqCAtVV1dZITgRuBZcA24IKq+kaSAFcD5wLfAn61qu5ur7UKeEd76f9RVdeOqm+AZWtuGeXLS9KcNsotjmeA366qU4EzgIuTnAqsAW6vquXA7W0c4BxgeXusBq4BaEFzOfAq4HTg8iTzR9i3JGkaIwuOqto5tcVQVU8DDwKLgfOBqS2Ga4E3teHzgeuq8wXghCSLgLOAjVX1ZFV9A9gInD2qviVJ05uVYxxJlgGvAO4AFlbVzjbpcbpdWdCFymMDi21vtQPV913H6iSbkmzavXv3zP4AkqS9Rh4cSV4EfBK4tKr2DE6rqqI7/nHYqmptVa2sqpULFiyYiZeUJO3HSIMjydF0ofHRqvpUKz/RdkHRnne1+g5g6cDiS1rtQHVJ0hiMLDjaWVIfBh6sqqsGJq0HVrXhVcBnB+pvSecM4Km2S+tW4Mwk89tB8TNbTZI0BiM7HRd4DfArwH1J7mm1twNXAjcleRvwKHBBm7aB7lTcrXSn474VoKqeTPIu4K423+9X1ZMj7FuSNI2RBUdVfR7IASa/fj/zF3DxAV5rHbBu5rqTJB0qvzkuSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0MFRxJfnLUjUiS5oZhtzj+OMmdSX4jyYtH2pEkaaINFRxV9dPALwFLgc1JPpbkDSPtTJI0kYY+xlFVDwPvAH4H+DfA+5J8Ocm/H1VzkqTJM+wxjpcneS/wIPBzwM9X1b9qw+8dYX+SpAkzb8j5/hfwIeDtVfXtqWJV/X2Sd4ykM0nSRBo2OM4Dvl1V3wdI8jzgmKr6VlV9ZGTdSZImzrDHOG4DXjgwfmyrSZKOMMMGxzFV9Y9TI2342NG0JEmaZMMGxzeTrJgaSfKvgW9PM78k6Tlq2OC4FPh4kr9J8nngRuCS6RZIsi7JriRbBmrvTLIjyT3tce7AtMuSbE3yUJKzBupnt9rWJGt6/XSSpBk31MHxqrorycuAl7bSQ1X1vYMs9mfAHwHX7VN/b1W9e7CQ5FTgQuAngJcAtyX58Tb5/cAbgO3AXUnWV9UDw/QtSZp5w55VBfBKYFlbZkUSqmrfUNirqv46ybIhX/t84Iaq+g7wSJKtwOlt2taq+gpAkhvavAaHJI3JsF8A/AjwbuC1dAHySmDlIa7zkiT3tl1Z81ttMfDYwDzbW+1A9f31uDrJpiSbdu/efYitSZIOZtgtjpXAqVVVh7m+a4B3AdWe3wP8p8N8TQCqai2wFmDlypWH26ck6QCGPTi+BfgXh7uyqnqiqr5fVf8EfJBnd0ftoLuA4pQlrXaguiRpTIbd4jgJeCDJncB3popV9cY+K0uyqKp2ttE30wUSwHrgY0muojs4vhy4EwiwPMnJdIFxIfCLfdYpSZpZwwbHO/u+cJLrgdcBJyXZDlwOvC7JaXS7qrYBvw5QVfcnuYnuoPczwMUDlze5BLgVOApYV1X39+1FkjRzhj0d96+S/CiwvKpuS3Is3X/k0y1z0X7KH55m/iuAK/ZT3wBsGKZPSdLoDXtW1a8BnwA+0EqLgc+MqCdJ0gQb9uD4xcBrgD2w96ZOPzKqpiRJk2vY4PhOVX13aiTJPLrjFJKkI8ywwfFXSd4OvLDda/zjwP8ZXVuSpEk1bHCsAXYD99GdCbWB7v7jkqQjzLBnVU19Ye+Do21HkjTphgqOJI+wn2MaVXXKjHckSZpofa5VNeUY4D8CJ858O5KkSTfUMY6q+vrAY0dV/SFw3mhbkyRNomF3Va0YGH0e3RZIn3t5SJKeI4b9z/89A8PP0F1n6oIZ70aSNPGGPavqZ0fdiCRpbhh2V9VvTTe9qq6amXYkSZOuz1lVr6S7bwbAz9PdL+PhUTQlSZpcwwbHEmBFVT0NkOSdwC1V9cujakySNJmGveTIQuC7A+PfbTVJ0hFm2C2O64A7k3y6jb8JuHYkHUmSJtqwZ1VdkeTPgZ9upbdW1RdH15YkaVINu6sK4FhgT1VdDWxPcvKIepIkTbBhbx17OfA7wGWtdDTwv0fVlCRpcg27xfFm4I3ANwGq6u+B40fVlCRpcg0bHN+tqqJdWj3JcaNrSZI0yYYNjpuSfAA4IcmvAbfhTZ0k6Yh00LOqkgS4EXgZsAd4KfC7VbVxxL1JkibQQYOjqirJhqr6ScCwkKQj3LC7qu5O8sqRdiJJmhOG/eb4q4BfTrKN7syq0G2MvHxUjUmSJtO0wZHkX1bVV4GzZqkfSdKEO9gWx2foror7aJJPVtV/mIWeJEkT7GDHODIwfMooG5EkzQ0HC446wLAk6Qh1sOD4qSR7kjwNvLwN70nydJI90y2YZF2SXUm2DNROTLIxycPteX6rJ8n7kmxNcm+SFQPLrGrzP5xk1eH8sJKkwzdtcFTVUVX1Q1V1fFXNa8NT4z90kNf+M+DsfWprgNurajlwexsHOAdY3h6rgWugCxrgcrqzuk4HLp8KG0nSePS5rHovVfXXwJP7lM/n2RtAXUt3Q6ip+nXV+QLdpU0W0Z3NtbGqnqyqb9B9AXHfMJIkzaKRBccBLKyqnW34cZ69/exi4LGB+ba32oHqPyDJ6iSbkmzavXv3zHYtSdprtoNjr8Gr7c7Q662tqpVVtXLBggUz9bKSpH3MdnA80XZB0Z53tfoOYOnAfEta7UB1SdKYzHZwrAemzoxaBXx2oP6WdnbVGcBTbZfWrcCZSea3g+JntpokaUyGvVZVb0muB14HnJRkO93ZUVfS3dvjbcCjwAVt9g3AucBW4FvAWwGq6skk7wLuavP9flXte8BdkjSLRhYcVXXRASa9fj/zFnDxAV5nHbBuBluTJB2GsR0clyTNTQaHJKkXg0OS1MvIjnFoblm25paxrHfbleeNZb2SDp1bHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvYwmOJNuS3JfkniSbWu3EJBuTPNye57d6krwvydYk9yZZMY6eJUmdcW5x/GxVnVZVK9v4GuD2qloO3N7GAc4BlrfHauCaWe9UkrTXJO2qOh+4tg1fC7xpoH5ddb4AnJBk0Rj6kyQxvuAo4C+SbE6yutUWVtXONvw4sLANLwYeG1h2e6v9M0lWJ9mUZNPu3btH1bckHfHmjWm9r62qHUl+BNiY5MuDE6uqklSfF6yqtcBagJUrV/ZaVpI0vLFscVTVjva8C/g0cDrwxNQuqPa8q82+A1g6sPiSVpMkjcGsB0eS45IcPzUMnAlsAdYDq9psq4DPtuH1wFva2VVnAE8N7NKSJM2yceyqWgh8OsnU+j9WVf83yV3ATUneBjwKXNDm3wCcC2wFvgW8dfZbliRNmfXgqKqvAD+1n/rXgdfvp17AxbPQmiRpCJN0Oq4kaQ4wOCRJvRgckqReDA5JUi8GhySpF4NDktSLwSFJ6sXgkCT1YnBIknoxOCRJvRgckqRexnU/DgmAZWtuGct6t1153ljWKz0XuMUhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqReDQ5LUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIvBockqRfvx6Ej0rjuAwLeC0Rzn1sckqReDA5JUi8GhySplzlzjCPJ2cDVwFHAh6rqyjG3JB0S77OuuW5ObHEkOQp4P3AOcCpwUZJTx9uVJB2Z5soWx+nA1qr6CkCSG4DzgQfG2pU0h4zzTLJxcStrNOZKcCwGHhsY3w68anCGJKuB1W30H5M8dAjrOQn42iF1OHqT3BtMdn/2dugmub+D9pY/mKVOftAk/97gwP396DALz5XgOKiqWgusPZzXSLKpqlbOUEszapJ7g8nuz94O3ST3Z2+H7nD7mxPHOIAdwNKB8SWtJkmaZXMlOO4Clic5OcnzgQuB9WPuSZKOSHNiV1VVPZPkEuBWutNx11XV/SNY1WHt6hqxSe4NJrs/ezt0k9yfvR26w9utX1Uz1Ygk6QgwV3ZVSZImhMEhSerF4GiSnJ3koSRbk6wZw/rXJdmVZMtA7cQkG5M83J7nt3qSvK/1em+SFSPubWmSzyV5IMn9SX5zUvpLckySO5N8qfX2e61+cpI7Wg83tpMqSPKCNr61TV82qt4GejwqyReT3DyBvW1Lcl+Se5JsarWxv69tfSck+USSLyd5MMmrJ6i3l7bf2dRjT5JLJ6i//9r+PWxJcn37dzJzn7uqOuIfdAfc/w44BXg+8CXg1Fnu4WeAFcCWgdr/BNa04TXAH7Thc4E/BwKcAdwx4t4WASva8PHA39Jd+mXs/bV1vKgNHw3c0dZ5E3Bhq/8J8F/a8G8Af9KGLwRunIX39reAjwE3t/FJ6m0bcNI+tbG/r2191wL/uQ0/HzhhUnrbp8+jgMfpvjw39v7ovjD9CPDCgc/br87k525WfrGT/gBeDdw6MH4ZcNkY+ljGPw+Oh4BFbXgR8FAb/gBw0f7mm6U+Pwu8YdL6A44F7qa7qsDXgHn7vr90Z+a9ug3Pa/NlhD0tAW4Hfg64uf3HMRG9tfVs4weDY+zvK/Di9p9fJq23/fR6JvD/JqU/nr3Sxontc3QzcNZMfu7cVdXZ3yVNFo+pl0ELq2pnG34cWNiGx9Zv24x9Bd1f9hPRX9sVdA+wC9hIt/X4D1X1zH7Wv7e3Nv0p4IdH1Rvwh8B/A/6pjf/wBPUGUMBfJNmc7rI9MBnv68nAbuBP226+DyU5bkJ629eFwPVteOz9VdUO4N3AV4GddJ+jzczg587gmCOq+3NgrOdOJ3kR8Eng0qraMzhtnP1V1fer6jS6v+5PB142jj72leTfAbuqavO4e5nGa6tqBd2Vpy9O8jODE8f4vs6j23V7TVW9Avgm3a6fSehtr3ac4I3Ax/edNq7+2nGV8+nC9yXAccDZM7kOg6MzqZc0eSLJIoD2vKvVZ73fJEfThcZHq+pTk9YfQFX9A/A5us3wE5JMfcF1cP17e2vTXwx8fUQtvQZ4Y5JtwA10u6uunpDegL1/nVJVu4BP0wXvJLyv24HtVXVHG/8EXZBMQm+DzgHurqon2vgk9PdvgUeqandVfQ/4FN1nccY+dwZHZ1IvabIeWNWGV9EdW5iqv6WdqXEG8NTA5vGMSxLgw8CDVXXVJPWXZEGSE9rwC+mOvTxIFyC/cIDepnr+BeAv21+GM66qLquqJVW1jO4z9ZdV9UuT0BtAkuOSHD81TLevfgsT8L5W1ePAY0le2kqvp7uNwth728dFPLubaqqPcff3VeCMJMe2f7tTv7uZ+9zNxsGjufCgO+vhb+n2j//3Maz/err9kd+j+2vrbXT7GW8HHgZuA05s84buxlZ/B9wHrBxxb6+l2+S+F7inPc6dhP6AlwNfbL1tAX631U8B7gS20u1GeEGrH9PGt7bpp8zS+/s6nj2raiJ6a318qT3un/rcT8L72tZ3GrCpvbefAeZPSm9tncfR/WX+4oHaRPQH/B7w5fZv4iPAC2byc+clRyRJvbirSpLUi8EhSerF4JAk9WJwSJJ6MTgkSb0YHJKkXgwOSVIv/x8zLm9mzPmCqQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "MAX_LEN = 300\n",
    "TRAIN_BATCH_SIZE = 64\n",
    "VALID_BATCH_SIZE = 32\n",
    "EPOCHS = 1\n",
    "LEARNING_RATE = 1e-05\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "train_dataset.comment_text.str.split().str.len().plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:14.997264Z",
     "start_time": "2020-12-23T03:50:14.986704Z"
    }
   },
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.comment_text = dataframe.comment_text\n",
    "        self.targets = self.data.list\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.comment_text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        comment_text = str(self.comment_text[index])\n",
    "        comment_text = \" \".join(comment_text.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            comment_text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            pad_to_max_length=True,\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True  #####   TODO\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:15.009112Z",
     "start_time": "2020-12-23T03:50:14.998961Z"
    }
   },
   "outputs": [],
   "source": [
    "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "testing_set = CustomDataset(test_dataset, tokenizer, MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:15.016152Z",
     "start_time": "2020-12-23T03:50:15.011212Z"
    }
   },
   "outputs": [],
   "source": [
    "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "testing_loader = DataLoader(testing_set, **test_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating the Neural Network for Fine Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:18.621968Z",
     "start_time": "2020-12-23T03:50:15.019422Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BERTClass(\n",
       "  (l1): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (l2): Dropout(p=0.3, inplace=False)\n",
       "  (l3): Linear(in_features=768, out_features=5, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.l2 = torch.nn.Dropout(0.3)\n",
    "        self.l3 = torch.nn.Linear(768, 5)\n",
    "    \n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids)\n",
    "        output_2 = self.l2(output_1)\n",
    "        output = self.l3(output_2)\n",
    "        return output\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:18.627214Z",
     "start_time": "2020-12-23T03:50:18.623099Z"
    }
   },
   "outputs": [],
   "source": [
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n",
    "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine Tuning the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:18.649298Z",
     "start_time": "2020-12-23T03:50:18.628130Z"
    }
   },
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    model.train()\n",
    "    for _,data in enumerate(training_loader, 0):\n",
    "        ids = data['ids'].to(device, dtype = torch.long)\n",
    "        mask = data['mask'].to(device, dtype = torch.long)\n",
    "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "        targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "        outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = loss_fn(outputs, targets)\n",
    "        if _%5000==0:\n",
    "            print(f'Epoch: {epoch}, Loss:  {loss.item()}')\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "def validation(epoch):\n",
    "    model.eval()\n",
    "    fin_targets=[]\n",
    "    fin_outputs=[]\n",
    "    with torch.no_grad():\n",
    "        for _, data in enumerate(testing_loader, 0):\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['targets'].to(device, dtype = torch.float)\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "            fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "            fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "    return fin_outputs, fin_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T03:50:18.658575Z",
     "start_time": "2020-12-23T03:50:18.650352Z"
    }
   },
   "outputs": [],
   "source": [
    "EPOCHS=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T04:27:46.144821Z",
     "start_time": "2020-12-23T03:50:24.764885Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch: 0, Loss:  0.8127931952476501\n",
      "Accuracy Score = 0.7427803895231699\n",
      "F1 Score (Micro) = 0.4612244897959183\n",
      "F1 Score (Macro) = 0.148440065681445\n",
      "\n",
      "\n",
      "Epoch: 1, Loss:  0.24262885749340057\n",
      "Accuracy Score = 0.8139691067830759\n",
      "F1 Score (Micro) = 0.7204066811909949\n",
      "F1 Score (Macro) = 0.4921214440475662\n",
      "\n",
      "\n",
      "Epoch: 2, Loss:  0.14457932114601135\n",
      "Accuracy Score = 0.8361316319677636\n",
      "F1 Score (Micro) = 0.7595959595959595\n",
      "F1 Score (Macro) = 0.6820137098339745\n",
      "\n",
      "\n",
      "Epoch: 3, Loss:  0.09324751794338226\n",
      "Accuracy Score = 0.8623237071860309\n",
      "F1 Score (Micro) = 0.7634112792297111\n",
      "F1 Score (Macro) = 0.7037231397921121\n",
      "\n",
      "\n",
      "Epoch: 4, Loss:  0.07992719858884811\n",
      "Accuracy Score = 0.8750839489590329\n",
      "F1 Score (Micro) = 0.7731811697574893\n",
      "F1 Score (Macro) = 0.7236598205009901\n",
      "\n",
      "\n",
      "Epoch: 5, Loss:  0.019926181063055992\n",
      "Accuracy Score = 0.8770987239758227\n",
      "F1 Score (Micro) = 0.7794221282593374\n",
      "F1 Score (Macro) = 0.7364736489786513\n",
      "\n",
      "\n",
      "Epoch: 6, Loss:  0.02823011390864849\n",
      "Accuracy Score = 0.8770987239758227\n",
      "F1 Score (Micro) = 0.7703488372093023\n",
      "F1 Score (Macro) = 0.7290575544072458\n",
      "\n",
      "\n",
      "Epoch: 7, Loss:  0.04823284596204758\n",
      "Accuracy Score = 0.8744123572867697\n",
      "F1 Score (Micro) = 0.772972972972973\n",
      "F1 Score (Macro) = 0.7431748745180767\n",
      "\n",
      "\n",
      "Epoch: 8, Loss:  0.01524769701063633\n",
      "Accuracy Score = 0.8757555406312961\n",
      "F1 Score (Micro) = 0.7656903765690377\n",
      "F1 Score (Macro) = 0.7348548309822751\n",
      "\n",
      "\n",
      "Epoch: 9, Loss:  0.01986449770629406\n",
      "Accuracy Score = 0.8656816655473473\n",
      "F1 Score (Micro) = 0.7678929765886288\n",
      "F1 Score (Macro) = 0.731794730185561\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    print('\\n')\n",
    "    train(epoch)\n",
    "        \n",
    "    outputs, targets = validation(epoch)\n",
    "    outputs = np.array(outputs) >= 0.5\n",
    "    accuracy = accuracy_score(targets, outputs)\n",
    "    f1_score_micro = f1_score(targets, outputs, average='micro')\n",
    "    f1_score_macro = f1_score(targets, outputs, average='macro')\n",
    "    print(f\"Accuracy Score = {accuracy}\")\n",
    "    print(f\"F1 Score (Micro) = {f1_score_micro}\")\n",
    "    print(f\"F1 Score (Macro) = {f1_score_macro}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T05:14:45.909966Z",
     "start_time": "2020-12-23T05:14:45.269240Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'save_reduced_bert/transformer.pth')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs, targets = validation(epoch)\n",
    "outputs = np.array(outputs)\n",
    "targets = np.array(targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:24:13.311246Z",
     "start_time": "2020-12-31T06:24:13.305189Z"
    }
   },
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# with open('save_reduced_bert/outputs.pt', 'wb') as file:\n",
    "#     pickle.dump(outputs, file)\n",
    "    \n",
    "# with open('save_reduced_bert/targets.pt', 'wb') as file:\n",
    "#     pickle.dump(targets, file)\n",
    "\n",
    "\n",
    "import pickle\n",
    "\n",
    "with open('save_reduced_bert/outputs.pt', 'rb') as file:\n",
    "    outputs = pickle.load(file)\n",
    "    \n",
    "with open('save_reduced_bert/targets.pt', 'rb') as file:\n",
    "    targets = pickle.load(file)\n",
    "    \n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:26:05.327906Z",
     "start_time": "2020-12-31T06:26:03.673977Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold 0.1\n",
      "threshold 0.2\n",
      "threshold 0.3\n",
      "threshold 0.4\n",
      "threshold 0.5\n",
      "threshold 0.6\n",
      "threshold 0.7\n",
      "threshold 0.8\n",
      "threshold 0.9\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(columns = ['threshold','type','accuracy', \n",
    "                             'recall(micro)', 'precision(micro)', 'f1(micro)', \n",
    "                             'recall(macro)', 'precision(macro)', 'f1(macro)', \n",
    "                             \n",
    "                             'accuracy(custom)', 'precision(custom)','recall(custom)', 'f1(custom)',  'EMR'\n",
    "                            ])\n",
    "\n",
    "Threshold = np.around(np.linspace(0.1, 0.9, 9),1)\n",
    "\n",
    "for threshold in Threshold:\n",
    "    print('threshold', threshold)\n",
    "    pred = outputs >= threshold\n",
    "\n",
    "#     accuracy = accuracy_score(targets, outputs)\n",
    "#     f1_score_micro = f1_score(targets, outputs, average='micro')\n",
    "#     f1_score_macro = f1_score(targets, outputs, average='macro')\n",
    "#     print(f\"Accuracy Score = {accuracy}\")\n",
    "#     print(f\"F1 Score (Micro) = {f1_score_micro}\")\n",
    "#     print(f\"F1 Score (Macro) = {f1_score_macro}\")\n",
    "\n",
    "\n",
    "    df.loc[len(df)] = [threshold, \n",
    "                       'all', \n",
    "                       accuracy_score(targets, pred),\n",
    "                       recall_score(targets, pred, average='micro'),\n",
    "                       precision_score(targets, pred, average='micro'),\n",
    "                       f1_score(targets, pred, average='micro'),\n",
    "                       recall_score(targets, pred, average='macro'),\n",
    "                       precision_score(targets, pred, average='macro'),\n",
    "                       f1_score(targets, pred, average='macro'),\n",
    "                       \n",
    "                       accuracy_multilabel(targets, pred),\n",
    "                       precision_multilabel(targets, pred),\n",
    "                       recall_multilabel(targets, pred),\n",
    "                       f1_multilabel(targets, pred),\n",
    "                       accuracy_score(targets, pred)\n",
    "                       ]\n",
    "    \n",
    "    \n",
    "    df.loc[len(df)] = [threshold, \n",
    "                       'multi', \n",
    "                       accuracy_score(targets[:,1:], pred[:,1:]),\n",
    "                       recall_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       precision_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       f1_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       recall_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       precision_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       f1_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       \n",
    "                        accuracy_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                       precision_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                       recall_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                       f1_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                       accuracy_score(targets[:,1:], pred[:,1:])\n",
    "                       ]\n",
    "    \n",
    "    \n",
    "    \n",
    "    df_target = pd.DataFrame(targets)\n",
    "    df_target.columns = ['is_relevant',\t'Armed Assault',\t'Bombing/Explosion',\t'Kidnapping',\t'Other']\n",
    "\n",
    "    df_pred = pd.DataFrame(pred)\n",
    "    df_pred.columns = df_target.columns\n",
    "    \n",
    "    for col in df_pred.columns:\n",
    "\n",
    "        df.loc[len(df)] = [threshold, \n",
    "                           col, \n",
    "                           accuracy_score(df_target[col], df_pred[col]),\n",
    "                           recall_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           precision_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           f1_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           recall_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           precision_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           f1_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           \n",
    "                           None, None, None,None, None\n",
    "                           ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T06:03:40.755042Z",
     "start_time": "2020-12-23T06:03:40.745488Z"
    }
   },
   "outputs": [],
   "source": [
    "# df.to_csv('save_reduced_bert/saved_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-29T21:46:34.067849Z",
     "start_time": "2020-12-29T21:46:34.058120Z"
    }
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv('save_reduced_bert/saved_df.csv')\n",
    "Threshold = np.around(np.linspace(0.1, 0.9, 9),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:26:12.805924Z",
     "start_time": "2020-12-31T06:26:12.778750Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "      <th>EMR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.5</td>\n",
       "      <td>all</td>\n",
       "      <td>0.865682</td>\n",
       "      <td>0.817664</td>\n",
       "      <td>0.723834</td>\n",
       "      <td>0.767893</td>\n",
       "      <td>0.750388</td>\n",
       "      <td>0.725284</td>\n",
       "      <td>0.731795</td>\n",
       "      <td>0.615355</td>\n",
       "      <td>0.713034</td>\n",
       "      <td>0.825277</td>\n",
       "      <td>0.634006</td>\n",
       "      <td>0.865682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.5</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.875084</td>\n",
       "      <td>0.768194</td>\n",
       "      <td>0.707196</td>\n",
       "      <td>0.736434</td>\n",
       "      <td>0.719707</td>\n",
       "      <td>0.721348</td>\n",
       "      <td>0.714327</td>\n",
       "      <td>0.607257</td>\n",
       "      <td>0.716265</td>\n",
       "      <td>0.792548</td>\n",
       "      <td>0.624721</td>\n",
       "      <td>0.875084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.5</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.903962</td>\n",
       "      <td>0.903962</td>\n",
       "      <td>0.903962</td>\n",
       "      <td>0.903962</td>\n",
       "      <td>0.892946</td>\n",
       "      <td>0.851405</td>\n",
       "      <td>0.869153</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.5</td>\n",
       "      <td>Armed Assault</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.886554</td>\n",
       "      <td>0.826781</td>\n",
       "      <td>0.853211</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.5</td>\n",
       "      <td>Bombing/Explosion</td>\n",
       "      <td>0.967092</td>\n",
       "      <td>0.967092</td>\n",
       "      <td>0.967092</td>\n",
       "      <td>0.967092</td>\n",
       "      <td>0.912488</td>\n",
       "      <td>0.867775</td>\n",
       "      <td>0.888545</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.5</td>\n",
       "      <td>Kidnapping</td>\n",
       "      <td>0.988583</td>\n",
       "      <td>0.988583</td>\n",
       "      <td>0.988583</td>\n",
       "      <td>0.988583</td>\n",
       "      <td>0.784003</td>\n",
       "      <td>0.876865</td>\n",
       "      <td>0.823629</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>0.5</td>\n",
       "      <td>Other</td>\n",
       "      <td>0.964406</td>\n",
       "      <td>0.964406</td>\n",
       "      <td>0.964406</td>\n",
       "      <td>0.964406</td>\n",
       "      <td>0.813244</td>\n",
       "      <td>0.840026</td>\n",
       "      <td>0.825997</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold               type  accuracy  recall(micro)  precision(micro)  \\\n",
       "28        0.5                all  0.865682       0.817664          0.723834   \n",
       "29        0.5              multi  0.875084       0.768194          0.707196   \n",
       "30        0.5        is_relevant  0.903962       0.903962          0.903962   \n",
       "31        0.5      Armed Assault  0.942915       0.942915          0.942915   \n",
       "32        0.5  Bombing/Explosion  0.967092       0.967092          0.967092   \n",
       "33        0.5         Kidnapping  0.988583       0.988583          0.988583   \n",
       "34        0.5              Other  0.964406       0.964406          0.964406   \n",
       "\n",
       "    f1(micro)  recall(macro)  precision(macro)  f1(macro) accuracy(custom)  \\\n",
       "28   0.767893       0.750388          0.725284   0.731795         0.615355   \n",
       "29   0.736434       0.719707          0.721348   0.714327         0.607257   \n",
       "30   0.903962       0.892946          0.851405   0.869153             None   \n",
       "31   0.942915       0.886554          0.826781   0.853211             None   \n",
       "32   0.967092       0.912488          0.867775   0.888545             None   \n",
       "33   0.988583       0.784003          0.876865   0.823629             None   \n",
       "34   0.964406       0.813244          0.840026   0.825997             None   \n",
       "\n",
       "   precision(custom) recall(custom) f1(custom)       EMR  \n",
       "28          0.713034       0.825277   0.634006  0.865682  \n",
       "29          0.716265       0.792548   0.624721  0.875084  \n",
       "30              None           None       None      None  \n",
       "31              None           None       None      None  \n",
       "32              None           None       None      None  \n",
       "33              None           None       None      None  \n",
       "34              None           None       None      None  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.threshold==0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:27:07.491397Z",
     "start_time": "2020-12-31T06:27:07.322111Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>f1(custom)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.718676</td>\n",
       "      <td>0.675054</td>\n",
       "      <td>0.598203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.742642</td>\n",
       "      <td>0.704107</td>\n",
       "      <td>0.6152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.753846</td>\n",
       "      <td>0.709475</td>\n",
       "      <td>0.626331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.765400</td>\n",
       "      <td>0.724456</td>\n",
       "      <td>0.632002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.767893</td>\n",
       "      <td>0.731795</td>\n",
       "      <td>0.634006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.773753</td>\n",
       "      <td>0.730628</td>\n",
       "      <td>0.641804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.774599</td>\n",
       "      <td>0.724652</td>\n",
       "      <td>0.645982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.767691</td>\n",
       "      <td>0.713337</td>\n",
       "      <td>0.637189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.743666</td>\n",
       "      <td>0.654940</td>\n",
       "      <td>0.6095</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   f1(micro)  f1(macro) f1(custom)\n",
       "0   0.718676   0.675054   0.598203\n",
       "1   0.742642   0.704107     0.6152\n",
       "2   0.753846   0.709475   0.626331\n",
       "3   0.765400   0.724456   0.632002\n",
       "4   0.767893   0.731795   0.634006\n",
       "5   0.773753   0.730628   0.641804\n",
       "6   0.774599   0.724652   0.645982\n",
       "7   0.767691   0.713337   0.637189\n",
       "8   0.743666   0.654940     0.6095"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEWCAYAAABv+EDhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABHpklEQVR4nO3deXhU1fnA8e+bnZBAgLCEhH0PSQAJQVQEFxBw30Gtu/6srUutC9ZaKWpF695a0VqXWitardYNFRXckTUkgBIWWbKwJ4EQQrb398e9CUMIMAlJZpJ5P88zz8zce+6ddwZy3nvPufccUVWMMcYEniBfB2CMMcY3LAEYY0yAsgRgjDEByhKAMcYEKEsAxhgToCwBGGNMgLIEYJqciMwTkWt9HUdzICLrReTUw6yfLSJXHGa9ikhfLz6np1s2pB4x1ntb41uWAExAao5JSESmici/PJep6kRVfcVXMZnmzRKAaVAt9SjQTRhjfR2HMQ3JEoA5am4zxV0ikgHsEZEQETlWRL4TkQIRWXa4ylNErhaRH0UkX0Q+EZEe7vJnReTRGmX/JyK3ua+nishaEdktIitF5FyPcleKyDci8qi7359FZKK77kFgNPBXESkSkb824G8xVkSyReROEdkqInkico6ITBKRLBHZKSK/8yj/sog8UHP7WvY7AfgdcLEb8zJ3uddnMiJyuogsFZFdIrJJRKbVUuxqEcl1477dY9sgj997h4i8KSLtvf9ljD+yBGAayhTgdCAG6Ax8CDwAtAduB94WkY41NxKRs3EqtvOAjsDXwOvu6tdxKjxxy7YDxgOz3PVrcSrytsAfgX+JSJzH7kcCq4BY4BHgHyIiqnqP+zm/VtUoVf11Q/wAHroAEUA88Afg78BlwHA33ntFpFdddqiqHwN/At5wYx5Sj7j2AJfj/BudDvxSRM6pUeYkoB/O73yXR//DTcA5wBigK5APPFOPGIwfsQRgGsrTqrpJVffiVHYfqepHqlqpqnOARcCkWra7AXhIVX9U1XKcSm6oexbwNaA4lSbABcD3qpoLoKr/UdVc9zPeAFYDaR773qCqf1fVCuAVIA4nOTW2MuBBVS3DSVaxwFOqultVVwArgfpU4EdFVeepaqb7e2XgJNgxNYr9UVX3qGom8BJOYgfn3+keVc1W1X3ANOCCltrkFygsAZiGssnjdQ/gQrf5p0BECoATcCrgmnoAT3mU2wkIEK/OSIWz2F8JXQK8VrWhiFwuIuke2ybhVLZVNle9UNVi92WUt1+olvg/8Fg29TCb7nCTDsBe93mLx/q9dYmjDvGucJuHikRkdC3rR4rIXBHZJiKFOJV6bI1inv+OG3CO9sH5d3rH4/f4EaigaRKqaSSWvU1D8RxWdhPwqqpe58V2m3COll87xPrXgU9FZAZOk865AO4Zwt+BU3DOCipEJB0nedQ13toLqMZUvRaRecA0VZ3n5f69tQeI9Hjf5XAhHW5Hqjr4CJ/1b+CvwERVLRGRJzk4AXQDfnJfdwdy3debgKtV9duaOxWRnkf4XOOn7AzANIZ/AWeKyGkiEiwiEW7nZkItZWcCd4vIYAARaSsiF1atVNWlwHbgBeATVS1wV7XGqRC3udtdhXMG4K0tQO86fq/GkA5MEpH2ItIFuPUwZbcAPUWkvn+30cBOt/JPwzmjquleEYl0/z2uAt5wl88EHvTooO/o9t+YZswSgGlwqroJqOrc3YZz9HgHtfx/U9V3gIeBWSKyC1gOTKxR7N/Aqe5z1XYrgceA73EqxmTgoKPTw3gKpw07X0SersN2De1VYBmwHviU/RVubf7jPu8QkSX1+Kwbgekishunc/rNWsp8CawBPgceVdVP3eVPAe/hnI3tBubjnJGZZkxsQhhjjAlMdgZgjDEByhKAMcYEKEsAxhgToCwBGGNMgGpW9wHExsZqz549fR2GMcY0K4sXL96uqgcNxdKsEkDPnj1ZtGiRr8MwxphmRUQ21LbcmoCMMSZAWQIwxpgAZQnAGGMClCUAY4wJUJYAjDEmQHmVAERkgoisEpE1tY2DLiJPuOOyp7vT3hW4y0/yWJ4uIiVVMxC5U+H97LFuaAN+L2OMMUdwxMtARSQYZ+q3cUA2sFBE3nNHYwRAVX/jUf4mYJi7fC4w1F3eHmeUwU/Z7w5Vfevov4Yxxpi68uY+gDRgjaquAxCRWThD/a48RPkpwH21LL8AmO0xM5MxphlQVSoqldKKSsrKnWfntfNcWl5JWfWzUlpRQWm5eixzHvuq1pdXEhIsJHZtQ0p8WzpEhfv6KwYsbxJAPAdOE5fNIcYBdyeL6AV8UcvqycDjNZY9KCJ/wBl7fKo712jNfV4PXA/QvXt3L8I1JnCpKtuK9pG1uYhVW3aTnV/sVLxVlXRFJaXlWl2B71/mVuoelXx1BV9RSWOOGh8f04qUhLYkJ7QlJT6G5Pi2tI0MbbwPNNUa+k7gycBbHvOhAiAicTgTdnzisfhunDlbw4DngbuA6TV3qKrPu+tJTU21yQuMcRXuLWP1lt2s2rKbrM27+WnzbrK27Ca/uKy6TFR4CBGhQYQGO4+wkP3PYcFCWEgQUREhHsuCCHWXhwY77z23cZZJLcv2v67a/sBlQQcsKy4tZ3nOLjJzCsjILiQzp5DZy6uncKZHh0iS49s6iSE+hqT4NkRHWFJoaN4kgByceUKrJLjLajMZ+FUtyy8C3lHV6v+ZqprnvtwnIi8Bt3sRizEBp6SsgjVbi1jlVvBVFX1eYUl1majwEPp3jmJCUhf6d45mQOdo+neJJtZPm1eiI0IZ1acDo/p0qF5WUFzK8pxdZOQUkJldyNKNBXyQ4VQTItA7tjUpCTHViSGxaxsiw5rVaDZ+x5tfbyHQT0R64VT8k6llLlERGQi0w5mir6YpOEf8nuXjVDVPRAQ4B2cqQGMCVnlFJet37GGV23yTtdk5ut+wYw+V7rlvWHAQfTtFcWzvDk5F3yWK/p2jiY9phfOn1HzFRIZxQr9YTui3f576HUX7yMgpJDO7kIzsQr5bu513ljrHn0EC/TpFO01HCW1Jjm/LoLg2RIQG++orNDtHTACqWi4iv8ZpvgkGXlTVFSIyHVikqu+5RScDs7TGHJMi0hPnDOLLGrt+TUQ6AoIzMfYNR/NFjGkuKiuVnIK9ZNVovlm3bQ+lFZWAU7n1jG3NwC7RnDWkKwO6RNO/czQ9O0QSEhw4t+90iArnpAGdOGlAp+plW3aVOAkhp5DM7ALm/rSVtxZnAxASJAzoEl3ddJSS0Jb+naMJCwmc36wumtWcwKmpqWqjgZrmQlXZXlTqVPQezTert+xmT+n+brL4mFb07xxF/y5u003naPp2irIjWS+pKnmFJW5fwv4+hQK3LyQsOIhBcdH7O5kT2tKvU1RAJVIRWayqqQcttwRgzNHbVeJ2yG4uciv6XWRtKWLnntLqMu1bhzGgc3T10fyALlH06xxNG+vcbHCqyqade6v7EzKyC1meU8jufeUARIQGkRjXhpQE5ywhJaEtvWKjCA5q3s1oh2IJwBgvqCq7SsopKC4lv7iM/OJSCopLKSguI7+4rHr5/mXOc5FbsQC0Dgs+4Gi+qsLvGO2fHbKBorJSWb9jD5k5TkLIzC5keW4hxe7ZWOuwYI7p0Y57z0ikf+doH0fbsCwBmIBTUlZBfnEp+XvKKNh7YIWdv6eUgr1lNSr6Mgr3llFReei/iTYRIbRrHUZMZBjtIkNpFxlG21ahdGoTXl3hx8e0IqiFHkm2NBWVyrptRdXNRu8vy6VoXzm/mzSIy0f1aPYd61UsAZhmb8++cn7avIv8PU6FXbjXea4+Mt9T5lGpl1JSVnnIfUWEBtEuskZFHhla/bpqeUxkqPvaqehbahOBcWzbvY8731rG3FXbGDugI49ckEKn6Ahfh3XULAGYZmlvaQVf/LSVDzJy+eKnrewrP7BSDw4SYlo5FXVVxR3jVuRVFXe7yFC3cg9zy4RaB6s5JFXl1fkbePDDH2kdHsIj56dwamJnX4d1VA6VAOwuCuN3Ssoq+DJrGx9k5PH5j1soLq0gNiqcySO6cWL/jsRGhTsVeetQosNDWsxpuvEPIsLlo3oyqncHbpmVzrX/XMQlI7vz+9MHtbgbz1rWtzHNVml5Jd+u2c77GbnMWbGF3fvKaRcZytlD4zkzJY6RvTtY84tpUv06R/POr47j8U+zeP7rdcxft4OnLh5GckJbX4fWYKwJyPhMeUUl36/bwQfL8vh4xWYK95bRJiKE0wZ34YwhXTmuTwdCA+habeO/vluzndveXMb2on3cNr4//3din2Z1QGJ9AMYvVFQqC37eyQcZuXy8fDM79pQSFR7CuMTOnJESx+h+He2uTeOXCopLueed5XyYmUdar/Y8cfFQ4mNa+Tosr1gCMD5TWaks3ZTP+8vy+Cgzj62799EqNJiTB3XizJQ4xg7oZJ2ypllQVd5eksN9/1tOUJDwwDlJnD003tdhHZF1ApsmpapkZBfyQUYuH2bkkVtYQlhIECcN6MgZKV05ZVCnFtehZlo+EeGC4Qmk9WzPrW8s5ZZZ6cz9aSvTz0lqlnd02xmAaTCqysq8XXyQkceHGXls3FlMaLBwYr+OnDEkjlMHdbYx3U2LUV5RyV/nruEvX6yhS5sInrh4KGm92vs6rFpZE5BpNKu37Ob9jDw+yMhl3bY9BAcJx/XpwJkpXTltcBeb3cm0aEs25nPrrHSy84v55dg+3Hpqf7+7eMESgGlQP2/fwwfLcvkgI49VW3YjAsf26sAZQ+KYMLiLzfNqAkrRvnL++N4K/rM4myEJbXni4qH07hjl67CqWQIwR23TzmI+cI/0V+TuAmBEz3ackdKVicldWsQt88YcjdmZeUz9byal5ZX84cxEJo/o5hc3KlonsKmXvMK9fJiRxwcZeaRvKgBgSLcYfn/6ICYlx9G1mVwGZ0xTmJgcx9DuMdz+n2Xc/d9M5v60lRnnp9C+dZivQ6uVnQGYg2zdXcLszM18kJHLwvX5AAzu2oYzUrpyRkoc3dpH+jhCY/xbZaXyj29+5s+frKJtZCiPXjiEMf07+iweawIyh7VlVwmzM/P4aPlmFq7fiSoM6BzNGSlxnJ4S51ftmcY0Fytzd3HLrKWs3lrElcf1ZOrEgT6558WagMxB8gr3MjtzMx9l5rF4Yz6q0L9zFLec0o9JyXEtblIMY5paYtc2vH/TCcyY/RMvf7ee79fu4MnJQxkU18bXoQFengGIyATgKZxJ4V9Q1Rk11j8BnOS+jQQ6qWqMu64CyHTXbVTVs9zlvYBZQAdgMfALVS3lMOwM4OjlFOx1jvQz81iysQCAgV2iOT05jonJXejbySp9YxrD3FVbueM/GezaW8adEwZw9fG9mmzioHo3AYlIMJAFjAOygYXAFFVdeYjyNwHDVPVq932Rqh7UfiAibwL/VdVZIjITWKaqzx4uFksA9bNpZzGzl+fxYeZmlrkduYlxbTg9JY6JSV2seceYJrKjaB93vZ3JZz9uYXS/WB69cAid2zT+1XNHkwBGAdNU9TT3/d0AqvrQIcp/B9ynqnPc9wclAHGui9oGdFHV8pqfcSiWALy3cUcxH2bmMXt5HhnZhQAkx7dlUrJT6feMbe3jCI0JTKrKvxds5P4PVhIRGsyM85KZkBTXqJ95NH0A8cAmj/fZwMhDfEgPoBfwhcfiCBFZBJQDM1T1XZxmnwJVrZpJO9v9nNr2eT1wPUD37t29CDdw/bx9Dx+5zTtV1+kP6RbD3RMHMinZrt4xxh+ICJeO7MGxvTtw66x0bvjXEi5KTeC+MwfTOrxpu2Ub+tMmA2+paoXHsh6qmiMivYEvRCQTKPR2h6r6PPA8OGcADRptC7B2WxEfZThX7/yY51T6w7o71+lPSOpCQjur9I3xR306RvH2L4/jyc+yePbLtSz4eSdPTh7G0G4xTRaDNwkgB+jm8T7BXVabycCvPBeoao77vE5E5gHDgLeBGBEJcc8CDrdPU8PqLbv5yL16Z9WW3QCk9mjHvWckMjGpi92cZUwzERYSxJ0TBjKmf0due3MZ5z/7Hbec0o8bx/YhpAnGE/ImASwE+rlX7eTgVPKX1CwkIgOBdsD3HsvaAcWquk9EYoHjgUdUVUVkLnABzpVAVwD/O9ov01KpKllbipw2/cw8Vm8tQgRG9GjPtDMTmZAUR5e2NgyDMc3VyN4d+OiW0fzhf8t5fE4WX2Vt44mLhzZ6s623l4FOAp7EuQz0RVV9UESmA4tU9T23zDQgQlWnemx3HPAcUAkEAU+q6j/cdb1xKv/2wFLgMlXdd7g4AqkTWFX5MW+3e/VOHuu27SFIIK1XeyYlOwOudWqCqweMMU3r3aU53PvuchT441mDOe+Y+KMeT8juBG4GVJUVubv4KDOP2cs38/N2p9If1acDE5PiOG1wFzpG2yibxrR0m3YWc9ub6Sxcn8/pKXH86ZzkoxpW3e4E9lOqSmZOIR9lbmb28jw27CiuHk//+hN7Mz6xsw2tbEyA6dY+klnXj2Lml2t5Yk4WSzbk8/JVaQzo0rA3aloC8JHKSuWl79bz0rc/k52/l5Ag4fi+sdw4tg/jE7vQzk9HDzTGNI3gIOFXJ/XlhL6xPPFZFgntGv7iDksAPpBbsJffvrmM79ftYFTvDtxySj/GJXYmJtIqfWPMgYZ0i+Hlq9IaZd+WAJrYBxm5/O6/mZRXKg+fn8xFqf4xYYQxJvBYAmgiu0vKuO+9Ffx3SQ5Du8Xw5MVDbTgGY4xPWQJoAovW7+TWN9LJLdjLzaf046aT+/rdpNHGmMBjCaARlVVU8vTnq3lm7hoS2kXynxuOY3iPdr4OyxhjAEsAjebn7Xu49Y10lm0q4ILhCUw7azBRTTzQkzHGHI7VSA1MVZm1cBPT319JWEgQf7v0GCYlN+5Qr8YYUx+WABrQzj2l3PV2BnNWbuH4vh147MKhNkaPMcZvWQJoIPNWbeWOtzIoLC7j96cPatLp3owxpj4sARylkrKK6gmf+3eO4pWr0kjs6h8TPhtjzOFYAjgKK3ILuXVWOqu3FnHV8T25a8JAIkKDfR2WMcZ4xRJAPVRWKi98s45HP8mibWQor1ydxpj+HX0dlmlqFeWwdSXkLIL8DRASAaERENLKfXYfoa08nsM91nuUC67/SI/G1JclgDryHMfntMGdeei8FNrbwG2BYVcuZC+E7EWQsxhyl0JZsbNOguGAmVDrSIIPkyiOkDwOl2QiYiC2P4TZ1KDmYJYA6sDG8QkgpXsgN905us9eCNmLYXeusy44DLokwzGXQ3wqJAyHdr1AK6F8H5SXQNle57m8BMpKoHzv/ufyffvXH7JcyYHri7a6+957YLmKw86h5BJo3xs6J0Knwfuf2/eCIGuyDGSWALywu6SM+/63gv8uzWGIO45PLxvHp+WorIQdqz2O7hfBlpX7j+jb9YQex0FCKiSMcCr/kFrmaJBg50i7KY+2KyudJHCoRFK8Hbb+CFtWOM1VP33oJCpwzhI6DoDOg6FT4v7EENUJ7MAmIFgCOAIbx6cF2rN9f0WfvRBylsK+QmddeBuIPwZG3+Ye3adC61jfxns4QUEQ1Mpp+jmUxLP3vy7bC9t+chLclhWwdQWsngPpr+0vE9nBTQhViSEJOg2EMDvoaWksARyC5zg+8e1a8Z8bRjG8R3tfh2XqqnwfbM50KvzshW6H7XpnnQQ5lVzSefuP7jv0cyrVliq0FXQd5jw87dm+/yyh6nnJP/f3cSDOmVDNs4X2vSHYqpHmyqt/ORGZADyFMyn8C6o6o8b6J4CT3LeRQCdVjRGRocCzQBugAnhQVd9wt3kZGAO4h15cqarpR/NlGsrP2/dw66ylLMsu5ILhCdx3ZiLREXaVht9TdSr36qP7RbA5AypKnfXRXZ32+tSrnaP7rkPtqLZK61joPcZ5VKmshIL1ztmCZ2JY9dH+ZqTg8NqbkaK7WDNSM3DESeFFJBjIAsYB2cBCYIqqrjxE+ZuAYap6tYj0B1RVV4tIV2AxMEhVC9wE8IGqvuVtsI09KXzNcXweOi/ZxvHxZyWFztU42Yv3H90X73DWhUY6R7nxw52j+/hUaBvv23hbirIS2L7KSQjVZw0roWjz/jKt2nl0OFc1Jw2C8Iad09Z452gmhU8D1qjqOndHs4CzgVoTADAFuA9AVbOqFqpqrohsBToCBXWKvgnsKNrH1P9mVo/j8+iFQ4hr2/BzcJqjsCsXsj52m3MWOZVQldgB0H/C/sq+U6I1TTSW0AiIG+I8PBXvPLgZKf3fUFq0v0xMdycxdB0KA093+hfsTMFnvPkLiQc2ebzPBkbWVlBEegC9gC9qWZcGhAFrPRY/KCJ/AD4HpqrqQde0icj1wPUA3bt39yLcurNxfPxczmL4/m+w8l2oLHc6KRNGQPKFTpNO12OgVYyvozSR7aHXaOdRpbISCje6zUgr9jcnrf4E5j0E7fvA4HNh8DmWDHygoQ+RJgNvqR54R4yIxAGvAleoVjUecjewGScpPA/cBUyvuUNVfd5dT2pq6uHbq+rIxvHxY5UV8NMHTsW/aT6ERUPa/0HqVdChr1UUzUVQkNN53K4nDJy0f3nRNvjpfVjxLnzzOHz9qCUDH/AmAeQA3TzeJ7jLajMZ+JXnAhFpA3wI3KOq86uWq2qe+3KfiLwE3O5t0A3BxvHxUyW7YOmr8MNMKNgIMT3gtIdg2GUQYcm5xYjq6HTGp15tycCHvOkEDsHpBD4Fp+JfCFyiqitqlBsIfAz0UnenIhIGzAbeV9Una5SPU9U8cW6lfQIoUdWph4ulITqBq8bx+fMnq4iJDOPRC4fYOD7+IH89/PAcLHkVSndD91Fw7I1OO7HdrRo4PJPB+q+dq40sGRy1Q3UCHzEBuBtPAp7EuQz0RVV9UESmA4tU9T23zDQgwrMSF5HLgJcAz2Rxpaqmi8gXOB3CAqQDN6iqR2/RwY42AXiO4zM+sTMzzrdxfHxKFTbOh/nPOHeoSpDzh37sjc7NWCawWTJoMEeVAPzF0SQAz3F87jsz0cbx8aWKMljxDsz/mzOgWkSM07afdj206err6Iw/2rMdfnzf+X9jyaDOAjYBqCp3/zeTWQs32Tg+vla8Exa/BAv+DrvznM7cY38JQ6bYDVnGe5YM6ixgEwDA81+tpaiknJtO6Wfj+PjC9tXO0X76684gZb3GwKhfQd9xLXvYBdP4LBl4JaATgPEBVVg3z6n4V3/qDKGcfJFzxN8lydfRmZbIksEhWQIwTaOsBDL/A/OfdW78iYyFEdfCiGucYYaNaQpVyWDlu/DzVwGfDCwBmMZVtBUW/gMW/QP2bHNu9x91IyRd4AwdYIyvWDKwBGAayZYVzt26mW86o272O82p+HuNafF/VKYZCtBkYAnANJzKSlgzB75/Bn7+0plZauglTvt+bD9fR2eMd2pLBp2T4Yr3nHGNWpCjGQ3UGEfpHlj2Osyf6UyhGN0VTrkPhl/Z4v5gTABoHevcf5J6lZMMVrwDs++CuX+C0x/1dXRNwhKAObJdubDgeVj0EpQUOOPsn/eCc8ocbBPlmBagdSykXQfbs2DhC05S6DzY11E1OksA5tByljiXca54xzk9Hng6HPsr6H5si2wnNYaxdztXsc2+C654v8X/P7cEYByVlVC4yblpa/sqp2104/fuMMzXw8j/c4b0NaYli2wPJ90DH90OP74HiWf7OqJGZQkg0JSVwM61zqnutiznefsq2L7GuUu3SkwPOO1PMOwXNgyzCSzDr3KaOz/5PfQbD6Etd2ZASwAtVfHO/UfznpV9wYb9E3qDM0VfbH/oeaJzBU9sf2eS78gOLf7015haBYfAxIfhlTPgu7/AmDt9HVGjsQTQnFVWwq7sGkfz7mPPtv3lgsOdgdfihkDKRU4lH9vfWRYW6bv4D6GsrIzs7GxKSkp8HUqLFRERQUJCAqGh1olfq16jneafrx93LnFum+DriBqFJYDmoHwf7Fh7YAW/bRXsWANlxfvLRcQ4R+/9J+w/ko/t5zTnNKNJVbKzs4mOjqZnz542ZHcjUFV27NhBdnY2vXr18nU4/mvc/ZD1Ccy5Dy74h6+jaRSWAPzJ3nyn2Wab22xT1YSTv/7AZpu23Z2KvecJbrPNAKfCbx3bIpptSkpKrPJvRCJChw4d2LZt25ELB7J2PeC4m+GrR5zxrHqM8nVEDc4SgC9t+N4ZQqGq+WbP1v3rgsOcJpouyc54OrH9oWNVs03LHzvfKv/GZb+vl064FdJfg9l3wvXzmtWZtDcsAfjK4pfhg9ucyrzjQOg/3m2bd5tt2vVscf/ZjGl2wlrDuOnw9jWw9FXnrvcWxGbjaGqVlfDpvfD+LdDnJPjNCrh2Dpz9DBx/CwyYAB36WOXvY08//TSDBg3i/PPPZ9SoUYSHh/PoowcOD7B3717GjBlDRUWF1/udOXMm//znP486vszMTK688sqj3o/xQtL50P04+Hw67C3wdTQNyqszABGZADyFMyn8C6o6o8b6J4CT3LeRQCdVjXHXXQH83l33gKq+4i4fDrwMtAI+Am7R5jQyXX2UFsM71zs3WY24FiY87FxyZvzO3/72Nz777DPCwsLYsGED77777kFlXnzxRc477zyCg71P1jfccIPXZcvLywkJqf3/R3JyMtnZ2WzcuJHu3bt7vU9TDyIwcQY8Nwa+fBgmPOTriBrMEc8ARCQYeAaYCCQCU0Qk0bOMqv5GVYeq6lDgL8B/3W3bA/cBI4E04D4Raedu9ixwHdDPfUxoiC/kt3ZvgZdPhx8/gNMegkmPWuXvp2644QbWrVvHxIkTee211xgxYkStl0u+9tprnH22c6fovHnzGDNmDGeffTa9e/dm6tSpvPbaa6SlpZGcnMzatWsBmDZtWvWZxJo1azj11FMZMmQIxxxzDGvXrmXevHmMHj2as846i8TEREpKSrjqqqtITk5m2LBhzJ07t/rzzzzzTGbNmtUEv4ghbggMv8IZE2vbKl9H02C8qYHSgDWqug5ARGYBZwMrD1F+Ck6lD3AaMEdVd7rbzgEmiMg8oI2qzneX/xM4B5hdv6/h57ashH9fBMU7YPK/YeAkX0fUbPzx/RWszN3VoPtM7NqG+8489EBfM2fO5OOPP2bu3LnExsbWWqa0tJR169bRs2fP6mXLli3jxx9/pH379vTu3Ztrr72WBQsW8NRTT/GXv/yFJ5988oB9XHrppUydOpVzzz2XkpISKisr2bRpE0uWLGH58uX06tWLxx57DBEhMzOTn376ifHjx5OVlUVERASpqanMmDGDO+9suTcq+ZWT74Xl78DHd8Nlb7eIK+686QOIBzZ5vM92lx1ERHoAvYAvjrBtvPvam31eLyKLRGRRs7xsbc1n8I/xUFkOV822yr+F2L59OzExMQcsGzFiBHFxcYSHh9OnTx/Gjx8POM0169evP6Ds7t27ycnJ4dxzzwWcG7MiI52b8tLS0qqvz//mm2+47LLLABg4cCA9evQgKysLgE6dOpGbm9tYX9HU1DoWxk6FtZ9D1se+jqZBNHQbxGTgLVX1vlfsCFT1eeB5cCaEaaj9NomF/4CP7oBOiXDJG9C21hxnDuNwR+q+1KpVq4PuVA4PD69+HRQUVP0+KCiI8vJyr/fdurV3l/mWlJTQqlXLHafGL6Vd51zB9/Hd0OdkCAk/4ib+zJszgBygm8f7BHdZbSYDr3uxbY772pt9Nj+VFfDJPfDhbdD3VLh6tlX+LUy7du2oqKio93AV0dHRJCQkVHcu79u3j+Li4oPKjR49mtdeew2ArKwsNm7cyIABA6rfJyUl1e8LmPoJDnU6gfN/doZKb+a8SQALgX4i0ktEwnAq+fdqFhKRgUA74HuPxZ8A40Wkndv5Ox74RFXzgF0icqw4d6RcDvzvKL+LfyjdA29eDt//FdL+z2nzD4/2dVSmnjZv3kxCQgKPP/44DzzwAAkJCeza5fRJjB8/nm+++abe+3711Vd5+umnSUlJ4bjjjmPz5s0HlbnxxhuprKwkOTmZiy++mJdffrn6zGLu3Lmcfvrp9f58U099T4EBk+CrR2H3wf9mzYqqHvEBTAKygLXAPe6y6cBZHmWmATNq2fZqYI37uMpjeSqw3N3nX3HnJz7cY/jw4erXduWpzjxRdVqM6vfP+jqaZmvlypW+DsErixcv1ssuu8wnn11SUqIjR47UsrKyeu+jufzOfmnHWtXpsar/vcHXkXgFWKS11Kle9QGo6kc41+p7LvtDjffTDrHti8CLtSxfBLSc89fNy+HfFzvj+Ux+3bmhy7RoxxxzDCeddBIVFRV1uhegIWzcuJEZM2Yc8j4B08ja94ZRv4JvnoAR10DCQfOtNwt2J3BDWD0HXpzgDNh29Wyr/API1Vdf3eSVP0C/fv0YO3Zsk3+u8TD6txDVxRknqLLyyOX9kCWAo7Xg7841/u17wXWfOzeMGGNavvBoOHUa5CyGjOZ5Q54lgPqqrHAuBfvoduh3mnONf5uuvo7KGNOUUi6G+FT4bBrs2+3raOrMEkB97CuCNy5zLgMb+UuY/BqER/k6KmNMUwsKgomPQNEW+OrPvo6mziwB1NWuPHhponMn4KRHnUGibOROYwJXwnAYeil8/zdn5r5mxBJAXWzOhBdOgZ3rYMobzl2BpkVqrOGgG0NpaSknnnhine42Ng3slPucu4I/ucfXkdSJXUPmraxP4K2rIaItXP2xM1OXabEaazjoo3Goy03DwsI45ZRTeOONN7j00kubJBZTQ3RnOPEO+Ow+Z/yvvqf6OiKv2BmAN354Dl6f7EzUcu3nVvm3cI05HPT777/PyJEjGTZsGKeeeipbtmwBoKioqHrY55SUFN5++20AoqKi+O1vf8uQIUP4/vvvefzxx0lKSiIpKemA0UXPOeec6iEjjI8c+0vn/oCP74aKMl9H4xU7Azicqit9FjwHA06H8/8eEPPx+pXZU52mt4bUJdnpuzmExhwO+oQTTmD+/PmICC+88AKPPPIIjz32GPfffz9t27YlM9P5rvn5+QDs2bOHkSNH8thjj7F48WJeeuklfvjhB1SVkSNHMmbMGIYNG0ZSUhILFy5suN/I1F1IuDPXx+sXO/MGjPqVryM6IksAh7KvyJkHNOtjGPVrZ15Q6+w1rsMNBw0cNBx01UQu2dnZXHzxxeTl5VFaWlo97PNnn312wOQu7do58yYFBwdz/vnnA87Q0Oeee271aKHnnXceX3/9NcOGDSM4OJiwsDB2795NdLSNPeUz/U9zmn/mzYDkiyCqo68jOixLALUpzHGy+JaVcPpjzvSNxjcOc6TuS/UdDvqmm27itttu46yzzmLevHlMmzbtsJ8TERHhdR/Dvn37iIiIqMO3MA1OxDkLeHYUfDEdzvqLryM6LOsDqClvmXulz3q45E2r/E2t6jscdGFhIfHxztDgr7zySvXycePG8cwzz1S/r2oC8jR69GjeffddiouL2bNnD++88w6jR48GYMeOHcTGxtbaV2GaWMf+MPIGWPIq5Kb7OprDsgTgadVseHEiSDBc8wn0ax49+abxNPRw0NOmTePCCy9k+PDhB/Qv/P73vyc/P5+kpCSGDBlywNy/VY455hiuvPJK0tLSGDlyJNdeey3Dhg0DbGhov3PiHRDZAWbfBerH81jVNkSovz4abTjoykrV7//mDOP83BhnWGfjM81lmGJfDgdd07nnnqurVq2q0zbN5Xdutha9rHpfG9WM//g6kkMOB21nABXlzrSNH091Jnm48iOI7uLrqEwz4DkctC+VlpZyzjnn0L9/f5/GYWoYdpkzOOSn9zoTRfmhwE4A+3Y71/cv/DscdxNc9CqERfo6KtOM+Go4aE9hYWFcfvnlPo3B1CIo2BknaHeuM2+AHwrcBFCY7Yzhv/YLOONJGP+AM7CTMcY0lO7HQvKF8O3TkL/e19EcJDBrvNyl8PdToGAjXPofSL3K1xEZY1qqU//onA18eq+vIzlI4CWAnz6ElyZBcBhc/YkzwbMxxjSWtvFwwm3w43vw81e+juYAXiUAEZkgIqtEZI2ITD1EmYtEZKWIrBCRf7vLThKRdI9HiYic4657WUR+9lg3tKG+VK1U4ftnYNal0HEgXPsZdE5s1I80xhgAjvs1xHR3Lgut8J9RW4+YAEQkGHgGmAgkAlNEJLFGmX7A3cDxqjoYuBVAVeeq6lBVHQqcDBQDn3psekfVelVNP/qvcwgV5fDhbfDJ72DQmXDlh87ofcYcgi+Ggy4oKOBvf/tbg+wL4K9//Ssvvvhig+3PHIXQVjD+Qdi6Eha/5OtoqnkzFEQasEZV1wGIyCzgbGClR5nrgGdUNR9AVbfWsp8LgNmqWnx0IdeRKrx5Oaz6EI6/1Rm32zp7zRH4YjjoqgRw4403Nsj+rr76ao4//niuvvrqBtmfOUqDzoReJ8IXD0DS+RDZ3tcRedUEFA9s8nif7S7z1B/oLyLfish8EZlQy34mA6/XWPagiGSIyBMiEl7LNojI9SKySEQWbdu2zYtwD9oBJJ0HZz4N4/5olb85ovoMBw3w8MMPk5yczJAhQ5g61WkpHTt2LIsWLQKcAeSqRg9dsWIFaWlpDB06lJSUFFavXs3UqVNZu3YtQ4cO5Y477kBVueOOO0hKSiI5OZk33ngD8H7o6cjISHr27MmCBQsa8+cy3hKBCQ87l5/PfdDX0QANNxhcCNAPGAskAF+JSLKqFgCISByQDHzisc3dwGYgDHgeuAuYXnPHqvq8u57U1NT63VOdfEG9NjO+9/CCh/lp508Nus+B7QdyV9pdh1xfn+GgZ8+ezf/+9z9++OEHIiMj2blz52FjmDlzJrfccguXXnoppaWlVFRUMGPGDJYvX056ejoAb7/9Nunp6Sxbtozt27czYsQITjzxRMC7oacBUlNT+frrr0lLS6vbj2QaR+dEGHENLHwBhl8FXZJ8Go43h8M5QDeP9wnuMk/ZwHuqWqaqPwNZOAmhykXAO6paPUuCqlaNt7APeAmnqcmYZqHmcNCfffYZV111FZGRzo2E7dsf/vR+1KhR/OlPf+Lhhx9mw4YNtGrV6qAy33zzDVOmTCE4OJjOnTszZsyY6jH/q4aeDg8PP2jo6fXr11fvo1OnTuTm5h7ltzUNauzdzsyCH0/1+ThB3pwBLAT6iUgvnIp/MnBJjTLvAlOAl0QkFqdJaJ3H+ik4R/zVRCROVfNERIBzgOX1+QKmZTvckbov1TYcdG1CQkKorKwEOKD8JZdcwsiRI/nwww+ZNGkSzz33HL179/b6870ZerrqM2tLLsaHItvDSffAR7c7l4Ymnn3kbRrJEc8AVLUc+DVO882PwJuqukJEpovIWW6xT4AdIrISmItzdc8OABHpiXMG8WWNXb8mIplAJhALPNAA38eYJlFzOOhx48bx0ksvUVzsXONQ1QTUs2dPFi9eDMBbb71Vvf26devo3bs3N998M2effTYZGRlER0eze/fu6jKjR4/mjTfeoKKigm3btvHVV1/VuSknKyuLpCTfNjOYWgy/CjoNhk9+D2V7fRaGVz2iqvqRqvZX1T6q+qC77A+q+p77WlX1NlVNVNVkVZ3lse16VY1X1coa+zzZLZukqpepalFDfjFjGoK3w0FPmDCBs846i9TUVIYOHVp9yejtt9/Os88+y7Bhw9i+fXv1ft98802SkpIYOnQoy5cv5/LLL6dDhw4cf/zxJCUlcccdd3DuueeSkpLCkCFDOPnkk3nkkUfo0qVuAxV+++23jBs3roF+DdNggkNg4sNQuBG+892kMaL+PFZ1DampqVp1RYVpuX788UcGDRrk6zCOaMmSJTzxxBO8+uqrvg6lVkuXLuXxxx8/ZHzN5Xdu0d68HLI+hZsWQduERvsYEVmsqqk1l9s1kcbUk78MB30o27dv5/777/d1GOZwxt0PKMz5g08+3hKAMUfBH4aDPpRx48ZVX6Zq/FS7HnDczbD8bdjwXZN/vCUA45eaU9Nkc2S/rx854VZoE++ME1TZtGeTlgCM34mIiGDHjh1WSTUSVWXHjh1ERET4OhQDENYaxk2HzRmwtGn7kxrqTmBjGkxCQgLZ2dnUa+gP45WIiAgSEhqv09HUUdL5sPAf8Pl0SDwHWsU0ycdaAjB+JzQ0lF69evk6DGOajghMnAHPjYEvH4YJDzXJx1oTkDHG+IO4ITD8CljwPGxb1SQfaQnAGGP8xcn3QmjrJhsnyBKAMcb4i9axMHYqrP0Csj5u9I+zBGCMMf4k7TqIHQAf3w3l+xr1oywBGGOMPwkOdTqB83+G+Q03RWhtLAEYY4y/6XsKDJgEXz0Kuzc32sdYAjDGGH902oNQUQqfTWu0j7AEYIwx/qh9bxj1K1j2OmQ3zijIlgCMMcZfjf4tRHWB2XdCZeWRy9eRJQBjjPFX4dFw6jTITYfcpQ2+exsKwhhj/FnKxdAtDTr0afBd2xmAMcb4s6CgRqn8wcsEICITRGSViKwRkamHKHORiKwUkRUi8m+P5RUiku4+3vNY3ktEfnD3+YaIhB391zHGGOOtIyYAEQkGngEmAonAFBFJrFGmH3A3cLyqDgZu9Vi9V1WHuo+zPJY/DDyhqn2BfOCao/omxhhj6sSbM4A0YI2qrlPVUmAWcHaNMtcBz6hqPoCqbj3cDkVEgJOBt9xFrwDn1CFuY4wxR8mbBBAPbPJ4n+0u89Qf6C8i34rIfBGZ4LEuQkQWucvPcZd1AApUtfww+wRARK53t19kE4QYY0zDaairgEKAfsBYIAH4SkSSVbUA6KGqOSLSG/hCRDKBQm93rKrPA88DpKam2hyBxhjTQLw5A8gBunm8T3CXecoG3lPVMlX9GcjCSQioao77vA6YBwwDdgAxIhJymH0aY4xpRN4kgIVAP/eqnTBgMvBejTLv4hz9IyKxOE1C60SknYiEeyw/Hlipzmzfc4EL3O2vAP53dF/FGOPP9pbvZV3BOr7O/ppP13/KtmJr0vW1IzYBqWq5iPwa+AQIBl5U1RUiMh1YpKrvuevGi8hKoAK4Q1V3iMhxwHMiUomTbGao6kp313cBs0TkAWAp8I8G/3bGmCZTVFpE7p5ccos8Hu77vD157CzZedA2vdv2Jq1LGiPjRjKiywjahrf1QeSBS7QJph1rKKmpqbpoUeMMimSMOTRVZVfproMqds/Xu0p3HbBNeHA4ca3j6BrV1Xm07lr9OkRCWLRlET9s/oElW5awt3wvgjCw/UBGxo1kZNxIjul0DJGhkT76xi2LiCxW1dSDllsCMMaoKjtLdpK3J4+cohzyitznPfuf95TtOWCbViGtiI+KP7CS96joO0R0wLni+/DKKsrI3J7JD5t/YEHeApZtW0ZZZRkhEkJyx+TqM4QhHYcQFmz3i9aHJQBjAlilVrJj744DKvUDmmiK8iipKDlgm+jQaLpGdSUuKq66oo+Pinfet46nbXhbryr4utpbvpelW5eyIG8BCzYvYMWOFVRqJeHB4QzrNMw5Q+gykkEdBhESZMOZecMSgDEBYsfeHaRvSyd9azqrdq6qruTLKssOKBcTHnNgpV6jkm8T1sZH3+BAu0p3sXjzYhZsXsD8vPmsKVgDQFRoFKmdUxkZN5K0uDT6xvQlSGx4s9pYAjCmBarUSn4u/JmlW5eSvjWd9G3pbNi1AYDQoFD6t+tPQnTCAe3vVa+ba/v69r3bWbR5EfPz5rNg8wI27XbuU20f0Z60LmmkxaUxsstIukV3a5QzlObIEoAxLcDe8r0s376cZduWVVf6VZ2vMeExDO00lGGdhjG041AGxw4mPDjcxxE3vtyiXBZsXsAPeU4fwta9zkg0ca3jqvsP0rqk0bl1Zx9H6juWAIxphrbv3c7SrUurK/sfd/xIuTuCSq+2vaor+6GdhtKzTc+AP+JVVdbvWu8kg81OH0LhPmfggZ5telYngxFdRtAuop2Po206lgCM8XOVWsmagjWkb02vrvRzipwb5MOCwkiKTao+wh/ScUhAVWD1VamVrNq5qvoMYfGWxRSXFwMwsP3A6jOE4Z2H0zq0tY+jbTyWAIzxM8VlxWRuz6w+us/YlsHust2A0549rNMw5wi/01AS2ycSGhzq44ibv7LKMlZsX1F9hrB061LKKssIlmCSYpMYHT+aK5OubHFNZ5YAjPGxzXs2V3fULt26lFU7V1GhFQD0jelbfXQ/rOMwEqITAr45pymUlJeQvi2dBXkL+GHzD2RsyyAlNoUnTnqCTpGdfB1eg7EEYEwTqqisICs/q7qyT9+aTt6ePAAigiNI7pjM0I5OhZ/SMcWGQPATn234jN998zuiQqN48qQnSemY4uuQGoQlAGMaUVFpERnbM6rb7zO2ZVS3NXdq1Wn/1TmdhjKg/QBCg6w5x19l5Wdx8xc3s614G/cddx9n9TnryBv5uUMlALuNzhhgX8U+dpfupqi0iKKyInaX7nbeu6+LyoooKq3xumx39TaFpYVUaiWC0L9df87sc2Z1pd+1dVdrzmlG+rfrz+unv87tX97OPd/cw6qdq/jN8N+0yLuOW943MgGntKL0kBVzrZW4x/qqdTXvkq1NVGgUUWFRRIVGER0WTYeIDvRo04Po0GhiI2MZEjuElI4pRIVFNcG3No2pXUQ7Zo6byZ8X/pl/rvwnawrW8MiJj7S4pjprAjJ+S1XZUryFtQVrnUfhWnKLcg+s1EuLKK0sPeK+Woe2rq64q5/DoogOdZ/d5Z7LokKjaBPWhqiwKFqHtrZhBgLU21lv88APDxAfFc/TJz1N75jevg6pzqwPwPgtVWVr8VbWFqxlTcEa1hY6Ff66gnXVl0WCc2lkQnQCbcLa7K+ka1TY0WHRB1XyrUNaExwU7MNvaJq7pVuXcuvcW9lXsY+HRz/MmG5jfB1SnVgCMD7nWdFXVfJrCtbUWtH3ielDn7Z9nOeYPvSN6Ws3Phmf2rxnMzd/cTM/7fyJm4+5mWuSrmk2fTvWCWyajKqybe8252i+qvnGfXhW9O3C29Enpg+Tek+ib0zf6sq+fUR7H0ZvTO26tO7CKxNf4b7v7uOpJU+RtTOLPx7/R1qFtPJ1aPVmCcDUm6qyfe/26oq+usIvXMvu0tor+qqjeavoTXPUKqQVD49+mAHtBvDUkqdYv2s9T530FHFRcb4OrV6sCcgckWdFv65w3QFH9p7TAMaEx1RX8L3b9q6u6Du06uDD6I1pHF9lf8VdX91FWHAYT4x9gmM6H+PrkA7pqPoARGQC8BTOpPAvqOqMWspcBEwDFFimqpeIyFDgWaANzmTxD6rqG275l4ExQKG7iytVNf1wcVgCaHyqyqr8VSzesri6fX5NwZoDKvq24W3p03b/kXzfmL70junt9RSAxrQU6wrWcfPcm8kpyuGekfdwQf8LfB1SreqdAEQkGMgCxgHZwEJgiqqu9CjTD3gTOFlV80Wkk6puFZH+gKrqahHpCiwGBqlqgZsAPlDVt7z9EpYAGoeqsnLHSj7d8ClzNsypnmDDs6LvHeNxRG8VvTHVCvcVctdXd/Ft7rdMHjCZO9Pu9Ls7vY+mEzgNWKOq69wdzQLOBlZ6lLkOeEZV8wFUdav7nFVVQFVzRWQr0BEoqOf3MA2kUivJ3J7JnPVzmLNhDrl7cgmREEbGjeSapGsYnTCajq06WkVvzBG0DW/LM6c8w5NLnuTlFS+ztnAtj415rFlcteZNAogHNnm8zwZG1ijTH0BEvsVpJpqmqh97FhCRNCAMWOux+EER+QPwOTBVVffV/HARuR64HqB79+5ehGsOpaKygvRt6Xy24TPmbJjDluIthAaFclzX47hx6I2M7Ta2xd3paExTCA4K5repv6V/u/5M+24aUz6cwlMnPcWA9gN8HdphNdRVQCFAP2AskAB8JSLJqloAICJxwKvAFapa6W5zN7AZJyk8D9wFTK+5Y1V93l1Pampq8+mx9hPlleUs2bKETzd8yucbP2f73u2EBYVxQvwJ3Dr8VsYkjCE6LNrXYRrTIpzZ50x6tunJrXNv5Rezf8GfTvgTp/Y41ddhHZI3CSAH6ObxPsFd5ikb+EFVy4CfRSQLJyEsFJE2wIfAPao6v2oDVc1zX+4TkZeA2+v5HUwNZZVlLNy8kDkb5vDFxi/YWbKTiOAIRieMZnyP8YxOGN2iZz8yxpeSOyYz64xZ3DrvVn4z7zf8csgvuWHIDX45lIg3CWAh0E9EeuFU/JOBS2qUeReYArwkIrE4TULrRCQMeAf4Z83OXhGJU9U8cRqZzwGWH80XCXRlFWXMz5vvVPqbvqBwXyGRIZGMSRjDuJ7jOL7r8USGRvo6TGMCQsfIjrx42ovc//39PLvsWbLys/jTCX/yu7/BIyYAVS0XkV8Dn+C077+oqitEZDqwSFXfc9eNF5GVOJd73qGqO0TkMuBEoIOIXOnusupyz9dEpCMgQDpwQ8N+tZZvX8U+vsv5js82fsbcjXPZXbabqNAoTup2Eqf2OJXjuh5HREiEr8M0JiCFB4dz//H3M6D9AB5d9CiXfnQpT5/8NN2iux154yZiN4I1M3vL9/Jtzrd8uuFTvtz0JcXlxbQJa8PJ3U9mXI9xHBt3LGHBYb4O0xjj4fvc77n9y9sRER4b8xgj42peR9O4bDC4Zqy4rJivcr5izvo5fJ3zNXvL99IuvB0ndz+Z8T3GMyJuhN9dd2yMOdDGXRu5+YubWb9rPXeOuJMpA6c02WXWNhhcM1NUWsSX2V8yZ8Mcvsn5hn0V++gQ0YGz+pzFuB7jGN55eIucociYlqp7m+78a9K/uPvru3lowUOsyl/FPSPv8ekZu9UgfqRwXyHzNs1jzoY5fJf7HWWVZXSK7MQF/S9gXI9xDO041Ma1N6YZiwqL4qmTn+KvS//K3zP/zrqCdTxx0hPEtor1STzWBORj+SX5zN00l083fMoPuT9QruXEtY5jXI9xjOsxjpSOKX55+Zgx5uh8vP5j7v3mXtqGt+Wpk59icIfBjfZZ1gfgZ1btXMVzGc/xxcYvqNAKEqISGNdzHON7jGdwh8E2BIMxAeCnnT9x8xc3s7NkJ9OPm86k3pMa5XOsD8BPrNq5ipnLZvLZxs+IDo3m8sGXM7HnRAa2H2iVvjEBZmD7gbx++uvcNu827vr6LrLys7hp2E1N1tRrCaCJVB3xz9kwh+jQaH455JdclngZbcLa+Do0Y4wPdWjVgRfGv8BDCx7iH8v/weqC1cwYPaNJhmixBNDIPCv+qNAoq/iNMQcJDQ7lD6P+wMD2A3noh4ecm8ZOepqebXs26udaH0AjqVnx/yLxF1w66FIbbdMYc1gLNy/kt/N+S3llOY+MeYQT4k846n1aJ3ATqVnxX5Z4GZcNuswqfmOM13KKcrjli1tYXbCa3xzzG64YfMVR9RFaJ3Ajy8rPYuaymdUV/w1DbrCK3xhTL/FR8fxz4j+599t7eWzxY6zKX8V9o+5r8LG9LAEcJav4jTGNITI0kkfHPMrzGc/zQuYLXDH4Cga2H9ign2FNQPVUs+K/dNCl/CLxF1bxG2Ma3NbirXSK7FTv7a0JqIGszl/NzGUz+XTDp7QObc3/pfyfVfzGmEZ1NJX/4VgC8JJV/MaYlsYSwBHUrPivT7meyxMvt4rfGNPsWQI4hNX5q3ku4zk+Xf8pkaGRVvEbY1ocSwA1rMlfw8yMmdUV/3Up11nFb4xpkSwBuDwr/lYhrbg2+VouT7ycmIgYX4dmjDGNwqsEICITgKdwJoV/QVVn1FLmImAaoMAyVb3EXX4F8Hu32AOq+oq7fDjwMtAK+Ai4RX1wTeqa/DU8l/Ecn6z/xCp+Y0xAOWICEJFg4BlgHJANLBSR91R1pUeZfsDdwPGqmi8indzl7YH7gFScxLDY3TYfeBa4DvgBJwFMAGY35Jc7HKv4jTGBzpszgDRgjaquAxCRWcDZwEqPMtcBz7gVO6q61V1+GjBHVXe6284BJojIPKCNqs53l/8TOIcmSABrC9Yyc9lMq/iNMQHPmwQQD2zyeJ8NjKxRpj+AiHyL00w0TVU/PsS28e4ju5blBxGR64HrAbp37+5FuLVbW7CW55Y9x8frP7aK3xhjaLhO4BCgHzAWSAC+EpHkhtixqj4PPA/OUBD12cf076fzVtZbVvEbY4wHbxJADtDN432Cu8xTNvCDqpYBP4tIFk5CyMFJCp7bznOXJxxhnw0mPiqea5Kv4fLEy2kX0a6xPsYYY5oVbxLAQqCfiPTCqaQnA5fUKPMuMAV4SURicZqE1gFrgT+JSFWtOx64W1V3isguETkWpxP4cuAvR/tlDuWa5Gsaa9fGGNNsHTEBqGq5iPwa+ASnff9FVV0hItOBRar6nrtuvIisBCqAO1R1B4CI3I+TRACmV3UIAzey/zLQ2TThFUDGGGNsOGhjjGnxDjUcdJAvgjHGGON7lgCMMSZAWQIwxpgAZQnAGGMClCUAY4wJUJYAjDEmQDWry0BFZBuwoZ6bxwLbGzCchmJx1Y3FVTcWV9201Lh6qGrHmgubVQI4GiKyqLbrYH3N4qobi6tuLK66CbS4rAnIGGMClCUAY4wJUIGUAJ73dQCHYHHVjcVVNxZX3QRUXAHTB2CMMeZAgXQGYIwxxoMlAGOMCVAtLgGIyAQRWSUia0Rkai3rTxSRJSJSLiIX+FFct4nIShHJEJHPRaSHn8R1g4hkiki6iHwjIon+EJdHufNFREWkSS7d8+L3ulJEtrm/V7qIXOsPcbllLnL/j60QkX/7Q1wi8oTHb5UlIgV+Eld3EZkrIkvdv8lJfhJXD7d+yBCReSKSUNt+vKaqLeaBM2HNWqA3EAYsAxJrlOkJpAD/BC7wo7hOAiLd178E3vCTuNp4vD4L+Ngf4nLLRQNfAfOBVH+IC7gS+GtT/L+qY1z9gKVAO/d9J3+Iq0b5m3AmnPJ5XDidrr90XycC6/0krv8AV7ivTwZePZrPbGlnAGnAGlVdp6qlwCzgbM8CqrpeVTOASj+La66qFrtv53PgnMm+jGuXx9vWQFNcNXDEuFz3Aw8DJU0QU13iamrexHUd8Iyq5gOo6lY/icvTFOB1P4lLgTbu67ZArp/ElQh84b6eW8v6OmlpCSAe2OTxPttd5mt1jesammaKTK/iEpFficha4BHgZn+IS0SOAbqp6odNEI/XcbnOd0/R3xKRbn4SV3+gv4h8KyLzRWSCn8QFOE0bQC/2V26+jmsacJmIZAMf4Zyd+ENcy4Dz3NfnAtEi0qG+H9jSEkCzJyKXAanAn30dSxVVfUZV+wB3Ab/3dTwiEgQ8DvzW17HU4n2gp6qmAHOAV3wcT5UQnGagsThH2n8XkRhfBlTDZOAtVa3wdSCuKcDLqpoATAJedf/f+drtwBgRWQqMAXJw5mGvF3/4Qg0pB/A84kpwl/maV3GJyKnAPcBZqrrPX+LyMAs4pzEDch0prmggCZgnIuuBY4H3mqAj+Ii/l6ru8Pi3ewEY3sgxeRUXztHke6papqo/A1k4CcHXcVWZTNM0/4B3cV0DvAmgqt8DETgDsvk0LlXNVdXzVHUYTl2BqhbU+xMbu2OjKR84RznrcE4lqzpRBh+i7Ms0XSfwEeMChuF0APXzp9/LMx7gTGCRP8RVo/w8mqYT2JvfK87j9bnAfD+JawLwivs6FqepoYOv43LLDQTW496Y6ie/12zgSvf1IJw+gEaNz8u4YoEg9/WDwPSj+sym+MGb8oFzupblVqb3uMum4xxVA4zAORraA+wAVvhJXJ8BW4B09/Gen8T1FLDCjWnu4SripoyrRtkmSQBe/l4Pub/XMvf3GugncQlOs9lKIBOY7A9xue+nATOaIp46/F6JwLfuv2M6MN5P4roAWO2WeQEIP5rPs6EgjDEmQLW0PgBjjDFesgRgjDEByhKAMcYEKEsAxhgToCwBGGNMgLIEYAKGiMSIyI3u67Ei8kEjfMbLdRllVkR6isjyQ6yb11SjnJrAZAnABJIY4Ma6bCAiwY0TijG+ZwnABJIZQB8RSccZaynKHbDtJxF5TUQEQETWi8jDIrIEuFBExovI9+48Ev8RkSi33AyPORwe9ficE0XkOxFZV3U2II4/i8hyd36Fi2sGJyKtRGSWiPwoIu8ArRr59zABLsTXARjThKYCSao6VETGAv8DBuPc5v8tcDzwjVt2h6oeIyKxwH+BU1V1j4jcBdwmIs/gDPUwUFW1xsBqccAJOEMcvAe8hTOC41BgCM7t/AtF5Ksa8f0SKFbVQSKSAixpyC9vTE12BmAC2QJVzVbVSpzb/Xt6rHvDfT4Wd1gA98zhCqAHUIgzD8E/ROQ8oNhj23dVtVJVVwKd3WUnAK+raoWqbgG+xBmWxNOJwL8A1JmzIqMhvqQxh2JnACaQeY64WsGBfw973GcB5qjqlJobi0gacArO+Cy/xpmhqeZ+pcGiNaaB2RmACSS7cYaSrov5wPEi0hdARFqLSH+3H6Ctqn4E/AanaedwvgYuFpFgEemIc7S/oEaZr4BL3M9Jwpm61JhGY2cAJmCo6g53RqzlwF6c0VePtM02EbkSeF1Ewt3Fv8dJJv8TkQico/zbjrCrd4BROKNLKnCnqm4WkZ4eZZ4FXhKRH4EfgcVefzlj6sFGAzXGmABlTUDGGBOgLAEYY0yAsgRgjDEByhKAMcYEKEsAxhgToCwBGGNMgLIEYIwxAer/AfuDrs1xaEHbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_select = df[df.type == 'all'][['f1(micro)','f1(macro)','f1(custom)' ]].reset_index(drop=True);\n",
    "ax = df_select.plot(xticks=df_select.index, title = 'relevant + multi-label', );\n",
    "ax.set_xticklabels(np.around(Threshold,1));\n",
    "ax.set_xlabel('threshold');\n",
    "df_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T06:30:23.960606Z",
     "start_time": "2020-12-31T06:30:23.932788Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "      <th>EMR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>70.0</td>\n",
       "      <td>allallallallallallallallallallallallallallalla...</td>\n",
       "      <td>87.239758</td>\n",
       "      <td>79.059829</td>\n",
       "      <td>75.923393</td>\n",
       "      <td>77.459874</td>\n",
       "      <td>70.531064</td>\n",
       "      <td>76.648329</td>\n",
       "      <td>72.465197</td>\n",
       "      <td>62.5298</td>\n",
       "      <td>74.734</td>\n",
       "      <td>80.0101</td>\n",
       "      <td>64.5982</td>\n",
       "      <td>87.2398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>70.0</td>\n",
       "      <td>multimultimultimultimultimultimultimultimultim...</td>\n",
       "      <td>88.179987</td>\n",
       "      <td>71.967655</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>73.452545</td>\n",
       "      <td>66.411565</td>\n",
       "      <td>76.610411</td>\n",
       "      <td>70.184896</td>\n",
       "      <td>60.823</td>\n",
       "      <td>75</td>\n",
       "      <td>74.8741</td>\n",
       "      <td>62.2634</td>\n",
       "      <td>88.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>70.0</td>\n",
       "      <td>is_relevantis_relevantis_relevantis_relevantis...</td>\n",
       "      <td>91.269308</td>\n",
       "      <td>91.269308</td>\n",
       "      <td>91.269308</td>\n",
       "      <td>91.269308</td>\n",
       "      <td>89.748055</td>\n",
       "      <td>86.470018</td>\n",
       "      <td>87.932286</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>70.0</td>\n",
       "      <td>Armed AssaultArmed AssaultArmed AssaultArmed A...</td>\n",
       "      <td>94.560107</td>\n",
       "      <td>94.560107</td>\n",
       "      <td>94.560107</td>\n",
       "      <td>94.560107</td>\n",
       "      <td>86.987135</td>\n",
       "      <td>83.941779</td>\n",
       "      <td>85.375782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>70.0</td>\n",
       "      <td>Bombing/ExplosionBombing/ExplosionBombing/Expl...</td>\n",
       "      <td>96.977837</td>\n",
       "      <td>96.977837</td>\n",
       "      <td>96.977837</td>\n",
       "      <td>96.977837</td>\n",
       "      <td>90.163658</td>\n",
       "      <td>88.587376</td>\n",
       "      <td>89.356193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>70.0</td>\n",
       "      <td>KidnappingKidnappingKidnappingKidnappingKidnap...</td>\n",
       "      <td>98.791135</td>\n",
       "      <td>98.791135</td>\n",
       "      <td>98.791135</td>\n",
       "      <td>98.791135</td>\n",
       "      <td>74.863107</td>\n",
       "      <td>88.413022</td>\n",
       "      <td>80.127825</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>70.0</td>\n",
       "      <td>OtherOtherOtherOtherOtherOtherOtherOtherOtherO...</td>\n",
       "      <td>96.709201</td>\n",
       "      <td>96.709201</td>\n",
       "      <td>96.709201</td>\n",
       "      <td>96.709201</td>\n",
       "      <td>77.549144</td>\n",
       "      <td>88.536802</td>\n",
       "      <td>82.002935</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold                                               type   accuracy  \\\n",
       "42       70.0  allallallallallallallallallallallallallallalla...  87.239758   \n",
       "43       70.0  multimultimultimultimultimultimultimultimultim...  88.179987   \n",
       "44       70.0  is_relevantis_relevantis_relevantis_relevantis...  91.269308   \n",
       "45       70.0  Armed AssaultArmed AssaultArmed AssaultArmed A...  94.560107   \n",
       "46       70.0  Bombing/ExplosionBombing/ExplosionBombing/Expl...  96.977837   \n",
       "47       70.0  KidnappingKidnappingKidnappingKidnappingKidnap...  98.791135   \n",
       "48       70.0  OtherOtherOtherOtherOtherOtherOtherOtherOtherO...  96.709201   \n",
       "\n",
       "    recall(micro)  precision(micro)  f1(micro)  recall(macro)  \\\n",
       "42      79.059829         75.923393  77.459874      70.531064   \n",
       "43      71.967655         75.000000  73.452545      66.411565   \n",
       "44      91.269308         91.269308  91.269308      89.748055   \n",
       "45      94.560107         94.560107  94.560107      86.987135   \n",
       "46      96.977837         96.977837  96.977837      90.163658   \n",
       "47      98.791135         98.791135  98.791135      74.863107   \n",
       "48      96.709201         96.709201  96.709201      77.549144   \n",
       "\n",
       "    precision(macro)  f1(macro) accuracy(custom) precision(custom)  \\\n",
       "42         76.648329  72.465197          62.5298            74.734   \n",
       "43         76.610411  70.184896           60.823                75   \n",
       "44         86.470018  87.932286              NaN               NaN   \n",
       "45         83.941779  85.375782              NaN               NaN   \n",
       "46         88.587376  89.356193              NaN               NaN   \n",
       "47         88.413022  80.127825              NaN               NaN   \n",
       "48         88.536802  82.002935              NaN               NaN   \n",
       "\n",
       "   recall(custom) f1(custom)      EMR  \n",
       "42        80.0101    64.5982  87.2398  \n",
       "43        74.8741    62.2634    88.18  \n",
       "44            NaN        NaN      NaN  \n",
       "45            NaN        NaN      NaN  \n",
       "46            NaN        NaN      NaN  \n",
       "47            NaN        NaN      NaN  \n",
       "48            NaN        NaN      NaN  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.threshold == 0.7]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-29T08:03:56.591953Z",
     "start_time": "2020-12-29T08:03:56.433841Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.893217</td>\n",
       "      <td>0.858940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.897246</td>\n",
       "      <td>0.862811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.900604</td>\n",
       "      <td>0.866450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.901948</td>\n",
       "      <td>0.868014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.903962</td>\n",
       "      <td>0.869153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.908664</td>\n",
       "      <td>0.874486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.912693</td>\n",
       "      <td>0.879323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.911350</td>\n",
       "      <td>0.876497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.911350</td>\n",
       "      <td>0.873696</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   f1(micro)  f1(macro)\n",
       "0   0.893217   0.858940\n",
       "1   0.897246   0.862811\n",
       "2   0.900604   0.866450\n",
       "3   0.901948   0.868014\n",
       "4   0.903962   0.869153\n",
       "5   0.908664   0.874486\n",
       "6   0.912693   0.879323\n",
       "7   0.911350   0.876497\n",
       "8   0.911350   0.873696"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEWCAYAAABollyxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxS0lEQVR4nO3deXzU1b3/8dcn+0I2krAlYQchBBQMoFVkURS3KqCCy/Xaat2q91ett9Vbb+XaRdqrrba1etVarRuKvVK9RVErVnFNQCAsiqxJCIFAwpZ9+fz+ON+QIQRJIMlMJp/n4+EjM9/vmZnPDPGdM+d7vucrqooxxpjgFeLvAowxxnQsC3pjjAlyFvTGGBPkLOiNMSbIWdAbY0yQs6A3xpggZ0Fvui0ReV9EbvB3HcZ0NAt6Y/zI/tiYzmBBb4KWiIT5uwZjAoEFvQkqIrJVRH4sIquBchE5U0Q+FpG9IrJKRKZ8w2O/KyLrRaRMRJaIyABv+2Mi8mCztn8TkTu923eLyCYROSAi60Rkpk+760RkmYg86D3vFhE539v3C2AS8AcROSgif2j3D8QYLOhNcLoSuBAYDPwN+DnQE7gL+KuIpDZ/gIhcAvwHMAtIBT4EXvJ2vwTMERHx2iYB5wILvP2bcIGdAPwX8LyI9PV5+onAV0AK8GvgTyIiqvoT73VuU9Ueqnpb+7x9Yw5nQW+C0e9UtQC4BlisqotVtUFV3wFygQtaeMzNwAOqul5V64BfAqd4vfoPAcWFOcBlwCeqWgSgqgtVtch7jZeBr4EJPs+9TVWfVNV64FmgL9C73d+1MUdhQW+CUYH3cwBwuTdss1dE9gJn4oK2uQHAIz7tSgEB0tSt/LcA900B4CrghcYHisi1IrLS57FZuN57o+LGG6pa4d3scWJv0ZjWs4NVJhg1LslaADynqt9rxWMKgF+o6gtH2f8S8LaIzMcNxcwE8Hr8TwJn43r59SKyEvdHoi21GtNhrEdvgtnzwMUicp6IhIpIlIhMEZH0Fto+DtwjIqMARCRBRC5v3KmqXwC7gaeAJaq619sViwvrEu9x38H16FtrJ+5YgjEdxoLeBC1vnL7xIGsJrtf+77Twe6+qrwG/AhaIyH5gDXB+s2YvAud4Pxsftw54CPgEF9qjgY/aUOYjwGXejJzfteFxxrSa2IVHjDEmuFmP3hhjgpwFvTHGBDkLemOMCXIW9MYYE+QCbh59SkqKDhw40N9lGGNMl7J8+fLdqnrE8h4QgEE/cOBAcnNz/V2GMcZ0KSKy7Wj7bOjGGGOCnAW9McYEOQt6Y4wJcgE3Rt+S2tpaCgsLqaqq8ncpQSEqKor09HTCw8P9XYoxphN0iaAvLCwkLi6OgQMH4l37wRwnVWXPnj0UFhYyaNAgf5djjOkEXWLopqqqiuTkZAv5diAiJCcn27cjY7qRLhH0gIV8O7LP0pjupUsM3RhjupaD1XUsztvB/spaMnrGkJEUQ0bPaOKi7LiQP1jQG2PahaqyIr+MBZ8X8Pe8HVTU1B/RJjEmnP5e8Kf3jCYjKcbd7xlDWmI0EWFdZpChS7Ggb4Pf/e53PPbYY2RmZlJUVMSKFSv4xS9+wV133XWoTWVlJTNmzOC9994jNDS0Vc/7+OOPExMTw7XXXntC9eXl5fHQQw/xzDPPnNDzGNMWew5W89oX21mQU8DGXQeJiQjl4jH9mDMhg0HJsRSUVVBQWun9rCC/tIJ1O/bzzrqd1NQ3HHoeEegTH3XYN4CMpBj6J7v7veIiCQmxYcfjYUHfBn/84x959913iYiIYNu2bSxatOiINk8//TSzZs1qdcgD3Hzzza1uW1dXR1hYy/9so0ePprCwkPz8fPr379/q5zSmreoblGUbd/NyTj7vrNtJbb0ytn8iv5o9mgvH9KNHZNPvaFJsBGPSE494joYGZeeBKvL3VFBQVklBaQUFZRUUllby0cbd7DxQhe91kSLCQkhPjHZ/CLw/Ahk9Yw59Q0iIsWGho+lyQf9fb6xlXdH+dn3OzH7x3HfxqG9sc/PNN7N582bOP/98vvvd73LHHXfw97///Yh2L7zwAi++6K409/7773PfffeRmJhIXl4eV1xxBaNHj+aRRx6hsrKSRYsWMWTIEObNm0ePHj2466672LhxIzfffDMlJSWEhoaycOFCCgoK+M///E+SkpL48ssvWb16Nbfccgu5ubmEhYXxm9/8hqlTpwJw8cUXs2DBAn70ox+162dkDEBhWQULcwt5dXkh2/dWkhQTzrWnD2TO+AyG945r03OFhAh9E6LpmxDNxBb2V9fVs72skoKySvJLKyj0/hAUlFayqnAveytqD2sfFxV26JtA43BQ4/30pBiiwlvf+Qo2XS7o/eXxxx/nrbfeYunSpaSkpLTYpqamhs2bN+O7+uaqVatYv349PXv2ZPDgwdxwww18/vnnPPLII/z+97/n4YcfPuw5rr76au6++25mzpxJVVUVDQ0NFBQUsGLFCtasWcOgQYN46KGHEBHy8vL48ssvOffcc9mwYQNRUVFkZ2czf/58C3rTbmrqGnh3/U4W5BTw4dclAJw5NIV7LhjB9MzeRIZ1TIBGhoUyOLUHg1N7tLh/f1Wt+xZQWklhmRsSKiitYFNJOe9/VUJ1XcNh7XvFRXrhH01iTESH1Hyi0pOiuWFS+18rvssF/bF63v60e/duEhMTD9s2fvx4+vbtC8CQIUM499xzATfMsnTp0sPaHjhwgO3btzNz5kzAncHaaMKECYdOcFq2bBm33347ACNGjGDAgAFs2LCBMWPG0KtXL4qKijrk/Znu5eudB3g5p4D//WI7peU19EuI4vZpw7j81HQyesb4uzzio8IZ1S+BUf0SjtinqpQcqG46PuB9G8gvrSBnaxkHqmpbeEb/G5OeaEEf6KKjo484ESkyMvLQ7ZCQkEP3Q0JCqKura/Vzx8bGtqpdVVUV0dHRrX5eY3yVV9fx99U7WJCTz4r8vYSFCNMzezNnfAaThqUS2kUOhooIveKj6BUfxakD/F2N/7Uq6EVkBvAIEAo8parzm+0fADwNpAKlwDWqWujtews4DVimqhe1Y+0BJykpifr6eqqqqg7rjbdWXFwc6enpLFq0iEsvvZTq6mrq64+cojZp0iReeOEFpk2bxoYNG8jPz+ekk04CYMOGDWRlZZ3wezHdh6qysmAvL+cU8MaqIspr6hmSGstPLhjJzHFppPSIPPaTmIB2zKAXkVDgUWA6UAjkiMjrqrrOp9mDwF9U9VkRmQY8APyLt++/gRjgpnat3I+Ki4vJzs5m//79hISE8PDDD7Nu3Tri4+M599xzWbZsGeecc85xPfdzzz3HTTfdxE9/+lPCw8NZuHDhEW1uvfVWbrnlFkaPHk1YWBjPPPPMoW8KS5cu5cILLzyh92e6h7LyGl77Yjsv5xTw1c4DRIeHctGYvswZn8GpA5LsDOogIuo7f6mlBiKnA/NU9Tzv/j0AqvqAT5u1wAxVLRD327FPVeN99k8B7mpNjz47O1ubX2Fq/fr1jBw5srXvya9WrFjBb3/7W5577rlOf+3q6momT57MsmXLjjoFs1FX+kxN+2loUD7etIcFOfm8vdbNYz85I5E52RlcfHJfO3O1CxOR5aqa3dK+1gzdpAEFPvcL4YjZUKuAWbjhnZlAnIgkq+qeVhZ4I3Aj0OXnf48bN46pU6dSX1/fprn07SE/P5/58+cfM+RN97NjXyULcwt5JbeAwrJKEqLDuWpif+aMz2Bk3/hjP4Hp0torEe4C/iAi1wEfANuBIweXj0JVnwCeANejb6ea/Oa73/2uX1532LBhDBs2zC+vbQJPbX0D/1i/i5dz8vnnhhIaFM4YmsyPZozg3Mze3XpeeXfTmqDfDmT43E/3th2iqkW4Hj0i0gOYrap726lGY0wbbCo5yCs5Bfx1RSG7D9bQOz6SW6cM5YrsDPon+39apOl8rQn6HGCYiAzCBfxc4CrfBiKSApSqagNwD24GjjGmk1TU1LE4r5iXc/LJ2VpGWIgwbUQv5k7I4KxhqYSF2mJh3dkxg15V60TkNmAJbnrl06q6VkTuB3JV9XVgCvCAiChu6Ob7jY8XkQ+BEUAPESkErlfVJe3/VozpfrbuLufpj7bw2ortHKiuY1BKLHefP4JZ49LoFdf2Kb4mOLVqjF5VFwOLm237qc/tV4FXj/LYSSdSoDHmSMu3lfHkB5tZsq6Y8JCQQ9MiJwzqadMizRHs+1wb/O53v2PkyJHMnj2b008/ncjISB588MHD2lRWVjJ58uQWT3TqTDU1NZx11lltOvvWBLb6BmXJ2mIue+xjZj/2MZ9s3sOtU4aw7O6p/GbOKUwcbJfbNC2zeXht0FHLFJ+Io03jjIiI4Oyzz+bll1/m6quv7pRaTMeoqq3nrysKeerDLWzZXU56UjTzLs7k8uwMYiPtf2FzbF3vt+TNu6E4r32fs89oOH/+NzbpyGWK33jjDX7+859TU1NDcnIyL7zwAr179+bgwYPcfvvt5ObmIiLcd999zJ49mx49enDTTTfx7rvv8uijj/L555/z9NPu+PcNN9zAD37wAwAuvfRS7rnnHgv6LmrPwWqe+3Qbf/lkG6XlNYxJT+APV41lxqg+dnDVtEnXC3o/6chlis8880w+/fRTRISnnnqKX//61zz00EP87Gc/IyEhgbw894etrKwMgPLyciZOnMhDDz3E8uXL+fOf/8xnn32GqjJx4kQmT57M2LFjycrKIicnp8M/G9O+tuwu50/LNrMwt5DqugbOHtGLG88abOPv5rh1vaA/Rs/bn453meLCwkLmzJnDjh07qKmpObQc8bvvvsuCBQsOPVdSUhIAoaGhzJ49G3BLFs+cOfPQ6pazZs3iww8/ZOzYsYSGhhIREcGBAweIi2vbRSFM51u+rYwnPtjE2+t2Eh4SwsyxaXzvrEEM7WX/dubEdL2gD2DHu0zx7bffzp133sm3v/1t3n//febNm/eNrxMVFdXqYwDV1dXHtZKm6Rz1Dco763by5IebWb6tjITocL4/ZSjXfmuATY807cYG+tqR7zLFbbFv3z7S0tIAePbZZw9tnz59Oo8++uih+41DN74mTZrEokWLqKiooLy8nNdee41Jk9yM1j179pCSkkJ4uC1UFWiqaut5/tNtnPObf3Lz88vZub+KeRdn8sk907jrvJMs5E27sh79cWjvZYrnzZvH5ZdfTlJSEtOmTWPLli0A3HvvvXz/+98nKyuL0NBQ7rvvPmbNmnXYY8eNG8d1113HhAkTAHcwduzYsYAtWRyImh9gPTk9gUevGsd5o3rbAVbTYY65THFns2WK28+sWbOYP38+w4cPP2JfV/pMg8GW3eU89eFmXl3uDrCeM7IX35tkB1hN+znRZYpNG/hzmWJfNTU1XHrppS2GvOk8y7eV8sQHmw8dYJ01Lo0bJtkBVtO5ukzQq2qX6fn4a5liXxEREVx77bUt7gu0b3HBpvEA6xMfbGJF/l47wGr8rksEfVRUFHv27CE52U7xPlGqyp49e2wmTgeorKnn1RWF/OnDzWzdU0FGz2j+69ujuDw7nZiILvG/mglSXeK3Lz09ncLCQkpKSvxdSlCIiooiPT3d32UEjT0Hq/nLJ9t47tPDD7DOyOpDaIh1TIz/dYmgDw8PP3QSkTGBYnPJQf60bMthB1hvPGsI4wfahbVNYOkSQW9MIFm+rZT/+edm3lm/k/DQEGaPS+P6MwcztFcPf5dmTIss6I05hvoGZcvuclYW7OXFz7axIn8viTHh3DZ1KNeePpDUuMhjP4kxfmRBb4yP2voGNu46yJrt+1hbtJ812/exbsd+Kmrc9QXsAKvpiuw31XRbVbX1fFV8gDVF+1izfT9ri/bxZfEBauoaAIiJCCWzbzxXZGcwql88o/olcFKfODvAarocC3rTLZRX17Fux37Wbt/HGq+n/vWug9Q3uHMK4qPCyEpL4LpvDWRUv3iy0hIYmBxroW6CggW9CTr7KmpZW7TvUE99TdE+tuwup/E8sZQeEWSlJXDOyN6HQj09KdpmypigZUFvurSSA9WsKdrneupeqBeWVR7an5YYzah+8VxychpZaS7Ue8VFWqibbsWC3nQJqkrRvip3kLTxQGnRPnburz7UZmByDCdnJHL1xAFkpbkx9Z6xEX6s2pjAYEFvAk5Dg5JfWnHYQdI12/dRVlELQIjA0F49OGNICqPSEsjqF8/IfvHER9m6+8a0xILe+MWBqloKSispKKugoLSCwrJKCkoryPduV9a66YzhocJJfeI4b1QfN/MlLYGRfeKJjvDfyqDGdDUW9KZD1NQ1sH2vC28X5r63Kw71zhv1iAwjPSmaQSmxnDU8leG9ezCqXwLDe8cREWYX5DDmRFjQm+PS0KDsOlB9KLjzSysO9dALSyvYsb8K39WQw0OF9KQY0pOiyRrdl4ykGPr3jCGjZzQZSTEkxoTbAVJjOogFvTmqfRW1FJQ1hvjhPfPCvZWHTixq1Cc+ioye0Zw2OJn0nl6QJ0WT0TOG3vFRNifdGD+xoO/GqmrrKSyrOGys3LdnfqCq7rD2CdHhZPSM5qQ+cUzP7E26T5CnJUYTFW7j5sYEolYFvYjMAB4BQoGnVHV+s/0DgKeBVKAUuEZVC719/wrc6zX9uao+2061m+Owr7KWf6zfyZtrivlgQwnVPr3yyLAQ0pOi6d8zhuyBSWQkuaGV9KQYMnrGkBBts1qM6YqOGfQiEgo8CkwHCoEcEXldVdf5NHsQ+IuqPisi04AHgH8RkZ7AfUA2oMBy77Fl7f1GzNGVltfwzrpiFucV8/Gm3dTWK33io5g7PoNT+id6QywxpPSIJMSGV4wJOq3p0U8ANqrqZgARWQBcAvgGfSZwp3d7KbDIu30e8I6qlnqPfQeYAbx0wpWbb7RrfxVL1hbz5ppiPttSSn2Dkp4UzXfOGMSMrD6ckp5ooW5MN9GaoE8DCnzuFwITm7VZBczCDe/MBOJEJPkoj01r/gIiciNwI0D//v1bW7tpprCsgrfWFPPWmmKW55ehCoNTY7l58mDOz+rLqH7xNrPFmG6ovQ7G3gX8QUSuAz4AtgP1rX2wqj4BPAGQnZ2tx2hufGzdXc6ba4p5c80OVhfuA2BEnzh+cPZwzh/dh2G9eli4G9PNtSbotwMZPvfTvW2HqGoRrkePiPQAZqvqXhHZDkxp9tj3T6Debk9V+XrXQd7Mc+H+ZfEBAE5OT+DHM0ZwflYfBqbE+rlKY0wgaU3Q5wDDRGQQLuDnAlf5NhCRFKBUVRuAe3AzcACWAL8UkSTv/rneftMGqsraov28uWYHb64pZnNJOSJwav8k7r1wJDOy+pCeFOPvMo0xAeqYQa+qdSJyGy60Q4GnVXWtiNwP5Krq67he+wMiorihm+97jy0VkZ/h/lgA3N94YNZ8s4YG5YuCvby1ZgdvrS2moLSS0BBh4qCefOdbAzlvVB96xUf5u0xjTBcgqoE1JJ6dna25ubn+LsMv6huUnK2lhw6oFu+vIjxUOGNoCudn9WF6Zh9bdtcY0yIRWa6q2S3tszNj/ay2voFPNu3hzTXFvLOumN0Ha4gMC2Hy8FR+PPokpo3obScqGWNOiAW9H1TV1rPs6928uaaYd9fvZF9lLTERoUwd0Yvzs/ow9aRexEbaP40xpn1YmnSSipo6/vlVCYvXFLP0y10crK4jLiqM6SN7MyOrD2cNT7W1YowxHcKCvgPVNygfbChh4fIC3vtyF1W1DfSMjeCiMX2ZkdWHbw1JsbXWjTEdzoK+AxSUVvBKbgGvLi9kx74qkmMjuCI7gxlZfZgwsCdhoRbuxpjOY0HfTqpq63l73U5eySlg2cbdiMDk4an89KJMzh7Z23ruxhi/saA/QV8W7+flnAJe+2I7eytqSUuM5s7pw7ns1HT6JUb7uzxjjLGgPx4Hq+t4Y1URC3IKWFWwl4jQEKaP6s3c8RmcMSTFVoU0xgQUC/pWUlVW5Jex4PMC/p63g4qaeob37sF/XpTJzLFpdiKTMSZgWdAfw56D1bz2xXYW5BSwcddBYiJC+fbJ/bhifAZjMxJtZUhjTMCzoG9BfYPy4dclvJJbwDvrdlJbr4zrn8ivZo/mwjH96GEnMxljuhBLLB+FZRUszC1kYW4BRfuqSIoJ59rTBzJnfAbDe8f5uzxjjDku3T7oq+vqeXfdLhbk5LNs424Azhyawk8uzOSczF5EhtnZqsaYrq3bBv2GnQcOTYssLa+hX0IU/zZtGJdnp9va7saYoNKtgr68uo6/r97Bgpx8VuTvJTxUmJ7ZmyuyM5g0LJVQmxZpjAlCQR/0qsrKgr28nFPAG6uKKK+pZ0hqLD+5YCQzx6WR0iPS3yUaY0yHCtqgLy2v4bUvtvNyTj4bdh4kOjyUi8b0Ze6EDMb1T7JpkcaYbiOogr6hQflo024W5BTwztqd1NQ3cHJGIg/MGs1FY/oSF2UX8DDGdD9BE/QFpRVc+eSnFJZVkhgTzlUT+zNnfAYj+8b7uzRjjPGroAn6fonRjO2fxI9mjODczN52EQ9jjPEETdCHhgi/v3Ksv8swxpiAY4ukG2NMkLOgN8aYIGdBb4wxQc6C3hhjgpwFvTHGBDkLemOMCXIW9MYYE+RaFfQiMkNEvhKRjSJydwv7+4vIUhH5QkRWi8gF3vYIEfmziOSJyCoRmdK+5RtjjDmWYwa9iIQCjwLnA5nAlSKS2azZvcArqjoWmAv80dv+PQBVHQ1MBx4SEfsWYYwxnag1oTsB2Kiqm1W1BlgAXNKsjQKNi8okAEXe7UzgPQBV3QXsBbJPsGZjjDFt0JqgTwMKfO4Xett8zQOuEZFCYDFwu7d9FfBtEQkTkUHAqUBG8xcQkRtFJFdEcktKStr4FowxxnyT9hpGuRJ4RlXTgQuA57whmqdxfxhygYeBj4H65g9W1SdUNVtVs1NTU9upJGOMMdC6Rc22c3gvPN3b5ut6YAaAqn4iIlFAijdcc0djIxH5GNhwQhUbY4xpk9b06HOAYSIySEQicAdbX2/WJh84G0BERgJRQImIxIhIrLd9OlCnquvarXpjjDHHdMwevarWichtwBIgFHhaVdeKyP1Arqq+DvwQeFJE7sAdmL1OVVVEegFLRKQB9y3gXzrsnRhjjGmRqKq/azhMdna25ubm+rsMY4zpUkRkuaq2OKvR5rQbY0yQs6A3xpggZ0FvjDFBzoLeGGOCnAW9McYEOQt6Y4wJchb0xhgT5CzojTEmyFnQG2NMkLOgN8aYIGdBb4wxQc6C3hhjgpwFvTHGBDkLemOMCXIW9MYYE+Qs6I0xJshZ0BtjTJCzoDfGmCBnQW+MMUHOgt4YY4KcBb0xxgQ5C3pjjAlyFvTGGBPkLOiNMSbIWdAbY0yQs6A3xpggZ0FvjDFBrlVBLyIzROQrEdkoIne3sL+/iCwVkS9EZLWIXOBtDxeRZ0UkT0TWi8g97f0GjDHGfLNjBr2IhAKPAucDmcCVIpLZrNm9wCuqOhaYC/zR2345EKmqo4FTgZtEZGA71W6MMaYVWtOjnwBsVNXNqloDLAAuadZGgXjvdgJQ5LM9VkTCgGigBth/wlUbY4xptdYEfRpQ4HO/0Nvmax5wjYgUAouB273trwLlwA4gH3hQVUubv4CI3CgiuSKSW1JS0rZ3YIwx5hu118HYK4FnVDUduAB4TkRCcN8G6oF+wCDghyIyuPmDVfUJVc1W1ezU1NR2KskYYwxAWCvabAcyfO6ne9t8XQ/MAFDVT0QkCkgBrgLeUtVaYJeIfARkA5tPtHBjTAAr+gI+fQzqqmHIVBgyDRL7+7uqbqs1QZ8DDBORQbiAn4sLcF/5wNnAMyIyEogCSrzt03A9/FjgNODh9indGBNQVGHLP2HZb2Hz+xCZABGxsG6R25881AX+kGkw8EyIjPNntd3KMYNeVetE5DZgCRAKPK2qa0XkfiBXVV8Hfgg8KSJ34A7AXqeqKiKPAn8WkbWAAH9W1dUd9m6MMZ2voR6+/D8X8EVfQI8+MP1+OPU7Lsx3b4BN77n/vngePn8CQsIgY6Lr7Q+eBv1OgZBQf7+ToCWq6u8aDpOdna25ubn+LsMYcyx11bBqAXz0CJRugp5D4Iz/ByfPhbDIoz+m4LOm4N+xym2PSoTBU5p6/IkZLT/eHJWILFfV7Bb3WdAbY9qkaj8s/zN88kc4WAx9T4Ez74CRF7e9V16+2w3zbFrqgv+ANzM7eVizYZ4e7f0ugo4FvTHmxB3cBZ89Dp8/BdX7XA/8zDtg0GQQOfHnV4WSr5p6+1uXQV0lhIQ3DfMMmQZ9T7ZhnhZY0Btjjl/pFvj49258vb4GMi+BM38A/cZ27OvWVUP+p03BX+wd3ovu6TPMMxUS0ju2ji7Cgt4Y03Y7VsNHD8Pa19zB05OvhG/9G6QM9U89B0u8YR4v+A8Wu+0pJzX19gec0W2HeSzojTGto+qGTD56GDa+CxFxMP67cNqtENfH39U1UYVd62GzN7a/9aOmYZ7+pzUFf5+TIaR7LNJrQW+M+WYNDfDVYjdFcnsuxKbCabdA9vUQnejv6o6ttgoKfId58tz2mOSmYZ7BUyGh+eotwcOC3hjTsroayHvFTZHcvQGSBrrhmVOugvBof1d3/A7uajbMs9NtTx3RFPoDz3AndAUJC3pjzOGqD8KKZ+HjP7gpjX1Gwxk/gMxLIbQ1J8x3Iaqwa50X+kth20dQV9U0zDN4ihvq6XtKl57NY0FvjHHKd8Nn/+POTq3aCwMnuRk0Q85unymSXUFtpZvN0zi+3zjME53kpooOmep6/EkD/FtnG1nQG9PdlW2DT/4AK55zBy1HXOTmwKe3mAvdy8ESt0ZPY4+/8aStnkOaQn/QJIhK8G+dx2BBb0x3tXOtG3/PexUkBMbMgTP+DVJP8ndlganxpK3NS13ob10GteUgoe6P4uCpLvzTsgNuiMuC3pjuZtsnbgbN10sgPBayv+OmSAbxrJMOUVcDhZ+70N+81C3apg0QGe+GvRqncfYc7PehLwt6Y7qDhgYX7Mt+6xYOi0mGibfA+Oshpqe/qwsOFaWw5YOm8f29+W57Qn8YMsX1+AdP8cvnbUFvTDCrr3VDMx89AiXrXeh863YYew1ExPi7uuClCqWbm4Z5tnwA1fsBccsuD/Z6+xkTjr6aZzuyoDcmGNXXweoF8M9fuZ5lr0x3gHXUTAgN93d13U99HRStaFqJszAHtB7CY9zSDI3DPKkjOmSYx4LemGDS0ABr/xeW/tKtA99vLEy+G4af5/dxYuOjar87mLvpPdfr37PRbY/r64Z3God54nq3y8t9U9AH1mFjY8zRqborOS39pTsBqNcomPsinHSBBXwgioqHERe4/wD2FjSN7W9YAqtectt7ZzUt0zDgWx1yRrL16I0JdKpugbH3fuauyJQ8DKbeA5kzu82CXUGnoQGKVzXN3S/4zC0B3WsU3PrxcT2l9eiN6aq2fADv/dwFQeIAuPQxGH1FwM3hNm0UEuKG3PqNhUk/hJpy2PaxO2u3A9hvizGBKP8z14Pf+iHE9YOLfgunXANhEf6uzHSEiFgYNr3Dnt6C3phAUvQFvPcL2PiOWyp4xnw49TsQHuXvykwXZkFvTCDYuQ6W/sIdbI1OgnPmwYQbg2oZXeM/FvTG+NPujfD+A7DmrxAZB1P+w13wIyre35WZIGJBb4w/lG2Df/4aVr0IYVHuRKdv3W5LFZgOYUFvTGfaXwQfPAgr/uJWk5x4i1sPvkcvf1dmgpgFvTGd4WCJW2ws5ym3+uG4a+GsuyC+n78rM92ABb0xHamiFD7+PXz2uLt83clXweR/d9dmNaaTWNAb0xGq9sOnj7mrOlUfgKzZMOVuSBnm78pMN9SqoBeRGcAjQCjwlKrOb7a/P/AskOi1uVtVF4vI1cC/+zQdA4xT1ZUnXroxAaim3F2P9aNHoLLMXbJv6n9A71H+rsx0Y8cMehEJBR4FpgOFQI6IvK6q63ya3Qu8oqqPiUgmsBgYqKovAC94zzMaWGQhb4JSbRUsfwY+fAjKd8HQ6TDtJ+4Ud2P8rDU9+gnARlXdDCAiC4BLAN+gV6Bx4m8CUNTC81wJLDj+Uo0JQHU1sPJ5N5Nm/3Z3ebk5z0H/0/xdmTGHtCbo04ACn/uFwMRmbeYBb4vI7UAscE4LzzMH9wfiCCJyI3AjQP/+/VtRkjF+1lAPq19xJzvt3QbpE9yCY4Mn+7syY47QXmucXgk8o6rpwAXAcyJy6LlFZCJQoaprWnqwqj6hqtmqmp2amtpOJRnTARoa3FmsfzwNFt0M0Ylw9atw/dsW8iZgtaZHvx3I8Lmf7m3zdT0wA0BVPxGRKCAF2OXtnwu8dGKlGuMHtVVuSGZ/EZRtgc/+B3augdSRMOd5d7DVLvphAlxrgj4HGCYig3ABPxe4qlmbfOBs4BkRGQlEASUAXs/+CmBSexVtTLuoqXAB3hjk+7c3u10EFXsOf0zPITD7T+66rCGh/qnbmDY6ZtCrap2I3AYswU2dfFpV14rI/UCuqr4O/BB4UkTuwB2YvU6bLl11FlDQeDDXmE5RffDo4b2/CPYVQtXeIx8Xk+zOVo1Pg/Tx7md8WtO2noMs4E2XY5cSNF1P1f5mAe4T4vu8n9X7jnxcbKoX2Onez35NIZ6Q5i7a3AHX6zSmM9ilBE3XUVcNpVtgX0Gz8PYJ9ZoDzR4k0KO3C+zkITDorKYQT/CCPK4vhEX65S0Z428W9KbzqbrA3vM17Nno1mRvvL033y361UhCoEcfF9apJ8GQaU3h3dgb79HHLrFnzDewoDcdp2q/C+/G/3Z/7QX6JqitaGoXHut64mmnwpi5kDwUkgY0hbhdCNuYE2L/B5kTU1/reuGHQtynh35wZ1M7CYHEAS7EB05yP5OHukW+4vraFEVjOpAFvTk2VSgvObJXvvtrN7e8oa6pbUwyJA9za72kDHW3k4e62So2Rm6MX1jQmyY1FVC66chx890bD5/FEhrphlp6jYCRF7teeWMP3S6FZ0zAsaDvrsr3wJf/B8V5TT30fQWHt4lPd4E+5nLXM0/xwjwhw+aSG9OFWNB3J9UH4avFkLcQNr3nhlwi41149z8dUq5t6pknD4GIWH9XbIxpBxb0wa6uxoV63kIX8rUVrqd++m0w+jLonWUHQo0Jchb0waihAfI/ceG+bpG70lF0Tzj5Shh9OWRMhJD2WrjUGBPoLOiDhaobb89b6JbR3b/dzU8fcaEL9yFTITTc31UaY/zAgr6rK90MeX91Ab/7KwgJg6HnwPT74aTzbZzdGGNB3yUd2AlrX3Phvt1bAG7AGXDazZB5qU1xNMYcxoK+q6ja76ZD5i2Eze+79WD6jHY996zZkJDu7wqNMQHKgj6Q1VbBxne8GTNvQX01JA2EST+ErMvcCUvGGHMMFvSBpqEetn7ozZh5w52RGpsK2d9xB1XTTrXpkMaYNrGgDwSqULQC8l51M2YO7oSIOLe8wOjLYNBkW8HRGHPcLD38affXrueet9DNngmNgGHnup778PPsakfGmHZhQd/Z9he5XnveQtixChB3RaQz73Q9+OhEf1dojAkyFvSdobIM1v3NDc1sXQYo9BsH5z0Ao2ZCfF9/V2iMCWIW9B2lvg42/QNWvgBfvQn1NW6xsCn3uHH35CH+rtAY001Y0Le3netcuK9+Bcp3uQtxZF8PJ8+BvqfYjBljTKezoG8PFaVuWGblC7BjpVuGYPgMOOUqd6Ulu3C1McaPLOiPV30tbHzXG5p5Cxpqoc8YmPErNzQTm+LvCo0xBrCgb7viNbDyRch7xV1HNSYFJtwIp1zpliQwxpgAY0HfGuV73HTIlS9A8WoICYeTZsApV7uVIm35X2NMALOgP5r6Wvj6bdd737DEDc30PQXO/7VbZyY22d8VGmNMq1jQN1ec58J99StQsRtie8HEm9yB1d6j/F2dMca0WauCXkRmAI8AocBTqjq/2f7+wLNAotfmblVd7O0bA/wPEA80AONVtaq93kC7OFjiDc28CDvz3FIEJ50PJ18FQ8+2oRljTJd2zKAXkVDgUWA6UAjkiMjrqrrOp9m9wCuq+piIZAKLgYEiEgY8D/yLqq4SkWSgtt3fxfGoq4Gvl8DKl9zPhjroNxYueNCt724X7zDGBInW9OgnABtVdTOAiCwALgF8g15xPXaABKDIu30usFpVVwGo6p72KPq4qbqDqY1DM5Wl0KM3nHarG5rpNdKv5RljTEdoTdCnAQU+9wuBic3azAPeFpHbgVjgHG/7cEBFZAmQCixQ1V83fwERuRG4EaB///5tqb91Du5ywb7qJdi5xhuaucDNmhkyzZYANsYEtfZKuCuBZ1T1IRE5HXhORLK85z8TGA9UAP8QkeWq+g/fB6vqE8ATANnZ2douFdXVwIa3XO/967dB691FO2xoxhjTzbQm6LcDGT73071tvq4HZgCo6iciEgWk4Hr/H6jqbgARWQyMA/5BR1B1SxCsfNEtSVBZCj36wLducwdW7dJ7xphuqDVBnwMME5FBuICfC1zVrE0+cDbwjIiMBKKAEmAJ8CMRiQFqgMnAb9up9sPtzYcX58CudRAaCSMudOPug6fa0Iwxpls7ZgKqap2I3IYL7VDgaVVdKyL3A7mq+jrwQ+BJEbkDd2D2OlVVoExEfoP7Y6HAYlX9e4e8k7h+kJAB42+ArFkQndQhL2OMMV2NuDwOHNnZ2Zqbm+vvMowxpkvxjn9mt7QvpLOLMcYY07ks6I0xJshZ0BtjTJCzoDfGmCBnQW+MMUHOgt4YY4KcBb0xxgQ5C3pjjAlyAXfClIiUANtO4ClSgN3tVE57srraxupqG6urbYKxrgGqmtrSjoAL+hMlIrlHOzvMn6yutrG62sbqapvuVpcN3RhjTJCzoDfGmCAXjEH/hL8LOAqrq22srraxutqmW9UVdGP0xhhjDheMPXpjjDE+LOiNMSbIdcmgF5EZIvKViGwUkbtb2H+WiKwQkToRuSyA6rpTRNaJyGoR+YeIDAig2m4WkTwRWSkiy0QkMxDq8mk3W0RURDplSlwrPq/rRKTE+7xWisgNgVCX1+YK7/dsrYi8GAh1ichvfT6rDSKyN0Dq6i8iS0XkC+//ywsCpK4BXkasFpH3RST9hF5QVbvUf7jLGW4CBgMRwCogs1mbgcAY4C/AZQFU11Qgxrt9C/ByANUW73P728BbgVCX1y4O+AD4FMgOhLqA64A/dMa/XxvrGgZ8ASR593sFQl3N2t+OuySp3+vCHfy8xbudCWwNkLoWAv/q3Z4GPHcir9kVe/QTgI2qullVa4AFwCW+DVR1q6quBhoCrK6lqlrh3f0UOLG/0u1b236fu7G4a/z6vS7Pz4BfAVWdUFNb6upsranre8CjqloGoKq7AqQuX1cCLwVIXQrEe7cTgKIAqSsTeM+7vbSF/W3SFYM+DSjwuV/obfO3ttZ1PfBmh1bUpFW1icj3RWQT8Gvg3wKhLhEZB2RoR11U/jjr8sz2vlq/KiIZAVLXcGC4iHwkIp+KyIwAqQtwQxLAIJpCzN91zQOuEZFCYDHu20Yg1LUKmOXdngnEiUjy8b5gVwz6Lk9ErgGygf/2dy2+VPVRVR0C/Bi419/1iEgI8Bvgh/6upQVvAANVdQzwDvCsn+tpFIYbvpmC6zk/KSKJ/iyombnAq6pa7+9CPFcCz6hqOnAB8Jz3e+dvdwGTReQLYDKwHTjuzywQ3lBbbQd8e0/p3jZ/a1VdInIO8BPg26paHUi1+VgAXNqRBXmOVVcckAW8LyJbgdOA1zvhgOwxPy9V3ePz7/cUcGoH19SqunC9w9dVtVZVtwAbcMHv77oazaVzhm2gdXVdD7wCoKqfAFG4hcX8WpeqFqnqLFUdi8sLVHXvcb9iRx946IADGWHAZtzXv8YDGaOO0vYZOu9g7DHrAsbiDsIMC7TPzLcm4GIgNxDqatb+fTrnYGxrPq++PrdnAp8GSF0zgGe92ym4IYJkf9fltRsBbMU7UTNAPq83geu82yNxY/QdWl8r60oBQrzbvwDuP6HX7IwPvAM+qAtwPZVNwE+8bffjeskA43E9m3JgD7A2QOp6F9gJrPT+ez2APrNHgLVeXUu/KXA7s65mbTsl6Fv5eT3gfV6rvM9rRIDUJbjhrnVAHjA3EOry7s8D5ndGPW34vDKBj7x/x5XAuQFS12XA116bp4DIE3k9WwLBGGOCXFccozfGGNMGFvTGGBPkLOiNMSbIWdAbY0yQs6A3xpggZ0Fvgo6IJIrIrd7tKSLyfx3wGs+0ZWVUERkoImuOsu/9zlqV03RPFvQmGCUCt7blASIS2jGlGON/FvQmGM0HhojIStx6Qj28hce+FJEXREQARGSriPxKRFYAl4vIuSLyiXctg4Ui0sNrN9/nOgIP+rzOWSLysYhsbuzdi/PfIrLGW99/TvPiRCRaRBaIyHoReQ2I7uDPw3RzYf4uwJgOcDeQpaqniMgU4G/AKNzp7R8BZwDLvLZ7VHWciKQA/wuco6rlIvJj4E4ReRS3xMEIVdVmC4T1Bc7Endr/OvAqbsXBU4CTcaex54jIB83quwWoUNWRIjIGWNGeb96Y5qxHb7qDz1W1UFUbcKe5D/TZ97L38zS80+G9bwL/CgwA9uHWwf+TiMwCKnweu0hVG1R1HdDb23Ym8JKq1qvqTuCfuCU5fJ0FPA+g7roJq9vjTRpzNNajN92B7yqh9Rz+e1/u/RTgHVW9svmDRWQCcDZu/ZHbcFf8af680m7VGtPOrEdvgtEB3BLHbfEpcIaIDAUQkVgRGe6N0yeo6mLgDtyQzDf5EJgjIqEikorrvX/erM0HwFXe62ThLntpTIexHr0JOqq6x7vC0hqgErdi6LEeUyIi1wEviUikt/le3B+Nv4lIFK7Xfucxnuo14HTcaogK/EhVi0VkoE+bx4A/i8h6YD2wvNVvzpjjYKtXGmNMkLOhG2OMCXIW9MYYE+Qs6I0xJshZ0BtjTJCzoDfGmCBnQW+MMUHOgt4YY4Lc/we5qlukqUzdhQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_select = df[df.type == 'is_relevant'][['f1(micro)','f1(macro)']].reset_index(drop=True)\n",
    "ax = df_select.plot(xticks=df_select.index, title = 'relevant');\n",
    "ax.set_xticklabels(np.around(Threshold,1));\n",
    "ax.set_xlabel('threshold');\n",
    "df_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-29T08:04:02.355725Z",
     "start_time": "2020-12-29T08:04:02.216724Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.661686</td>\n",
       "      <td>0.646467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.697892</td>\n",
       "      <td>0.681614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.713592</td>\n",
       "      <td>0.687115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.732323</td>\n",
       "      <td>0.705297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.736434</td>\n",
       "      <td>0.714327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.740346</td>\n",
       "      <td>0.711037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.734525</td>\n",
       "      <td>0.701849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.724679</td>\n",
       "      <td>0.688949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.681682</td>\n",
       "      <td>0.617492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   f1(micro)  f1(macro)\n",
       "0   0.661686   0.646467\n",
       "1   0.697892   0.681614\n",
       "2   0.713592   0.687115\n",
       "3   0.732323   0.705297\n",
       "4   0.736434   0.714327\n",
       "5   0.740346   0.711037\n",
       "6   0.734525   0.701849\n",
       "7   0.724679   0.688949\n",
       "8   0.681682   0.617492"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEWCAYAAABollyxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABAz0lEQVR4nO3dd3hU1dbA4d9KBxISAgktQAgkQOgSmtIERSxAAAsoer1eu+h39aJiu2Ln2huKHcWCgoqgKIqCAtJCL4FQQgk1hBpC+v7+OAOEECAJMzkzk/U+T57MnNlzzpohrNmzzj57izEGpZRS3svH7gCUUkq5liZ6pZTycprolVLKy2miV0opL6eJXimlvJwmeqWU8nKa6FWlJSKjReTzszx+g4j8epbHx4vIs6U81mwRubWccZb7uUqBJnqlABCRaBExIuJ3fJsx5gtjTF8741LKGTTRK6WUl9NErzyOiGwRkQdFZKWIHBWRj0Sktoj8LCJHRGSmiNQQkV4iklbCcy8pYbd/OX4fFJFMEekqIjeLyNxSxlRDRH4UkXQROeC4HVWsWRMRWSQih0XkBxEJL/L8LiLyt4gcFJEVItKrDG+JUmeliV55qiHApUAc0B/4GXgUiMD6u76vjPvr4fgdZowJNsbML+PzfYBPgEZAQ+AY8HaxNjcBtwB1gXzgTQARqQ/8BDwLhAMjgW9FJKKMMShVIk30ylO9ZYzZY4zZAcwBFhpjlhljsoHvgfYVGYwxJsMY860xJssYcwR4DuhZrNkEY8xqY8xR4AngWhHxBYYD040x040xhcaY34Ak4IqKfA3Ke2miV55qT5Hbx0q4H+zMg4nIo46STqaIjCvh8aoi8p6IbBWRw1iloDBHIj9ue5HbWwF/oBbWt4BrHGWbgyJyEOiG1fNX6rz5nbuJUh7rKFD1+B1H0j1TOeSs07gaY54Hnj9Lk/8AzYDOxpjdItIOWAZIkTYNitxuCOQB+7A+ACYYY247WwxKlZf26JU3SwGCRORKEfEHHgcCz9A2HSgEYsp5rBCsbxIHHSdZnyyhzXARiReRqsDTwGRjTAHwOdBfRC4TEV8RCXKcSC5+MlepctFEr7yWMeYQcDfwIbADq4efdoa2WVh19XmO8kmXMh7udaAKVg99AfBLCW0mAOOB3UAQjhPGxpjtwECsk8npWD38B9H/n8pJRBceUUop76Y9BqWU8nKa6JVSystpoldKKS+niV4ppbyc242jr1WrlomOjrY7DKWU8ihLlizZZ4wp8ToRt0v00dHRJCUl2R2GUkp5FBHZeqbHtHSjlFJeThO9Ukp5OU30Sinl5dyuRl+SvLw80tLSyM7OtjsUrxAUFERUVBT+/v52h6KUqgAekejT0tIICQkhOjoaETn3E9QZGWPIyMggLS2Nxo0b2x2OUqoCeETpJjs7m5o1a2qSdwIRoWbNmvrtSKlKpFSJXkT6ich6EdkoIqNKePw1EVnu+ElxLJxQ9PHqIpImIsWXVis1TfLOo++lUpXLOUs3jsUaxmKtz5kGLBaRqcaYtcfbGGPuL9L+Xk5fxu0ZTi6+rJQ6T4WFhp2HjrFhTyYb92YSWsWfzjHhNAyvqh/k6jSlqdF3AjYaYzYDiMhErLmz156h/TCKLLogIh2A2ljzcyecV7RKVTKFhYYdB4+RsucIG/ZmsmFPJhv2HmHj3kyycgtOa1+nehCdY8Lp3LgmnWPCialVTRO/KlWir8+pa12mAZ1LaigijYDGwB+O+z7AK1iLH19ypgOIyO3A7QANGzYsTdy2ePPNN3n33XeJj49n586dLF26lOeee46RI0eeaHPs2DH69evHH3/8ga+v71n2dtK4ceOoWrUqN91003nFt2rVKl555RXGjx9/XvtRFa+g0JB2IIuU44l8TyYpjoSenVd4ol3t6oHERoZwXccGxEaGEFs7mKYRwezLzGFB6n4Wbs7g700Z/LB8JwARIYF0ahxOl8bhdI6pSWxksCb+SsjZo26GcnJ5NLBW95lujEk72x+XMeZ94H2AhIQEt10J5Z133mHmzJkEBASwdetWpkyZclqbjz/+mMGDB5c6yQPceeedpW6bn5+Pn1/J/2ytW7cmLS2Nbdu2ufUHZmVWUGjYtj+LlD1WEt+w5wgpezLZlJ5JTv7JhF43NIimkcHc0LkRsZHBVkKPDCG0SslDYmtUCyC2dgg3dmmEMYbUfUdZ6Ej8C1P389PKXQCEVwugU3T4iV5/8zoh+Pho4vd2pUn0Ozh1UeMox7aSDAXuKXK/K9BdRO4GgoEAEck0xpx2Qre0npq2hrU7D5f36SWKr1edJ/u3PGubO++8k82bN3P55Zdzyy23cP/99/PTTz+d1u6LL77gyy+/BGD27Nk8+eSThIWFsWrVKq699lpat27NG2+8wbFjx5gyZQpNmjRh9OjRBAcHM3LkSDZu3Midd95Jeno6vr6+TJo0ie3bt/PEE09Qo0YN1q1bx8qVK7nrrrtISkrCz8+PV199lYsvvhiA/v37M3HiRB566CGnvkeqbPILCtm6P4sNe444yi2ZpOw5wuZ9R8ktktDrh1WhaWQwFzapSVztEJrWDqZpZDDVg8p/jYOIEBMRTExEMMM6NcQYw/b9x1iQmsHCzftZmJrBL2t2AxBaxZ+O0eF0iQmnS0xNWtStjq8mfq9TmkS/GIgVkcZYCX4ocH3xRiLSHKgBzD++zRhzQ5HHbwYSzifJ22ncuHH88ssvzJo1i1q1apXYJjc3l82bN1N09s0VK1aQnJxMeHg4MTEx3HrrrSxatIg33niDt956i9dff/2Ufdxwww2MGjWKQYMGkZ2dTWFhIdu3b2fp0qWsXr2axo0b88orryAirFq1inXr1tG3b19SUlIICgoiISGBMWPGaKKvIHkFhWzNOGqVXBxllw17Mkndd5TcgpMJPapGFWIjg+kRF+HooYfQNDKY4EDXX8oiIjSsWZWGNatybYLVZ9tx8JjV23ck/pnJewAICfKjY3Q4nR2lnlb1quPn6xGjsNVZnPOvzBiTLyIjgBmAL/CxMWaNiDwNJBljpjqaDgUmGhcvQnuunred9u3bR1hY2CnbOnbsSN26dQFo0qQJffv2Bawyy6xZs05pe+TIEXbs2MGgQYMA6wrW4zp16nTiAqe5c+dy7733AtC8eXMaNWpESkoKbdq0ITIykp07d7rk9VVmeQWFpO47ap0UdYx0SdlzhNR9R8kvPPkn3yC8CnGRIfRqHkFsZAhxtYNpEhFMtQpI6GVRP6wKgy+IYvAFUQDsPpTNwtQMFjgS/x/r9gJQLcCXDo7E3yUmnNb1wwjw08TvaUr112eMmQ5ML7btv8Xujz7HPsYD48sUnYepUqXKaRciBQYGnrjt4+Nz4r6Pjw/5+fml3ne1atVK1S47O5sqVaqUer/qVMYY0g5Yo1zW7T5Cyp4jrN99hE3pmeQVWAldBBqGVyU2MoRL4msTGxlMXO0QYiKqUTXAvRJ6adUJDWJgu/oMbFcfgL1HslmUuv9Ej/+lGesBCPL3oUOjGtaonsbhtG0QRpB/6c9HKXt45l+lm6pRowYFBQVkZ2ef0hsvrZCQEKKiopgyZQqJiYnk5ORQUHD6ELru3bvzxRdf0Lt3b1JSUti2bRvNmjUDICUlhVatWp33a6kMMjJzWO9I5McT+4Y9mWTmnPwArhcaRLM6IfRqFkmzOsHERlolF29PbpEhQVzVph5XtakHwP6juSw60ePfz2szUzAGAvx8aN8gjM4xNenSOJz2DWtQJcC73xtPpIm+HHbv3k1CQgKHDx/Gx8eH119/nbVr11K9enX69u3L3LlzueSSM44mPasJEyZwxx138N///hd/f38mTZp0Wpu7776bu+66i9atW+Pn58f48eNPfFOYNWsWV1555Xm9Pm+TlZtPyp5MUnZbyXz9nsOs353JvsycE23CqvrTrHYIQy6oT1ydEJrXCSG2dsh5nRT1JuHVAujXqi79WlllyENZeSzacnJUz9t/bOBNA/6+QtuosBOjejo1Dvf6D0VPIC4uqZdZQkKCKb7CVHJyMi1atLAporJZunQpr732GhMmTKjwY+fk5NCzZ0/mzp17xiGYx3nSe1pax+vo63dbvfTjvfVt+7NOtAny9yGudghxta1kfvx3REigji8/D4ez81iy5cCJkT2rdhyioNBQPciPaxIaMLxLIxrXKl35UZWPiCwxxpR4Uar26J3sggsu4OKLL6agoKBMY+mdYdu2bYwZM+acSd7TlaaO7usjNK5Vjdb1Q7m6Q9SJhN4gvKoOH3SB6kH+XNw8koubRwJwNCefxVv28+3SHXz69xY+mptK99ha3NilEX1a1NZ/gwqmPfpKylPe09LU0euHVSGudjDN6lSnWZ1gmtWuTkxENS0ZuIm9R7KZuGg7Xy7cxu7D2dQPq8L1nRtyXccG1AoOPPcOVKloj155jLyCQt7/azN/b9qndXQvERkSxH19Yrm7VxNmJu9hwoKtvDRjPW/M3MAVretwY9doLmgYpqUzF9JEr9xG+pEc7vlyKYtS99O6figXN4ugWZ0Q66e21tE9nZ+vz4kTuhv3ZvL5gq18uySNKct3El+3Ojd1bcSAdvU8doiqO9N3VLmFpdsOcNfnSzh0LI83hrY7MZ5beaemkcGMHtCSBy9rxpTlO5gwfyujvlvFc9OTuaZDA4Z3aUhMRLDdYXoNTfTKdl8u3MboqWuoHRrId3ddRHy96naHpCpItUA/bujciOs7NSRp6wE+m7+VCQu28PG8kydvezeP1GkYzpO+e2Xw5ptv0qJFC4YMGULXrl0JDAzk5ZdfPqXNsWPH6NmzZ4kXOlWk3NxcevToUaarbytaTn4Bo75dyaPfr6JLk5pMG9FNk3wlJSJ0jA7nrWHtmTeqN/+5NI6NezO5fcISerw4i7GzNpJ+JOfcO1Il0h59GbhqmuLzcaZhnAEBAfTp04evv/6aG264oYRn2mvXoWPc+flSVmw/yIiLm3L/pXE65E4B1snbe/vEclevJsxM3suEBVt4acZ6Xp+ZwhWt63Jjl0Z0aFRDz9eUgecl+p9Hwe5Vzt1nndZw+ZizNnHlNMXTpk3j2WefJTc3l5o1a/LFF19Qu3ZtMjMzuffee0lKSkJEePLJJxkyZAjBwcHccccdzJw5k7Fjx7Jo0SI+/vhjAG699Vb+/e9/A5CYmMgjjzzidol+weYM7vliKTn5hYwb3oF+rerYHZJyQ9bJ2zr0a1XnlJO3PyzfSQvHyduBevK2VLR0U0rjxo2jXr16zJo1i/vvv7/ENmeapnjcuHEkJyczYcIEUlJSWLRoEbfeeitvvfUWAN26dWPBggUsW7aMoUOH8uKLLwLwzDPPEBoayqpVq1i5ciW9e/cG4OjRo3Tu3JkVK1ZQpUoVPvnkExYuXMiCBQv44IMPWLZsGQCtWrVi8eLFLnxXysYYw0dzU7nhw4WEVvVnyj0XaZJXpXL85O2CR/vw/KDWGGN45LtVdH7+d56atobN6Zl2h+jWPO+j8Bw9bzuVd5ritLQ0rrvuOnbt2kVubu6J6YhnzpzJxIkTT+yrRo0aAPj6+jJkyBDAmrJ40KBBJ2a3HDx4MHPmzKF9+/b4+voSEBDAkSNHCAkJcd0LL4VjuQWM+m4lPyzfSd/42rxybVtCdPy7KqNqgX5c37khwzo1YInj5O3nC7byybwtdGtaixu7NqKPnrw9jeclejdW3mmK7733Xh544AEGDBjA7NmzGT169FmPExQUVOpzADk5OeWaSdOZtmVkccfnS1i3+zAj+8Zxd6+munydOi8iQkJ0OAnR4aQfiefrxdv4YuE27piwhHqhQY4rbxsSEaJX3oKWbpyq6DTFZXHo0CHq17fGjX/66acntl966aWMHTv2xP0DBw6c9tzu3bszZcoUsrKyOHr0KN9//z3du3cHICMjg1q1auHvb1/Pefb6vfR/ey47Dx7jk5s7MqJ3rCZ55VQRIYGM6B3LnIcu5r0bOxATEczLv6Zw4Zjfue+rZSRt2Y+7TfVS0TTRl8Pu3buJiori1Vdf5dlnnyUqKorDh611bI9PU1wWo0eP5pprrqFDhw6nLFP4+OOPc+DAAVq1akXbtm1PW5EKrEnUbr75Zjp16kTnzp259dZbad++PWDvlMXGGMbO2sg/xy+mbmgQ00Z0o1ezSFtiUZWDn68Pl7Wsw+e3dub3//RkeJdGzFq/l6vHzefyN+bw5cJtHM1x3+HGrqSTmjmZndMUFzd48GDGjBlDXFzcaY+58j09kp3HyEkrmLFmDwPa1mPMkNY6MkLZIis3nx+W7+Sz+VtJ3nWYkEA/hnSI4rYeMdQP866V2HRSswpk5zTFReXm5pKYmFhikneljXszuWNCElsysnjiqnhuuShaxzsr21QN8GNYp4YM7diApdusk7dfLNzKnynp/Hp/D/wryUlbj3mV7vbN42xuueUWW5M8WBdM3XTTTSU+5qr3csaa3SSOncfBrDw+/1dn/tWtsSZ55RZEhA6NwnljaHvGDe9A6r6jfLVom91hVRiPSPRBQUFkZGR4VLJ3V8YYMjIynDoSp6DQ8PKM9dwxYQlNIqox7d5udG1S02n7V8qZejePpEtMOG/M3MCR7Dy7w6kQHlG6iYqKIi0tjfT0dLtD8QpBQUFERUU5ZV8Hs3L5v4nL+TMlnesSGvDUwJa64IdyayLCY1fE0//tuYz7cxMPXtbc7pBcziMSvb+//4mLiJT7WLvzMHd+voRdh47x/KDWXN+5od0hKVUqraNCGdiuHh/OSWV4l0bUDfWuE7PFlap0IyL9RGS9iGwUkVElPP6aiCx3/KSIyEHH9nYiMl9E1ojIShG5zsnxK5v8sHwHg9+dR05+AV/f0VWTvPI4I/s2wxh45dcUu0NxuXP26EXEFxgLXAqkAYtFZKoxZu3xNsaY+4u0vxdo77ibBdxkjNkgIvWAJSIywxhz0ImvQVWgvIJCxvy8jo/mptIpOpy3b2hPZIi9V94qVR4Nwqvyjwsb8eHcVP7VrTEt6nrvFNml6dF3AjYaYzYbY3KBicDAs7QfBnwFYIxJMcZscNzeCewFIs4vZGWXfZk5DP9wIR/NTeXmC6P54rbOmuSVRxtxcSzVg/x54ed1dofiUqVJ9PWB7UXupzm2nUZEGgGNgT9KeKwTEABsKuGx20UkSUSS9ISre1q+/SD935rL8u0HefXatowe0LLSjEFW3iu0qj/39m7KXynpzNngvbnH2f9ThwKTjTGnLK8kInWBCcA/jTGFxZ9kjHnfGJNgjEmIiNAOv7v5evE2rh03H18f4du7LmTwBc4ZsaOUO7ixayOialTh+enrKCj0ziHcpUn0O4AGRe5HObaVZCiOss1xIlId+Al4zBizoDxBKnvk5Bfw6PerePjbVXSOCWfaiG60qh9qd1hKOVWgny8PXtaM5F2HmbLsTKnNs5Um0S8GYkWksYgEYCXzqcUbiUhzoAYwv8i2AOB74DNjzGTnhKwqwu5D2Qx9fwFfLtzGXb2aMP6fnahRLcDusJRyif5t6tEmKpRXfl1Pdp696z27wjkTvTEmHxgBzACSgW+MMWtE5GkRGVCk6VBgojn18tVrgR7AzUWGX7ZzXvjKFRal7ueqt+ayfvcR3r3hAh7u11zXc1VezcdHePSKFuw8lM3H81LtDsfpPGL2SlUxjDGM/3sLz/2UTIPwqrx/Ywdia9u7MpVSFenWTxezcPN+Zj/Yi5rBnrVoydlmr9RhEwqwlvr7zzcreGraWno1i+SHERdpkleVzsP9mnM0N5+3/thodyhOpYlesX1/FkPe/Zvvl+/ggUvjeP/GDlTX9VxVJRRbO4TrOjbk8wVb2bLvqN3hOI1HzHWjXGfOhnTu/WoZBYWGj//RkYub6ypQJdq9ClZ+DQX5EB4D4Y2t32ENwVc/FL3J/ZfG8sPyHbw4Yx3v3NDB7nCcQhN9JVVQaBj35yZe+XU9sZEhvHdjB6JrVbM7LPeStR9WfwvLJsCuFeDjbyX1vKyTbcQHQhucmvzDY6BGY6gRDQFVbQtflU9kSBC394jh9ZkbWLL1AB0a1bA7pPOmib4S2pSeyUOTV7Jk6wGualOX/w1pQ7VA/VMAoLAANs+GZZ/Duh+hIBfqtIbLX4TW10CVGpC5F/Zvtn4OpDpup1ofCtmHTt1fSD3HB0BjK/kf/yAIbwxBek2Cu7qtewxfLNzG89OTmXxnV49fQEf/d1ciBYWGj+em8vKv6wny9+W169qS2K6+x/8RO8X+VFj+BSz/Cg6nQVAYdPgntL8B6rY9tW1IbeunUdfT95O135H8j/84PhA2/AaZe05tW7Xmyd5/0Q+A8BjrMf13sU21QD/uvySOR79fxYw1e+jXqo7dIZ0XHV5ZSWzcm8mDk1ewbNtBLmlRm+cHtSKyeiWfkCw3C5KnWr33LXMAgaZ9oN0N0OwK8Hfy+5OTWeRDoNi3gUNpQJH/iwEhRUpBxUpCIXXBR8dRuFp+QSH93phDQaHxiPVldXHwSqyg0PDhnM288lsKVQN8eWNoOwa0rVd5e/HGQFqSVXdf/R3kHrGSZ+/Hoe0wCHXhPD6BwVYZqE7r0x/Lz4EDW4skf8cHwO5VVgmpMP9kW78gx7eAIh8EjS6CyBaui70S8vP14ZHLm/OvT5P4atE2buoabXdI5aaJ3ott3HuEkZNWsnz7QfrG1+bZQa0q77TCmXthxUSr975vPfhXhfhEaD8cGl1of5nELxAi4qyf4gryrXLS8eR//PeBVNg0C/KPWe3qJ8AFN0KrIRCo10A4Q9H1ZQe1r0+Ihw471tKNF8ovKOSDOam8NjOFagG+PDWwFf3b1K18vfiCPNjwq5XcU2aAKYAGna3STMtBEOQFC00YY5V9kqfC0gmQngz+1aDVIGh/EzToZP+HmIdbmXaQAW/P456Lm7j1+rJnK91oovcyKXuO8OCkFaxIO0S/lnV4JrEVESGedSn3edu7zirNrPwajqZDcG1oOxTaDS+5x+wtTpSlPnOUpTKhVjOrl99mKATrFODldd9Xy5ixZjezH+zltuvLaqKvBPILCnnvr828MXMDwUF+PD2wJVe2rkS9+OxDVnJb9jnsSAIfP4jrB+1vhKaXgG8lq1LmZMKa72HpZ5C2yHo/ml0BF9wETXqDj6/dEXqU7fuz6PPKnwxsV4+Xrml77ifYQE/Gerl1uw/z4KSVrNpxiCtb1+WpgS2p5WETMpVLYSFsnWsl97VTrVp1RAu47HlofW3l7sEGBls9+QtuPPkNZ8VXVomnen2rfNV+ONRoZHekHqHo+rK3eOD6stqj92B5BYWMm72JN//YQPUgf54e2Ior29S1OyzXO7jdSlrLPoeDWyEwFFoPsRJXvQu0Jn0m+bmwfrqV9Df+bm2L6Wn18ptfZZ0QVmd0KCuPHi/Nom2DMD67pZPd4ZxGe/ReKHnXYUZOWsGanYe5qk1dnhrQ0uOmVS2TvGxrmOGyz60rVzHQuCf0fgJaXAX+7lk3dSt+AdAy0fo5uB2Wf2m9n5Nvsa74bTPU+gZQu6Xdkbql0Kr+jLi4Kc9NT2bOhnS6x3rON0bt0XuYvIJC3pm1ibdnbSC0ij/PDGzF5a29tBdvDOxaDsu+gFXfWHX40IbQ7nrrR8sO56+wEFJnWyN2jk/5UL+DdW6j1RDvGJnkRDn5BfR55U+qB/nz473d8HGjBXn0ZKyXWLPzEA9OWsnaXYcZ0LYeowe0JNwbl/c7mmEl9mWfw57V1gVCLfpbpZnoHnpVqKtk7bdGKi39DPauta41aDnIKu006KwlMYcflu/g/yYu55Vr2jKkgwsvsCsjTfQeLje/kLGzNjJ21kbCqgbwbGIrj59744ySf4Qpd0POIave3n641bOsEmZ3ZJWHMbBjiZXwV3/rGKYZZ/Xy2w6r3Ce5gcJCQ+I789h3JIc/RvYiyN89RjBpovdgq3ccYuSkFazbfYRB7evz36vivXOR7oJ8+H00/P2WleAHvAV1WtkdlcrJhLVTrNLO9gWOYZqXWxdjNe1TaYdpzt+UwbAPFvBQv2bc3aup3eEAejLWI+XmF/L2Hxt4Z/YmalQL4IObErg0vrbdYbnGkd0w6Z+w7W/oeKs1PFJHgLiHwGDrW1X74ZC+3hqxs/wrSJ7mGKZ5vWOYZrTdkVaork1q0qd5JO/O2sTQjg3dvoSqPXo3tCrtEA9Otnrxgy+wevFhVd37D6ncUudYoz5yM6H/G9DmWrsjUueSnwspv1ilnU2/gym0RkAdH6bp7Fk/3dSGPUe47PW/uKlrNKMH2D9SSXv0HiInv4C3ft/Iu39uoma1AD76RwJ9WnhpL76wEP5+A35/GsKbwD+m6uyLnsIvAOIHWD+H0hzDNCfAt/+y5vFvO9Sq53t56a3o+rI3Xxjt1iu0aY/eTaxMO8jISStI2ZPJ1R2ieOLKeEKreuZMeed07IB1wnX9dGtUx4C3dLZFT1dYCKl/Wgk/eZo1TLPeBdD5TmtlLi8dKbX3cDa9Xp5Nr2YRtq8ve7YefanefRHpJyLrRWSjiIwq4fHXRGS54ydFRA4WeewfIrLB8fOPcr8KL5WTX8CLv6xj0Dt/c+hYHp/c3JGXr2nrvUl+1wp4r6c1q2S//8HVn2iS9wY+PtDkYrj6Y/jPeuvfNj8bvr8d3utx8kpcLxNZPYjbuscwfdVulm47YHc4Z3TOHr2I+AIpwKVAGrAYGGaMWXuG9vcC7Y0xt4hIOJAEJGAtn7ME6GCMOeM7Upl69Mu3H+TBSSvYsDeTaxOieOzKeEKreGmCN8aq6U5/EKrVgmvGW1PoKu9VWAhrvrPKcwe3QkwvuPTp05dm9HBHc/Lp9fJsGoVXZZKN68ueb4++E7DRGLPZGJMLTAQGnqX9MOArx+3LgN+MMfsdyf03oF/pQ/dO2XkFjPl5HYPfmUdmTj7j/9mRF69u671JPjcLfrgHpt1nLfJxx1+a5CsDHx9ofTWMWAyXveD4NtcDvrsdDm6zOzqnOb6+bNLWA8xYs+fcT7BBaRJ9fWB7kftpjm2nEZFGQGPgj7I8V0RuF5EkEUlKT08vTdwea+m2A1z55hzG/bmJaxMaMOP+HvRqFml3WK6TsQk+utQ6YdfzYRj+rdWjV5WHXyB0vRvuWw7d7oe1P8BbHWDGY9bVuF7g2oQomkYG879f1pFXUGh3OKdx9hmSocBkY0xBWZ5kjHnfGJNgjEmIiPDOq+6y8wp4fnoyV7/7N8dyC/j0lk6MGdKG6h66NFmpJE+D93vB4R1ww2S4+NFKe4GNwrq6+ZLRcO9Saxrp+WPhzXYw7w1r0joP5ufrw6h+zUndd5SvFrnft5XSJPodQIMi96Mc20oylJNlm7I+12st2XqAK96cw/t/bea6jg2ZcX8PesZ55wcaYC3h9+vj8PVwqNnUKtXEXmJ3VMpdhNaHxLFw1zxrDp3f/gtvJ1hr+ha6X2+4tPq0iKRzY2t92SPZeXaHc4rSJPrFQKyINBaRAKxkPrV4IxFpDtQA5hfZPAPoKyI1RKQG0NexrdJYt/swQ9+fT05eIRP+1YkXBrf22AWGS+XwLvh0gDWVQcfb4JZfIKyh3VEpd1S7JdwwCW6aClVrwvd3ePQIHRHhsStbkHE0l/f+3Gx3OKc4Z6I3xuQDI7ASdDLwjTFmjYg8LSIDijQdCkw0RYbxGGP2A89gfVgsBp52bKsU8gsKeWjySqoH+TN1xEUeNX91uaTOsf6j7loOgz+EK1/WqQzUucX0hNtmwZCPIOcwfD4YPku0Tt56mDZRYQxoW48P525m9yH3KUfpBVMu9N6fm3jh53W8fX17rmpTz+5wXKewEOa9Dn88Y13let0EvcpVlU9+Diz+CP560bqwrs110Ptxj/pWaNf6sud9wZQqu83pmbz6Wwp942tzpbcuDALWf8aJ18PvT0F8Itw+S5O8Kj8vGKHTILwqN3VtxOSlaazbfdjucABN9C5RWGh4+NuVBPr58GxiK9suoHC5ncutq1w3zoTLX7SuitSrXJUznBihs6TYCJ03PWKEzojeTQkJ9OOF6evsDgXQRO8Sny/cyuItB3j8qngiq3vhTH7GwJLx8FFfKMyHf/4Mne/QFYiU84VGWSN07pwLUZ3gtyc8YoROWNUA7u0dy58p6czdsM/ucDTRO1vagSz+9/M6usfW4ho3WmbMaXKzrAnJpv1fkatcO9odlfJ2dVrB8MmnjtB5371H6NzYtRH1w6rw/PRkCgvtPReqid6JjDE88t0qAF4Y3Nr7SjbHr3Jd8RX0HKVXuaqKV3SETrZ7j9AJ8vfloX7NWLvrMN8vs/fyIU30TjRpSRpzNuzj4cubE1Wjqt3hONfaqVY9/sRVro/oVa7KHqfNobPc+tt0wzl0+repR+v6obzy63qy88o0YYBTaaJ3kr2Hs3n2x7V0ig5neOdGdofjPAV51oiHb26EiDi4Y45e5arcwykjdP59coTOr49bo8HcgI+P8OgVLdh5KJtP5m2xLw7bjuxFjDE8PmU1OfmFjBnSGh8fLynZHN4Fn/aH+W9bV7n+82cIa3Du5ylVkYqP0Pn7bXijrduM0Dm+vuw7szay/2iuLTFooneCn1bt4te1e3jg0jhiIoLtDsc5Uv+C97rDrpVWPVSvclXuzo1H6Iy6vDlHc/N58/cNthxfE/152n80lyd/WEObqFD+1a2x3eGcv8JCmPMKfDYQqtSA2/6w6qFKeYozjdDZ9Me5n+si1vqyDfh8wVa27Dta4cfXRH+enpq2hsPZebx4dRv8fD387Tx2ACYOs1YEik+0knxkc7ujUqp8io/QmTDI1hE6918SR4CfDy/NWF/hx/bwzGSv35P38MPyndzdqynN61S3O5zzc+Iq19/h8pf0KlflHc40Quevlys8lOPry/60aleFry+rib6cDmfn8dj3q2lWO4R7Lm5qdzjlZwwkfeK4yrXAcZXr7XqVq/IuRUfoNOltnajNz6nwMG7vEUOt4ECe/ymZipxQUhN9Ob0wPZm9R7J58eo2BPh56NuYmwVT7oIf/w3RF+lVrsr7VQmzpuvIOQSbZ1f44asF+nH/pbEVvr6sh2Yoe83buI+vFm3ntu4xtG0QZnc4pWeMdUHJym/gxwfgnc7WiISeo6yLoKrVtDtCpVwv5mIIDLXG3dvguoQGNI0M5sUKXF/Wr0KO4kWycvMZ9d1KGteqxv2XxtkdztkVFsCe1bBtwcmfIzutxwJCrN77Va9D0z62hqlUhfILgOZXwLofIf91635FHt6xvuytnyUxcdE2buwa7fpjuvwIXualGevZvv8YX9/ehSB/N5sCICcTdiSdTOppiyE303qsepQ1CVnDLtZPZLxOYaAqr/hEa86m1D8h9tIKP3yfFpF0ahzO6zM3kNi+vsuXF9VEXwZLtu5n/N9buLFLIzrHuEGZ4/Au2L4Ati2EbfNh9yowBYBA7VbQdig07GotwKxXtCp1UpOLIbA6rJliS6IXER67ogUDx87jvT83M/KyZi49nib6UsrOK+ChySupF1qFhy+3YWx5YSHsW28l9OOJ/eBW6zG/KhCVAN0fsHrrUR0hKLTiY1TKU/gFQrPLrfJNwevg69oedUnaNgijv2N92eFdGlEn1HVrV2iiL6W3/tjApvSjfHpLJ4IDK+Bty8uGnUtPlmG2L4Tsg9Zj1SKhYWdr9ECDLlC3jS1/qEp5tPhEWPm1Vb5pas9EfQ9d1owZq3fz6m/refFq160vq4m+FFbvOMS4PzdzdYcoesZFuOYgRzMcZRjHz67lUOCYAKlWHMQPsJJ6wy4QHqPj3JU6X016W4MS1kyxLdEfX1/2o3mp3NKtscsuvNREfw55BYU8NHkl4dUCeOLKeOfs1BjYv9lRhnEk9gzHZEe+AVCvPXS5y0rsDTrrsEelXME/CJr1s8o3V71m27fiEb2b8k3Sdl6Yvo5Pb+nkkmNooj+H9/7cxNpdhxk3vAOhVcv5h5CfC7tXOpL6fKsMczTdeiwozOqlt7veOnFar731B6iUcr34RFg1CbbMsXr4NgirGsCI3k15fvo65m7YR7dY56/aVqpELyL9gDcAX+BDY8yYEtpcC4wGDLDCGHO9Y/uLwJVYF2f9Bvyfqchrf8/Dhj1HePP3jVzZui79WtUp304O74IPep8cv14j2vqa2KCzldhrxVnzcSilKl7TPhAQbJVvbEr0ADd1jebTv7fy/PRkfrqvm9OXIT1nohcRX2AscCmQBiwWkanGmLVF2sQCjwAXGWMOiEikY/uFwEVAG0fTuUBPYLYzX4QrFBQaHvp2JdUCfRk9oGX5dmKMtYj2sQPWDHrR3SCknB8YSinn868CcZdZ5ZsrXwVfe4ocQf6+vDC4NVUCfF2y1nRpupKdgI3GmM3GmFxgIjCwWJvbgLHGmAMAxpi9ju0GCAICgEDAH6i4CR7Ow/i/t7Bs20Ge7N+SiJByLrix/EvYMAMuedKaQU+TvFLuJz4RsjJg61xbw+gRF0HH6HCX7Ls0ib4+sL3I/TTHtqLigDgRmSciCxylHowx84FZwC7HzwxjTHLxA4jI7SKSJCJJ6enp5XkdTrU14ygvzVhH7+aRDGxXr3w7ObQDfhkFDS+ETnc4N0CllPPEXgr+1azyjZdyVnHYD4gFegHDgA9EJExEmgItgCisD4feItK9+JONMe8bYxKMMQkRES4avlhKxhhGfbsKfx8fnhvUqnxfo4yBqSOgMN9a2kxr8Eq5L/8qENcXkqdBQb7d0bhEaTLQDqDo9fNRjm1FpQFTjTF5xphUIAUr8Q8CFhhjMo0xmcDPQNfzD9t1vlq0nfmbM3jkihbUDa1Svp0s/dRatuzSp60x70op9xafCFn7YNvfdkfiEqVJ9IuBWBFpLCIBwFBgarE2U7B684hILaxSzmZgG9BTRPxExB/rROxppRt3sevQMZ6fnsyFTWoyrFM554Y5uA1mPAaNe0DCv5wboFLKNWL7gn9Vry3fnDPRG2PygRHADKwk/Y0xZo2IPC0iAxzNZgAZIrIWqyb/oDEmA5gMbAJWASuwhl1Oc8HrOG/GGB79bhUFhYYxg9uUr2RTWAg/3GPdHvC2lmyU8hQBVa1knzzNmt7by5RqLJExZjowvdi2/xa5bYAHHD9F2xQAHnEmcsryHcxan84TV8XTsGbV8u0k6SNI/cua471GI6fGp5RysfiBsHYKbP0bGp92KtGjaZcTSD+Sw1PT1nJBwzBuvjC6fDvZnwq//de66KLDzc4MTylVEeIus2aCtWnlKVfSRA+MnrqGrJwCXry6Db4+51Gy8fGDAW/phGNKeaKAatZQy+SpXle+qfSJ/pfVu/lp1S7u69OUppEh5dvJovdh6zzo9wKERjk3QKVUxYkfCJl7rHmpvEilTvQHs3J54ofVxNetzh09m5RvJxmbYOZo60ROuxucGp9SqoLF9QO/IK8r31TqRP/Mj8nsP5rLi1e3wd+3HG9FYQFMuctaXLj/m1qyUcrTBQZbkw4mT7VKsl6i0ib62ev38u3SNO7sGUOr+uVcdm/BO9aUw5e/BNXrOjdApZQ9Wg6CI7us/9teolIm+sycfB77fjVNIqpxb+/Y8u0kPQV+fwaaXQltrnVugEop+8RdBr6B1lBLL1EpE/3/fl7HzkPHePHqtgT5+5Z9BwX5MOVO6yKLq17Tko1S3iQwxCrfrPWe8k2lS/QLN2cwYcFW/nlhYzo0qlG+ncx/C3YsgStehpDazg1QKWW/lonWYkFpi+2OxCkqVaI/llvAw9+upGF4VUZeFle+nexZC7OehxYDoNUQ5waolHIPcZdZ6zd7SfmmUiX612amsCUjizGDW1M1oBwryRTkWaNsAqtryUYpbxYUCk36WMMsvaB8U2kS/YrtB/lwzmaGdWrAhU3Lufju3Ndh13K46lWo5vwFfJVSbqRlIhzeYZVpPVylSPS5+YU8NHklkSFBPHJFi/LtZPcq+PN/VrkmvvhKikoprxPXD3z8vaJ8UykS/dhZG1m/5wjPDWpF9SD/su8gP9cq2VSpYZ2AVUp5vyph1iSFa3+wVo3zYF6f6JN3HWbsrI0ktqtHnxblHCEz52WrR9//dajqmsV7lVJuqGUiHNoOO5baHcl58epEn19glWxCq/jz3/4ty7eTncvhr5ehzVBofqVT41NKublmlzvKN9/bHcl58epE/+HcVFbtOMRTA1sSXi2g7DvIz7FKNtUi4PIxzg9QKeXeqtSAmF6wxrPLN16b6DenZ/Labyn0ja/Nla3LOQ/Nn/+DvWutOearlPPiKqWUZ2uZCIe2wc5ldkdSbl6Z6AsLDQ9/u5JAPx+eTWxVvvVfdyyBua9B++EQ19f5QSqlPEOzK6xFhTx49I1XJvoJC7ayeMsBnrgqnsjqQWXfQV42fH8XhNSFy553foBKKc9RNRwa94Q1Uzy2fON1iX77/iz+98s6esRFcHWHcq72NOs52LfeKtkElXMKY6WU92iZCAe3wq4VdkdSLl6V6I0xPPr9KgR4flA5SzbbFsLfb1kLfDft4+wQlVKeqPlVIL4eW77xqkQ/aUkaczbs4+HLmxNVo2rZd5CbZY2yCW0AfZ91foBKKc9UNRxiPLd8U6pELyL9RGS9iGwUkVFnaHOtiKwVkTUi8mWR7Q1F5FcRSXY8Hu2k2E+x53A2z/64lk7R4Qzv3Kh8O/njWdi/CQa+bc1JrZRSx8UPhAOpsHul3ZGU2TkTvYj4AmOBy4F4YJiIxBdrEws8AlxkjGkJ/LvIw58BLxljWgCdgL3OCf1UVQN8SWxfnzFDWuPjU46Szda/raUBO95mfXIrpVRRzfs7yjeet3B4aXr0nYCNxpjNxphcYCJQfFav24CxxpgDAMaYvQCODwQ/Y8xvju2Zxpgsp0VfREiQP08PbEVMRHDZn5x71CrZ1GgEl4x2emxKKS9QrSY07u6R5ZvSJPr6wPYi99Mc24qKA+JEZJ6ILBCRfkW2HxSR70RkmYi85PiG4F5mjoYDW2HgO9Yq8EopVZL4gVZ5d89quyMpE2edjPUDYoFewDDgAxEJc2zvDowEOgIxwM3Fnywit4tIkogkpaenOymkUkr9Cxa9D13uguiLKvbYSinP0rw/iI/HlW9Kk+h3AA2K3I9ybCsqDZhqjMkzxqQCKViJPw1Y7ij75ANTgAuKH8AY874xJsEYkxAREVGOl1FOOUfgh3sgvAn0fqLijquU8kzBERDdzePKN6VJ9IuBWBFpLCIBwFBgarE2U7B684hILaySzWbHc8NE5Hj27g2sPf+wneTXJ+Dgdkh8FwLKMRxTKVX5xCdCxgZrHiwPcc5E7+iJjwBmAMnAN8aYNSLytIgMcDSbAWSIyFpgFvCgMSbDGFOAVbb5XURWAQJ84IoXUmYbf4cln8CFI6BhZ7ujUUp5ihaO8s2aKXZHUmpi3OzrR0JCgklKSnLtQbIPwTsXWr34O+aAfznmw1FKVV7jr4LMvTBikd2RnCAiS4wxCSU95lVXxpbajMfgyE5IHKdJXilVdvEDrfmw9ibbHUmpVL5En/IrLJsAF/0bojrYHY1SyhO1GACIx5RvKleiP3YApt0HES2gV4kzOSil1LmF1IZGF3rMMMvKleh/ecSqqw16F/wC7Y5GKeXJ4hMhPRnS19sdyTlVnkS/bjqs+Ap6jIR67e2ORinl6Vr0x1PKN5Uj0Wfthx//DbVbQ/eRdkejlPIG1etCwy4eMUd95Uj00x+ErAxHySbA7miUUt4iPtG6cCo9xe5Izsr7E/3aH2D1ZOj5MNRpbXc0SilvEu+4ZtTNT8p6d6I/ug9+fADqtoNu99sdjVLK21SvBw06u335xrsT/U//gZzD1lw2vv52R6OU8kbxida0xfs22h3JGXlvol/9nfUp2+sRqB1/zuZKKVUuJ8o3U2wN42y8M9Fn7rV68/U7wIX32R2NUsqbhUZBVEdN9BXKGJj2b2t5wMR3wdfP7oiUUt4uPhF2r4KMTXZHUiLvS/SrJsH6n6DPExDRzO5olFKVQbxjGW03HX3jXYn+8C5rzHyDztDlbrujUUpVFmENoH6C25ZvvCfRG2Nd/ZqfYy3y7eN+a5ArpbxY/EDYtQL2p9odyWm8J9FnbLIW+r7kSajV1O5olFKVjRuXb7wn0ddqCvcsgk532B2JUqoyqtEI6l3gluUb70n0YNXJfLzrJSmlPEj8QNi5DA5ssTuSU2hWVEopZ2mZaP12s/KNJnqllHKWGtHW3Fqa6JVSyou1TIQdS+DgNrsjOUETvVJKOZMbjr7RRK+UUs4UHgN12nheoheRfiKyXkQ2isioM7S5VkTWisgaEfmy2GPVRSRNRN52RtBKKeXWWiZC2mI4lGZ3JEApEr2I+AJjgcuBeGCYiMQXaxMLPAJcZIxpCfy72G6eAf5yRsBKKeX24hOt327Sqy9Nj74TsNEYs9kYkwtMBAYWa3MbMNYYcwDAGLP3+AMi0gGoDfzqnJCVUsrN1WwCtVt7VKKvD2wvcj/Nsa2oOCBOROaJyAIR6QcgIj7AK8DIsx1ARG4XkSQRSUpPTy999Eop5a5aDoTtC+HQDrsjcdrJWD8gFugFDAM+EJEw4G5gujHmrIUqY8z7xpgEY0xCRESEk0JSSikbxQ+yfidPtTcOrAR9LjuABkXuRzm2FZUGLDTG5AGpIpKClfi7At1F5G4gGAgQkUxjTIkndJVSymvUagqRLWHNFOhyl62hlKZHvxiIFZHGIhIADAWKf0RNwerNIyK1sEo5m40xNxhjGhpjorHKN59pkldKVRotE2H7Aji809YwzpnojTH5wAhgBpAMfGOMWSMiT4uIY1VcZgAZIrIWmAU8aIzJcFXQSinlEY6PvkmeZmsYYoyxNYDiEhISTFJSkt1hKKWUc4ztAlVqwC0/u/QwIrLEGJNQ0mN6ZaxSSrlSy0TYNh+O7LYtBE30SinlSvGJgLG1fKOJXimlXCmyOUQ0t0bf2EQTvVJKuVr8QNg6D47sseXwmuiVUsrVjpdv1tlTvtFEr5RSrhbZAmrF2Va+0USvlFKuJnKyfJNZ8fN5aaJXSqmKEJ8IptCW8o0meqWUqgi1W0LNpraUbzTRK6VURThevtkyB47uq9BDa6JXSqmKcrx8U8EXT2miV0qpilKntbV4eAWvPKWJXimlKoqI1atP/QuOVtwEv5rolVKqIsUPBFMA636ssENqoldKqYpUty3UiK7Q8o0meqWUqkgnyjd/Qtb+CjmkJnqllKpo8QOhMB/W/VQhh9NEr5RSFa1eewhrCGunVMjhNNErpVRFO16+2Twbjh1w+eE00SullB1aJjrKN9NdfihN9EopZYd6F0BoxZRvNNErpZQdRCB+AGyaBccOuvRQmuiVUsouLQdBYR6s/9mlhylVoheRfiKyXkQ2isioM7S5VkTWisgaEfnSsa2diMx3bFspItc5M3illPJo9TtA9SiXl2/8ztVARHyBscClQBqwWESmGmPWFmkTCzwCXGSMOSAikY6HsoCbjDEbRKQesEREZhhjDjr7hSillMc5PnXx4g8g+xAEhbrkMKXp0XcCNhpjNhtjcoGJwMBibW4DxhpjDgAYY/Y6fqcYYzY4bu8E9gIRzgpeKaU8XstEKMiF9b+47BClSfT1ge1F7qc5thUVB8SJyDwRWSAi/YrvREQ6AQHAphIeu11EkkQkKT294tdTVEop29RPgOr1XVq+cdbJWD8gFugFDAM+EJGw4w+KSF1gAvBPY0xh8ScbY943xiQYYxIiIrTDr5SqRHx8oMUA2Pg7ZB92zSFK0WYH0KDI/SjHtqLSgKnGmDxjTCqQgpX4EZHqwE/AY8aYBecfslJKeZmWiVCQAykzXLL70iT6xUCsiDQWkQBgKDC1WJspWL15RKQWVilns6P998BnxpjJzgpaKaW8SlQnCKnrsvLNOUfdGGPyRWQEMAPwBT42xqwRkaeBJGPMVMdjfUVkLVAAPGiMyRCR4UAPoKaI3OzY5c3GmOUueC1KKeWZfHyg462Ql+WS3YsxxiU7Lq+EhASTlJRkdxhKKeVRRGSJMSahpMf0ylillPJymuiVUsrLaaJXSikvp4leKaW8nCZ6pZTycprolVLKy2miV0opL6eJXimlvJzbXTAlIunA1vPYRS1gn5PCcSaNq2w0rrLRuMrGG+NqZIwpcVZIt0v050tEks50dZidNK6y0bjKRuMqm8oWl5ZulFLKy2miV0opL+eNif59uwM4A42rbDSustG4yqZSxeV1NXqllFKn8sYevVJKqSI00SullJfzyEQvIv1EZL2IbBSRUSU83kNElopIvohc7UZxPSAia0VkpYj8LiKN3Ci2O0VklYgsF5G5IhLvDnEVaTdERIyIVMiQuFK8XzeLSLrj/VouIre6Q1yONtc6/s7WiMiX7hCXiLxW5L1KEZGDbhJXQxGZJSLLHP8vr3CTuBo5csRKEZktIlHndUBjjEf9YC1nuAmIAQKAFUB8sTbRQBvgM+BqN4rrYqCq4/ZdwNduFFv1IrcHAL+4Q1yOdiHAX8ACIMEd4gJuBt6uiH+/MsYVCywDajjuR7pDXMXa34u1JKntcWGd/LzLcTse2OImcU0C/uG43RuYcD7H9MQefSdgozFmszEmF5gIDCzawBizxRizEih0s7hmGWOOLwq5ADi/T2nnxna4yN1qQEWcpT9nXA7PAP8DsisgprLEVdFKE9dtwFhjzAEAY8xeN4mrqGHAV24SlwGqO26HAjvdJK544A/H7VklPF4mnpjo6wPbi9xPc2yzW1nj+hfws0sjOqlUsYnIPSKyCXgRuM8d4hKRC4AGxpifKiCeUsflMMTx1XqyiDRwk7jigDgRmSciC0Skn5vEBVglCaAxJ5OY3XGNBoaLSBowHevbhjvEtQIY7Lg9CAgRkZrlPaAnJnqPJyLDgQTgJbtjKcoYM9YY0wR4GHjc7nhExAd4FfiP3bGUYBoQbYxpA/wGfGpzPMf5YZVvemH1nD8QkTA7AypmKDDZGFNgdyAOw4Dxxpgo4ApgguPvzm4jgZ4isgzoCewAyv2eucMLKqsdQNHeU5Rjm91KFZeIXAI8BgwwxuS4U2xFTAQSXRmQw7niCgFaAbNFZAvQBZhaASdkz/l+GWMyivz7fQh0cHFMpYoLq3c41RiTZ4xJBVKwEr/dcR03lIop20Dp4voX8A2AMWY+EIQ1sZitcRljdhpjBhtj2mPlC4wxB8t9RFefeHDBiQw/YDPW17/jJzJanqHteCruZOw54wLaY52EiXW396xoTEB/IMkd4irWfjYVczK2NO9X3SK3BwEL3CSufsCnjtu1sEoENe2Oy9GuObAFx4WabvJ+/Qzc7LjdAqtG79L4ShlXLcDHcfs54OnzOmZFvOEueKOuwOqpbAIec2x7GquXDNARq2dzFMgA1rhJXDOBPcByx89UN3rP3gDWOOKadbaEW5FxFWtbIYm+lO/XC473a4Xj/WruJnEJVrlrLbAKGOoOcTnujwbGVEQ8ZXi/4oF5jn/H5UBfN4nramCDo82HQOD5HE+nQFBKKS/niTV6pZRSZaCJXimlvJwmeqWU8nKa6JVSystpoldKKS+niV55HREJE5G7Hbd7iciPLjjG+LLMjCoi0SKy+gyPza6oWTlV5aSJXnmjMODusjxBRHxdE4pS9tNEr7zRGKCJiCzHmk8o2DHx2DoR+UJEBEBEtojI/0RkKXCNiPQVkfmOtQwmiUiwo92YIusIvFzkOD1E5G8R2Xy8dy+Wl0RktWN+/+uKByciVURkoogki8j3QBUXvx+qkvOzOwClXGAU0MoY005EegE/AC2xLm+fB1wEzHW0zTDGXCAitYDvgEuMMUdF5GHgAREZizXFQXNjjCk2QVhdoBvWpf1TgclYMw62A9piXca+WET+KhbfXUCWMaaFiLQBljrzxStVnPboVWWwyBiTZowpxLrMPbrIY187fnfBcTm845vAP4BGwCGsefA/EpHBQFaR504xxhQaY9YCtR3bugFfGWMKjDF7gD+xpuQoqgfwOYCx1k1Y6YwXqdSZaI9eVQZFZwkt4NS/+6OO3wL8ZowZVvzJItIJ6IM1/8gIrBV/iu9XnBatUk6mPXrljY5gTXFcFguAi0SkKYCIVBOROEedPtQYMx24H6skczZzgOtExFdEIrB674uKtfkLuN5xnFZYy14q5TLao1dexxiT4VhhaTVwDGvG0HM9J11Ebga+EpFAx+bHsT40fhCRIKxe+wPn2NX3QFes2RAN8JAxZreIRBdp8y7wiYgkA8nAklK/OKXKQWevVEopL6elG6WU8nKa6JVSystpoldKKS+niV4ppbycJnqllPJymuiVUsrLaaJXSikv9//IoFVYhJoRnAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_select = df[df.type == 'multi'][['f1(micro)','f1(macro)']].reset_index(drop=True)\n",
    "ax = df_select.plot(xticks=df_select.index, title = 'multi-label');\n",
    "ax.set_xticklabels(np.around(Threshold,1));\n",
    "ax.set_xlabel('threshold');\n",
    "df_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T06:01:53.252773Z",
     "start_time": "2020-12-23T06:01:53.135480Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                type  f1(micro)  f1(macro)\n",
      "0      Armed Assault   0.942915   0.853211\n",
      "1  Bombing/Explosion   0.967092   0.888545\n",
      "2         Kidnapping   0.988583   0.823629\n",
      "3              Other   0.964406   0.825997\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAFJCAYAAACLh9YgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtyUlEQVR4nO3deZyVZf3/8deb3QWFBDcGBQ01AhVDNHMhJUNNUNyXFJfcNU39ZWlqlmWW5RKKWi4huX81LEwzIXcFNdxIQlQY3JBQEWX18/vjugcP4wxzmDnMYe7zfj4ePJhzzj3nXOeemfe57mtVRGBmZi1fq3IXwMzMSsOBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFAty+QdKGkW5bz+GGSHlzO4zdJ+nmRrzVe0rGNLGejv7ee53tD0qBSPd9yXqcp77neMkoaKKm6aaWzlsyBbsslqYekkNSm5r6IGB0Ru5ezXE21Ih86lU7S1pKelfRJ9v/Wyzl2vKT5kj7O/r3ajEWteA50s0Yo/IDLM0ntgL8AtwCdgZuBv2T31+eUiFgz+7d5c5TTEgd6TmSX4mdLekHSPEl/lLSepPslzZX0kKTO2bFfuDRfzqX8I9n/H2Q1rq9LGi7psSLL1VnSXyXNkjQn+7qq1mGbSnpG0keS/iLpSwXfv72kJyR9IGmSpIH1vM5G2TEbFVGm44DDgP+Xvaf7Ch7eOjuHH0q6XVKH7HsGSqqW9ENJ7wA3Smol6RxJr0maLemOmrJL6iDpluz+DyRNkLRewetsLOnx7GfzoKQuBeUbIunl7PvGS/pKPe9jtexKY46kV4BtG3rvjTAQaANcHhELIuJKQMCuK+G1rIkc6PmyH/AtYDNgb+B+4MdAV9LP+rRGPOfO2f+dshrXkyv4/a2AG4GNgY2AT4Hf1zrmCOBoYANgMXAlgKRuwN+AnwNfAs4C7pbUtfaLRMT0iOgUEdMbKlBEXAeMBi7N3tPeBQ8fCAwGegJbAsMLHls/K8fGwHHAqcA+wC7AhsAcYER27JHA2kB3YB3ghOy91zgUOApYF2iXvTckbQbcCpxO+rmNBe6rp0Z8AbBp9u/b2WvWK/ug+qCef1fX821fBV6IZdcIeSG7vz6/lPR+9oE1cHllstJyoOfLVRHxbkTMBB4Fno6I5yNiPnAP0K+5CxQRsyPi7oj4JCLmAheTArDQqIh4KSLmAT8BDpTUGjgcGBsRYyPis4j4BzAR2HMlFvnKiHgrIv4H3AdsXfDYZ8AFWU31U1JInxsR1RGxALgQ2D9rjllECvIvR8SSiHg2Ij4qeK4bI2JK9jx3FLzOQcDfIuIfEbEI+A2wGrBDHWU9ELg4Iv4XETPIPgjrExFbZh96df07qZ5vWxP4sNZ9HwId6zn+h8AmQDfgOtKH0abLK5eVjgM9X94t+PrTOm6vWeoXlPTjgg6wkXU8vrqkayW9KekjUhNOpyywa8wo+PpNoC3QhVQTPqCwJgnsSKrJryzvFHz9Ccues1nZh2ONjYF7Cso2GVgCrAeMAh4AbpP0lqRLJbUt4nU2JJ0DACLiM9L56VZHWTfki+eu1D4G1qp131rA3LoOjoinI2Ju9qF3M/A4K/cD2Ao40CvTPGD1mhtZuH6hGSOz3OU4I+IXBR1gJ9RxyJnA5sB2EbEWnzfhqOCY7gVfb0Sq3b5PCqtRtWqSa0TEJcsrU5Eas8xo7e+ZAexRq3wdImJmRCyKiJ9GRG9S7fo7pKalhrxF+qAAQJJI52dmHce+zRfPXb2ydvmP6/n3hQ/jzMvAllk5amyZ3V+MYNmfta1EDvTKNAXoIGmvrNZ4HtC+nmNnkZoaNmnka3UkXR18kHUYXlDHMYdL6i1pdeAi4K6IWEIaWbG3pG9Lap11NA6so1O1cHhljyLL9S6Nf081RgIXS9o4K0NXSUOzr78pqW/2YfkR6UPqsyKe8w5gL0m7ZT+bM4EFwBP1HPsjpY7nKlKbfr0i4qsFH761/9X1YQwwnnTVcZqk9pJOye5/uPaBkjplP6sOktpIOoz0Af73It63lYADvQJFxIfAScAfSDW/eUCdE1Ii4hNSu/fjWdPC9iv4cpeT2oDfB56i7j/uUcBNpGaIDmSdt1m78FBSx+4sUo34bOr+ve1OanKoqyZblz8CvbP3dG+R31PbFcAY4EFJc0nvb7vssfWBu0hhPhn4F+l9LldEvErqO7iKdM72BvaOiIV1HP5T0nt+HXiwmOdfUdnr7kO6uviA1Hm9T015sia3+7PD25I6sGdlZT81O3ZKqctldZM3uLA8kHQeqY372nKXxaxcHOhmZjnhJhczs5xwoJuZ5YQD3cwsJxzoZmY5UbYV47p06RI9evQo18ubmbVIzz777PsRUedEwLIFeo8ePZg4cWK5Xt7MrEWSVO8SD25yMTPLCQe6mVlONBjokm6Q9J6kl+p5XJKulDQ1W295m9IX08zMGlJMG/pNpA0J/lTP43sAvbJ/2wHX8Pl6FmZWIRYtWkR1dTXz589v+GBrUIcOHaiqqqJt27YNH5xpMNAj4pEGVrAbCvwp29HkqWzFtQ0i4u2iS2FmLV51dTUdO3akR48eLLvarq2oiGD27NlUV1fTs2fPor+vFG3o3Vh2kf1q6l6M38xybP78+ayzzjoO8xKQxDrrrLPCVzvN2ikq6ThJEyVNnDVrVnO+tJk1A4d56TTmXJYi0Gey7K4pVdSzJnVEXBcR/SOif9eu9W2QY2ZmjVGKiUVjgFMk3UbqDP3Q7edmy+pxzt9K/pxvXLJXyZ+zlEr9not5v1deeSXXXHMNvXv35q233uK5557j4osv5qyzzlp6zKeffsrgwYN5+OGHad269XKe7XMjR45k9dVX54gjitlFsH4vvvgil112GTfddFOTnqc+DQa6pFuBgUAXSdWkLcTaAkTESGAsaRPYqaTNbo9aKSU1M2vA1VdfzUMPPUS7du148803uffee79wzA033MCwYcOKDnOAE06ob4e+L1q8eDFt2tQdrX379qW6uprp06ez0UbL3QK2URpscomIQyJig4hoGxFVEfHHiBiZhTmRnBwRm0ZE34jwfH4za3YnnHAC06ZNY4899mD06NFsu+22dQ75Gz16NEOHDgVg/Pjx7LLLLgwdOpRNNtmEc845h9GjRzNgwAD69u3La6+9BsCFF17Ib37zGwCmTp3KoEGD2Gqrrdhmm2147bXXGD9+PDvttBNDhgyhd+/ezJ8/n6OOOoq+ffvSr18/xo0bt/T19957b2677baVcg48U9TMcmHkyJFsuOGGjBs3jjPOOKPOYxYuXMi0adMoXBhw0qRJjBw5ksmTJzNq1CimTJnCM888w7HHHstVV131hec47LDDOPnkk5k0aRJPPPEEG2ywAQDPPfccV1xxBVOmTGHEiBFI4sUXX+TWW2/lyCOPXDpipX///jz66KOlPwE40M2sgrz//vt06tRpmfu23XZbNthgA9q3b8+mm27K7rvvDqTmkTfeeGOZY+fOncvMmTPZd999gTT5Z/XVVwdgwIABS8eMP/bYYxx++OEAbLHFFmy88cZMmZL2yl533XV56623Vsr7K9tqi7bqq8SOPGsZXqj+oM77Fy35jJff+pDO81O0vfvRfOYubr30+I8+WMjceZ8svf3arI9ZEK14ofoDtqzqRKtWrWjfvj0ArVq1YvHixUWXaY011ijquPnz57PaaqsV/bwrwjV0M6sYa3XqxJIlS1jQyOUJOnbsSFVV1dLO1gULFvDJJ5984biddtqJ0aNHAzBlyhSmT5/O5ptvvvR2nz59GvcGGuAaupmtFOW8Gnv/vXc5ZK9dmffxXFq1Erf8cST3PPwka3Zci6/vvCvPT3iK7Xca2KjnHjVqFMcffzznn38+bdu25c477/zCMSeddBInnngiffv2pU2bNtx0001La/7jxo1jr71WzrlRWoKl+fXv3z+8wcWqzU0upVMJ53Ly5Ml85StfaZbXqq/JpRiTX5zEqD9czS+uuHaZ+7es6tS0QhVhwYIF7LLLLjz22GP1Dm0sVNc5lfRsRPSv63g3uZhZRflK363Y9us7sWTJkmZ/7enTp3PJJZcUFeaNkasml0qoBZlZ0+178OFled1evXrRq1evlfb8rqGbmeWEA93MLCcc6GZmOeFANzPLiVx1iprZKuTCtUv8fB82eMjoG67lzlE3sEmvzZn17jtMfmkSp559HkeecOrSY+Z/+iknfXd/rr99zAqtuFhqCxcuZNCgQTz88MMlG/XiQDez3LjjT3/kulvvoU3bdrxdPYNxD3xx5Nu9t9/Cbnvs3WxhvmTJkjpfq127duy2227cfvvtHHbYYSV5LTe5mFku/OxHZ1A9/Q1O+u4BjL3nTvpsvQ1t6lg+d+y9dzJw9z0BmPDkYxy9/158/+hDl7t87n333cd2221Hv379GDRoEO+++y4AH3/88dJlcrfcckvuvvtuANZcc03OPPNMttpqK5588kl++9vf0qdPH/r06cPll1++tCz77LPP0iUCSsGBbma58JNf/o5111ufP9xxH9/93kl1HrNo4UKqp79Jt+6fby4xZfJLnPfL3y53+dwdd9yRp556iueff56DDz6YSy+9FICf/exnrL322rz44ou88MIL7LrrrgDMmzeP7bbbjkmTJrHaaqtx44038vTTT/PUU09x/fXX8/zzzwPQp08fJkyYULJz4CYXM6sYc/43m45rLdu2/9WttqHreuvXuXxuzcYU1dXVHHTQQbz99tssXLhw6TK5Dz300DKbVXTu3BmA1q1bs99++wFpKd1999136WqMw4YN49FHH6Vfv360bt2adu3aMXfuXDp27Njk9+cauplVjA4dVmPhgmVXWmzbrt3Sr+tbPvfUU0/llFNO4cUXX+Taa69dullF/a/Toeg2+gULFtChQ4cVeRv1cg3dmlepRz5AUaMfzGDZ5XPbr0CIfvjhh3Tr1g2Am2++een93/rWtxgxYsTSdvE5c+YsraXX2GmnnRg+fDjnnHMOEcE999zDqFGjAJg9ezZdunSpc6u8xnCgm9nKUcYP2lIvn3vhhRdywAEH0LlzZ3bddVdef/11AM477zxOPvlk+vTpQ+vWrbngggsYNmzYMt+7zTbbMHz4cAYMGADAscceS79+/YDSL6Wbq+VzvThXaa2U89nh0JI/Z0uooVfC76aXz11xw4YN45JLLmGzzTar83Evn2tmthzlXD630MKFC9lnn33qDfPGcJOLmVWcci2fW6hdu3YcccQRJX1O19DNrGTK1YSbR405lw50MyuJDh06MHv2bId6CUQEs2fPXuHhjG5yaYiH2ZkVpaqqiurqambNmrXSX+vdOZ+W/Dknz12t5M/ZFB06dKCqqmqFvseBbmYl0bZt26UzKFe2PSpg1FBjuMnFzCwnHOhmZjnhQDczywkHuplZTrhT1MwMcjGizYFu1lLlIICstIpqcpE0WNKrkqZKOqeOxzeSNE7S85JekLRn6YtqZmbL02CgS2oNjAD2AHoDh0jqXeuw84A7IqIfcDBwdakLamZmy1dMDX0AMDUipkXEQuA2YGitYwJYK/t6beCt0hXRzMyKUUwbejdgRsHtamC7WsdcCDwo6VRgDWBQSUpnZmZFK9WwxUOAmyKiCtgTGCXpC88t6ThJEyVNbI71HszMKkkxgT4T6F5wuyq7r9AxwB0AEfEk0AHoUvuJIuK6iOgfEf27du3auBKbmVmdign0CUAvST0ltSN1eo6pdcx0YDcASV8hBbqr4GZmzajBQI+IxcApwAPAZNJolpclXSRpSHbYmcD3JE0CbgWGhxdFNjNrVkVNLIqIscDYWvedX/D1K8A3Sls0MzNbEV7LxcwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCeKCnRJgyW9KmmqpHPqOeZASa9IelnSn0tbTDMza0ibhg6Q1BoYAXwLqAYmSBoTEa8UHNML+BHwjYiYI2ndlVVgMzOrWzE19AHA1IiYFhELgduAobWO+R4wIiLmAETEe6UtppmZNaSYQO8GzCi4XZ3dV2gzYDNJj0t6StLgUhXQzMyK02CTywo8Ty9gIFAFPCKpb0R8UHiQpOOA4wA22mijEr20mZlBcTX0mUD3gttV2X2FqoExEbEoIl4HppACfhkRcV1E9I+I/l27dm1smc3MrA7FBPoEoJeknpLaAQcDY2odcy+pdo6kLqQmmGmlK6aZmTWkwUCPiMXAKcADwGTgjoh4WdJFkoZkhz0AzJb0CjAOODsiZq+sQpuZ2RcV1YYeEWOBsbXuO7/g6wB+kP0zM7My8ExRM7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznCgq0CUNlvSqpKmSzlnOcftJCkn9S1dEMzMrRoOBLqk1MALYA+gNHCKpdx3HdQS+Dzxd6kKamVnDiqmhDwCmRsS0iFgI3AYMreO4nwG/AuaXsHxmZlakYgK9GzCj4HZ1dt9SkrYBukfE35b3RJKOkzRR0sRZs2atcGHNzKx+Te4UldQK+C1wZkPHRsR1EdE/Ivp37dq1qS9tZmYFign0mUD3gttV2X01OgJ9gPGS3gC2B8a4Y9TMrHkVE+gTgF6SekpqBxwMjKl5MCI+jIguEdEjInoATwFDImLiSimxmZnVqcFAj4jFwCnAA8Bk4I6IeFnSRZKGrOwCmplZcdoUc1BEjAXG1rrv/HqOHdj0YpmZ2YryTFEzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OcKCrQJQ2W9KqkqZLOqePxH0h6RdILkv4paePSF9XMzJanwUCX1BoYAewB9AYOkdS71mHPA/0jYkvgLuDSUhfUzMyWr5ga+gBgakRMi4iFwG3A0MIDImJcRHyS3XwKqCptMc3MrCHFBHo3YEbB7ersvvocA9xf1wOSjpM0UdLEWbNmFV9KMzNrUEk7RSUdDvQHfl3X4xFxXUT0j4j+Xbt2LeVLm5lVvDZFHDMT6F5wuyq7bxmSBgHnArtExILSFM/MzIpVTA19AtBLUk9J7YCDgTGFB0jqB1wLDImI90pfTDMza0iDgR4Ri4FTgAeAycAdEfGypIskDckO+zWwJnCnpH9LGlPP05mZ2UpSTJMLETEWGFvrvvMLvh5U4nKZmdkK8kxRM7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznCgq0CUNlvSqpKmSzqnj8faSbs8ef1pSj5KX1MzMlqvBQJfUGhgB7AH0Bg6R1LvWYccAcyLiy8DvgF+VuqBmZrZ8xdTQBwBTI2JaRCwEbgOG1jpmKHBz9vVdwG6SVLpimplZQ9oUcUw3YEbB7Wpgu/qOiYjFkj4E1gHeLzxI0nHAcdnNjyW92phCNydBF2q9jyb7aeV+1vl8lo7PZWm1oPO5cX0PFBPoJRMR1wHXNedrNpWkiRHRv9zlyAufz9LxuSytPJzPYppcZgLdC25XZffVeYykNsDawOxSFNDMzIpTTKBPAHpJ6impHXAwMKbWMWOAI7Ov9wcejogoXTHNzKwhDTa5ZG3ipwAPAK2BGyLiZUkXARMjYgzwR2CUpKnA/0ihnxctqomoBfD5LB2fy9Jq8edTrkibmeWDZ4qameWEA93MLCcc6GZm9WhpEyQd6M1IUudsWCeSvlzu8pjZ8tWM1pO0oaRVPi+bdWJRJcs+6XcCtpU0B+gh6ccR8XGZi5ZbkhQRIaldtmyF1SJptYj4tNzlWJVJOhLYlTTLfUGZi7NcHuXSDCT1i4jnJXUAHictlTAwIv4jqU1ELC5zEXNL0neAE4FxwLMRMa7MRSq7gg+6zYFhwKiIqC53uVZFko4GtgGuiohVfqmSVf4SIieukfRQRMwHHgUeAk6W1MlhvvJkgXUs8HegHTBc0l7lLVX5ZWG+O3AZ8D3gcC95ndS0mRe0ne8InFTw+CrdquEa+kokqVVEfJZ9/SjwZkQcLqkj8GugdUR8T9LXgQ6uPZaOpL7Aw8C5EXFdFli7kZq97omIv5SzfOUkqR9wO3AQaaG9r5IW3bslImov61Exaq5csq83jog3s69/D+wO9ImIhavyVbUDvRlJehJ4LyKGSuoGXAD0A9oC+0TEG+UsX95IuhfYKiJ6ZrergCGkWtf3I2JWGYtXNpIOAPaPiIOy23uQfhf/AtwcEW+Vs3zlls2MHwz8F/hvRFwt6UbS3+p2EbHKtqO7yWUlk7SfpCsAIuLrQFdJ90bEzIg4jrQhyP4O86YpuFT+qqTdJXWJiH2AxyVNBMjaie8DflhJYV5wbmqaCyYCHbMgJyLuB14AtiC1F7e44XqlImkIcCBwCLAlsDVARBwFTAb+lR23Sp4f19BXMkkbA/8H/D0izs3uewQgInYuZ9nyRtJQ4Kek5oOPgQkRcZmkm4FtI6L2TlsVQ9IgUnPT6xFxk6QzgXVJK6U+DVxB6t/ZICIOL19Jy0vSYcASYDVSqH8na2bpERFvSNogIt4ubynr5xr6SiJpK0kbZe1w+wDflPRrWBrkn0jaqJxlbOkkrVkwrn8N4GjgiIj4DnAt0E3SkIg4Epgm6RtlLG6zy7aPRNIOwFXAHOAXkk4D7iAF+UBSc8uJwN+AVtlorIpSMMZ8OvBz4NiI2D0L81OBM7O281U2zMHj0FcKST2BU4BXJd0WETMkHQRMlLRWRBwfEYPLXMwWLetY/g1wLmmXmc9INc6NSc0Hj5M6/AYDY7KQrwiS1ift8bsgG+lzHPDLiPiTpLHALUDbiLgM+D9Jq5HGWV8MHJmNxso1SRsA70bEZ5IOAXpKegh4AriH9MG2J9AVGE46L6tkR2gh19BLoHZ7WkS8DtxJ2vRjmKTuETGDtDzn1pK+VIZi5kpEzAV+Aqwl6YBscsw1wBBJ22cTiZ4F1s5q8hXxuy6pLXAEsEl214bABqR9fjeMiCnAocAxkn4KkJ27LYBDImJSGYrdrCR1B34E7C9pOPBjoANpX4ddScuBvwScAOwCDI+Il8pT2hXjNvQmqjXU6Rg+30v1T8A3SBM3grSD09eAU7Nwt0aqNRx0KHA+cBHwKvAt0tjq+0mdWydHxNhylbUcJK0OdCZdvXwf6A8cBvwHuCsi3lFaemK9iHi8fCUtj+yK5ARgfaAncGFEvJKN/jkP+HFE/K2mOa8l1MxrVEStZWUqCPPTgcOB10htuVcBz5Dact8kjR44z2HeNNkH6GeSBko6KBtP/kvgNKAXcCOpiWECcHAlhXnNlWJEfAKsAXQkTR6aQOqY34w0iWiDiJhaaWEuqZOkdbMrkoeBd4D1SDX19hFxJ6licK2k70TE4pYU5uA29EarVTPvCfQlTT74AWmERWtSqJ8REb+TNMLriTRdNstxMOncHpPdd5ek+cDpQMeI+HMZi1gWNb+P2Vh7RcQUSeeTfh+vINXU2wD7kmbNVqJ+wA6SOgPtgQuBeaTmpv0k3RkRd0taBLxSvmI2nptcGqFWmK+fXcKuD3wF+Blp5MC3SX9I40k1RmqaCaxxshroOsC9pLHkj0saSDrvY0gfqucCB0TEO2UqZtlI2ptUI38HmEVqG24FnExqIz4BWDsiKmoD9+xD7mNgK9KH/gDSKJb7s871I0l9Di+TJla1qFp5ITe5NEJBmJ8I/DwbufIO6Y/mqewXoivwZ+D8iPjMYd50kbxPWgvnJ5JuIg236w/8JCL+DhxYoWG+GXAU6f3vDLwNnEVq7vstaRTQ5hUY5kOBu4AbSDXyIaROz29K+lrWuT4CeA/YCFi9TEUtCdfQG0lpFbYTgf0iYrqk9qSOqH8BT5KaX3aNiP+UsZgtXkFTQm/SKpUvZf/3ByZFxJOSajpCDwMWRwX9UhdctVxNqmUeUzNSRdKDwDMRcZ6kNSJiXhmL2uwkfZPUh3UI8AapT+EW4DngddL5+iVpuGsP4JGI+F85yloqrqEXqXBoYjZhYwBp3G5kHaL/IrVPbgtcD+zgMG+6LMyHAHcDp5KGJn6dNFrjyeyP9lLSwlKLKiXMCzpAa65afk2aFLNj1qcDqVa6ODuuosI8swNwZUQ8C8yLtLzGoaT5CTuQRkX9mdSE9++WHubgQC9KrTbzr5E6lf5GunS7KjvsHFINsV1EPB5em6XRJK1V8PXapIkdh0XEENIolo1IG4VsCOwNXBARY2rPB8iz7INuL0nXS7qE1Ln3E9LCYxdns0EvIq3bUlEKfg+qgC7Z1wsktY6I6aTO9LWBf5L6GXbNy9+rA70IBWF+Jqk2uF5E3EcaZ35oRFxOGkEgwCNZmiCbdj5O0vHZXZ8Ca5KWeCUbpjiXNAnmLVLb+ZjCD908qwkrSVuQxt9PJE3pf5RU0fgpqWmhL3B6RPy1TEUtm4Lfg7uAb2Rt5UG6mm4LfAB8BFRHxBMRMa1MRS05B3qRJA0D9gP2irRIT0/S784nSstt/g44MSI+KmtBW7hI087/H/ADSUdnQz1HA1+RtGN22D9Jf5yr1TQlVEKYw9Ka+QDS1eFtEXFtRPyKtAnDLaQRLleSlmTeOBt9VameIi0BcZCk/tnghEWk5pZ1yOGw7dy9oZVoPjAJOExps4SdSZdx55Nm4B3gNvOmyWqfioh/ZjX0GyQtAB4kLaNwgaTXgEHAD6Jy98J8lfS3O1hpaWZFxO3Z+PzOEfEPpcXK9gBuLWdByyki5km6nrRr1WVK+xEsBPYnXeHNKWsBVwKPcqlD9ofxVeBfETExu29T0rCwr5OaXV4jte2OjYgnylTU3CgYzdID+Dgi3pfUn7SzzjmkmY59ST+XVyNiYiU1s2TnpidpUa0pktYEHiBVMkYAa5GaGPYsGOVScSNb6pJN9e9PmhvyPnB/tID9QRvDgV4HSVeSpvFPIQ1BvAaYFhGLle0gL+lAUifU0Dy1wZVDQWDtQRo59CFpadcbSetS3w5cFhHXlbGYZZWNp/4R8C5pFuPvSB2hfyVNX78P+GdEPKiCtW6ssrgNvW53kWo/+5MmGhwLjJK0HrCG0h6g55Iu2xzmTVTQLnwy8F3gbNKEmLNI44ePA86TVFVJI1lqZL9vPwT2JC3vejjpHHUE9gLeAtaJiAezb3EtrUI50OsQEY+QZnoeFRHHkyazHEQa13sJaZGjXaOFLKm5qsuGKR4D9I6Il7NmrgdJozYGRcS/gH4RUV0JTSx1aEWasr4dqWP+CFITwuWkFQP3BXbOhi9WTAexfZEDvZZs0hCkxYzWkHQkaUeXQaQxqxNIbesVNYV6ZZG0FalZ5RpgpqTLASLiNT5fpRLSULOKUDA08UsAkVZFnAjsBlwcEeNIH3hrkNrU55ICfmR5SmyriooPdH2+TVfNzLsl2UOzSMux/p5UU38462y6IS+TEMpJUs2Kf1cB346If5NqoV0kjZW0F2nS0HhY5ueSe1kT1HeA2yWNkdQ7axOvBi5S2mFnX+BXEfGq0tZoH/n30io60CVtCTyYLa4Vhe2zEfEeaTr1VOC/Bfe7s6mRtOyuQR2z/8cAC7KvXwJ+RRqxcT5wWtbJVxHDawtq5l8DziB1uk8hDdfcGRhF2jjlUOCiiHgMWtYGDLZyVfwoF0m3ktZGHh4RHxWMuGgVaSOFkaSRLrdUUi2x1CT1IjUL3EVaXOs+0vIJO5PGBh9LmgH6Hmlnp+HAoog4oxzlbU7ZUM2vRtolpzvpQ+2ziDg8e/zHQB/g+ogYJ2n1bEJbRQzbtOJVZA1dSc17vxbYEnhAUseamnpBTfxR0nAwh3kjKS3tei/QJpup9zZpV6e/kj4sdyatencTqeP5deBmoJ2kdctQ5ObWm2yXHFKzyjPAl5W2RCMifkG6SjxNUudIOxK589O+oKJr6JJOBfYhbURxNmktlj0iYm7WLulL2SZSWvb2JtI48tuzD9L+EfFMwTFXkqarzwY2zSYNtSZ9ACyo63nzouCK8CjSkgcnRsT4bKZsf+DvEXF3duymWWexWZ0qsoYOS9srtweuiYgxEbETaR2MhySt7TBvumyG3qmkkRi3Z3ePI3V2IqmN0obG/YHuETGnYAbokgoI81ZZmO9NWiVxOnCNpMERcS2ppj4sm8SGw9waUhGdTbDsEriwdCTBDNLi9jVOJF3aXiHpKF/SNp7Svo1nA7cBi7LhiJuSNhH4CSztzFss6a+kFQPJ7s/1ec+a9uZmfTTdSNvGHUXqgN8J+LWkBRFxfdYhPLmc5bWWoyICvTDMlVZNhNRWeQdws6Q3SKuy7UQaRXB53kOlGaxG2lzhHVI/xSWk5pS9aw6QtB1pfZYbokK2jcs+6I6W9KeImEXqCJ6cjTVH0j2kCUS3SDoiIq4pY3GthamIJpeCMD+RNJ28PWlpzTmkTZ1PJIXOz4GrPZ2/JGoC+tCIeJk0df1JSb+GpUNGRwHTKyjMNyQta3s70EbSsEjLLbeTdCMsHW8/iTRxaFHZCmstUkV0imbt5d1JnZ/DgYNJU/l3zxbc6kiqUbaqlHBpDkqbVdwN3BkRN2WzQk8g/Sw2BM6NiPvLWcbmkjWdPAL8KCL+JekHpNEtfwaeJTVNLQbuIa0ueUhEPOuhibYicltDrzVJKID/kVapO5/UKbdXFuanAlUR8Z7DvDQk9ZS0eaTNKi4AqrL1Wl4G/kAaTXRRpYR5phVpAtX72e0bgBeAoaTO+b1JV43rAWdE2gcz9/0JVlq5bUMvaGbZHHg3Ij6Q1AU4MCI6Z48dSFrd796yFTQnCobf9QdOA2ZIGkFaN747sG2kjSsmAQdnQ0NzX/uUtAkwJyLmSPqI1GZO9vt4M2lRsr1IQzQvLvi+3J8bK73c1tABlDbK/Ttpt5KzI62cOEHSvVmb5VnAMRExo6wFzYGC9UeuJjUdfJPUvAVpa7RfStoiIhZHWkyqUmqfmwDTJHUCPiZtfQZARHxIWsumGthNBdvFVci5sRLLVRt6rdEsG5DGQN8AfIm01varEfELSTuQ1gv5T3hBo5LImlRuBn4dEU9I+jbwHdJkoUtIy75+Cbg20j6hFUNpB6yRpNr5naSlDmZk/88jdX6+5s54a6rcNLnUCvMjSUO/upB2eHmDtMPL6ZKujIjTylbQnIq0Ds4S0ga8T0TEA9m6JGeQVq58n9T0VVFhDhARf5c0HHgYeI4U5oNI7eWdgLMd5lYKuWlyKQjzA0gjKV4GtiablRhpedbfA62Udh6yJqjpdFbaRWiz7O6bgXWVlr6FNNOxmtRG/HRE3NH8JV01RMR4UohvD1wQEUdFxJ4RsUPNGHSzpmrxTS61aua9ScO+jo2IR7M23TOBPwK3R8QiSe3zPqW8uWTn91ekisGfSef5IGAgqR29LynMzwL+GhH3VXpnn9K+qaOAXllHaUWfDyutFh3otcL8JNKl7AmkYXH7RcSn2R/QL0ibAdxWvtK2fLXO9xbApaTp/R+RRgrdSZqg9SVgAPA8aYu0G4BvRcSbZSj2KkfSnsAnWa3drGRadKDXkLQvabeb75L6Bc4gzQb9fhbqu5M6RB0ojZQ1Ux0PXExq9/0FqUlrWETMVFrTezTwaESck33PNqS+i1Mi4sUyFHuV5tq5lVqLD/Sspng38OeIuFhSe9IiUMeTFt46KpvgYk0gaSPSh+Q8Ugfn10ijiJ4hzQSdKaknaX2cwyJiSvZ960XEu2UqtllFaXGdonV0aH5EutT/rqTts/bx/5Dac2cAnZu5iLkUEdNJmzafRzq3z5KG4m0K7Cepe0S8DuwUEVOU7dXqMDdrPi2qhp7Vxl8hrckyOSKuy+5vD5wE7EraFf0ppY0U2lTiMLmVSdLGwA9ItfXTSMNDjyT9XEaQto3zvqtmZdDSAr2KtIjRfcBupDHmdwAPR8Q8SSeTRlmcFQU74tiKk7Q2sHpEvJ11LH+dtFLgL0gLa52Y3T6DNPZ8VraqopmVSYsKdABJvyVtMnwYsD9pB/S1ScMTO5DC5klP5288pV2ELiON5Z9KuiL6KanT+R3SUMVPSQtvLQaOd+eeWfm1mEAvWPypHWkTitOBLUhD4v5B6gCdSVqS9cOyFTQnsjHm+5Ompf8nIi7L7r8c2CQihkj6MtDeNXOzVUOLmfqfhXnNkrj/JdUgvwb8ICLuzWYrznKYN43SPpefRcRfJX0CfB9YW9kGxRFxuqTxknpGxNRyl9fMPtdiAh2WTu9fKOkW4F/AiIi4N3tsSjnLlgfZVdBnkrYGPgSeJPVTnAvsIelxUo29CnDHp9kqpsU0udSWLXbUA7g0Ij4pb2nyQ9I3SR3NY0mdnj8m9VGcR1oK9g1gVM0HqZmtOlrcOPQCTwHblLsQeVCw0FYnYDNgCGnS0ETgStJY//OB14HzsiYu1f1sZlYuLbaGDmk0hmvnpZGtL3II0Av4cUQ8nO3w9F3SAlvHA+/VbE5hZquellxDx2FeGpK2Iy2ydRdpQ4r9JG0YEe+Tdht6AOjsMDdbtbXoGro1Xbaz05+BFyLi+5LWAK4D5pBWqJwhqW1ELCprQc2sQS26hm4l8TFwP7CnpMERMQ84ljR567xs/XiHuVkL0KKGLVrTFUzQ6gdsAEwnLbL1DnCypM8i4kFJhwKbezMQs5bDgV5hsjAfDFxDGpq4M2kkyzOkK7YfZaH/APDvshXUzFaYA73CZEMTjwaOyUay7EpaD+d90tZo7Ukdo2bWwrgNPeckbSppmKShABHxAWkWaF9JrSPiYWA8aW2cVsAfImJimYprZk3gQM+xbH2bvwDfAH4o6YTsocdIq1Jun91+AXiPtH78kmYvqJmVhIct5pSk3qQ9Ps+PiPskHQ6sRVqZchpp5udmQGvSqpUXRsT/lau8ZtZ0DvSckrQj8EhEtMpuv0BaXrgb8Chpan8PoA8wIyKe96bFZi2bAz3Hsp2GRpBq5I9ExEXZevIvAX+MiF+VtYBmVlIO9JyTtBtp6n67mr0+JR0DdKrZtMLM8sGdojkXEf8krZ44BSDbZehs4MVylsvMSs/j0CtARIyV9Fm2A9HrwOkR8WC5y2VmpeUmlwqSNb+sFRH3lLssZlZ6DvQK5NEsZvnkQDczywl3ipqZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McuL/A5FsRPzQSy+oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_select = df[df.type.isin(['Armed Assault','Bombing/Explosion','Kidnapping','Other']) ]\n",
    "df_select = df_select[df_select.threshold == 0.5][['type','f1(micro)','f1(macro)']]\n",
    "df_select = df_select.reset_index(drop=True)\n",
    "print(df_select)\n",
    "ax = df_select.plot.bar(xticks=df_select.index, title = 'multi-label;  threshold = 0.5', rot= 45);\n",
    "ax.set_xticklabels(['Armed Assault','Bombing/Explosion','Kidnapping','Other']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-23T06:02:10.770025Z",
     "start_time": "2020-12-23T06:02:10.622813Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                type  f1(micro)  f1(macro)\n",
      "0      Armed Assault   0.945601   0.853758\n",
      "1  Bombing/Explosion   0.969778   0.893562\n",
      "2         Kidnapping   0.987911   0.801278\n",
      "3              Other   0.967092   0.820029\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAFJCAYAAACLh9YgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtiUlEQVR4nO3deZyVZf3/8debHRWVBFdQ0HAhUDFEs1BTMtQExX3JPXctt9/X0pQszTTLJRQ1txB3k7AwzYTcE8xwIwlRYXBDQkWQ1c/vj+sePA4zzGHmMIe5z/v5ePDgLPc55zr3zLzPda5VEYGZmTV/LcpdADMzKw0HuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3ZYhaaikO5Zz/+GSHl3O/bdJ+kWRrzVO0vENLGeDH1vH870laUCpnm85r9OY91xnGSXtKqmqcaWz5syBbsslqZukkNSq+raIGBkRe5SzXI21Ih86lU7StpJekDQv+3/b5Rz7aY1/SyRd24TFrWgOdLMGKPyAyzNJbYA/AXcAHYHbgT9lty8jItao/gesD3wG3NdU5a10DvScyL6KnyvpJUlzJd0saT1JD0uaI+kxSR2zY5f5ar6cr/JPZP9/lNW4viHpaElPFVmujpL+LGmmpNnZ5S41DttM0vOSPpH0J0lfKXj8jpKekfSRpImSdq3jdTbOjtm4iDKdABwO/L/sPT1UcPe22Tn8WNI9ktplj9lVUpWk/5P0HnCrpBaSzpP0hqRZku6tLrukdpLuyG7/SNJ4SesVvM4mkp7OfjaPSupUUL5Bkl7NHjdO0lZ1vI/22TeN2ZJeA7av7703wK5AK+CqiFgQEdcAAnYr4rH7Ax8AT66EclktHOj5sj/wHWBzYB/gYeAnQGfSz/qMBjznztn/a2c1r2dX8PEtgFuBTYCNSTW239U45kjgWGADYDFwDYCkjYC/AL8AvgKcAzwgqXPNF4mIaRGxdkRMq69AEXEjMBK4PHtP+xTcfRAwEOgObA0cXXDf+lk5NgFOAE4H9gV2ATYEZgPDsmOPAtYCugLrACdl773aYcAxwLpAm+y9IWlz4C7gR6Sf2xjgoTpqxBcBm2X/vpu9Zp2yD6qP6vh3XR0P+xrwUnx5jZCXstvrcxTwh/D6Ik3GgZ4v10bE+xExg1Qr+mdEvBgR84EHgT5NXaCImBURD0TEvIiYA1xCCsBCIyLilYiYC/wUOEhSS+AIYExEjImIzyPib8AEYK+VWORrIuKdiPgf8BCwbcF9nwMXZTXVz0ghfX5EVEXEAmAocEDWHLOIFORfjYglEfFCRHxS8Fy3RsTk7HnuLXidg4G/RMTfImIR8GugPbBTLWU9CLgkIv4XEdPJPgjrEhFbZx96tf07pY6HrQF8XOO2j4EOy3stSZuQfs63L+84Ky0Her68X3D5s1qur1HqF5T0k4IOsOG13L+apBskvS3pE1ITztpZYFebXnD5baA10IlUEz6wsCYJfItUk19Z3iu4PI8vn7OZ2YdjtU2ABwvKNglYAqwHjAAeAe6W9I6kyyW1LuJ1NiSdAwAi4nPS+dmolrJuyLLnrtQ+BdascduawJx6Hvd94KmIeHMllMnq4ECvTHOB1aqvZOG6TDNGZrlflyPi0oKOsJNqOeRsYAtgh4hYky+acFRwTNeCyxuTarcfksJqRI2a5OoRcdnyylSkhjQD1HzMdGDPGuVrFxEzImJRRPwsInqSatffIzUt1ecd0gcFAJJEOj8zajn2XZY9d3XK2uVrjkKp88M48yqwdVaOaltnty/Pkbh23uQc6JVpMtBO0t5ZrfECoG0dx84kNTVs2sDX6kD6dvBR1mF4US3HHCGpp6TVgIuB+yNiCWlkxT6SviupZdbRuGstnaqFwyu7FVmu92n4e6o2HLgka15AUmdJg7PL35bUO/uw/IT0IfV5Ec95L7C3pN2zn83ZwALgmTqO/XHW8dyF1KZfp4j4WuEolBr/avswBhhH+tZxhqS2kk7Lbn+8rteRtBPpG4VHtzQxB3oFioiPgVOA35NqfnOBWiekRMQ8Urv301nTwo4r+HJXkdqAPwSeA/5ayzEjgNtIzRDtyDpvs3bhwaSO3ZmkGvG51P5725XU5FBbTbY2NwM9s/c0qsjH1HQ1MBp4VNIc0vvbIbtvfeB+UphPAv5Bep/LFRGvk/oOriWds32AfSJiYS2H/4z0nt8EHi3m+VdU9rr7kmrcH5E6r/etLk/W5PZwjYcdBfwx6zOxJiR3QFseSLqA1MZ9Q7nLYlYuDnQzs5xwk4uZWU440M3McsKBbmaWEw50M7OcKNuKcZ06dYpu3bqV6+XNzJqlF1544cOIqHUiYNkCvVu3bkyYMKFcL29m1ixJqnOJBze5mJnlhAPdzCwn6g10SbdI+kDSK3XcL0nXSJqSrbe8XemLaWZm9SmmDf020oYEf6jj/j2BHtm/HYDr+WI9CzOrEIsWLaKqqor58+fXf7DVq127dnTp0oXWrVvXf3Cm3kCPiCfqWcFuMF/sSvKcpLUlbRAR7xZdCjNr9qqqqujQoQPdunXjy6vt2oqKCGbNmkVVVRXdu3cv+nGlaEPfiC8vsl9F7Yvxm1mOzZ8/n3XWWcdhXgKSWGeddVb4206TdopKOkHSBEkTZs6c2ZQvbWZNwGFeOg05l6UI9Bl8edeULtSxJnVE3BgRfSOib+fOdW2QY2ZmDVGKiUWjgdMk3U3qDP3Y7edmX9btvL+U/Dnfumzvkj9nKZX6PRfzfq+55hquv/56evbsyTvvvMO//vUvLrnkEs4555ylx3z22WcMHDiQxx9/nJYtWy7n2b4wfPhwVlttNY48sphdBOv28ssvc+WVV3Lbbbc16nnqUm+gS7oL2BXoJKmKtIVYa4CIGA6MIe3CPoW02e0xK6WkZmb1uO6663jsscdo06YNb7/9NqNGjVrmmFtuuYUhQ4YUHeYAJ51U1w59y1q8eDGtWtUerb1796aqqopp06ax8cbL3QK2QeptcomIQyNig4hoHRFdIuLmiBiehTmRnBoRm0VE74jwfH4za3InnXQSU6dOZc8992TkyJFsv/32tQ75GzlyJIMHDwZg3Lhx7LLLLgwePJhNN92U8847j5EjR9KvXz969+7NG2+8AcDQoUP59a9/DcCUKVMYMGAA22yzDdtttx1vvPEG48aNo3///gwaNIiePXsyf/58jjnmGHr37k2fPn0YO3bs0tffZ599uPvuu1fKOfBMUTPLheHDh7PhhhsyduxYzjzzzFqPWbhwIVOnTqVwYcCJEycyfPhwJk2axIgRI5g8eTLPP/88xx9/PNdee+0yz3H44Ydz6qmnMnHiRJ555hk22GADAP71r39x9dVXM3nyZIYNG4YkXn75Ze666y6OOuqopSNW+vbty5NPPln6E0AZF+eyVV8ltvta8/BS1Ue13r5oyee8+s7HdJyfou39T+YzZ3HLpcd/8N67tF9jzaXX35j5KVtt3YeZS9qzQdu2bLbZZuyxxx5Aah4prFkDzJkzhxkzZrDffvsBafJPtX79+i0dM/7UU09x+umnA7DllluyySabMHnyZLbeemvWXXdd3nnnnZKch5pcQzezitGuXXsWLvjy2O7WbdosvdyiRQvatm279PLixYuLfu7VV1+9qOPmz59P+/bti37eFeFAN7OKsebaa7NkyRIWNHB5gg4dOtClS5elna0LFixg3rx5yxzXv39/Ro4cCcDkyZOZNm0aW2yxxdLrvXr1atgbqIebXMxspShn89qHH7zPoXvvxtxP59Cihbjj5uE8+PizrNFhTb6x8268OP45duy/a4Oee8SIEZx44olceOGFtG7dmvvuu2+ZY0455RROPvlkevfuTatWrbjtttuW1vzHjh3L3nuvnHOjtARL0+vbt294g4tVm9vQS6cSzuWkSZPYaqutmuS16mpDL8aklycy4vfXcenVN3zp9q27rN24QhVhwYIF7LLLLjz11FN1Dm0sVNs5lfRCRPSt7fhc1dAr4Y/GzBpnq97bsP03+rNkyZIVGoteCtOmTeOyyy4rKswbIleBbmZWjP0OOaIsr9ujRw969Oix0p7fnaJmZjnhQDczywkHuplZTjjQzcxywp2iZrZyDF2rxM/3cb2HjLzlBu4bcQub9tiCme+/x6RXJnL6uRdw1EmnLz1m/mefccr3D+Cme0Y3+SiXQgsXLmTAgAE8/vjjJRv14kA3s9y49w83c+NdD9KqdRverZrO2EeWHco86p472H3PfZoszOsaHtmmTRt233137rnnHg4//PCSvJabXMwsF37+4zOpmvYWp3z/QMY8eB+9tt2OVrUsnztm1H3susdeAIx/9imOPWBvfnjsYctdPvehhx5ihx12oE+fPgwYMID3338fgE8//XTpMrlbb701DzzwAABrrLEGZ599Nttssw3PPvssv/nNb+jVqxe9evXiqquuWlqWfffdd+kSAaXgQDezXPjpL3/Luuutz+/vfYjv/+CUWo9ZtHAhVdPeZqOuX2wuMXnSK1zwy98sd/ncb33rWzz33HO8+OKLHHLIIVx++eUA/PznP2ettdbi5Zdf5qWXXmK33XYDYO7cueywww5MnDiR9u3bc+utt/LPf/6T5557jptuuokXX3wRgF69ejF+/PiSnQM3uZhZxZj9v1l0WPPLbftf22Y7Oq+3Pm2Xs3xuVVUVBx98MO+++y4LFy5cukzuY4899qXNKjp27AhAy5Yt2X///YG0lO5+++23dDXGIUOG8OSTT9KnTx9atmxJmzZtmDNnDh06dGj0+3OgW9MqdUcZFNVZZgYNXz739NNP56yzzmLQoEGMGzeOoUOH1vM67Ypuo1+wYMGX1lVvDDe5mFnFaOjyuR9//DEbbbQRALfffvvS27/zne8wbNiwpddnz569zGP79+/PqFGjmDdvHnPnzuXBBx+kf//+AMyaNYtOnTrVulVeQ7iGbmYrRxm/OZV6+dyhQ4dy4IEH0rFjR3bbbTfefPNNAC644AJOPfVUevXqRcuWLbnooosYMmTIlx673XbbcfTRR9OvXz8Ajj/+ePr06QOUfindXC2f69UWS2ulnM92h5X8OZtDk0sl/G56+dwVN2TIEC677DI233zzWu9f0eVz3eRiZhWlcPncclq4cCH77rtvnWHeEG5yMbOKU67lcwu1adOGI488sqTP6Rq6mZVMuZpw86gh59KBbmYl0a5dO2bNmuVQL4GIYNasWSs8nNFNLvXxuGmzonTp0oWqqipmzpy50l/r/dmflfw5J81pX/LnbIx27drRpUuXFXqMA93MSqJ169ZLZ1CubHtWwKihhnCTi5lZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YRHuZiZQS6GKBdVQ5c0UNLrkqZIOq+W+zeWNFbSi5JekrRX6YtqZmbLU2+gS2oJDAP2BHoCh0rqWeOwC4B7I6IPcAhwXakLamZmy1dMDb0fMCUipkbEQuBuYHCNYwJYM7u8FvBO6YpoZmbFKKYNfSNgesH1KmCHGscMBR6VdDqwOjCgJKUzs7rloM3XSqtUo1wOBW6LiC7AXsAIScs8t6QTJE2QNKEp1nswM6skxQT6DKBrwfUu2W2FjgPuBYiIZ4F2QKeaTxQRN0ZE34jo27lz54aV2MzMalVMoI8HekjqLqkNqdNzdI1jpgG7A0jaihToroKbmTWhegM9IhYDpwGPAJNIo1lelXSxpEHZYWcDP5A0EbgLODq8KLKZWZMqamJRRIwBxtS47cKCy68B3yxt0czMbEV46r+ZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOFBXokgZKel3SFEnn1XHMQZJek/SqpDtLW0wzM6tPq/oOkNQSGAZ8B6gCxksaHRGvFRzTA/gx8M2ImC1p3ZVVYDMzq10xNfR+wJSImBoRC4G7gcE1jvkBMCwiZgNExAelLaaZmdWnmEDfCJhecL0qu63Q5sDmkp6W9JykgbU9kaQTJE2QNGHmzJkNK7GZmdWqVJ2irYAewK7AocBNktaueVBE3BgRfSOib+fOnUv00mZmBsUF+gyga8H1LtlthaqA0RGxKCLeBCaTAt7MzJpIMYE+HughqbukNsAhwOgax4wi1c6R1InUBDO1dMU0M7P61BvoEbEYOA14BJgE3BsRr0q6WNKg7LBHgFmSXgPGAudGxKyVVWgzM1tWvcMWASJiDDCmxm0XFlwO4Kzsn5mZlYFnipqZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeVEUYEuaaCk1yVNkXTeco7bX1JI6lu6IpqZWTHqDXRJLYFhwJ5AT+BQST1rOa4D8EPgn6UupJmZ1a+YGno/YEpETI2IhcDdwOBajvs58CtgfgnLZ2ZmRSom0DcCphdcr8puW0rSdkDXiPjL8p5I0gmSJkiaMHPmzBUurJmZ1a3RnaKSWgC/Ac6u79iIuDEi+kZE386dOzf2pc3MrEAxgT4D6FpwvUt2W7UOQC9gnKS3gB2B0e4YNTNrWsUE+nigh6TuktoAhwCjq++MiI8jolNEdIuIbsBzwKCImLBSSmxmZrWqN9AjYjFwGvAIMAm4NyJelXSxpEEru4BmZlacVsUcFBFjgDE1bruwjmN3bXyxzMxsRXmmqJlZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU4UFeiSBkp6XdIUSefVcv9Zkl6T9JKkv0vapPRFNTOz5ak30CW1BIYBewI9gUMl9axx2ItA34jYGrgfuLzUBTUzs+UrpobeD5gSEVMjYiFwNzC48ICIGBsR87KrzwFdSltMMzOrTzGBvhEwveB6VXZbXY4DHq7tDkknSJogacLMmTOLL6WZmdWrpJ2iko4A+gJX1HZ/RNwYEX0jom/nzp1L+dJmZhWvVRHHzAC6Flzvkt32JZIGAOcDu0TEgtIUz8zMilVMDX080ENSd0ltgEOA0YUHSOoD3AAMiogPSl9MMzOrT72BHhGLgdOAR4BJwL0R8aqkiyUNyg67AlgDuE/SvyWNruPpzMxsJSmmyYWIGAOMqXHbhQWXB5S4XGZmtoI8U9TMLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY54UA3M8sJB7qZWU440M3McsKBbmaWEw50M7OccKCbmeWEA93MLCcc6GZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznHCgm5nlhAPdzCwnigp0SQMlvS5piqTzarm/raR7svv/KalbyUtqZmbLVW+gS2oJDAP2BHoCh0rqWeOw44DZEfFV4LfAr0pdUDMzW75iauj9gCkRMTUiFgJ3A4NrHDMYuD27fD+wuySVrphmZlafVkUcsxEwveB6FbBDXcdExGJJHwPrAB8WHiTpBOCE7Oqnkl5vSKGbkqATNd5Ho/2scj/rfD5Lx+eytJrR+dykrjuKCfSSiYgbgRub8jUbS9KEiOhb7nLkhc9n6fhcllYezmcxTS4zgK4F17tkt9V6jKRWwFrArFIU0MzMilNMoI8HekjqLqkNcAgwusYxo4GjsssHAI9HRJSumGZmVp96m1yyNvHTgEeAlsAtEfGqpIuBCRExGrgZGCFpCvA/UujnRbNqImoGfD5Lx+eytJr9+ZQr0mZm+eCZomZmOeFANzPLCQe6mVkdmtsESQd6E5LUMRvWiaSvlrs8ZrZ81aP1JG0oaZXPyyadWFTJsk/6/sD2kmYD3ST9JCI+LXPRckuSIiIktcmWrbAaJLWPiM/KXY5VmaSjgN1Is9wXlLk4y+VRLk1AUp+IeFFSO+Bp0lIJu0bEfyS1iojFZS5ibkn6HnAyMBZ4ISLGlrlIZVfwQbcFMAQYERFV5S7XqkjSscB2wLURscovVbLKf4XIieslPRYR84EngceAUyWt7TBfebLAOh74K9AGOFrS3uUtVfllYb4HcCXwA+AIL3mdVLeZF7Sdfws4peD+VbpVwzX0lUhSi4j4PLv8JPB2RBwhqQNwBdAyIn4g6RtAO9ceS0dSb+Bx4PyIuDELrN1JzV4PRsSfylm+cpLUB7gHOJi00N7XSIvu3RERNZf1qBjV31yyy5tExNvZ5d8BewC9ImLhqvyt2oHehCQ9C3wQEYMlbQRcBPQBWgP7RsRb5Sxf3kgaBWwTEd2z612AQaRa1w8jYmYZi1c2kg4EDoiIg7Pre5J+F/8E3B4R75SzfOWWzYwfCPwX+G9EXCfpVtLf6g4Rscq2o7vJZSWTtL+kqwEi4htAZ0mjImJGRJxA2hDkAId54xR8Vf6apD0kdYqIfYGnJU0AyNqJHwL+r5LCvODcVDcXTAA6ZEFORDwMvARsSWovbnbD9UpF0iDgIOBQYGtgW4CIOAaYBPwjO26VPD+uoa9kkjYB/gj8NSLOz257AiAidi5n2fJG0mDgZ6Tmg0+B8RFxpaTbge0jouZOWxVD0gBSc9ObEXGbpLOBdUkrpf4TuJrUv7NBRBxRvpKWl6TDgSVAe1Kofy9rZukWEW9J2iAi3i1vKevmGvpKImkbSRtn7XD7At+WdAUsDfJ5kjYuZxmbO0lrFIzrXx04FjgyIr4H3ABsJGlQRBwFTJX0zTIWt8ll20ciaSfgWmA2cKmkM4B7SUG+K6m55WTgL0CLbDRWRSkYYz4N+AVwfETskYX56cDZWdv5Khvm4HHoK4Wk7sBpwOuS7o6I6ZIOBiZIWjMiToyIgWUuZrOWdSz/GjiftMvM56Qa5yak5oOnSR1+A4HRWchXBEnrk/b4XZCN9DkB+GVE/EHSGOAOoHVEXAn8UVJ70jjrS4CjstFYuSZpA+D9iPhc0qFAd0mPAc8AD5I+2PYCOgNHk87LKtkRWsg19BKo2Z4WEW8C95E2/RgiqWtETCctz7mtpK+UoZi5EhFzgJ8Ca0o6MJsccz0wSNKO2USiF4C1spp8RfyuS2oNHAlsmt20IbABaZ/fDSNiMnAYcJyknwFk525L4NCImFiGYjcpSV2BHwMHSDoa+AnQjrSvw26k5cBfAU4CdgGOjohXylPaFeM29EaqMdTpOL7YS/UPwDdJEzeCtIPT14HTs3C3BqoxHHQwcCFwMfA68B3S2OqHSZ1bp0bEmHKVtRwkrQZ0JH17+SHQFzgc+A9wf0S8p7T0xHoR8XT5Sloe2TeSk4D1ge7A0Ih4LRv9cwHwk4j4S3VzXnOomVeriFrLylQQ5j8CjgDeILXlXgs8T2rLfZs0euACh3njZB+gn0vaVdLB2XjyXwJnAD2AW0lNDOOBQyopzKu/KUbEPGB1oANp8tB4Usf85qRJRBtExJRKC3NJa0taN/tG8jjwHrAeqabeNiLuI1UMbpD0vYhY3JzCHNyG3mA1aubdgd6kyQdnkUZYtCSF+pkR8VtJw7yeSONlsxwHks7tcdlt90uaD/wI6BARd5axiGVR/fuYjbVXREyWdCHp9/FqUk29FbAfadZsJeoD7CSpI9AWGArMJTU37S/pvoh4QNIi4LXyFbPh3OTSADXCfP3sK+z6wFbAz0kjB75L+kMaR6oxUt1MYA2T1UDXAUaRxpI/LWlX0nkfTfpQPR84MCLeK1Mxy0bSPqQa+XvATFLbcAvgVFIb8UnAWhFRURu4Zx9ynwLbkD70+5FGsTycda4fRepzeJU0sapZ1coLucmlAQrC/GTgF9nIlfdIfzTPZb8QnYE7gQsj4nOHeeNF8iFpLZyfSrqNNNyuL/DTiPgrcFCFhvnmwDGk978z8C5wDqm57zekUUBbVGCYDwbuB24h1cgHkTo9vy3p61nn+jDgA2BjYLUyFbUkXENvIKVV2E4G9o+IaZLakjqi/gE8S2p+2S0i/lPGYjZ7BU0JPUmrVL6S/d8XmBgRz0qq7gg9HFgcFfRLXfCt5TpSLfO46pEqkh4Fno+ICyStHhFzy1jUJifp26Q+rEOBt0h9CncA/wLeJJ2vX5KGu3YDnoiI/5WjrKXiGnqRCocmZhM2+pHG7UbWIfoPUvvk9sBNwE4O88bLwnwQ8ABwOmlo4jdIozWezf5oLyctLLWoUsK8oAO0+lvLFaRJMd/K+nQg1UoXZ8dVVJhndgKuiYgXgLmRltc4jDQ/YSfSqKg7SU14/27uYQ4O9KLUaDP/OqlT6S+kr27XZoedR6ohtomIp8NrszSYpDULLq9FmthxeEQMIo1i2Zi0UciGwD7ARRExuuZ8gDzLPuj2lnSTpMtInXs/JS08dkk2G/Ri0rotFaXg96AL0Cm7vEBSy4iYRupMXwv4O6mfYbe8/L060ItQEOZnk2qD60XEQ6Rx5odFxFWkEQQCPJKlEbJp52MlnZjd9BmwBmmJV7JhinNIk2DeIbWdjy780M2z6rCStCVp/P0E0pT+J0kVjZ+RmhZ6Az+KiD+XqahlU/B7cD/wzaytPEjfplsDHwGfAFUR8UxETC1TUUvOgV4kSUOA/YG9Iy3S0530uzNPabnN3wInR8QnZS1oMxdp2vn/A86SdGw21HMksJWkb2WH/Z30x9m+uimhEsIcltbM+5G+Hd4dETdExK9ImzDcQRrhcg1pSeZNstFXleo50hIQB0vqmw1OWERqblmHHA7bzt0bWonmAxOBw5U2S9iZ9DXuQtIMvAPdZt44We1TEfH3rIZ+i6QFwKOkZRQukvQGMAA4Kyp3L8zXSX+7A5WWZlZE3JONz+8YEX9TWqxsT+Cucha0nCJirqSbSLtWXam0H8FC4ADSN7zZZS3gSuBRLrXI/jC+BvwjIiZkt21GGhb2DVKzyxuktt0xEfFMmYqaGwWjWboBn0bEh5L6knbWOY8007E36efyekRMqKRmluzcdCctqjVZ0hrAI6RKxjBgTVITw14Fo1wqbmRLbbKp/n1Jc0M+BB6OZrA/aEM40Gsh6RrSNP7JpCGI1wNTI2Kxsh3kJR1E6oQanKc2uHIoCKw9SSOHPiYt7XoraV3qe4ArI+LGMhazrLLx1D8G3ifNYvwtqSP0z6Tp6w8Bf4+IR1Ww1o1VFreh1+5+Uu3nANJEg+OBEZLWA1ZX2gP0fNLXNod5IxW0C58KfB84lzQh5hzS+OETgAskdamkkSzVst+3/wP2Ii3vegTpHHUA9gbeAdaJiEezh7iWVqEc6LWIiCdIMz2PiYgTSZNZDiaN672MtMjRbtFMltRc1WXDFI8DekbEq1kz16OkURsDIuIfQJ+IqKqEJpZatCBNWd+B1DF/JKkJ4SrSioH7ATtnwxcrpoPYluVAryGbNARpMaPVJR1F2tFlAGnM6nhS23pFTaFeWSRtQ2pWuR6YIekqgIh4gy9WqYQ01KwiFAxN/ApApFURJwC7A5dExFjSB97qpDb1OaSAH16eEtuqouIDXV9s01U9825JdtdM0nKsvyPV1B/POptuycskhHKSVL3i37XAdyPi36RaaCdJYyTtTZo0NA6+9HPJvawJ6nvAPZJGS+qZtYlXARcr7bCzH/CriHhdaWu0T/x7aRUd6JK2Bh7NFteKwvbZiPiANJ16CvDfgtvd2dRA+vKuQR2y/0cDC7LLrwC/Io3YuBA4I+vkq4jhtQU1868DZ5I63SeThmvuDIwgbZxyGHBxRDwFzWsDBlu5Kn6Ui6S7SGsjHx0RnxSMuGgRaSOF4aSRLndUUi2x1CT1IDUL3E9aXOsh0vIJO5PGBh9PmgH6AWlnp6OBRRFxZjnK25SyoZpfi7RLTlfSh9rnEXFEdv9PgF7ATRExVtJq2YS2ihi2acWryBq6kur3fgOwNfCIpA7VNfWCmviTpOFgDvMGUlradRTQKpup9y5pV6c/kz4sdyatencbqeP5TeB2oI2kdctQ5KbWk2yXHFKzyvPAV5W2RCMiLiV9SzxDUsdIOxK589OWUdE1dEmnA/uSNqI4l7QWy54RMSdrl/RX2UZSWvb2NtI48nuyD9K+EfF8wTHXkKarzwI2yyYNtSR9ACyo7XnzouAb4TGkJQ9Ojohx2UzZvsBfI+KB7NjNss5is1pVZA0dlrZX7ghcHxGjI6I/aR2MxySt5TBvvGyG3umkkRj3ZDePJXV2IqmV0obGfYGuETG7YAbokgoI8xZZmO9DWiVxGnC9pIERcQOppj4km8SGw9zqUxGdTfDlJXBh6UiC6aTF7audTPpqe7WkY/yVtuGU9m08F7gbWJQNR9yMtInAT2FpZ95iSX8mrRhIdnuuz3vWtDcn66PZiLRt3DGkDvj+wBWSFkTETVmH8KRylteaj4oI9MIwV1o1EVJb5b3A7ZLeIq3K1p80iuCqvIdKE2hP2lzhPVI/xWWk5pR9qg+QtANpfZZbokK2jcs+6I6V9IeImEnqCJ6UjTVH0oOkCUR3SDoyIq4vY3GtmamIJpeCMD+ZNJ28LWlpzdmkTZ1PJoXOL4DrPJ2/JKoD+rCIeJU0df1ZSVfA0iGjI4BpFRTmG5KWtb0HaCVpSKTllttIuhWWjrefSJo4tKhshbVmqSI6RbP28q6kzs+jgUNIU/n3yBbc6kCqUbaolHBpCkqbVTwA3BcRt2WzQk8i/Sw2BM6PiIfLWcamkjWdPAH8OCL+Ieks0uiWO4EXSE1Ti4EHSatLHhoRL3hooq2I3NbQa0wSCuB/pFXqLiR1yu2dhfnpQJeI+MBhXhqSukvaItJmFRcBXbL1Wl4Ffk8aTXRxpYR5pgVpAtWH2fVbgJeAwaTO+X1I3xrXA86MtA9m7vsTrLRy24Ze0MyyBfB+RHwkqRNwUER0zO47iLS636iyFTQnCobf9QXOAKZLGkZaN74rsH2kjSsmAodkQ0NzX/uUtCkwOyJmS/qE1GZO9vt4O2lRsr1JQzQvKXhc7s+NlV5ua+gAShvl/pW0W8m5kVZOHC9pVNZmeQ5wXERML2tBc6Bg/ZHrSE0H3yY1b0HaGu2XkraMiMWRFpOqlNrnpsBUSWsDn5K2PgMgIj4mrWVTBeyugu3iKuTcWInlqg29xmiWDUhjoG8BvkJaa/v1iLhU0k6k9UL+E17QqCSyJpXbgSsi4hlJ3wW+R5osdBlp2devADdE2ie0YijtgDWcVDu/j7TUwfTs/7mkzs833BlvjZWbJpcaYX4UaehXJ9IOL2+Rdnj5kaRrIuKMshU0pyKtg7OEtAHvMxHxSLYuyZmklSs/JDV9VVSYA0TEXyUdDTwO/IsU5gNI7eVrA+c6zK0UctPkUhDmB5JGUrwKbEs2KzHS8qy/A1oo7TxkjVDd6ay0i9Dm2c23A+sqLX0LaaZjFamN+J8RcW/Tl3TVEBHjSCG+I3BRRBwTEXtFxE7VY9DNGqvZN7nUqJn3JA37Oj4inszadM8GbgbuiYhFktrmfUp5U8nO769IFYM7Sef5YGBXUjt6b1KYnwP8OSIeqvTOPqV9U0cAPbKO0oo+H1ZazTrQa4T5KaSvsieRhsXtHxGfZX9Al5I2A7i7fKVt/mqc7y2By0nT+z8hjRS6jzRB6ytAP+BF0hZptwDfiYi3y1DsVY6kvYB5Wa3drGSadaBXk7Qfabeb75P6Bc4kzQb9YRbqe5A6RB0oDZQ1U50IXEJq972U1KQ1JCJmKK3pPRJ4MiLOyx6zHanv4rSIeLkMxV6luXZupdbsAz2rKT4A3BkRl0hqS1oE6kTSwlvHZBNcrBEkbUz6kJxL6uD8OmkU0fOkmaAzJHUnrY9zeERMzh63XkS8X6Zim1WUZtcpWkuH5iekr/rfl7Rj1j7+H1J77nSgYxMXMZciYhpp0+YLSOf2BdJQvM2A/SV1jYg3gf4RMVnZXq0Oc7Om06xq6Flt/DXSmiyTIuLG7Pa2wCnAbqRd0Z9T2kihVSUOk1uZJG0CnEWqrZ9BGh56FOnnMoy0bZz3XTUrg+YW6F1Iixg9BOxOGmN+L/B4RMyVdCpplMU5UbAjjq04SWsBq0XEu1nH8jdIKwVeSlpY6+Ts+pmkseczs1UVzaxMmlWgA0j6DWmT4cOBA0g7oK9FGp7YjhQ2z3o6f8Mp7SJ0JWks/xTSN6KfkTqd3yMNVfyMtPDWYuBEd+6ZlV+zCfSCxZ/akDah+BGwJWlI3N9IHaAzSEuyfly2guZENsb8ANK09P9ExJXZ7VcBm0bEIElfBdq6Zm62amg2U/+zMK9eEve/pBrk14GzImJUNltxpsO8cZT2ufw8Iv4saR7wQ2AtZRsUR8SPJI2T1D0ippS7vGb2hWYT6LB0ev9CSXcA/wCGRcSo7L7J5SxbHmTfgj6XtC3wMfAsqZ/ifGBPSU+TauxdAHd8mq1imk2TS03ZYkfdgMsjYl55S5Mfkr5N6mgeQ+r0/Ampj+IC0lKwbwEjqj9IzWzV0ezGoRd4Dtiu3IXIg4KFttYGNgcGkSYNTQCuIY31vxB4E7gga+JS7c9mZuXSbGvokEZjuHZeGtn6IocCPYCfRMTj2Q5P3yctsHUi8EH15hRmtuppzjV0HOalIWkH0iJb95M2pNhf0oYR8SFpt6FHgI4Oc7NVW7OuoVvjZTs73Qm8FBE/lLQ6cCMwm7RC5XRJrSNiUVkLamb1atY1dCuJT4GHgb0kDYyIucDxpMlbF2TrxzvMzZqBZjVs0RqvYIJWH2ADYBppka33gFMlfR4Rj0o6DNjCm4GYNR8O9AqThflA4HrS0MSdSSNZnid9Y/txFvqPAP8uW0HNbIU50CtMNjTxWOC4bCTLbqT1cD4kbY3WltQxambNjNvQc07SZpKGSBoMEBEfkWaB9pbUMiIeB8aR1sZpAfw+IiaUqbhm1ggO9BzL1rf5E/BN4P8knZTd9RRpVcods+svAR+Q1o9f0uQFNbOS8LDFnJLUk7TH54UR8ZCkI4A1SStTTiXN/NwcaElatXJoRPyxXOU1s8ZzoOeUpG8BT0REi+z6S6TlhTcCniRN7e8G9AKmR8SL3rTYrHlzoOdYttPQMFKN/ImIuDhbT/4V4OaI+FVZC2hmJeVAzzlJu5Om7rep3utT0nHA2tWbVphZPrhTNOci4u+k1RMnA2S7DJ0LvFzOcplZ6XkcegWIiDGSPs92IHoT+FFEPFrucplZabnJpYJkzS9rRsSD5S6LmZWeA70CeTSLWT450M3McsKdomZmOeFANzPLCQe6mVlOONDNzHLCgW5mlhMOdDOznPj/fxcy5Nyuq5oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_select = df[df.type.isin(['Armed Assault','Bombing/Explosion','Kidnapping','Other']) ]\n",
    "df_select = df_select[df_select.threshold == 0.7][['type','f1(micro)','f1(macro)']]\n",
    "df_select = df_select.reset_index(drop=True)\n",
    "print(df_select)\n",
    "ax = df_select.plot.bar(xticks=df_select.index, title = 'multi-label;  threshold = 0.7', rot= 45);\n",
    "ax.set_xticklabels(['Armed Assault','Bombing/Explosion','Kidnapping','Other']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}