notebooks/snli_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e72127e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "if os.getcwd().endswith(\"notebooks\"):\n",
    "    os.chdir(\"..\")\n",
    "\n",
    "import importlib\n",
    "from IPython.display import display, Markdown, Latex\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "from src.utils import tree_utils\n",
    "from src.features import feature_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f278530a",
   "metadata": {},
   "source": [
    "### Load the maximum likelihood subtrees for SNLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e65db185",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parsed 65532/65532 trees\n",
      "parsed 16380/16380 trees\n"
     ]
    }
   ],
   "source": [
    "data = feature_utils.load_trees(output_dir=\"data/snli\", dataset=\"snli\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06e4c5ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9\n",
      "  (1\n",
      "    (14\n",
      "      (14 (41 a/a) (32 girl/kid))\n",
      "      (10 (16 (75 in/) (94 a/)) (11 (70 green/) (70 shirt/))))\n",
      "    (31 (77 jumps/sits) (2 (0 (59 in/in) (38 the/a)) (83 air/chair))))\n",
      "  (90 ./.))\n"
     ]
    }
   ],
   "source": [
    "print(data[\"dev\"][\"tree\"][333])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8827c741",
   "metadata": {},
   "source": [
    "### Enumerate all of the subtrees and sort by mutual information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e7dc121d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "683820 Subtrees features\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Root</th>\n",
       "      <th>Subtree</th>\n",
       "      <th>Yield</th>\n",
       "      <th>MI</th>\n",
       "      <th>Count</th>\n",
       "      <th>Entailment</th>\n",
       "      <th>Contradiction</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Majority label</th>\n",
       "      <th>% majority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49</td>\n",
       "      <td>(49 *)</td>\n",
       "      <td>ϵ/to</td>\n",
       "      <td>0.009520</td>\n",
       "      <td>2373.0</td>\n",
       "      <td>297.0</td>\n",
       "      <td>471.0</td>\n",
       "      <td>1605.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.675926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>(49 *)</td>\n",
       "      <td>ϵ/for</td>\n",
       "      <td>0.006778</td>\n",
       "      <td>963.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>762.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.789855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>(41 *)</td>\n",
       "      <td>a/the</td>\n",
       "      <td>0.006350</td>\n",
       "      <td>13054.0</td>\n",
       "      <td>2999.0</td>\n",
       "      <td>5127.0</td>\n",
       "      <td>4928.0</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.392740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32</td>\n",
       "      <td>(32 *)</td>\n",
       "      <td>man/woman</td>\n",
       "      <td>0.005654</td>\n",
       "      <td>482.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>444.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.917526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>(14 (41 *) (32 *))</td>\n",
       "      <td>a man/a woman</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>276.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>267.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.960573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>49</td>\n",
       "      <td>(49 *)</td>\n",
       "      <td>ϵ/the</td>\n",
       "      <td>0.003427</td>\n",
       "      <td>2757.0</td>\n",
       "      <td>470.0</td>\n",
       "      <td>955.0</td>\n",
       "      <td>1332.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.482971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>49</td>\n",
       "      <td>(49 *)</td>\n",
       "      <td>ϵ/a</td>\n",
       "      <td>0.003208</td>\n",
       "      <td>3117.0</td>\n",
       "      <td>624.0</td>\n",
       "      <td>964.0</td>\n",
       "      <td>1529.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.490385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>32</td>\n",
       "      <td>(32 *)</td>\n",
       "      <td>woman/man</td>\n",
       "      <td>0.002678</td>\n",
       "      <td>265.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.880597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>89</td>\n",
       "      <td>(89 *)</td>\n",
       "      <td>a/there</td>\n",
       "      <td>0.002424</td>\n",
       "      <td>956.0</td>\n",
       "      <td>586.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.612096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>35</td>\n",
       "      <td>(35 *)</td>\n",
       "      <td>ϵ/[UNK]</td>\n",
       "      <td>0.002176</td>\n",
       "      <td>955.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>349.0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.522965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>49</td>\n",
       "      <td>(49 *)</td>\n",
       "      <td>ϵ/his</td>\n",
       "      <td>0.002166</td>\n",
       "      <td>646.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>167.0</td>\n",
       "      <td>409.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.631741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>(4 (89 *) (43 *))</td>\n",
       "      <td>a /there is</td>\n",
       "      <td>0.001872</td>\n",
       "      <td>725.0</td>\n",
       "      <td>446.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.614011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>32</td>\n",
       "      <td>(32 *)</td>\n",
       "      <td>man/person</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>412.0</td>\n",
       "      <td>289.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.698795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>(14 (41 *) (32 *))</td>\n",
       "      <td>a woman/a man</td>\n",
       "      <td>0.001840</td>\n",
       "      <td>149.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.927632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>85</td>\n",
       "      <td>(85 *)</td>\n",
       "      <td>ϵ/tall</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>163.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.903614</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Root             Subtree          Yield        MI    Count  Entailment  \\\n",
       "0    49              (49 *)           ϵ/to  0.009520   2373.0       297.0   \n",
       "1    49              (49 *)          ϵ/for  0.006778    963.0        64.0   \n",
       "2    41              (41 *)          a/the  0.006350  13054.0      2999.0   \n",
       "3    32              (32 *)      man/woman  0.005654    482.0        15.0   \n",
       "4    14  (14 (41 *) (32 *))  a man/a woman  0.003906    276.0         8.0   \n",
       "5    49              (49 *)          ϵ/the  0.003427   2757.0       470.0   \n",
       "6    49              (49 *)            ϵ/a  0.003208   3117.0       624.0   \n",
       "7    32              (32 *)      woman/man  0.002678    265.0        12.0   \n",
       "8    89              (89 *)        a/there  0.002424    956.0       586.0   \n",
       "9    35              (35 *)        ϵ/[UNK]  0.002176    955.0       106.0   \n",
       "10   49              (49 *)          ϵ/his  0.002166    646.0        70.0   \n",
       "11    4   (4 (89 *) (43 *))    a /there is  0.001872    725.0       446.0   \n",
       "12   32              (32 *)     man/person  0.001848    412.0       289.0   \n",
       "13   14  (14 (41 *) (32 *))  a woman/a man  0.001840    149.0         6.0   \n",
       "14   85              (85 *)         ϵ/tall  0.001815    163.0         7.0   \n",
       "\n",
       "    Contradiction  Neutral Majority label  % majority  \n",
       "0           471.0   1605.0        Neutral    0.675926  \n",
       "1           137.0    762.0        Neutral    0.789855  \n",
       "2          5127.0   4928.0  Contradiction    0.392740  \n",
       "3           444.0     23.0  Contradiction    0.917526  \n",
       "4           267.0      1.0  Contradiction    0.960573  \n",
       "5           955.0   1332.0        Neutral    0.482971  \n",
       "6           964.0   1529.0        Neutral    0.490385  \n",
       "7           235.0     18.0  Contradiction    0.880597  \n",
       "8           199.0    171.0     Entailment    0.612096  \n",
       "9           349.0    500.0        Neutral    0.522965  \n",
       "10          167.0    409.0        Neutral    0.631741  \n",
       "11          155.0    124.0     Entailment    0.614011  \n",
       "12           44.0     79.0     Entailment    0.698795  \n",
       "13          140.0      3.0  Contradiction    0.927632  \n",
       "14            7.0    149.0        Neutral    0.903614  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_utils.add_features(data, \"Subtrees\", feature_utils.subtrees)\n",
    "subtrees = feature_utils.get_subtree_feature_table(data)\n",
    "subtrees.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df3fcc8d",
   "metadata": {},
   "source": [
    "### Create aggregated features by merging subtrees by root non-terminal and majority class label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5032af04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Root</th>\n",
       "      <th>Examples</th>\n",
       "      <th>MI</th>\n",
       "      <th>Count</th>\n",
       "      <th>Entailment</th>\n",
       "      <th>Contradiction</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Majority label idx</th>\n",
       "      <th>Majority label</th>\n",
       "      <th>% majority</th>\n",
       "      <th>Support count</th>\n",
       "      <th>Counter count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49</td>\n",
       "      <td>ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after</td>\n",
       "      <td>0.021651</td>\n",
       "      <td>14478.0</td>\n",
       "      <td>2753.0</td>\n",
       "      <td>4408.0</td>\n",
       "      <td>7317.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.505352</td>\n",
       "      <td>7317.0</td>\n",
       "      <td>7161.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day</td>\n",
       "      <td>0.015840</td>\n",
       "      <td>4675.0</td>\n",
       "      <td>593.0</td>\n",
       "      <td>1118.0</td>\n",
       "      <td>2964.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.633818</td>\n",
       "      <td>2964.0</td>\n",
       "      <td>1711.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>85</td>\n",
       "      <td>ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old</td>\n",
       "      <td>0.010309</td>\n",
       "      <td>1945.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.721253</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>541.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>17</td>\n",
       "      <td>ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]</td>\n",
       "      <td>0.007060</td>\n",
       "      <td>1411.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>271.0</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>402.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>32</td>\n",
       "      <td>man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man</td>\n",
       "      <td>0.014281</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1127.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.911147</td>\n",
       "      <td>1127.0</td>\n",
       "      <td>108.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy</td>\n",
       "      <td>0.010892</td>\n",
       "      <td>1351.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>1116.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.824963</td>\n",
       "      <td>1116.0</td>\n",
       "      <td>235.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>78</td>\n",
       "      <td>standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing</td>\n",
       "      <td>0.008478</td>\n",
       "      <td>695.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>645.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.925501</td>\n",
       "      <td>645.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>41</td>\n",
       "      <td>a/the, ϵ/the</td>\n",
       "      <td>0.007769</td>\n",
       "      <td>15436.0</td>\n",
       "      <td>3553.0</td>\n",
       "      <td>6055.0</td>\n",
       "      <td>5828.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.392253</td>\n",
       "      <td>6055.0</td>\n",
       "      <td>9381.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>35</td>\n",
       "      <td>ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap</td>\n",
       "      <td>0.004991</td>\n",
       "      <td>887.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>608.0</td>\n",
       "      <td>248.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.684270</td>\n",
       "      <td>608.0</td>\n",
       "      <td>279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>65</td>\n",
       "      <td>men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs</td>\n",
       "      <td>0.004058</td>\n",
       "      <td>377.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>339.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>339.0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>85</td>\n",
       "      <td>black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot</td>\n",
       "      <td>0.004004</td>\n",
       "      <td>560.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>432.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.769094</td>\n",
       "      <td>432.0</td>\n",
       "      <td>128.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>44</td>\n",
       "      <td>walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside</td>\n",
       "      <td>0.005579</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>908.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>270.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.683973</td>\n",
       "      <td>908.0</td>\n",
       "      <td>418.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal</td>\n",
       "      <td>0.005367</td>\n",
       "      <td>9009.0</td>\n",
       "      <td>4115.0</td>\n",
       "      <td>2316.0</td>\n",
       "      <td>2578.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.456724</td>\n",
       "      <td>4115.0</td>\n",
       "      <td>4894.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are</td>\n",
       "      <td>0.004985</td>\n",
       "      <td>1725.0</td>\n",
       "      <td>1087.0</td>\n",
       "      <td>342.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>1087.0</td>\n",
       "      <td>638.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>83</td>\n",
       "      <td>beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen</td>\n",
       "      <td>0.004354</td>\n",
       "      <td>3312.0</td>\n",
       "      <td>1687.0</td>\n",
       "      <td>604.0</td>\n",
       "      <td>1021.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.509201</td>\n",
       "      <td>1687.0</td>\n",
       "      <td>1625.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Root  \\\n",
       "0    49   \n",
       "1    35   \n",
       "4    85   \n",
       "7    17   \n",
       "2    32   \n",
       "3    14   \n",
       "5    78   \n",
       "6    41   \n",
       "10   35   \n",
       "13   65   \n",
       "14   85   \n",
       "8    44   \n",
       "9    14   \n",
       "11    4   \n",
       "12   83   \n",
       "\n",
       "                                                                                                                                     Examples  \\\n",
       "0                                                                                      ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after   \n",
       "1                                                                        ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day   \n",
       "4                                                                     ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old   \n",
       "7                                                ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]   \n",
       "2                                                            man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man   \n",
       "3                       a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy   \n",
       "5   standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing   \n",
       "6                                                                                                                                a/the, ϵ/the   \n",
       "10                                                                                 ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap   \n",
       "13                                             men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs   \n",
       "14                                                            black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot   \n",
       "8    walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside   \n",
       "9                 a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal   \n",
       "11                          a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are   \n",
       "12                       beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen   \n",
       "\n",
       "          MI    Count  Entailment  Contradiction  Neutral  Majority label idx  \\\n",
       "0   0.021651  14478.0      2753.0         4408.0   7317.0                   2   \n",
       "1   0.015840   4675.0       593.0         1118.0   2964.0                   2   \n",
       "4   0.010309   1945.0       163.0          378.0   1404.0                   2   \n",
       "7   0.007060   1411.0       131.0          271.0   1009.0                   2   \n",
       "2   0.014281   1235.0        37.0         1127.0     71.0                   1   \n",
       "3   0.010892   1351.0        99.0         1116.0    136.0                   1   \n",
       "5   0.008478    695.0        15.0          645.0     35.0                   1   \n",
       "6   0.007769  15436.0      3553.0         6055.0   5828.0                   1   \n",
       "10  0.004991    887.0        31.0          608.0    248.0                   1   \n",
       "13  0.004058    377.0        12.0          339.0     26.0                   1   \n",
       "14  0.004004    560.0        18.0          432.0    110.0                   1   \n",
       "8   0.005579   1326.0       908.0          148.0    270.0                   0   \n",
       "9   0.005367   9009.0      4115.0         2316.0   2578.0                   0   \n",
       "11  0.004985   1725.0      1087.0          342.0    296.0                   0   \n",
       "12  0.004354   3312.0      1687.0          604.0   1021.0                   0   \n",
       "\n",
       "   Majority label  % majority  Support count  Counter count  \n",
       "0         Neutral    0.505352         7317.0         7161.0  \n",
       "1         Neutral    0.633818         2964.0         1711.0  \n",
       "4         Neutral    0.721253         1404.0          541.0  \n",
       "7         Neutral    0.714286         1009.0          402.0  \n",
       "2   Contradiction    0.911147         1127.0          108.0  \n",
       "3   Contradiction    0.824963         1116.0          235.0  \n",
       "5   Contradiction    0.925501          645.0           50.0  \n",
       "6   Contradiction    0.392253         6055.0         9381.0  \n",
       "10  Contradiction    0.684270          608.0          279.0  \n",
       "13  Contradiction    0.894737          339.0           38.0  \n",
       "14  Contradiction    0.769094          432.0          128.0  \n",
       "8      Entailment    0.683973          908.0          418.0  \n",
       "9      Entailment    0.456724         4115.0         4894.0  \n",
       "11     Entailment    0.629630         1087.0          638.0  \n",
       "12     Entailment    0.509201         1687.0         1625.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_utils.add_merges(data, \"Subtrees\", K=1000, merge_name=\"Subtree groups\")\n",
    "subtree_groups = feature_utils.get_merged_feature_table(data, merge_name=\"Subtree groups\")\n",
    "subtree_groups.head(15).sort_values(by=[\"Majority label idx\", \"MI\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0731d34b",
   "metadata": {},
   "source": [
    "To group the top 1000 subtrees, restricted to subtrees with at least two leaves:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "de955755",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtering\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Root</th>\n",
       "      <th>Examples</th>\n",
       "      <th>MI</th>\n",
       "      <th>Count</th>\n",
       "      <th>Entailment</th>\n",
       "      <th>Contradiction</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Majority label idx</th>\n",
       "      <th>Majority label</th>\n",
       "      <th>% majority</th>\n",
       "      <th>Support count</th>\n",
       "      <th>Counter count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy</td>\n",
       "      <td>0.013349</td>\n",
       "      <td>2232.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>1690.0</td>\n",
       "      <td>307.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.756600</td>\n",
       "      <td>1690.0</td>\n",
       "      <td>542.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]</td>\n",
       "      <td>0.011597</td>\n",
       "      <td>2434.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>467.0</td>\n",
       "      <td>1715.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.704144</td>\n",
       "      <td>1715.0</td>\n",
       "      <td>719.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal</td>\n",
       "      <td>0.009757</td>\n",
       "      <td>13399.0</td>\n",
       "      <td>6212.0</td>\n",
       "      <td>3354.0</td>\n",
       "      <td>3833.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.463588</td>\n",
       "      <td>6212.0</td>\n",
       "      <td>7187.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>in a/on the, on a/in a, on the/in the, on a/in the, on the/in a, on /in a, in the/on a, in a/on a</td>\n",
       "      <td>0.006443</td>\n",
       "      <td>3158.0</td>\n",
       "      <td>555.0</td>\n",
       "      <td>1810.0</td>\n",
       "      <td>793.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.572920</td>\n",
       "      <td>1810.0</td>\n",
       "      <td>1348.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are</td>\n",
       "      <td>0.005765</td>\n",
       "      <td>2207.0</td>\n",
       "      <td>1356.0</td>\n",
       "      <td>449.0</td>\n",
       "      <td>402.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.614027</td>\n",
       "      <td>1356.0</td>\n",
       "      <td>851.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>on a/on a, at a/at a, in a/in a, on the/on the, in a/wearing a, of a/by a, in the/in the, of a/of a</td>\n",
       "      <td>0.004712</td>\n",
       "      <td>6473.0</td>\n",
       "      <td>3074.0</td>\n",
       "      <td>1703.0</td>\n",
       "      <td>1696.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.474830</td>\n",
       "      <td>3074.0</td>\n",
       "      <td>3399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>17</td>\n",
       "      <td>ϵ/at home, ϵ/in bed, ϵ/' t, ϵ/ice cream, ϵ/watching tv, ϵ/a [UNK], ϵ/a nap, ϵ/of [UNK]</td>\n",
       "      <td>0.004281</td>\n",
       "      <td>863.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>597.0</td>\n",
       "      <td>210.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.690531</td>\n",
       "      <td>597.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>in the grass/ outside, down the street/ outside, in the snow/ outside, on the sidewalk/ outside, in the snow/in the snow, on a bench/on a bench, on the beach/on the beach, at a table/at a table</td>\n",
       "      <td>0.004128</td>\n",
       "      <td>1026.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>230.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.670554</td>\n",
       "      <td>689.0</td>\n",
       "      <td>337.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>on a/for a, in a/for a, on the/for a, in the/ a, of a/for a, in a/for the, holding a/holding her, on the/ a</td>\n",
       "      <td>0.004096</td>\n",
       "      <td>1131.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>752.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.664021</td>\n",
       "      <td>752.0</td>\n",
       "      <td>379.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/the man, a man/a tall human, a man/a couple, a woman/the woman, a woman/a tall human, a man/an old man, a woman/a mother, a woman/ she</td>\n",
       "      <td>0.003176</td>\n",
       "      <td>5354.0</td>\n",
       "      <td>1214.0</td>\n",
       "      <td>1765.0</td>\n",
       "      <td>2375.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.443532</td>\n",
       "      <td>2375.0</td>\n",
       "      <td>2979.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>24</td>\n",
       "      <td>black dog/ cat, little girl/a boy, young woman/a man,  man/naked man, brown dog/ cat, young girl/a man, young girl/a boy, young boy/a girl</td>\n",
       "      <td>0.002437</td>\n",
       "      <td>347.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>278.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.797143</td>\n",
       "      <td>278.0</td>\n",
       "      <td>69.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>24</td>\n",
       "      <td>man/tall human,  woman/tall human,  man/old man,  man/tall person,  man/construction worker,  woman/sad woman, group of people/group of friends,  man/homeless man</td>\n",
       "      <td>0.001719</td>\n",
       "      <td>586.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>368.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.626486</td>\n",
       "      <td>368.0</td>\n",
       "      <td>218.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>24</td>\n",
       "      <td>group of people/there are people, little boy/a boy, black dog/ animal, group of people/several people, young boy/a child, little girl/a girl, young girl/a girl, young girl/a child</td>\n",
       "      <td>0.001707</td>\n",
       "      <td>1649.0</td>\n",
       "      <td>836.0</td>\n",
       "      <td>367.0</td>\n",
       "      <td>446.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.506659</td>\n",
       "      <td>836.0</td>\n",
       "      <td>813.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>16</td>\n",
       "      <td>in the/ϵ, on a/ϵ, down a/ϵ, through a/ϵ, through the/ϵ, to a/ϵ, at the/ϵ, on the/ϵ</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>24726.0</td>\n",
       "      <td>9099.0</td>\n",
       "      <td>7829.0</td>\n",
       "      <td>7798.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.367989</td>\n",
       "      <td>9099.0</td>\n",
       "      <td>15627.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>10</td>\n",
       "      <td>towards the camera/ϵ, in the sand/ϵ, down a road/ϵ, with red - hair/ϵ, on the street/ϵ, at the top/ϵ, in a marathon/ϵ, in a green shirt/ϵ</td>\n",
       "      <td>0.001377</td>\n",
       "      <td>2242.0</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>621.0</td>\n",
       "      <td>573.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.467261</td>\n",
       "      <td>1048.0</td>\n",
       "      <td>1194.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Root  \\\n",
       "0    14   \n",
       "1    17   \n",
       "2    14   \n",
       "3     0   \n",
       "4     4   \n",
       "5     0   \n",
       "6    17   \n",
       "7     2   \n",
       "8     0   \n",
       "9    14   \n",
       "10   24   \n",
       "11   24   \n",
       "12   24   \n",
       "13   16   \n",
       "14   10   \n",
       "\n",
       "                                                                                                                                                                                             Examples  \\\n",
       "0                                                                               a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy   \n",
       "1                                                                                                        ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]   \n",
       "2                                                                         a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal   \n",
       "3                                                                                                   in a/on the, on a/in a, on the/in the, on a/in the, on the/in a, on /in a, in the/on a, in a/on a   \n",
       "4                                                                                   a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are   \n",
       "5                                                                                                 on a/on a, at a/at a, in a/in a, on the/on the, in a/wearing a, of a/by a, in the/in the, of a/of a   \n",
       "6                                                                                                              ϵ/at home, ϵ/in bed, ϵ/' t, ϵ/ice cream, ϵ/watching tv, ϵ/a [UNK], ϵ/a nap, ϵ/of [UNK]   \n",
       "7   in the grass/ outside, down the street/ outside, in the snow/ outside, on the sidewalk/ outside, in the snow/in the snow, on a bench/on a bench, on the beach/on the beach, at a table/at a table   \n",
       "8                                                                                         on a/for a, in a/for a, on the/for a, in the/ a, of a/for a, in a/for the, holding a/holding her, on the/ a   \n",
       "9                                                        a man/the man, a man/a tall human, a man/a couple, a woman/the woman, a woman/a tall human, a man/an old man, a woman/a mother, a woman/ she   \n",
       "10                                                         black dog/ cat, little girl/a boy, young woman/a man,  man/naked man, brown dog/ cat, young girl/a man, young girl/a boy, young boy/a girl   \n",
       "11                                 man/tall human,  woman/tall human,  man/old man,  man/tall person,  man/construction worker,  woman/sad woman, group of people/group of friends,  man/homeless man   \n",
       "12                group of people/there are people, little boy/a boy, black dog/ animal, group of people/several people, young boy/a child, little girl/a girl, young girl/a girl, young girl/a child   \n",
       "13                                                                                                                 in the/ϵ, on a/ϵ, down a/ϵ, through a/ϵ, through the/ϵ, to a/ϵ, at the/ϵ, on the/ϵ   \n",
       "14                                                          towards the camera/ϵ, in the sand/ϵ, down a road/ϵ, with red - hair/ϵ, on the street/ϵ, at the top/ϵ, in a marathon/ϵ, in a green shirt/ϵ   \n",
       "\n",
       "          MI    Count  Entailment  Contradiction  Neutral  Majority label idx  \\\n",
       "0   0.013349   2232.0       235.0         1690.0    307.0                   1   \n",
       "1   0.011597   2434.0       252.0          467.0   1715.0                   2   \n",
       "2   0.009757  13399.0      6212.0         3354.0   3833.0                   0   \n",
       "3   0.006443   3158.0       555.0         1810.0    793.0                   1   \n",
       "4   0.005765   2207.0      1356.0          449.0    402.0                   0   \n",
       "5   0.004712   6473.0      3074.0         1703.0   1696.0                   0   \n",
       "6   0.004281    863.0        56.0          597.0    210.0                   1   \n",
       "7   0.004128   1026.0       689.0          107.0    230.0                   0   \n",
       "8   0.004096   1131.0       157.0          222.0    752.0                   2   \n",
       "9   0.003176   5354.0      1214.0         1765.0   2375.0                   2   \n",
       "10  0.002437    347.0        34.0          278.0     35.0                   1   \n",
       "11  0.001719    586.0        83.0          135.0    368.0                   2   \n",
       "12  0.001707   1649.0       836.0          367.0    446.0                   0   \n",
       "13  0.001627  24726.0      9099.0         7829.0   7798.0                   0   \n",
       "14  0.001377   2242.0      1048.0          621.0    573.0                   0   \n",
       "\n",
       "   Majority label  % majority  Support count  Counter count  \n",
       "0   Contradiction    0.756600         1690.0          542.0  \n",
       "1         Neutral    0.704144         1715.0          719.0  \n",
       "2      Entailment    0.463588         6212.0         7187.0  \n",
       "3   Contradiction    0.572920         1810.0         1348.0  \n",
       "4      Entailment    0.614027         1356.0          851.0  \n",
       "5      Entailment    0.474830         3074.0         3399.0  \n",
       "6   Contradiction    0.690531          597.0          266.0  \n",
       "7      Entailment    0.670554          689.0          337.0  \n",
       "8         Neutral    0.664021          752.0          379.0  \n",
       "9         Neutral    0.443532         2375.0         2979.0  \n",
       "10  Contradiction    0.797143          278.0           69.0  \n",
       "11        Neutral    0.626486          368.0          218.0  \n",
       "12     Entailment    0.506659          836.0          813.0  \n",
       "13     Entailment    0.367989         9099.0        15627.0  \n",
       "14     Entailment    0.467261         1048.0         1194.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_utils.add_merges(data, \"Subtrees\", K=1000, merge_name=\"Subtree groups (len >= 2)\",\n",
    "                         filter_=lambda w: w[0].count(\"*\") >= 2)\n",
    "subtree_groups_len2 = feature_utils.get_merged_feature_table(data, merge_name=\"Subtree groups (len >= 2)\")\n",
    "subtree_groups_len2.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6942f2a5",
   "metadata": {},
   "source": [
    "### Do State-of-the-art Classifiers Use these Features?\n",
    "Compare the accuracy of BERT and RoBERTa classifiers on supporting and counter-examples for each feature.\n",
    "\n",
    "The predictions of the classifiers we trained in the paper are available at [this link](https://huggingface.co/princeton-nlp/shortcut_grammar/resolve/main/classifier_predictions.tar.gz):\n",
    "```\n",
    "wget https://huggingface.co/princeton-nlp/shortcut_grammar/resolve/main/classifier_predictions.tar.gz\n",
    "tar -xzvf classifier_predictions.tar.gz\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7eb1ced8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in (\"BERT\", \"RoBERTa\"):\n",
    "    feature_utils.add_classifier_accuracy(\n",
    "        data, f\"classifier_predictions/snli/{model.lower()}/predictions.snli.end.json\", model\n",
    "    )\n",
    "    feature_utils.get_support_counter_accuracy(\n",
    "        data, model, \"Subtree groups\", subtree_groups\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c4b6e767",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Majority label</th>\n",
       "      <th>Root</th>\n",
       "      <th>Examples</th>\n",
       "      <th>Count</th>\n",
       "      <th>% majority</th>\n",
       "      <th>BERT Support</th>\n",
       "      <th>BERT Counter</th>\n",
       "      <th>RoBERTa Support</th>\n",
       "      <th>RoBERTa Counter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>49</td>\n",
       "      <td>ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after</td>\n",
       "      <td>14478.0</td>\n",
       "      <td>0.505352</td>\n",
       "      <td>0.866270</td>\n",
       "      <td>0.810073</td>\n",
       "      <td>0.892521</td>\n",
       "      <td>0.816369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>35</td>\n",
       "      <td>ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day</td>\n",
       "      <td>4675.0</td>\n",
       "      <td>0.633818</td>\n",
       "      <td>0.889896</td>\n",
       "      <td>0.760965</td>\n",
       "      <td>0.930052</td>\n",
       "      <td>0.760965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>85</td>\n",
       "      <td>ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old</td>\n",
       "      <td>1945.0</td>\n",
       "      <td>0.721253</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.691892</td>\n",
       "      <td>0.941520</td>\n",
       "      <td>0.740541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>17</td>\n",
       "      <td>ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]</td>\n",
       "      <td>1411.0</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.928870</td>\n",
       "      <td>0.769912</td>\n",
       "      <td>0.949791</td>\n",
       "      <td>0.769912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>32</td>\n",
       "      <td>man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>0.911147</td>\n",
       "      <td>0.969283</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.972696</td>\n",
       "      <td>0.769231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>14</td>\n",
       "      <td>a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy</td>\n",
       "      <td>1351.0</td>\n",
       "      <td>0.824963</td>\n",
       "      <td>0.982143</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.982143</td>\n",
       "      <td>0.825397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>78</td>\n",
       "      <td>standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing</td>\n",
       "      <td>695.0</td>\n",
       "      <td>0.925501</td>\n",
       "      <td>0.993789</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.987578</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>41</td>\n",
       "      <td>a/the, ϵ/the</td>\n",
       "      <td>15436.0</td>\n",
       "      <td>0.392253</td>\n",
       "      <td>0.862139</td>\n",
       "      <td>0.821518</td>\n",
       "      <td>0.889038</td>\n",
       "      <td>0.829024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>35</td>\n",
       "      <td>ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap</td>\n",
       "      <td>887.0</td>\n",
       "      <td>0.684270</td>\n",
       "      <td>0.880503</td>\n",
       "      <td>0.808989</td>\n",
       "      <td>0.937107</td>\n",
       "      <td>0.797753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>65</td>\n",
       "      <td>men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs</td>\n",
       "      <td>377.0</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.975610</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>0.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>85</td>\n",
       "      <td>black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot</td>\n",
       "      <td>560.0</td>\n",
       "      <td>0.769094</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.785714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>44</td>\n",
       "      <td>walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.683973</td>\n",
       "      <td>0.986425</td>\n",
       "      <td>0.836207</td>\n",
       "      <td>0.986425</td>\n",
       "      <td>0.836207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>14</td>\n",
       "      <td>a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal</td>\n",
       "      <td>9009.0</td>\n",
       "      <td>0.456724</td>\n",
       "      <td>0.925150</td>\n",
       "      <td>0.802621</td>\n",
       "      <td>0.924152</td>\n",
       "      <td>0.838088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>4</td>\n",
       "      <td>a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are</td>\n",
       "      <td>1725.0</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>0.971660</td>\n",
       "      <td>0.778481</td>\n",
       "      <td>0.963563</td>\n",
       "      <td>0.791139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>83</td>\n",
       "      <td>beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen</td>\n",
       "      <td>3312.0</td>\n",
       "      <td>0.509201</td>\n",
       "      <td>0.935065</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.914286</td>\n",
       "      <td>0.802721</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Majority label Root  \\\n",
       "0         Neutral   49   \n",
       "1         Neutral   35   \n",
       "4         Neutral   85   \n",
       "7         Neutral   17   \n",
       "2   Contradiction   32   \n",
       "3   Contradiction   14   \n",
       "5   Contradiction   78   \n",
       "6   Contradiction   41   \n",
       "10  Contradiction   35   \n",
       "13  Contradiction   65   \n",
       "14  Contradiction   85   \n",
       "8      Entailment   44   \n",
       "9      Entailment   14   \n",
       "11     Entailment    4   \n",
       "12     Entailment   83   \n",
       "\n",
       "                                                                                                                                     Examples  \\\n",
       "0                                                                                      ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after   \n",
       "1                                                                        ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day   \n",
       "4                                                                     ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old   \n",
       "7                                                ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]   \n",
       "2                                                            man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man   \n",
       "3                       a man/a woman, a woman/a man, a man/ nobody, a boy/a girl, a dog/a cat, two women/two men,  man/a woman, a girl/a boy   \n",
       "5   standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing   \n",
       "6                                                                                                                                a/the, ϵ/the   \n",
       "10                                                                                 ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap   \n",
       "13                                             men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs   \n",
       "14                                                            black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot   \n",
       "8    walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside   \n",
       "9                 a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal   \n",
       "11                          a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are   \n",
       "12                       beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen   \n",
       "\n",
       "      Count  % majority  BERT Support  BERT Counter  RoBERTa Support  \\\n",
       "0   14478.0    0.505352      0.866270      0.810073         0.892521   \n",
       "1    4675.0    0.633818      0.889896      0.760965         0.930052   \n",
       "4    1945.0    0.721253      0.944444      0.691892         0.941520   \n",
       "7    1411.0    0.714286      0.928870      0.769912         0.949791   \n",
       "2    1235.0    0.911147      0.969283      0.615385         0.972696   \n",
       "3    1351.0    0.824963      0.982143      0.777778         0.982143   \n",
       "5     695.0    0.925501      0.993789      0.555556         0.987578   \n",
       "6   15436.0    0.392253      0.862139      0.821518         0.889038   \n",
       "10    887.0    0.684270      0.880503      0.808989         0.937107   \n",
       "13    377.0    0.894737      0.975610      0.625000         0.926829   \n",
       "14    560.0    0.769094      0.916667      0.714286         0.958333   \n",
       "8    1326.0    0.683973      0.986425      0.836207         0.986425   \n",
       "9    9009.0    0.456724      0.925150      0.802621         0.924152   \n",
       "11   1725.0    0.629630      0.971660      0.778481         0.963563   \n",
       "12   3312.0    0.509201      0.935065      0.777778         0.914286   \n",
       "\n",
       "    RoBERTa Counter  \n",
       "0          0.816369  \n",
       "1          0.760965  \n",
       "4          0.740541  \n",
       "7          0.769912  \n",
       "2          0.769231  \n",
       "3          0.825397  \n",
       "5          0.666667  \n",
       "6          0.829024  \n",
       "10         0.797753  \n",
       "13         0.625000  \n",
       "14         0.785714  \n",
       "8          0.836207  \n",
       "9          0.838088  \n",
       "11         0.791139  \n",
       "12         0.802721  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subtree_groups.head(15).sort_values(\n",
    "    by=[\"Majority label idx\", \"MI\"], ascending=False\n",
    ")[[\"Majority label\", \"Root\", \"Examples\", \"Count\", \"% majority\",\n",
    "   \"BERT Support\", \"BERT Counter\", \"RoBERTa Support\", \"RoBERTa Counter\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4219896",
   "metadata": {},
   "source": [
    "### Comparing Tree Features to Simpler Features\n",
    "\n",
    "The SCFG can express that patterns like \"Changing the subject noun from `man` to `human`\" are correlated with class labels. However, we know from prior work ([Poliak et al., 2018](https://arxiv.org/abs/1805.01042); [Gururangan et al., 2018](https://arxiv.org/abs/1803.02324)) that simpler features are correlated with labels as well. For example, just knowing that the word `human` appears in the hypothesis sentence is evidence that the label is `Entailment`. We can also consider a slightly more sophisticated baseline consisting of pairs of n-grams---for example, if `man` appears in the premise and `human` appears in the hypothesis, but possibly occupying a different syntactic role.\n",
    "\n",
    "The tokenizer we used for SNLI can be downloaded from [this repository](https://huggingface.co/princeton-nlp/shortcut_grammar/).\n",
    "```\n",
    "wget -P models/snli/ https://huggingface.co/princeton-nlp/shortcut_grammar/resolve/main/snli/tokenizer.json\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "da0bbd36",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.utils import tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4fdbc43a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tokenizer = tokenizers.get_tokenizer(load_tokenizer_from=\"models/snli/tokenizer.json\")\n",
    "feature_utils.add_tokens(data, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "510b9522",
   "metadata": {},
   "outputs": [],
   "source": [
    "subtree_features = [sts for _, sts in data[\"idx_w\"][\"Subtree groups\"]]\n",
    "ngram_features = [feature_utils.get_ngrams_from_subtrees(sts) for sts in subtree_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d4e66b0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 3\n",
      "141 N-gram pairs features\n"
     ]
    }
   ],
   "source": [
    "feature_utils.add_equivalent_ngram_pairs(data, ngram_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "15059d32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 3\n",
      "141 Hypothesis n-grams features\n"
     ]
    }
   ],
   "source": [
    "feature_utils.add_equivalent_hypothesis_ngrams(data, ngram_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c6e4be5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Majority label</th>\n",
       "      <th>Root</th>\n",
       "      <th>Examples</th>\n",
       "      <th>Count</th>\n",
       "      <th>% majority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>6</td>\n",
       "      <td>a/the, ϵ/the</td>\n",
       "      <td>28273.0</td>\n",
       "      <td>0.382091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>4</td>\n",
       "      <td>ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old</td>\n",
       "      <td>13342.0</td>\n",
       "      <td>0.452979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>7</td>\n",
       "      <td>ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]</td>\n",
       "      <td>3659.0</td>\n",
       "      <td>0.597215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after</td>\n",
       "      <td>61115.0</td>\n",
       "      <td>0.343990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Neutral</td>\n",
       "      <td>1</td>\n",
       "      <td>ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day</td>\n",
       "      <td>46134.0</td>\n",
       "      <td>0.354228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>10</td>\n",
       "      <td>ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap</td>\n",
       "      <td>3465.0</td>\n",
       "      <td>0.645040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>2</td>\n",
       "      <td>man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man</td>\n",
       "      <td>3299.0</td>\n",
       "      <td>0.659903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>5</td>\n",
       "      <td>standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing</td>\n",
       "      <td>1322.0</td>\n",
       "      <td>0.836226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>3</td>\n",
       "      <td>a man/a woman, a woman/a man, a man/nobody, a boy/a girl, a dog/a cat, two women/two men, man/a woman, a girl/a boy</td>\n",
       "      <td>2932.0</td>\n",
       "      <td>0.608177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>14</td>\n",
       "      <td>black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot</td>\n",
       "      <td>2664.0</td>\n",
       "      <td>0.587177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Contradiction</td>\n",
       "      <td>13</td>\n",
       "      <td>men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs</td>\n",
       "      <td>1078.0</td>\n",
       "      <td>0.634598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>8</td>\n",
       "      <td>walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside</td>\n",
       "      <td>6141.0</td>\n",
       "      <td>0.513021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>9</td>\n",
       "      <td>a man/a person, a man/a man, a woman/a person, man/a man, a man/a human, a man/two people, a man/someone, a dog/an animal</td>\n",
       "      <td>11353.0</td>\n",
       "      <td>0.458348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>11</td>\n",
       "      <td>a/there is, ϵ/there are, two/there are, a/there are, ϵ/there is, the/there is, two/a couple, four/there are</td>\n",
       "      <td>2794.0</td>\n",
       "      <td>0.601716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Entailment</td>\n",
       "      <td>12</td>\n",
       "      <td>beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen</td>\n",
       "      <td>4988.0</td>\n",
       "      <td>0.484472</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Majority label  Root  \\\n",
       "6         Neutral     6   \n",
       "4         Neutral     4   \n",
       "7         Neutral     7   \n",
       "0         Neutral     0   \n",
       "1         Neutral     1   \n",
       "10  Contradiction    10   \n",
       "2   Contradiction     2   \n",
       "5   Contradiction     5   \n",
       "3   Contradiction     3   \n",
       "14  Contradiction    14   \n",
       "13  Contradiction    13   \n",
       "8      Entailment     8   \n",
       "9      Entailment     9   \n",
       "11     Entailment    11   \n",
       "12     Entailment    12   \n",
       "\n",
       "                                                                                                                                     Examples  \\\n",
       "6                                                                                                                                a/the, ϵ/the   \n",
       "4                                                                     ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old   \n",
       "7                                                ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]   \n",
       "0                                                                                      ϵ/to, ϵ/for, ϵ/the, ϵ/a, ϵ/his, ϵ/at, ϵ/their, ϵ/after   \n",
       "1                                                                        ϵ/[UNK], ϵ/work, ϵ/get, ϵ/friends, ϵ/park, ϵ/mother, ϵ/school, ϵ/day   \n",
       "10                                                                                 ϵ/home, ϵ/bed, ϵ/tv, ϵ/alone, ϵ/t, ϵ/lunch, ϵ/cream, ϵ/nap   \n",
       "2                                                            man/woman, woman/man, boy/girl, dog/cat, girl/boy, man/girl, man/lady, child/man   \n",
       "5   standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing   \n",
       "3                         a man/a woman, a woman/a man, a man/nobody, a boy/a girl, a dog/a cat, two women/two men, man/a woman, a girl/a boy   \n",
       "14                                                            black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot   \n",
       "13                                             men/women, women/men, dogs/cats, people/person, people/man, men/woman, girls/boys, people/dogs   \n",
       "8    walking/walking, running/running, playing/playing, sitting/sitting, jumping/jumping, working/working, standing/standing, outside/outside   \n",
       "9                   a man/a person, a man/a man, a woman/a person, man/a man, a man/a human, a man/two people, a man/someone, a dog/an animal   \n",
       "11                                a/there is, ϵ/there are, two/there are, a/there are, ϵ/there is, the/there is, two/a couple, four/there are   \n",
       "12                       beach/beach, water/water, building/building, snow/snow, street/street, pool/pool, sidewalk/sidewalk, kitchen/kitchen   \n",
       "\n",
       "      Count  % majority  \n",
       "6   28273.0    0.382091  \n",
       "4   13342.0    0.452979  \n",
       "7    3659.0    0.597215  \n",
       "0   61115.0    0.343990  \n",
       "1   46134.0    0.354228  \n",
       "10   3465.0    0.645040  \n",
       "2    3299.0    0.659903  \n",
       "5    1322.0    0.836226  \n",
       "3    2932.0    0.608177  \n",
       "14   2664.0    0.587177  \n",
       "13   1078.0    0.634598  \n",
       "8    6141.0    0.513021  \n",
       "9   11353.0    0.458348  \n",
       "11   2794.0    0.601716  \n",
       "12   4988.0    0.484472  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngram_pair_groups = feature_utils.get_merged_feature_table(data, merge_name=\"N-gram pairs\")\n",
    "hypothesis_ngram_groups = feature_utils.get_merged_feature_table(data, merge_name=\"Hypothesis n-grams\")\n",
    "\n",
    "ngram_pair_groups.head(15).sort_values(\n",
    "    by=[\"Majority label idx\", \"MI\"], ascending=False\n",
    ")[[\"Majority label\", \"Root\", \"Examples\", \"Count\", \"% majority\"]]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "20db3577",
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in (\"BERT\", \"RoBERTa\"):\n",
    "    feature_utils.get_support_counter_accuracy(\n",
    "        data, model, \"N-gram pairs\", ngram_pair_groups\n",
    "    )\n",
    "    feature_utils.get_support_counter_accuracy(\n",
    "        data, model, \"Hypothesis n-grams\", hypothesis_ngram_groups\n",
    "    )    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b8abc866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Root</th>\n",
       "      <th>Examples</th>\n",
       "      <th>MI</th>\n",
       "      <th>Count</th>\n",
       "      <th>Entailment</th>\n",
       "      <th>Contradiction</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Majority label idx</th>\n",
       "      <th>Majority label</th>\n",
       "      <th>% majority</th>\n",
       "      <th>Support count</th>\n",
       "      <th>Counter count</th>\n",
       "      <th>BERT Support</th>\n",
       "      <th>BERT Counter</th>\n",
       "      <th>RoBERTa Support</th>\n",
       "      <th>RoBERTa Counter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>14</td>\n",
       "      <td>a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal</td>\n",
       "      <td>0.005367</td>\n",
       "      <td>9009.0</td>\n",
       "      <td>4115.0</td>\n",
       "      <td>2316.0</td>\n",
       "      <td>2578.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.456724</td>\n",
       "      <td>4115.0</td>\n",
       "      <td>4894.0</td>\n",
       "      <td>0.925150</td>\n",
       "      <td>0.802621</td>\n",
       "      <td>0.924152</td>\n",
       "      <td>0.838088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are</td>\n",
       "      <td>0.004985</td>\n",
       "      <td>1725.0</td>\n",
       "      <td>1087.0</td>\n",
       "      <td>342.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Entailment</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>1087.0</td>\n",
       "      <td>638.0</td>\n",
       "      <td>0.971660</td>\n",
       "      <td>0.778481</td>\n",
       "      <td>0.963563</td>\n",
       "      <td>0.791139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>78</td>\n",
       "      <td>standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing</td>\n",
       "      <td>0.008478</td>\n",
       "      <td>695.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>645.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.925501</td>\n",
       "      <td>645.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.993789</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.987578</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>85</td>\n",
       "      <td>black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot</td>\n",
       "      <td>0.004004</td>\n",
       "      <td>560.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>432.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Contradiction</td>\n",
       "      <td>0.769094</td>\n",
       "      <td>432.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.785714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>85</td>\n",
       "      <td>ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old</td>\n",
       "      <td>0.010309</td>\n",
       "      <td>1945.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.721253</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>541.0</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.691892</td>\n",
       "      <td>0.941520</td>\n",
       "      <td>0.740541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>17</td>\n",
       "      <td>ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]</td>\n",
       "      <td>0.007060</td>\n",
       "      <td>1411.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>271.0</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>402.0</td>\n",
       "      <td>0.928870</td>\n",
       "      <td>0.769912</td>\n",
       "      <td>0.949791</td>\n",
       "      <td>0.769912</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Root  \\\n",
       "9    14   \n",
       "11    4   \n",
       "5    78   \n",
       "14   85   \n",
       "4    85   \n",
       "7    17   \n",
       "\n",
       "                                                                                                                                     Examples  \\\n",
       "9                 a man/a person, a man/a man, a woman/a person,  man/a man, a man/a human, a man/two people, a man/ someone, a dog/an animal   \n",
       "11                          a /there is, ϵ/there are, two /there are, a /there are, ϵ/there is, the /there is,  two/a couple, four /there are   \n",
       "5   standing/sitting, walking/sitting, sitting/standing, walking/running, running/sitting, sitting/running, jumping/sitting, standing/playing   \n",
       "14                                                            black/white, red/blue, ϵ/empty, ϵ/living, white/black, ϵ/naked, blue/red, ϵ/hot   \n",
       "4                                                                     ϵ/tall, ϵ/sad, ϵ/[UNK], ϵ/new, ϵ/big, ϵ/professional, ϵ/favorite, ϵ/old   \n",
       "7                                                ϵ/to work, ϵ/to get, ϵ/to buy, ϵ/the park, ϵ/on vacation, ϵ/to go, ϵ/first time, ϵ/the [UNK]   \n",
       "\n",
       "          MI   Count  Entailment  Contradiction  Neutral  Majority label idx  \\\n",
       "9   0.005367  9009.0      4115.0         2316.0   2578.0                   0   \n",
       "11  0.004985  1725.0      1087.0          342.0    296.0                   0   \n",
       "5   0.008478   695.0        15.0          645.0     35.0                   1   \n",
       "14  0.004004   560.0        18.0          432.0    110.0                   1   \n",
       "4   0.010309  1945.0       163.0          378.0   1404.0                   2   \n",
       "7   0.007060  1411.0       131.0          271.0   1009.0                   2   \n",
       "\n",
       "   Majority label  % majority  Support count  Counter count  BERT Support  \\\n",
       "9      Entailment    0.456724         4115.0         4894.0      0.925150   \n",
       "11     Entailment    0.629630         1087.0          638.0      0.971660   \n",
       "5   Contradiction    0.925501          645.0           50.0      0.993789   \n",
       "14  Contradiction    0.769094          432.0          128.0      0.916667   \n",
       "4         Neutral    0.721253         1404.0          541.0      0.944444   \n",
       "7         Neutral    0.714286         1009.0          402.0      0.928870   \n",
       "\n",
       "    BERT Counter  RoBERTa Support  RoBERTa Counter  \n",
       "9       0.802621         0.924152         0.838088  \n",
       "11      0.778481         0.963563         0.791139  \n",
       "5       0.555556         0.987578         0.666667  \n",
       "14      0.714286         0.958333         0.785714  \n",
       "4       0.691892         0.941520         0.740541  \n",
       "7       0.769912         0.949791         0.769912  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subtree_groups.iloc[[9, 11, 5, 14, 4, 7]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ee669161",
   "metadata": {},
   "outputs": [],
   "source": [
    "idxs = [9, 11, 5, 14, 4, 7]\n",
    "a = subtree_groups.iloc[idxs].copy(deep=True)\n",
    "a[\"Feature\"] = \"SCFG\"\n",
    "b = ngram_pair_groups.iloc[idxs].copy(deep=True)\n",
    "b[\"Feature\"] = \"N-gram pairs\"\n",
    "c = hypothesis_ngram_groups.iloc[idxs].copy(deep=True)\n",
    "c[\"Feature\"] = \"Hypothesis n-grams\"\n",
    "\n",
    "a[\"Shortcut\"] = b[\"Shortcut\"] = c[\"Shortcut\"] =[\n",
    "    \"Subj. hypernym\", \"Expletive construction\",\n",
    "    \"Verb antonym\", \"Adj. antonym\",\n",
    "    \"Add adj.\", \"Add PP phrase\"\n",
    "]\n",
    "table = pd.concat([a, b, c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a296db0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Shortcut</th>\n",
       "      <th>Feature</th>\n",
       "      <th>Support count</th>\n",
       "      <th>Counter count</th>\n",
       "      <th>BERT Support</th>\n",
       "      <th>BERT Counter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Subj. hypernym</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>4115.0</td>\n",
       "      <td>4894.0</td>\n",
       "      <td>0.925150</td>\n",
       "      <td>0.802621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Expletive construction</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>1087.0</td>\n",
       "      <td>638.0</td>\n",
       "      <td>0.971660</td>\n",
       "      <td>0.778481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Verb antonym</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>645.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.993789</td>\n",
       "      <td>0.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adj. antonym</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>432.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Add adj.</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>541.0</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.691892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Add PP phrase</td>\n",
       "      <td>SCFG</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>402.0</td>\n",
       "      <td>0.928870</td>\n",
       "      <td>0.769912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Subj. hypernym</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>5204.0</td>\n",
       "      <td>6149.0</td>\n",
       "      <td>0.920499</td>\n",
       "      <td>0.807644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Expletive construction</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>1682.0</td>\n",
       "      <td>1112.0</td>\n",
       "      <td>0.966746</td>\n",
       "      <td>0.738971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Verb antonym</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>1107.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>0.978182</td>\n",
       "      <td>0.681818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adj. antonym</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>1565.0</td>\n",
       "      <td>1099.0</td>\n",
       "      <td>0.923483</td>\n",
       "      <td>0.800633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Add adj.</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>6044.0</td>\n",
       "      <td>7298.0</td>\n",
       "      <td>0.841173</td>\n",
       "      <td>0.831494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Add PP phrase</td>\n",
       "      <td>N-gram pairs</td>\n",
       "      <td>2186.0</td>\n",
       "      <td>1473.0</td>\n",
       "      <td>0.897248</td>\n",
       "      <td>0.775061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Subj. hypernym</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>8698.0</td>\n",
       "      <td>11427.0</td>\n",
       "      <td>0.930243</td>\n",
       "      <td>0.818119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Expletive construction</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>1782.0</td>\n",
       "      <td>1368.0</td>\n",
       "      <td>0.964206</td>\n",
       "      <td>0.742331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Verb antonym</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>6104.0</td>\n",
       "      <td>7188.0</td>\n",
       "      <td>0.930659</td>\n",
       "      <td>0.825688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adj. antonym</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>2262.0</td>\n",
       "      <td>2658.0</td>\n",
       "      <td>0.900885</td>\n",
       "      <td>0.840683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Add adj.</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>6044.0</td>\n",
       "      <td>7298.0</td>\n",
       "      <td>0.841173</td>\n",
       "      <td>0.831494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Add PP phrase</td>\n",
       "      <td>Hypothesis n-grams</td>\n",
       "      <td>2186.0</td>\n",
       "      <td>1473.0</td>\n",
       "      <td>0.897248</td>\n",
       "      <td>0.775061</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Shortcut             Feature  Support count  Counter count  \\\n",
       "9           Subj. hypernym                SCFG         4115.0         4894.0   \n",
       "11  Expletive construction                SCFG         1087.0          638.0   \n",
       "5             Verb antonym                SCFG          645.0           50.0   \n",
       "14            Adj. antonym                SCFG          432.0          128.0   \n",
       "4                 Add adj.                SCFG         1404.0          541.0   \n",
       "7            Add PP phrase                SCFG         1009.0          402.0   \n",
       "9           Subj. hypernym        N-gram pairs         5204.0         6149.0   \n",
       "11  Expletive construction        N-gram pairs         1682.0         1112.0   \n",
       "5             Verb antonym        N-gram pairs         1107.0          215.0   \n",
       "14            Adj. antonym        N-gram pairs         1565.0         1099.0   \n",
       "4                 Add adj.        N-gram pairs         6044.0         7298.0   \n",
       "7            Add PP phrase        N-gram pairs         2186.0         1473.0   \n",
       "9           Subj. hypernym  Hypothesis n-grams         8698.0        11427.0   \n",
       "11  Expletive construction  Hypothesis n-grams         1782.0         1368.0   \n",
       "5             Verb antonym  Hypothesis n-grams         6104.0         7188.0   \n",
       "14            Adj. antonym  Hypothesis n-grams         2262.0         2658.0   \n",
       "4                 Add adj.  Hypothesis n-grams         6044.0         7298.0   \n",
       "7            Add PP phrase  Hypothesis n-grams         2186.0         1473.0   \n",
       "\n",
       "    BERT Support  BERT Counter  \n",
       "9       0.925150      0.802621  \n",
       "11      0.971660      0.778481  \n",
       "5       0.993789      0.555556  \n",
       "14      0.916667      0.714286  \n",
       "4       0.944444      0.691892  \n",
       "7       0.928870      0.769912  \n",
       "9       0.920499      0.807644  \n",
       "11      0.966746      0.738971  \n",
       "5       0.978182      0.681818  \n",
       "14      0.923483      0.800633  \n",
       "4       0.841173      0.831494  \n",
       "7       0.897248      0.775061  \n",
       "9       0.930243      0.818119  \n",
       "11      0.964206      0.742331  \n",
       "5       0.930659      0.825688  \n",
       "14      0.900885      0.840683  \n",
       "4       0.841173      0.831494  \n",
       "7       0.897248      0.775061  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table[[\"Shortcut\", \"Feature\", \"Support count\", \"Counter count\", \"BERT Support\", \"BERT Counter\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b8e87ff4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.melt(table[[\"Shortcut\", \"Feature\", \"Support count\", \"Counter count\", \"BERT Support\", \"BERT Counter\"]],\n",
    "             id_vars=[\"Shortcut\", \"Feature\"])\n",
    "df[\"Metric\"] = np.array([\"Count\" if v.endswith(\"count\") else \"BERT accuracy\" for v in df[\"variable\"]])\n",
    "df[\"Subset\"] = np.array([\"Supporting\" if \"Support\" in v else \"Counter\" for v in df[\"variable\"]])\n",
    "df[\"Feature/Subset\"] = np.array([f\"{a}/{b}\" for a, b in zip(df[\"Feature\"], df[\"Subset\"])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c0824ae2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.ticker as mtick\n",
    "import seaborn as sns\n",
    "hls = sns.color_palette(\"hls\", 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f43356d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TOKENIZERS_PARALLELISM=true\n"
     ]
    }
   ],
   "source": [
    "%env TOKENIZERS_PARALLELISM=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "80aa3376",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1kAAAFCCAYAAAADjxdCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAACKWUlEQVR4nOzdd1gUV9sG8HupipSl92bvDXvF8iIRxRJRERRQYoy9NyygRqNGY0mU2EBULEk0KkoSTcTEV6MCFixEg1KkCQICSme+P/yY15Ui6CKo9++69rqcOWfOPDPLrPvsOXNGIgiCACIiIiIiIpILhZoOgIiIiIiI6EPCJIuIiIiIiEiOmGQRERERERHJEZMsIiIiIiIiOWKSRUREREREJEdMsoiIiIiIiOSISRYREREREZEcMckiIiIiIiKSIyZZREREREREcsQki6iWiI6OhkQiwb///lvToQAAzpw5gyZNmkBDQwMLFiyo6XDkxt/fH2ZmZjUdBhEREX3AmGQRvcTW1hYSiQQnT56UWe/q6gp3d/eaCaqGTJ06FZ6ensjKysLatWtrOhyiamVrawsVFRWoq6tDXV0dpqammDp1KnJycsQ67u7uUFZWFuuUvNatWwcACAkJgUQiEdcbGRlhxIgRiImJwYEDB2S2UVBQQJ06dcTlFi1a1NShExFRNWCSRfQKPT09zJkzB/n5+TUdylt7m2OIiopCu3bt5BgNUe02f/58ZGdnIzs7G//973/x+++/Y8WKFTJ1Ro0aJdYpec2fP1+mTkZGBrKzs3Hjxg2kpKTA3d0dLi4uMtuYmJjA19dXXL59+/a7PNQ3IggCCgsLazoMIqL3ApMsole4u7ujuLgYW7ZsKbeOlZUVdu3aJbNOIpHg7NmzAP73i/bhw4fRuHFjqKmpYfDgwcjIyMDSpUthbGwMPT09LF++vFTb586dQ/PmzaGpqYl+/fohKipKLCsqKsKGDRvQrFkzaGlpwcbGBr///rtYXjIU7rvvvoOVlRV0dXXLjL+oqAjr169H48aNoaWlhQ4dOiA4OBgAcO/ePairq6OoqAiDBw+Guro6/vrrrzLbiYyMxKBBg2BoaAhTU1NMnjwZz549AwDs378furq6iI6OBgDk5uaiffv2mDp1KgAgMTFR3FZDQwOtW7fGDz/8UOqcbt68GV27dkW9evXQpk0bRERE4IcffkCTJk2gqamJESNGIDs7W+a9Wb58Ofr27Qt1dXW0bNkSv/76a5nxV+ac3rhxA71794ZUKoW2tjZsbGzwzz//lNsefRisrKxgb2+PmzdvvnEbhoaGcHJywtWrV9+4jR9++AE2NjbQ1taGnp4eHB0d8fDhQ5k6p06dQpcuXaCtrQ1dXV2MGDFCLHv06BFcXFxgZmYGTU1NtG3bFuHh4QBe9N4tWbJEpq2XP9tKhjDv3r0bbdq0gZqaGkJDQxESEoJu3bpBV1cX2tra6Nu3L65fvy7TzqVLl9C3b1/o6elBR0cHffr0QU5ODpYuXYo+ffrI1E1KSoKKikqpNoiI3mdMsoheoaqqiq+//horV67E48eP36qtU6dOISwsDNHR0fjnn3/QuXNnGBoaIi4uDkFBQfjyyy9x6dIlmW127tyJX3/9FUlJSbC2tsbgwYPFX49XrlyJffv24eeff0Z6ejqWLFkCR0dHmUQsKSkJN27cwK1bt5CcnFxmXJs2bcLmzZtx6NAhPHnyBHPnzsWQIUMQHh6Oxo0bi0nLyZMnkZ2djZ49e5ZqIzU1FT179kS/fv0QGxuLGzdu4N69e5g5cyaAF0MsR48ejREjRiAvLw9TpkyBiooKNm7cCOBFcjN+/HhERUUhLS0NM2bMwJgxY0r9ou/v74/AwECkp6ejSZMmGDp0KE6fPo3Q0FDcv38f165dw9atW2W22bZtG1auXImMjAzMmjWrzC+mJV53TidPnox+/fohNTUVKSkp2L17N6RSaZlt0YcjKioKwcHB6NWr1xu3kZCQgMOHD6NZs2Zv3IaGhgb27NmD1NRUREZGQhAEjBkzRiw/c+YMRowYgTlz5iA5ORnx8fH44osvAAA5OTno27evmMBkZGQgMDCw3B9fyrNnzx4EBQUhOzsb7dq1g7KyMr7++mskJiYiNjYWDRs2xJAhQ8Se89u3b6Nv37749NNPERsbi6SkJCxfvhwKCgqYOHEiLly4gPv374vt7969G+3bt0fbtm3f+DwREdU6AhGJevfuLXh5eQmCIAh9+/YVJkyYIAiCILi4uAhubm5iPUtLS2Hnzp0y2wIQzpw5IwiCIJw7d04AIMTGxorlM2fOFBo3biyzTcuWLYVNmzYJgiAIDx8+FAAIJ06cEMszMzMFRUVF4c8//xQEQRA0NTWFX375RaaN/v37CytXrhQEQRD8/PwERUVF4dmzZxUeZ+PGjcX9lnB0dBQ+//zzMo+nLBs2bBC6dOkis+7ChQuCioqKUFhYKAiCIOTl5QldunQR2rVrJ+jr6wtxcXEVxtW6dWthy5YtMjEEBASIyz///LMAQEhISBDXTZ06VRg6dKi4bGlpKcyePVum3U6dOgkrVqwQBOHFOTI1NRXLXndObW1thQkTJgj//vtvhbHT+613796CqqqqoKWlJdSrV08AIPTs2VPIzMwU67i5uQnKysqClpaWzOvs2bOCIPzvun+5jXbt2gn37t0rtT9TU1PBz8+vynGGh4cLAMS4HBwchClTppRZ94cffhB0dHSE3Nzcco+55POuxMufbSWfSa9eH69KS0sTAAg3b94UBEEQpkyZIjg4OJRb39HRUZg7d64gCIJQVFQkWFpavtG5ICKqzdiTRVSOzZs3Y9++fbh27dobt2FsbCz+u169ejLLJeuysrJk1llbW4v/1tDQgJ6eHuLi4pCcnIzMzEw4OTlBKpWKr4sXLyI+Pl7cxsDAAGpqahXGFRcXhwYNGsisa9iwIWJjYyt9bPfv30dYWJhMLAMHDoREIkFSUhIAQEVFBdOmTcO1a9fg6ekpM6tfeno6PvvsM1hbW0NTUxNSqRS3b98u1Xv46jksa11F57BkOS4urtQxVOac+vv7QyKRoG/fvjAzM8PMmTNlhifSh2Pu3Lni/VSPHz+GoaEhBgwYIFNn5MiRyMjIkHn169dPpk5qaiqysrJw4cIFJCYm4sGDB28c0/nz59GvXz8YGxtDU1MTvXv3BgDxOnn48CGaNGlS5rYPHz6ElZUVVFVV33j/QOnr6ebNmxg8eDBMTU2hqakpllcmJuBF77C/vz/y8/Px66+/4unTpxg1atRbxUhEVNswySIqR8uWLeHp6YkZM2aUKtPQ0BDvPQJeDAuSl5J7mAAgOzsbqampMDMzg1QqRZ06dRAUFCTzBe/Zs2fYvn27uI2Cwusva3Nzc5khhsCL4VEWFhaVjtPIyAg9evSQieXp06fIzc2FqakpACA2NhYzZszA1KlTsXnzZoSFhYnbL1y4EJGRkTh//jyePn2KjIwMtGjRAoIgVDqG8rx8DkuWy5q2vTLn1NLSEjt37kRMTAxCQkJw5swZrFmz5q1jpNpNX18fbm5uuHTpEp48eVLl7SUSCbp3744VK1ZgwoQJeP78eZXbyM/Px6BBg2Bvb4979+4hMzMT58+fBwDxOrGyssK9e/fK3N7KygrR0dHlToDz6udYYWFhmUOkX/1McXJyQoMGDXDr1i1kZmaKQ3ErExMA2NnZQUtLC0ePHsX333+PcePGoW7duuXWJyJ6HzHJIqrAihUrcOvWrVITJ3To0AEHDx5ERkYGMjMzsXDhQrntc9WqVXj06BGeP3+OOXPmoGHDhujWrRtUVVUxadIkzJ8/H3fv3oUgCMjJycGff/5Z4Reasnh6euLrr7/G9evXUVhYiCNHjuD06dPw9PSsdBseHh64du0atm3bhufPn0MQBMTFxeHnn38GAOTl5WHEiBFwcnLC1q1bsWTJEowYMQJpaWkAgKdPn0JNTQ26urooKCjA1q1b5TbDWkBAAC5duoTCwkL4+/vj2rVrcHFxKVWvMufU398fjx49giAI0NTUhJKSEpSUlOQSJ9Ve6enp2LdvH8zNzat8D9PLPDw8ULduXXzzzTdV3jY/Px85OTnQ1taGhoYGEhISSk1UMWPGDOzevRs//fQT8vPzkZubK07cMmjQIGhra2Py5MlITU2FIAi4c+cOYmJiALz4HDtx4gQSEhKQk5ODhQsXoqCg4LVxPX36FJqamtDS0kJaWhrmzJkjU/7FF1/gzJkz8PX1RU5ODgoKCnD+/Hnk5eUBeJGATpo0CWvWrMGpU6fw+eefV/ncEBHVdkyyiCqgq6sLb29vpKamyqxftWoVNDU1YW5uDhsbGwwbNkxu+5wwYQL+85//wNDQEPfu3cPJkyfFL/Vff/01nJ2dxeFtVlZWWLNmTaW+GL1s9uzZmDJlCkaMGAEdHR2sXbsWR48eRYcOHSrdhoWFBS5duoQzZ86gQYMGkEqlGDBgACIiIgC8+PInkUiwadMmAC96rlq3bg1XV1cIgoBVq1YhJycHhoaGsLKyQnJyMrp3716l4yjPpEmT4OXlBalUivXr1+PYsWOlhkeWeN05PXfuHDp16gR1dXW0adMGXbt2/aAezkz/s27dOvG5VY0bN8bz58/FWTdLHD58uNRzskpmzCyLkpISli1bhnXr1pX6HHkddXV17Nq1C6tWrYK6ujo++eQTODk5ydSxs7PDwYMH8dVXX0FfXx9mZmb4/vvvAQB169bFH3/8gezsbLRq1QpaWlpwcXERf+iYNWsWbGxs0KxZMzRp0gQNGzYUe6ErsmfPHvzwww/Q0NBAly5d8Mknn8iUt2zZEmfPnsXBgwdhYmICQ0NDrFixAsXFxWIdDw8P/PPPP+jatSuaN29epfNCRPQ+kAjyGJtDRFRLWFlZYcmSJVXqlSOid6uoqAgWFhZYt25dmb3MRETvO/ZkERER0Tu1Y8cOKCgolOqZIyL6UPDGAiIiInonnj59CjMzM2hpacHPzw8qKio1HRIRUbXgcEEiIiIiIiI54nBBIiIiIiIiOWKSRUREREREJEdMsoiIiIiIiOSISRbViOfPnyM8PBzPnz+v6VCIqBy8TomIiN4MkyyqEZGRkbCxsUFkZGRNh0JE5eB1SkRE9GaYZBEREREREckRkywiIiIiIiI5YpJFREREREQkR0o1HQB93L787g70jYprOoxay/fLDjUdAhGvU6p1+NlIRLUde7KIiIiIiIjkiEkWERERERGRHDHJIiIiIiIikiMmWURERERERHLEiS+oZpkDsKrpIGqvSf6hcmvL1503itMb4nVKtQg/y4jofcCeLCIiIiIiIjlikkVERERERCRHTLKIiIiIiIjkiEkWERERERGRHHHiC6pZlgAa13QQH4dJF+U3icb7xLcbb5J/a7xOqRZ5+bOM1zcR1VbsySIiIiIiIpIjJllERERERERyxCTrJd7e3hg9enS55Z988gl2795dqbbc3d2xcOFCeYVGRERERETviQ8uybp48SJ69uwJqVQKqVSKDh064PTp03JpOzg4GBMmTJBLW0RERERE9GH6oCa+yMzMhIODAzZt2gQXFxcUFRXhypUrkEgkNR1ajSksLISSUu19m73aNEf79u1rOgwiqgCvUyIioqr5oHqy7t27h4KCAri5uUFJSQmqqqro2bMnevToAQDw9/dHly5dZLYxMjJCSEiIuJyfnw9XV1doaGigRYsWOH/+vFhma2sLX1/fSseTmZmJ4cOHQ0NDA61bt8b169cBABs2bMDAgQNl6q5duxaDBg0C8GKo4eeff46BAwdCQ0MDnTp1wq1bt8S6SUlJGDVqFAwNDWFubg5vb28UFxfLHOOCBQtgYGCAKVOmwNvbG59++ik+++wzaGlpoWHDhjh79iwA4KeffkLz5s1lYjl8+DBatWoFAOK248ePh6amJho1aoQrV64gICAAlpaW0NPTwzfffFPuOUhMTER4eHip1927dyt9HomoevE6JSIikq8PKslq3Lgx6tSpgzFjxiAoKAgpKSlVbuPEiRMYMGAA0tPTMX/+fAwZMgTp6elvFE9gYCBmz56NjIwM9O3bF9OnTwcAuLq64ty5c0hOThbr7tu3D+7u7jLLs2fPRlpaGhwdHTF06FAUFhaiuLgYjo6OaNiwIWJiYnD58mUcP35c5l6x0NBQ6OnpIT4+Hps2bQIABAUFYeDAgUhLS8OUKVMwfvx4AMDgwYPx+PFjhIb+b0rcgIAAuLm5ictBQUEYOnQo0tPTMXToUDg5OeHixYuIjIzEqVOnsGDBAsTFxZV5Dr7//nvY2NiUerm6ur7ROSUi+eN1SkREJF8fVJKlqamJixcvQk1NDVOmTIGRkRH69OmDf//9t9JttGnTBmPHjoWSkhLc3NxgbW2NU6dOvVE8Q4cORY8ePaCoqIhx48YhPDwcAGBoaAg7OzscOHAAABAWFoaEhAQMHjxY3PaTTz5B//79oaysjEWLFuHp06f4+++/ERoairi4OKxatQp16tSBiYkJZs+ejYMHD4rbGhgYYO7cuVBWVkbdunUBAF27dsWwYcOgqKgINzc3xMXFITU1FSoqKnBxcUFAQAAAIDk5Gb///jtcXFzE9rp27QpHR0coKirC2dkZsbGx8Pb2Rt26ddG5c2dYWVnhxo0bZZ6Dzz//HGFhYaVe+/fvf6NzSkTyx+uUiIhIvmrvzTpvqHHjxti1axcAICYmBhMnTsTYsWNx6dKlSm1vYWEhs2xpaYn4+Pg3isXIyEj8t5qaGp49eyYue3h4wNvbG7Nnz0ZAQABGjx4NVVXVMuNQVFSEqakp4uPjIZFIkJKSAm1tbbG8uLgY5ubm4rK5uXmp+9BejQUAsrOzoaenBw8PD9jZ2WHDhg0IDAxE3759YWxsXOG2r67Lzs4u8xwYGxvLtEVEtQ+vUyIiIvn64JKsl1laWmLatGlwdnYGAKirq+P58+dieUFBAdLS0mS2iY2NLbU8YsQIucfm4OCAiRMnIjw8HAcPHsTJkyfLjaOoqAjx8fEwNTWFkpISzMzMEB0dXW7bVZ3oo23btjA1NUVwcDACAgLe6dTzd9Z9iWID/Xe2v49Bhy2Vv2+QqDJ4nX54+DlBRFS9PqjhgpGRkVi/fj1iY2MhCAIeP36MXbt2oWvXrgBeDAWMjIzElStXkJ+fj2XLlokTRpS4ceMGAgMDUVhYiH379iEqKqrUJBXyoKysDFdXV3h4eEBXVxedO3eWKf/ll1/wxx9/oKCgAGvXroWGhgY6d+6Mjh07Ql9fHytXrsSzZ89QXFyM+/fvy0zQ8SY8PDywdOlSREdHY8iQIW/VFhERERHRx+yDSrI0NDQQGhqKbt26QV1dHW3btoW6ujr27t0LAGjUqBFWrVoFBwcHWFpawtzcHHp6ejJtODo64vTp09DW1saaNWtw7Ngx6OjolNrXX3/9BXV19beK18PDAzdv3pSZZKKEq6srvv76a2hra+PYsWM4duwYlJWVoaioiJMnT+L+/fto1KgRtLW1MXLkSCQmJr5VLC4uLoiMjMSoUaNQp06dt2qLiIiIiOhjJhEEQajpIN4XPXv2hIeHhzgz39tKS0uDsbExoqKiYGZmJq53d3eHkZERvvrqK7nspzIKCwthYmKC48ePiz1/1Sk8PBw2NjbYN2o4mnIYklxxGBDJC6/TDxc/J4iIqtcH1ZNVnZ49e4YHDx7A2tpaLu0JgoBNmzbB3t5eJsGqKXv37oWJick7SbCIiIiIiD5kH/TEF/ISGxuLli1bYtCgQejdu/dbt5eXlwddXV0YGxsjKChIDhG+nYYNGyInJweHDx9+9zu3BGD+2lpUBaHfTqq2tjtM5a/fHyVepx+cks8JXtNERNWDSVYlWFhYIDMzU27tqaqqljvlOQD4+/vLbV+VUZXniBERERERUcU4XJCIiIiIiEiOmGQRERERERHJEZMsIiIiIiIiOeI9WVSzzAE0qukgqLJCf62+STWobB0G1IKJCXidfrBevaZrxd8bEdEHgD1ZREREREREcsQki4iIiIiISI6YZBEREREREckRkywiIiIiIiI54sQXVKOad/NC+/btazoMIqoAr1MiIqKqYU8WERERERGRHDHJIiIiIiIikiMmWURERERERHLEJIuIiIiIiEiOmGQRERERERHJEZMsIiIiIiIiOWKSRUREREREJEd8ThbVqC+/uwN9o+K3bsf3yw5yiIaIyiKv65SoNuL/H0RUHdiTRUREREREJEdMsoiIiIiIiOSISRYREREREZEcMckiIiIiIiKSI058QTXLHIDV2zczyT+0UvV83XmDM1GVyek6JapN+P8BEVUn9mQRERERERHJEZMsIiIiIiIiOWKSRUREREREJEdMsoiIiIiIiOSIE19QzbIE0Pjd7W7SxRcTZPh24w3PRJX2jq9TourG/wOIqLqxJ4uIiIiIiEiOmGQRERERERHJUa1IsqKjoyGRSJCbm/tG2x84cAB9+/aVc1Qfj9WrV8Pd3b2mwyAiIiKiKvD394eZmVlNh0FlqFKSZWtrizp16kBdXV182djYVFdsZQoJCYGRkZHMOhcXF/zxxx/vNI7qYmVlhV9++aXa2vf390eXLl1k1i1evBj+/v7Vtk8iIiKiN2VrawsVFRWZ75+9evV663ZrIkERBAFmZma4cuUKiouLsXr1ajRp0gQaGhrQ0dFBx44dcfjw4XcaU0VsbW2xZMmSmg7jvVTliS82bdqESZMmVUcsVAmFhYVQUvpw5ivxatMc7du3r+kwiKgCvE6JqKbNnz8fq1atqukwypSfnw8VFZVK1b18+TIUFBTQsWNHrFu3Dnv27MHRo0fRqlUr5OTk4OrVq8jLy6vmiOldkNtwwWPHjsHExASpqakAgLCwMGhra+Off/4B8KKHZs2aNWjVqhW0tLTw6aefIiMjo8y2MjMzMWnSJJiZmcHIyAhTp05Fbm4unj59ik8++QSPHz8Wf8m4e/euTO/M5MmTMWXKFJn2Jk+ejMmTJ1fYdnkCAgLQsmVLaGhooGHDhmIvU2ZmJiZMmAAjIyOYmZlh9uzZ4kVRMvxx3759sLa2hra2NmbNmiW2+eDBA/Tt2xdaWlrQ1dUVf41xdnZGbGwshg0bBnV1dXh5eQEAJBIJtm3bhqZNm0IqlYrrIiMjxTYXLlwoM+TvypUr6NWrF7S1tWFkZIQ1a9YgIiICkyZNwtWrV8Xz9+zZM3h7e2P06NHitsHBwWjTpg20tLTQuXNnXLp0SSxzd3fH5MmTMXz4cGhoaKB169a4fv16ueePiIiIqDpkZGTgiy++gKWlJXR1dTFw4EA8ePBALP/hhx9gY2MDbW1t6OnpwdHREQ8fPgQA/PXXX5g0aRISEhLE70QHDhwQv8P9+++/YjshISGQSCQoLCwEAHh7e6NHjx5YunQpTExM0LZtWwBAZGQkBg0aBENDQ5iammLy5Ml49uyZTMw//fQThg0bBolEggsXLmDgwIFo3bo1JBIJ1NTU0Lt3b9jZ2Yn1JRIJzp49Ky6XFR8AbN68GWZmZtDV1cX48eORnZ0N4EXP2bJly2BmZgYNDQ2YmZlh8eLF4nbx8fEYM2YMTE1NYWBgAGdnZ6SkpAAAJk2ahL/++gvr1q0TzxFVntySrGHDhmHgwIHw9PTE8+fP4eLigrVr16JJkyZiHX9/fxw/fhyPHj1CXl4epk+fXmZbHh4eyM3NxZ07dxAZGYn79+9j5cqV0NLSQnBwMAwMDJCdnY3s7Gw0a9ZMZltXV1ccOXJEvBAKCgpw5MgRuLq6Vth2WX7++WfMnz8fO3bsQGZmJs6dOwdLS0sAwPTp0xEfH4/IyEiEhobiv//9L1asWCGz/dmzZ3Hr1i2EhYXBz89PHNLo5eWFxo0bIzU1FUlJSfjyyy8BAAcPHoSFhQWOHTuG7OxscT0AHDlyBH/++af4h1+RR48e4T//+Q/Gjx+P5ORk3Lt3D3379kWrVq3g6+uLjh07iuevXr16Mtvev38fn376KVavXo0nT55gypQp+OSTT8TkGQACAwMxe/ZsZGRkoG/fvuW+jwCQmJiI8PDwUq+7d+++9jiI6N3gdUpE7xtBEDBs2DBkZmbi2rVrSEhIQKtWrTBo0CAUFBQAADQ0NLBnzx6kpqYiMjISgiBgzJgxAICePXvC19cXJiYm4nciFxeXSu//77//hrKyMh48eIDQ0FCkpqaiZ8+e6NevH2JjY3Hjxg3cu3cPM2fOlNnu6NGj+PTTTwG8GIq3d+9erFy5EiEhIcjMzHyjc5GUlITr16/jn3/+wc2bNxERESH+uH/27Fns2bMHFy9eRFZWFm7evInBgwcDAPLy8tCvXz+YmJjg3r17ePDgAZSUlMRz5Ovri549e2L+/PniOaIqEKqgd+/eQt26dQUtLS3xNW7cOLE8OztbaNSokdCqVSth6NChMttaWloKW7ZsEZevX78uqKioCEVFRcLDhw8FAEJOTo6QnJwsKCkpCZmZmWLdP//8U7C2thYEQRDOnTsnGBoayrTt5+cndO7cWVyuX7++EBQUJAiCIJw8eVKoX7++IAjCa9t+1YABA4T169eXWl9YWCioqKgI169fF9cFBwcLZmZmgiAI4vE8fPhQLB88eLDY1rhx4wRHR0fh/v37pdq2tLQUgoODZdYBKHPd3bt3xeUFCxYIbm5ugiAIwpo1awQHB4cyj+nVcyUIgrB8+XJh1KhRgiAIwsqVK0u9d507dxZ27dolCIIguLm5ifsRBEEICwsT6tWrV+a+StoGUO4rLCys3G2J6N3gdUpEtVnv3r0FVVVVme+f3t7egrKyspCVlSXWKywsFOrUqSP89ddfZbYTHh4uABC/B/r5+QmmpqYydUq+w738He3cuXMCAKGgoEAQhBefmSYmJkJxcbFYZ8OGDUKXLl1k2rpw4YKgoqIiFBYWCoIgCNeuXRMMDQ2FoqIiQRAEobi4WAgMDBQGDx4s6OnpCYqKikK/fv2E27dvi20AEM6cOVNufH5+foKCgoKQkZEh1jl9+rSgrKwsFBYWCiEhIYKurq5w+vRp4fnz5zLx/fTTT6WO49GjRwIAIS4uTjz3Xl5eZZ5PqliVe7I2btyIjIwM8bV3716xrF69enB1dUVERATmzZtXalsLCwvx35aWlsjPzy/VMxMdHY2ioiKYm5tDKpVCKpVi0KBBePz4caVjdHFxwYEDBwC8mHmw5JeJqrYdGxuLhg0bllqfmpqK/Px8WFlZieusrKyQmJgIQRDEdS9P0KGmpib+ArB+/XqYmJigT58+qF+/Pr766qvXHlNJD1pllBd3ZcTHx8scF/Di2OLj48XlV4/r1a7wl33++ecICwsr9dq/f/8bxUdE8sfrlIhqu7lz58p8/2zatCkKCwthZmYmfqfT1dUFAMTFxQEAzp8/j379+sHY2Biampro3bs3AFTpO2V5LC0tIZFIxOX79+8jLCxMjEUqlWLgwIGQSCRISkoC8GKo4JAhQ6Cg8OLrt0QigbOzM06cOIGUlBTcvHkTxcXFcHBwkPk++Tra2trQ0tISl62trVFQUIDk5GT07t0b69atw1dffQVDQ0P06tULZ86cEWNOTk6Gtra2GHOLFi2gqqqK2NjYtz5HHzu5zqAQGRmJjRs3wt3dHdOnT8elS5egrKwslr/8hsXGxkJZWRn6+voy6y0sLKCkpITHjx+XeRPhy3/Q5XF1dUW7du2QmJiIEydO4Nq1a5Vq+1UWFhalxrwCgJ6eHlRUVBAdHY02bdoAeJHAGRsbVyo+AwMDbN++Hdu3b8f169fRr18/dOzYEf369St3+1fX16tXD8+fPxeXSy7gkrgvXLhQqXZeZWpqirCwMJl10dHR+M9//lPhduUxNjaGsbFxueV31n2JYgP917bTYYvvG+2fiF5PXtcp1Q78vKSPgZGREVRUVJCSkiLzXbNEfn4+Bg0ahGXLluHnn3+GhoYGrl27hvbt24sJTEmy8zINDQ0AkPkBOSEhoVS9V7c1MjJCjx49Kpzt+ujRo/jmm2/KLW/evDlmzZoFR0dHpKenQ0dHR7x/vqJY0tPT8fTpUzHRio6OhrKyMgwNDQEA48ePx/jx45GXl4fvvvsOgwcPRkpKCoyMjGBpaYmoqKhyYyrrHFHlyO3M5efnw8XFBQsXLsTu3buhrq6OZcuWydTZtm0bHj58iKysLCxZsgSjRo0q84/UwcEBM2bMQHp6OgRBQFxcnDjhhKGhIdLT05Genl5uLI0bN0aLFi3g5uaGFi1aoHHjxpVq+1UTJ07Ehg0b8Pfff0MQBDx69AiRkZFQVFTE6NGjsWjRImRkZCA5ORk+Pj4YO3Zspc7VkSNHxF9ZpFIpFBUVoaioKB5fRX/sJdq1a4d9+/ahqKgIFy9exPHjx8UyFxcX/Pnnn9i3bx8KCgqQmZmJy5cvi+3Hx8eXO3PNyJEj8euvvyI4OBiFhYXYv38/IiMjMWTIkEodGxEREVF169GjB1q2bIkvvvhC7JlKT0/HTz/9hOfPnyM/Px85OTnQ1taGhoYGEhISSk1FbmRkhNTUVDx58kRcp6urC2tra+zatQuFhYV48OABvv7669fG4+HhgWvXrmHbtm14/vy5+B3z559/BvCiIyIxMRF9+vQRt9mwYQNOnTqFtLQ0AC86ILZt24YWLVpAR0cHANChQwf4+/sjNzdX/L75KolEgjlz5uDZs2dISEjA8uXLMXbsWCgqKuLKlSv4888/kZOTAxUVFWhoaEAikUBRURHDhw9HQUEBli5diqdPnwJ40cv38hTyRkZGuHfvXmXeEnpFlZOsmTNnyjynoGTo2NKlS6GhoYH58+dDQUEBAQEB2LFjB86fPy9uO27cODg6OsLMzAyKiorYvHlzmfvYu3cvlJWV0bZtW2hpaWHAgAHiG9y0aVO4urqiYcOGkEql5d6Y7erqijNnzogTXlSm7VcNHz4cq1atwvjx46GpqYk+ffogJiYGALBlyxYYGhqiSZMmaNeuHTp16lQqqSxPWFgYunbtinr16qFXr16YOXMmbG1tAQCLFi3CunXrIJVKsXTp0nLb2LJlC37//XdIpVJs2LABzs7OYpm5uTl++eUX+Pr6Ql9fH02aNEFISAgAoG/fvmjbti2MjY0hlUpLDfVr3Lgxjhw5ggULFkBXVxebN2/GqVOnoKenV6ljIyIiIqpuioqKOHPmDNTU1NC5c2doaGigTZs2OHbsGCQSCdTV1bFr1y6sWrUK6urq+OSTT+Dk5CTTRt++fTFkyBA0btwYUqkUgYGBAF7MLB0SEgKpVApXV1d4enq+Nh4LCwtcunQJZ86cQYMGDSCVSjFgwABEREQAeDFUcPDgwTK9blpaWlizZg0aNWoEdXV1dOvWDXp6eggKChLrfPfdd0hKSoKenh7+85//lPmDvpGREVq1aoXGjRujZcuWaNasGTZt2gQAyM7OxuzZs2FgYACpVIodO3bg2LFjUFNTg4aGBi5duoTY2Fi0atUKmpqa6NatG/7880+x7Tlz5uCff/4RhxRS5UmEqgz6fAtWVlbw9fWFvb39u9gd1XLh4eGwsbHBvlHD0ZTDBYlqpapep1Q78POSqPaxsbHB8uXL4ejoWNOh0DvCgZZERERERNUkPz8fjo6OMs+/og+fXCe+IKoySwDmr68W+u2kCss7TOUvt0TVppLXKckHP8+IPiwqKipYvnx5TYdB79g7S7Kio6Pf1a6IiIiIiIhqDIcLEhERERERyRGTLCIiIiIiIjlikkVERERERCRHnPiCapY5gEZv30zor6UnxugwgDePE8mFnK5TqpySzzN+hhERvb/Yk0VERERERCRHTLKIiIiIiGpAixYtcPbs2ZoOg6oBhwsSERER1VKTvEKrtX3fLztUqf7FixexYMECREREAAAaNmyIFStWYODAgcjOzoaPjw9+/PFHPH78GPr6+ujXrx+WLl0KKysruLu7IzAwECoqKmJ7q1evxvTp0wEA586dw+rVq3HlyhVIJBKYmJhg6NChmD17NvT09MRt5s+fD6lUisWLF2PdunX4/vvv8fjxY9SrVw82NjY4dOgQNDQ05HB25MvW1hajR4/GpEn/u8Xh9u3bNRgRVSf2ZBERERHRa2VmZsLBwQGenp5ITU1FcnIyvvnmG2hqaiI/Px/9+/dHWFgYTpw4gczMTFy7dg1t2rSR6amZPXs2srOzxVdJgnX06FE4Ojpi6NChiIqKQkZGBoKDg6GiooLw8HCZOIKCgjBo0CAEBARg586dOH36NLKysnDr1i04OTm903NSGcXFxRAEoabDoHeMPVlUo5p380L79u1rOgwiqgCvUyICgHv37qGgoABubm4AACUlJfTs2RMAsGfPHjx48ABRUVFiL5K2traYRFVEEATMnDkTS5YswZQpU8T1lpaW8Pb2lqn74MEDPHv2DK1bt8aOHTtgZ2eHJk2aAAD09PTg7u4u1n215+iXX37BpEmTEB0dDQCwsrLC559/jsDAQMTGxqJ///7YvXs3pFIpoqOjYW1tjZ07d8LHxwe5ubn47LPPsGrVKigoKEAQBKxfvx6+vr54+vQpevbsiW3btsHExERse/LkyTh8+DBu374NNzc3/PXXX/j7778xd+5cDBs2DPv27YOVlRV8fX1hb28Pb29vREREQEdHB0eOHIG+vj58fX3Rv39/AEBsbCzc3d1x9epVtGnTBr169cLFixcREhJStTeS3gn2ZBERERHRazVu3Bh16tTBmDFjEBQUhJSUFLHst99+g729/RsN0/vnn38QFxdXqV6ooKAgODg4AAC6deuG/fv3Y82aNfj777+Rn59f5X37+/vj+PHjePToEfLy8kolhcHBwbhz5w7+/vtvHDp0CP7+/gCAvXv3Ytu2bTh9+jQePXoEY2NjjBw5UmbbgIAA/PDDD8jKysJ3332Hnj17YtOmTcjOzsa+ffvKPb6BAwciLS0NU6ZMwfjx48UyZ2dnNG/eHCkpKfj222/h5+dX5eOld4dJFhERERG9lqamJi5evAg1NTVMmTIFRkZG6NOnD/799188efJE7MWpyKZNmyCVSsVXWloaUlNTAUBm++nTp0MqlaJevXpYunSpuL5kqCAAjBkzBjt27MD58+dhZ2cHPT09zJ07F0VFRZU+pqlTp6J+/frQ0NDAl19+icOHD6O4uFgs9/b2hoaGBho0aIAZM2YgMDAQALB//37MmjULTZs2Rd26dbF+/XpcvnwZUVFRpdpWVlaGklLlBo917doVw4YNg6KiItzc3BAXF4fU1FTExsbi0qVLWL16NerUqYO2bdvCxcWl0sdJ7x6TLCIiIiKqlMaNG2PXrl2IiYnBgwcPoKKigrFjx0JXVxcJCQmv3X7mzJnIyMgQXzo6OuKkFi9vv2XLFmRkZMDJyQkFBQUAgOzsbISGhqJv375ivVGjRuGXX35Beno6fvzxR+zevRu7du2q9PFYWFiI/7a0tER+fr5MD92r5fHx8QCA+Ph4WFlZiWXq6urQ1dUVy0vqV5WRkZH4bzU1NQAvjjshIQFaWlrQ1NQUy83NzavcPr07TLKIiIiIqMosLS0xbdo03Lp1C3Z2dvj111+RnZ1d5XaaNGkCMzMz/PTTTxXWO3PmDLp37446deqUKlNUVISdnR369euHW7duAXiR+Dx//lysk5SUVGq72NhYmX8rKytDX1+/3HJTU1MAgKmpqXhvF/AiEXry5IlYDgASiURmX68uV4WJiQmePn2KrKwscV1cXNwbt0fVj0kWEREREb1WZGQk1q9fj9jYWAiCgMePH2PXrl3o2rUrXF1dYWVlhaFDh+L27dsoKirC06dP8d1332H37t0VtiuRSLBx40asXLkS27dvx5MnTwAAjx49woMHD8R6Lw8VBAA/Pz+cPHkST58+hSAIuHTpEkJCQtC1a1cAQLt27fDjjz8iOzsbcXFx2Lp1a6l9b9u2DQ8fPkRWVhaWLFmCUaNGQUHhf1+PfXx8kJWVhQcPHmDz5s1wdnYGALi4uGDTpk24d+8ecnNzsWDBAnTq1AkNGjQo9zgNDQ1lhhNWhYWFBbp06QIvLy/k5eXh5s2b4tBFqp04uyARERFRLVXV51hVJw0NDYSGhmLz5s1IT0+HlpYW+vbti+3bt0NFRQVnz56Fj48PHBwckJKSAj09PdjZ2cncU1UeJycnaGtr46uvvsKCBQugoKAAU1NTODo6Yvbs2RAEAcHBwVixYoW4jVQqxdq1a+Hm5obCwkKYmJhg8eLFGDNmDABg1qxZCA0NhbGxMZo0aQIXFxds2bJFZr/jxo2Do6MjYmNj0bdvX2zevFmm3N7eHs2bN0dubi4mTJgADw8PAICbmxuSkpIwYMAAPH36FD169MDhw4crPMYZM2bA3d0du3btgqOjI/bu3Vup814iMDAQ7u7u0NPTQ+vWreHi4oKbN29WqQ16dyQCJ+6nGhAeHg4bGxuEhYVxamiiWorXKRHVFlevXsXEiRNx7do1ubX58vTpryqZwj0nJ6fM4Ym1wYwZM/Ds2bMq3YNG7w6HCxIRERFRrSYIAlatWlXTYdSosLAw3L9/H4Ig4MKFC9i7dy+GDx9e02FROThckGrUl9/dgb5R8Wvr1abhEkQfm8pep0QfIv7/Uzt06tSppkOoccnJyRg+fDhSUlJgZGSEZcuWYeDAgTUdFpWDSRYRERERfXRenh3wVVZWVqhtd9QMHDgQMTExNR0GVRKHCxIREREREckRkywiIiIiIiI5YpJFREREREQkR7wni2qWOQCr11eb5B9aYbmvO29MJqo2lbxOid5n/H+EiOSJPVlERERERERyxCSLiIiIiKgSJk2ahOXLl9d0GLVCbGws1NXVkZeXV9Oh1EocLkhERERUS71uuPzbqsowSVtbW1y4cAF37txB48aNAQCRkZFo1qxZrZvuvLr4+vq+dRudOnXCpk2b0KJFC8ydOxenTp1CVlYW9PT0MHToUHzzzTdyiFT+JBIJ7t69i6ZNmwIALCwskJ2dXcNR1V7sySIiIiKiStHS0sLSpUurrf3CwsJqa7u6VSb25ORkxMTEoEuXLpg1axaePHmCmzdvIjMzE3/88QfatWv3DiKtmvf5PalJ7MmimmUJoPHbNzPpYvX+0keAbzfeFP7RktN1SlQd+Nn0bk2bNg0bNmzAtWvXKp0QHDx4EF5eXkhPT8eECRNw4cIFTJo0Ce7u7vD394evry969+4NPz8/DBs2DOvXr4eLiwsuX76MgoICdO3aFb6+vrCwsADwokete/fuuHDhAkJDQ9GxY0ccOXIEa9euhZ+fH6RSKfbs2QNbW9sy47GyssLnn3+OwMBAxMbGon///ti9ezekUikAwNnZGSEhIXj+/Dlat26Nbdu2oVWrVgAAd3d3GBkZ4auvvkJISAhGjx6NhQsXYv369WjVqhV+/vlnTJo0CSdPnkRhYSGsrKxw8OBBNG/eHABw6tQpDBgwAAoKCrhy5QqWL18OPT09AIC1tTWsra3FOF/tOVq4cCGSkpLg7++P6OhoWFtbY+fOnfDx8UFubi4+++wzrFq1CgoKCuJ57datG/bs2QNdXV2sWbMGI0eOBABkZmZi1qxZOHXqFJSUlDBy5EisWbMGqqqqYtt+fn7w8fFB3bp1xXNjY2MDiUSCjRs3ws7ODtbW1sjJyUGdOnVga2uLnj17iu9L69atERgYCEtLSwDAH3/8galTpyIuLg7Dhg1DdnY2WrduDW9v78r98b1n2JNFRERERJViZGSE6dOnY/HixZWqHxkZiQkTJmDXrl14/Pgx9PX1ERYWJlMnNDQUenp6iI+Px6ZNm1BcXAw3NzdER0cjLi4OmpqamDx5ssw2Bw4cwPfff4+UlBQUFRWhS5cuaNGiBVJSUjB58mRMnDixwrj8/f1x/PhxPHr0CHl5eZg+fbpYZmdnh3/++QePHz9Gp06d4OzsXG47qampePjwIaKionDs2DHs3bsXERERiIqKQkZGBg4dOgQdHR2xflBQEAYNGgQA6NatG5YuXYpdu3bhzp07lTqfrwoODsadO3fw999/49ChQ/D39xfLSs5rSkoKtm/fDjc3N9y/fx8AMH36dMTHxyMyMhKhoaH473//ixUrVsi0/csvv+DGjRsICwvDxYsXAQBhYWHIzs4u9/wGBARg69atePLkCYyNjcVez7S0NAwdOhRLlixBeno67O3tcfLkyTc65vcFkywiIiIiqrR58+bhypUr+PPPP19b98iRIxg4cCD69u0LZWVlzJs3D9ra2jJ1DAwMMHfuXCgrK4u9JiNGjICamhrU1dWxaNEinD9/XmYbd3d3NG3aFGpqahg2bBhUVFQwfvx4KCoqYsyYMbh//36F9wtNnToV9evXh4aGBr788kscPnwYxcXFAAAPDw9oampCVVUVy5Ytw+3bt/HkyZMy2xEEAWvWrEGdOnVQt25dqKioICsrC5GRkRAEAc2aNYORkREAID8/HyEhIRgwYAAAYMuWLfDw8MCOHTvQtm1bmJmZISAg4LXn9GXe3t7Q0NBAgwYNMGPGDAQGBoplOjo6WLhwIZSVlWFnZwd7e3scPnwYRUVFOHjwINauXQupVAojIyP4+PiU2re3tzc0NTVRt27dSsfj4eGBli1bQkVFBWPGjEF4eDiAF8ll48aNMWbMGCgpKWHMmDFo06ZNlY71fcMkS06srKzwyy+/1HQYRERERNVKKpViwYIFWLRokcz6ktnmSl6xsbFISEiAubm5WEdBQQGmpqYy25mbm0MikYjLz58/x+effw5LS0toamqiZ8+eyM7OlpnFriRxAQA1NbVSywAqTLJKhh4CgKWlJfLz88VesYULF6JBgwbQ1NQUh7qlpqaW2Y6enp64PwAYO3Ys3NzcMHHiRBgYGGDixInIzMwEAJw/fx5t27aFlpYWAKBOnTpYsGABrly5goyMDMybNw8eHh64e/duuXG/7jji4+PFZVNTUygoKJQqT01NRX5+PqysrMQyKysrJCYmykxgUnLsVfHq+1DyHrz6dwCg1PKH5qO8J8ve3h6tWrXC+vXrZdbfunULbdu2RWxsLExMTGooutd7dYzu+8yrTXO0b9++psMgogrwOiWiV02bNg2bN29GUFCQuK6s2eZMTExw8+ZNcbm4uFgmEQAgk2ABwIYNG8QhcMbGxrhx4wbatm0r1xkMY2NjZf6trKwMfX19HDhwAEePHsWZM2dgbW2NzMxMSKXScvf9auxKSkrw8vKCl5cXEhMT4eTkhPXr12PlypUICgqCg4NDme2oqalhxowZWLlyJe7cuYNmzZqhXr16eP78uVgnKSmpzOMouV8sNjZWJoGNj49HcXGxmGjFxsaiffv20NPTg4qKCqKjo8XepOjoaBgbG8scz6vH9jZMTEwQFxcnsy4uLu6D7s36KHuyPDw8sH//fhQVFcms9/f3h52dXZUSLM64QkRERB+bunXrYtmyZfjqq68qrOfk5ITTp08jJCQEhYWF2LBhA9LT0yvcJisrSxw2mJ6ejpUrV8ozdADAtm3b8PDhQ2RlZWHJkiUYNWoUFBQUkJWVBVVVVejq6iInJwdLliypUrvnzp3DjRs3UFRUBHV1daiqqkJRURHAi0kvSu7HAoAVK1bg8uXLyMvLQ0FBAfbs2YPs7GzxR6127dph3759KCoqwsWLF3H8+PFS+/Px8UFWVhYePHiAzZs3y9w/lpaWhnXr1qGgoABnz55FcHAwnJycoKioiNGjR2PRokXIyMhAcnIyfHx8MHbs2AqPzdDQEFFRUVU6HyUcHBxw7949HD58GIWFhTh06BBu3LjxRm29Lz7KJGvo0KHIy8vDr7/+Kq4rKirCgQMH4OHhAUEQsHHjRjRu3Bg6OjoYOHAgHj16JNaVSCTYtm0bmjZtKs62AgDXrl1Dq1atoKWlhU8//RQZGRll7j8zMxODBw+GgYEBtLW1MXDgQJlfVGxtbbF06VL06dMHGhoa6N69O2JiYgC8uEkSeDG7i7q6Onbs2AHgxY2Pbdq0gZaWFjp37oxLly6J7bm7u2Py5MkYPnw4NDQ00Lp1a1y/fh3Ai1+LBg4cKBPf2rVrxQ8Bd3d3fPHFFxg6dCjU1dXRrl07REVFYd26dTAyMoKxsTEOHz5c7rlOTExEeHh4qVdVusKJqHrxOiWiNzFhwoRS91e9qlmzZti5cyc8PDygr6+P5ORktG7dGqqqquVuM3PmTOTn50NfXx+dO3fGf/7zH3mHjnHjxsHR0RFmZmZQVFTE5s2bxfX169eHqakpmjVrho4dO1ap3aSkJIwaNQpaWlpo0KABTE1NMW/ePNy9excSiQRNmjQR6yooKMDT0xP6+vowMDDAzp07cezYMXGGwS1btuD333+HVCrFhg0bypyAw97eHs2bN0fnzp3h5OQEDw8PsaxDhw7iZCMTJ07Enj17xP1v2bIFhoaGaNKkCdq1a4dOnTph2bJlFR6bj48PPD09IZVKsXPnziqdF11dXRw9ehTe3t7Q1tbG6dOnYW9vX+HfwXtP+EhNnjxZcHJyEpeDgoIEHR0dITc3V9iyZYvQvn174eHDh0J+fr7g5eUldO/eXawLQOjdu7eQnJwsPH/+XBAEQbC0tBQaN24sREVFCZmZmYKDg4MwduzYMvednp4u/PDDD8KzZ8+ErKwsYdSoUYKDg4NY3rt3b8HCwkKIiIgQ8vLyhE8//VSmLQDC3bt3xeV79+4JdevWFYKCgoSCggJh7969gpaWlpCSkiIIgiC4ubkJWlpawl9//SUUFhYKM2bMEHr27CkIgiAkJSUJderUEZKSksT2WrRoIfzwww/itlKpVLh06ZKQn58vjBw5UrC2tha8vb2F/Px84ciRI4KWlpZ4Hl61fPlyAUC5r7CwsMq9YURUbXidEtG7UlBQIOjq6gp//fVXjcVgaWkpBAcHv9N9rlu3Tpg5c6bc2nv48KEAQMjJySmz3M/PT+jcubPc9lcdWrduLezbt6+mw6g2H22SdfXqVUFVVVVIS0sTBEEQnJychKlTpwqCIAjNmjUTTp8+LdYtKCgQVFRUhJiYGEEQXiQ5r16clpaWwpYtW8Tl69evCyoqKkJRUdFrY7l+/bqgrq4uLvfu3VtYvny5uPzTTz8JLVq0EJdfTbJWrlwpDB06VKbNzp07C7t27RIE4UWi5ObmJpaFhYUJ9erVE5cdHR2FDRs2CIIgCKGhoYK2traQm5tb5rY//fSTUK9ePaGwsFBcp6KiIkRERJR5bAkJCUJYWFip1/79+/nljaiW4HVKRNXpxIkTQlZWlpCTkyMsXrxYMDMzE/Ly8mosnppIsg4fPizcunVLbu29j0nWL7/8Ijx58kTIz88Xtm/fLtSrV094/PhxTYdVbT7KiS+AF12ojRo1wqFDhzB69GicOHFCfAZAdHS0ODa3hIKCAh49eiTO4lLWjCvlzVRjaGgoU+/58+eYNWsWfvnlF3FccsmsOSXdpuXNzlKW+Ph4mRligBezxLx8Y+mr7T179kxc9vDwgLe3N2bPno2AgACMHj1apvv21W319PTE8cXAi3HZ5cVnbGwMY2PjcmMnopr3uuv0zrovUWygL9d9dtjiK9f2iKj2On36NMaNG4eioiLxgb0qKio1HdY7VfIQ4I/Z1atX4eLigtzcXDRq1AjHjh2Dvr58/2+pTT7aJAt4kVzs3bsXANCkSRPxRkMLCwv4+vqW+6RwoOwZV8qbqeZV8p41x9TUtNSD/aKjoys9htnBwQETJ05EeHg4Dh48+ME/HI6IiIjene3bt2P79u01HYYoOjq6pkN4a1ZWVhV+b3R3d4e7u/u7C6gSlixZUuWJRN5nH+XEFyVcXV0RHh6OtWvXytwo+MUXX8DLy0ucQSU9PR1Hjhx5bXvlzVTzqredNefV2V1GjhyJX3/9FcHBwSgsLMT+/fsRGRmJIUOGVKo9ZWVluLq6wsPDA7q6uujcuXOV4iEiIiIiov/5qJMsAwMDDBw4EAkJCXBxcRHXT5s2DaNHj8agQYOgqamJNm3ayMxEWJ7yZqp51dvOmvPq7C6NGzfGkSNHsGDBAujq6mLz5s04deoU9PT0Kt2mh4cHbt68CTc3tyrFQkREREREsiTCm45Row9KWloajI2NERUVBTMzs2rfX3h4OGxsbBAWFsaHnBLVUiXX6b5Rw9GU92QRERFV2kd9Txa9IAgCNm3aBHt7+3eSYL3szuEvUXxRHx2m8gsXUa1lCcBcvk2GfjtJvg3WAH5uERFReZhkfeTy8vKgq6sLY2NjBAUF1XQ4RERERETvvY/6niwCVFVVkZ2djfv378s8hZyIiIioNgsJCZF5zIy8rF69utbNzEdvJzY2Furq6sjLy3tn+2RPFhEREVEtNeliaLW279utQ6Xr2traYvTo0Zg06X/DfaOjo2FtbY2cnBzUqVOnOkJ8p/sBgMWLF1dr+++TTp06YdOmTWjRogXmzp2LU6dOISsrC3p6ehg6dCi++eabmg6xTBKJBHfv3kXTpk0BvHg8U0XPnK0OTLKIiIiIiGoxQRBQXFwMRUXFd7bP5ORkxMTEoEuXLvD09ERGRgZu3rwJXV1dREdH46+//npnsVRWYWEhlJRqR3pTO6Kgj5c5gEZA6K/v/03wVHkdBnDCgPfK/1+nJOtD/9zidUpVFRYWhv79+yMpKQmqqqoAgMuXL2PgwIFITExEYGAgfH190a1bN+zZswe6urpYs2YNRo4cCQDIzMzErFmzcOrUKSgpKWHkyJFYs2YNVFVV0a1bNwAQH0/z448/ij1aW7duxerVq1FUVIR58+Zh3rx5AF4kJt988w18fX2RmpqKLl26YMeOHTAzM4MgCJg/fz4CAgKQk5MDExMT+Pr6wtbWFt7e3oiMjMShQ4eQm5uLSZMm4eTJkygsLISVlRUOHjyI5s2blzp+W1tb9OzZExcuXEBoaChat26NwMBAWFpalnm+cnNzMXXqVBw9ehS6urqYOXMmpk6dKj5k2NbWFt26dcPFixdx5coV/P7778jIyICXlxf+/fdfaGlpwd3dXXzeaklvn7+/P5YvX460tDTMmzcPrq6uGDt2LG7evIkePXrg0KFD0NTUfO2xnTp1CgMGDICCggKuXLmC5cuXi+ff2toa1tbW4rG82nO0cOFCJCUlwd/fX4xr586d8PHxQW5uLj777DOsWrUKCgoK8Pf3f+O/i5K2/fz84OPjIz6HFgBsbGwgkUiwceNG2NnZyfSEvu69+uOPPzB16lTExcVh2LBhyM7ORuvWreHt7V3p64H3ZBERERHRW7OxsYG5uTlOnjwprgsICICzszNUVFQAAKGhodDT00NKSgq2b98ONzc33L9/HwAwffp0xMfHIzIyEqGhofjvf/+LFStWAAAuXrwIAEhNTUV2djbs7e3F5bi4OMTExCAoKEhMQADg22+/xYEDB/Dbb78hOTkZ7du3x+jRowEAv/32Gw4dOoTr168jMzMTwcHBsLCwKHVMe/fuRUREBKKiopCRkYFDhw5BR0en3HMQEBCArVu34smTJzA2NsbSpUvLrbty5Urcvn0b//zzDy5evIjDhw+XquPv749NmzYhOzsb7du3R7169eDv74+MjAycPn0aO3fuxI8//iizzfnz53Hnzh2cP38eK1aswIQJE+Dn54fExEQ8efIE3377baWOLSgoCIMGDQIAdOvWDUuXLsWuXbtw586dco+pIsHBwbhz5w7+/vtvHDp0CP7+/mLZm/5dlPjll19w48YNhIWFiX8rYWFhyM7OxsSJE8uMp7z3Ki0tDUOHDsWSJUuQnp4Oe3t7mb/pymKSRURERESVMnv2bEilUvHVunVrmXIPDw8EBAQAAPLz83H48GG4ubmJ5To6Oli4cCGUlZVhZ2cHe3t7HD58GEVFRTh48CDWrl0LqVQKIyMj+Pj4iG2VR0FBAatWrYKKigo6deqEpk2b4vr16wCA7du3Y9WqVbCysoKysjK8vb1x9epVxMbGQkVFBbm5ubh9+zYKCgpgbW2N+vXrl2pfRUUFWVlZiIyMhCAIaNasWYWTbXh4eKBly5ZQUVHBmDFjEB4eXm7dgwcPwsvLC/r6+tDX18f8+fNL1Rk3bhzatm0LBQUFqKqqolevXmjdujUUFBTQqlUrODs74/z58zLbLFu2DGpqamjXrh2aNWsGOzs7NGrUCPXq1cOQIUPEmCo6tvz8fISEhGDAgAEAgC1btsDDwwM7duxA27ZtYWZm9tr35lXe3t7Q0NBAgwYNMGPGDAQGBoplb/t34e3tDU1NTdStW7fS8ZT3XgUFBaFx48YYM2YMlJSUMGbMGLRp06ZKxwowySIiIiKiStq4cSMyMjLE182bN2XKXV1d8fvvvyMlJQWnTp2CgYEBOnbsKJabmppCQeF/Xz8tLS0RHx+P1NRU5Ofnw8rKSiyzsrJCYmKiOHyuLDo6OmIvGQCoqamJExxER0dj1KhRYkKop6cHBQUFPHr0CH369IGPjw8WL14MfX19jB49GgkJCaXaHzt2LNzc3DBx4kQYGBhg4sSJyMzMLDeelxOwl2NZvXo11NXVoa6ujk8++QQAkJCQAHPz/z2E8OV/v3x+Xnb58mX06dMH+vr60NLSwvbt25GamlphDOXFVNGxnT9/Hm3btoWWlhYAoE6dOliwYAGuXLmCjIwMzJs3Dx4eHrh792655+JVL/cUlrzvJd7276K8IZkVKe+8vPq+AGW/N6/DJIuIiIiI5EJfXx92dnY4ePAgAgICZHqxACA+Ph7FxcXicmxsLExNTaGnpwcVFRVER0eLZdHR0TA2NoZEIoFEIqlyLBYWFjhx4oRMUpiTkyPe3zV58mRcuXIFDx8+RGFhIRYsWFCqDSUlJXh5eeHmzZuIiIjAnTt3sH79+irHsnjxYmRnZyM7OxvBwcEAABMTE8TFxYl1Xv53iVePe8yYMXBwcEBsbCyePn2KL774osIktCIVHVtQUBAcHBzK3E5NTQ0zZsyAtra2OHSwXr16eP78uVgnKSmp1HaxsbEy/zY1NRWX3/TvosSb/H2U59X3BSj7vXkdTnxBNap5Ny+0b9++psMgogrwOiWiqvDw8MCCBQsQExOD7777TqYsLS0N69atw5w5c3D+/HkEBwdjzZo1UFRUxOjRo7Fo0SIEBgYiLy8PPj4+GDt2LIAXyZuCggKioqLQokWLSsXxxRdfwMvLCwEBAWjQoAHS09Nx5swZjBw5ElevXkVhYSFsbGygpqYGNTU1mZ6UEufOnYOOjg5atmwJdXV1qKqqym2Gv1GjRmHNmjXo0qULAODrr79+7TZZWVnQ1tZG3bp1ERoaisDAQPTp0+eN9l/RsZ06dQqnTp0S665YsQIDBgwQhy7u27dPvE8MANq1a4d9+/ahTZs2uHz5Mo4fP44hQ4bI7M/Hxwd+fn5ISUnB5s2bZabKf9O/i/IYGhoiKipKnIijKhwcHDB16lQcPnwYn376KX788UfcuHFDvD+tstiTRURERERyM3DgQKSnp6NXr14wMTGRKevQoQMeP34MfX19TJw4EXv27EGTJk0AvLjvx9DQEE2aNEG7du3QqVMnLFu2DMCL3pMlS5agT58+kEql+PXXX18bx7Rp0zB69GgMGjQImpqaaNOmjbhdZmYmJk2aBF1dXZiYmODp06f46quvSrWRlJSEUaNGQUtLCw0aNICpqak4e+HbWrZsGRo3boxGjRqha9euGDp0qMzQx7Js27YNK1asgIaGBry9veHk5PTG+y/v2O7evQuJRCK+L8CLe988PT2hr68PAwMD7Ny5E8eOHRNnGNyyZQt+//13SKVSbNiwAc7OzqX2Z29vj+bNm6Nz585wcnKCh4eHWPamfxfl8fHxgaenJ6RSKXbu3Fml86Krq4ujR4/C29sb2traOH36NOzt7cUZMytLIrxpHyPRWwgPD4eNjQ3CwsL4CzlRLcXrlIjeVPv27TFv3jyZL9slU3X//fffNRhZ7XXs2DHMmzdPnB2xpqxfvx4JCQlye9Dw6x4k/T78XbRp00acDr+y2JNFRERERHLz+++/Iz4+HsOHD6/pUGq1pKQk/PnnnyguLkZMTAxWrlxZK86ZpaUlPD09azqMGvXrr78iLS0NBQUF8PX1RVRUlDjTYmXxniwiIiIikov+/fvj2rVr+P7776s8vOpjU1hYiKlTp+LBgwdQV1eHo6Pja4fBvQslDwH+mF29ehUuLi7Izc1Fo0aNcOzYMejr61epDQ4XpBrBYUhEtR+vUyIiojfD4YJERERERERyxCSLiIiIiIhIjphkERERERERyREnvqAa9eV3d6BvVFxqve+XHWogGiIqS3nXKdG7xP8XiOh9wp4sIiIiIiIiOWKSRUREREREJEdMsoiIiIiIiOSISRYREREREZEcceILqlnmAKxKr57kHyrX3fi684ZpojdWznVK9C69/P8CP9OJqLZjTxYREREREZEcMckiIiIiIiKSIyZZREREREREcsQki4iIiIiISI448QXVLEsAjat/N5MuVn0iDd9uvLGaCMA7u06JAH72EtGHgT1ZREREREREcsQki4iIiIiISI6YZL0hX19f2Nraisvq6uq4d+9ezQVERERERES1ApOsVwwdOhQqKipISUmp0nbZ2dlo3Lj6b1pwd3fHwoULq30/RERERET0ZjjxxUseP36MU6dOQVNTEwcOHMDMmTNrOqQPnleb5mjfvn1Nh0FEFeB1SkREVDXsyXrJ/v370bBhQ8ybNw9+fn4yZbGxsejXrx80NDTQvXt3xMTEyJRLJBJERkZWaj/r169Hw4YNoaGhgWbNmuGnn34Sy/z9/dGlSxd4eXlBV1cXpqamOHDgAABg27ZtOHDgADZu3Ah1dXX06tULAJCUlIRPP/0Uenp6sLa2xurVq1FcXAwACAkJgZGREbZu3QpjY2MYGBhg/fr1AIDk5GTUqVMHiYmJ4v4TExNRp04dJCUlidtu3rwZxsbG0NXVxa5duxAWFoa2bdtCS0sLY8eORWFhYbnHmpiYiPDw8FKvu3fvVupcEVH143VKREQkX+zJeomfnx9cXV3h4uKCxYsXIzw8XPz11tnZGW3atMGpU6dw+/ZtfPLJJ2jevPkb7cfa2hrnz5+HsbExjh49irFjx6Jz584wMzMDAISFhcHFxQXJyck4ceIExo0bh8GDB2Py5Mm4cuUKjIyM8NVXX4ntOTs7w9raGrGxsUhISIC9vT309fXx2WefAQBSU1MRFxeHmJgYXL9+HT169MCwYcPQsGFD2NnZ4cCBA5g7dy6AF4lm//79YWRkhMjISKSmpiI5ORkxMTH49ddf4eTkBDs7OwQHB0NJSQkdO3bEkSNHMGbMmDKP9fvvv4ePj88bnSciejd4nRIREckXe7L+X2hoKG7fvg0XFxeYm5ujV69eYm9WbGwsLl26hDVr1qBOnTqwsbGBi4vLG+9rxIgRMDU1hYKCAkaMGIEmTZrg8uXLYrmpqSmmTZsGJSUlDB8+HAoKCuVOqvHo0SOcP38eGzZsgJqaGho2bIi5c+ciICBArKOgoIBVq1ZBRUUFnTp1QtOmTXH9+nUAgIeHB/bt2yfW3bdvH9zc3GS29fHxgYqKCgYPHgwVFRWMGTMGxsbG0NfXh52dHcLDw8s91s8//xxhYWGlXvv373/T00dEcsbrlIiISL7Yk/X//Pz80KNHD1hZWQEAxo4di/nz5+Prr79GQkICtLS0oKWlJda3tLTEtWvX3mhfAQEB2LhxI6KjowG8mDQjNTVVLDcyMpKpr6amhuzs7DLbio+Ph5aWFrS1tcV1VlZWiI+PF5d1dHSgoqJSZnuDBg3CxIkTcf36dQiCgEePHsHR0VFmW2VlZZltX46votgAwNjYGMbGxuWWE1HN43VKREQkX0yyAOTl5eHgwYPIy8sTE4jCwkKkpaXhxIkT6Ny5M54+fYrMzExoamoCeNG79SZiYmLg6emJs2fPonv37lBUVES7du0gCEKltpdIJDLLpqamePr0KTIyMiCVSgEA0dHRMDU1rVR7ysrKcHV1RUBAAARBwKhRo6CqqlqlY3obd9Z9ifaHfnp9RSKqMXfWfYliA/2aDuOD0GGLb02HQERE7wCHCwL4+eefkZ+fj4iICFy/fh3Xr1/HrVu34OLiAj8/P1hYWKBLly5YvHgx8vLycO3aNXEyiqp69uwZAEBf/8UXloCAANy6davS2xsaGuLBgwfispmZGXr27Im5c+ciJycHUVFR2LBhA8aOHVvpNj08PBAYGIiDBw/C3d290tsREREREVFp7MnCi6GC48aNQ/369WXWz5o1C507d0ZCQgICAwPh4eEBPT09tG7dGuPHj8elS5fKbG/16tX466+/EBwcXKqsefPmmDdvHrp37w4FBQWMGzcO3bp1q3SsEyZMwMiRI6GtrY22bdvi3LlzOHjwIKZMmQIzMzOoq6vD09MTnp6elW6zdevWMDU1xbNnz9C5c+dKb0dERETVK3T6pGptn72rRNVDIlR2nBqVq6ioCEpKSoiKiiqVqL0vhg8fjo4dO2LRokXvZH/h4eGwsbHBvlHD4crhgkS10svXaVMOF5QLfqGlqqptSdbFixexYMECREREAAAaNmyIFStWYODAgcjOzoaPjw9+/PFHPH78GPr6+ujXrx+WLl0KKysruLu7IzAwUOY+8dWrV2P69OkAgHPnzmH16tW4cuUKJBIJTExMMHToUMyePRt6enriNvPnz4dUKsXixYtfu8/qZGtri9GjR2PSpOp9j+j9xOGCchAREQFVVdX39sbxu3fv4syZMxg/fnxNh0JERES1VGZmJhwcHODp6Sk+4uWbb76BpqYm8vPz0b9/f4SFheHEiRPIzMzEtWvX0KZNG5w9e1ZsY/bs2cjOzhZfJQnW0aNH4ejoiKFDhyIqKgoZGRkIDg6GiopKqVmMg4KCMGjQoErvs7aq6Dmj9P7jcMG3dODAAcyYMQNr1qxB3bp1azqcKpswYQJ++OEHrF69GoaGhu8+AEsg9Nu3/wWow1T+OkxUbSwBmNd0EB8GeXzeyQM/M+lN3Lt3DwUFBeKjXpSUlNCzZ08AwJ49e/DgwQNERUVBQ0MDAKCtrS0mURURBAEzZ87EkiVLMGXKFHG9paUlvL29Zeo+ePAAz549Q+vWrSu1z6SkJEyZMgXnz5+HhoYGPvvsMyxcuBAKCgrw9/eHr68v/v77b7G+kZERDh06BFtbW3h7eyMiIgI6Ojo4cuQI9PX14evri/79+2PBggX466+/8Pfff2Pu3LkYNmwY9u3bh6SkJMyYMQMhISFQUVHBhAkTsGzZMpn99e7dG35+fhg2bBi+//77N3szqNZjT9ZbcnFxQWpqKmbNmlXTobyR3bt3IzMzE9OmTavpUIiIiKgWa9y4MerUqYMxY8YgKCgIKSkpYtlvv/0Ge3t7Mdmpin/++QdxcXFwcnJ6bd2goCA4ODhUep/Ozs7Q0tJCbGwszpw5gz179mD37t2Vji0oKAgDBw5EWloapkyZIo76Wbt2LXr27IlNmzYhOzsb+/btQ3FxMRwdHdGwYUPExMTg8uXLOH78uMz+QkNDoaenh/j4eGzatKnScdD7h0kWEREREb2WpqYmLl68CDU1NUyZMgVGRkbo06cP/v33Xzx58gQmJiavbWPTpk2QSqXiKy0tTXxW6MvbT58+HVKpFPXq1cPSpUvF9SVDBQG8dp+PHj3C+fPnsWHDBqipqaFhw4aYO3cuAgICKn3MXbt2xbBhw6CoqAg3NzfExcXJPNv0ZaGhoYiLi8OqVatQp04dmJiYYPbs2Th48KBYx8DAAHPnzoWysvJ7OQKKKo9JFhERERFVSuPGjbFr1y7ExMTgwYMHUFFRwdixY6Grq4uEhITXbj9z5kxkZGSILx0dHXFSi5e337JlCzIyMuDk5ISCggIAQHZ2NkJDQ9G3b18AeO0+4+PjoaWlBW1tbXGdlZUV4uPjK328Jc9PBQA1NTUxjrJER0cjJSUF2traYhI5ZcoUJCcni3XMzc1LPfOUPkxMsoiIiIioyiwtLTFt2jTcunULdnZ2+PXXX8tNQCrSpEkTmJmZ4aefKp5t+MyZM+jevTvq1KkDAK/dp6mpKZ4+fYqMjAxxXXR0NExNTQEA6urqeP78uVhWUFCAtLS0Ssf9arJkYWEBMzMzmSQyMzMTt2/fLncb+nBx4guqWeYAGr19M6G/vtubyTsM4E3j9BGR03VKtcfbfGby8+/jFRkZiZMnT2LUqFEwNzdHSkoKdu3aha5du8LV1RXff/89hg4dis2bN6Np06bIzs7G/v37UadOHUyYMKHcdiUSCTZu3IgJEyZAXV0dI0eOhK6uLh49eoQHDx6IvUkvDxUEUKl99uzZE3PnzsXWrVuRkJCADRs2YN68eQCANm3aIDIyEleuXEHbtm2xfPlyFBcXV/p8GBoaIioqSlzu2LEj9PX1sXLlSsyePRt169ZFVFQUEhIS0Lt376qebnrPMckiIiIiqqVq07PVNDQ0EBoais2bNyM9PR1aWlro27cvtm/fDhUVFZw9exY+Pj5wcHBASkoK9PT0YGdnJ3NPVXmcnJygra2Nr776CgsWLICCggJMTU3h6OiI2bNnQxAEBAcHY8WKFeI2ldnnwYMHMWXKFJiZmUFdXR2enp7w9PQEADRq1AirVq2Cg4MDlJSUsHTpUpnncb3OjBkz4O7ujl27dsHR0RF79+7FyZMnMX/+fDRq1AjPnj1D/fr1sWDBgiqeafoQ8GHEVCPEh5xuHY6mjd6/h5zyl1z6GLzv1ylVD37+UU24evUqJk6ciGvXrtV0KESVwnuyiIiIiKhWEwQBq1atqukwiCqNwwWJiIiIqFbr1KlTTYdAVCVMsqhGNe/mhfbt29d0GERUAV6nREREVcPhgkRERERERHLEJIuIiIiIiEiOmGQRERERERHJEZMsIiIiIiIiOWKSRUREREREJEecXZCIiIiolgr9dlK1tt9hKh8uXRWTJk2CoaEhfHx8ajoUquXYk0VEREREr2VrawslJSXcu3dPXBcZGQmJRFKDUb1bvr6+b51gderUCRcvXgQAREVFYcyYMTAwMICmpiaaNWsGb29vPHv2TB7hVkgikSAyMrLa9/OxYk8W1agvv7sDfaPiamnb98sO1dIu0cemOq9TourAz//qo6WlhaVLl+Lw4cPV0n5hYSGUlN7Pr6eViT05ORkxMTHo0qULHjx4gE6dOmHMmDEIDw+HmZkZoqOj8fXXX+Pff/9FmzZt3lHkb+Z9fq/eBfZkEREREVGlTJs2DadPn8a1a9cqvc3BgwdRv359aGtrY+7cuejSpQv8/f0BAP7+/ujSpQsWLFgAAwMDTJkyBZmZmRg8eDAMDAygra2NgQMHIjY2VmzP1tYWXl5e6N27N+rVqwdbW1s8fvwYc+bMgY6ODurXr4+QkJBy47GyssKaNWvQqlUraGlp4dNPP0VGRoZY7uzsDGNjY2hpaaFnz56IiIgQy9zd3bFw4UIAQEhICIyMjLBp0yaYmppi0KBByM3Nhbu7O3R1daGlpYU2bdrgzp074vanTp3CgAEDoKCgAG9vb3Tq1Albt26FmZmZGNu3334rJlhXrlxB165doaWlhVatWiEoKKjMWIDSvYq2trZYunQp+vTpAw0NDXTv3h0xMTEAgG7dugEAbGxsoK6ujh07dgAArl69il69ekFbWxvNmjXD0aNHZfY3adIkDBs2DOrq6vjhhx8qeNeJSRYRERERVYqRkRGmT5+OxYsXV6p+ZGQkJkyYgF27duHx48fQ19dHWFiYTJ3Q0FDo6ekhPj4emzZtQnFxMdzc3BAdHY24uDhoampi8uTJMtscOHAA33//PVJSUlBUVIQuXbqgRYsWSElJweTJkzFx4sQK4/L398fx48fx6NEj5OXlYfr06WKZnZ0d/vnnHzx+/BidOnWCs7Nzue2kpqbi4cOHiIqKwrFjx7B3715EREQgKioKGRkZOHToEHR0dMT6QUFBGDRoEADgt99+g5OTU7ltp6enw97eHuPHj8eTJ0+wYcMGjBo1Cnfv3q3w2F4WEBCArVu34smTJzA2NsbSpUsBQByuGBYWhuzsbEycOBGJiYmwt7fH7NmzkZqaCn9/f3h6esrsb//+/Zg+fTqysrIwdOjQSsfxMWKSRURERESVNm/ePFy5cgV//vnna+seOXIEAwcORN++faGsrIx58+ZBW1tbpo6BgQHmzp0LZWVl1K1bF1KpFCNGjICamhrU1dWxaNEinD9/XmYbd3d3NG3aFGpqahg2bBhUVFQwfvx4KCoqYsyYMbh//z6ys7PLjWvq1KmoX78+NDQ08OWXX+Lw4cMoLn4xLNrDwwOamppQVVXFsmXLcPv2bTx58qTMdgRBwJo1a1CnTh3UrVsXKioqyMrKQmRkJARBQLNmzWBkZAQAyM/PR0hICAYMGAAAePLkCUxMTMqN8dSpU7C0tMRnn30GJSUl2NnZYfDgwQgMDHzteS/h4eGBli1bQkVFRRyWWJ59+/ahf//+GDp0KBQVFdG5c2cMGzZMpsdq0KBB6NOnDyQSCerWrVvpOD5GTLKIiIiIqNKkUikWLFiARYsWyayPjY2Furq6+IqNjUVCQgLMzc3FOgoKCjA1NZXZztzcXGaY2/Pnz/H555/D0tISmpqa6NmzJ7Kzs5GXlyfWKUlcAEBNTa3UMoAKkywLCwvx35aWlsjPzxd7xRYuXIgGDRpAU1MTlpaWAF70WJVFT09P3B8AjB07Fm5ubpg4cSIMDAwwceJEZGZmAgDOnz+Ptm3bQktLCwCgq6uLhISEcmOMj4+HlZWVzDorKyvEx8eXu82rXj0vFZ2T6OhoHD9+HFKpVHwdPnwYiYmJYp2S80Gvx7vVqGaZA7CqnqYn+YdWT8MfIV933kT+UavG65TeT/xMoGnTpmHz5s0y9whZWFiU+hJvYmKCmzdvisvFxcWlkoRXZyfcsGED7ty5g7///hvGxsa4ceMG2rZtC0EQ5Bb/y/d4xcbGQllZGfr6+jhw4ACOHj2KM2fOwNraGpmZmZBKpeXu+9XYlZSU4OXlBS8vLyQmJsLJyQnr16/HypUrERQUBAcHB7GunZ0dfvzxR4wfP77Mtk1NTREdHS2zLjo6Go0aNQIAqKur4/nz52JZUlJSlc7BqywsLDB69GjxfrmyfEwzSb4t9mQRERERUZXUrVsXy5Ytw1dffVVhPScnJ5w+fRohISEoLCzEhg0bkJ6eXuE2WVlZ4rDB9PR0rFy5Up6hAwC2bduGhw8fIisrC0uWLMGoUaOgoKCArKwsqKqqQldXFzk5OViyZEmV2j137hxu3LiBoqIiqKurQ1VVFYqKigBeDP8ruR8LALy9vXH58mXMnDlT7NGKjY3FjBkzcPPmTQwcOBDR0dHYs2cPCgsLcfbsWZw8eRJjxowBALRr1w6nT59GSkoK0tLSsHbt2irFamhoiKioKHHZ1dUVwcHBOHnyJAoLC5Gfn4/Lly9X6R4w+h/2ZBERERHVUrX5YcETJkzA119/Xe79SgDQrFkz7Ny5Ex4eHsjIyMCECRPQunVrqKqqlrvNzJkzMWbMGOjr68PExARz5szBTz/9JNfYx40bB0dHR8TGxqJv377YvHmzuP7XX3+FqakpdHV1q5zgJSUl4YsvvsCjR4+gpqYGe3t7zJs3D3fv3oVEIkGTJk3EuvXr18fly5exdOlStGnTBnl5eTA1NYWzszMaNmwINTU1nD59GjNnzsSsWbNgbm6OwMBANGvWDMCLpOiPP/5Aw4YNYWJigtmzZ+OXX36pdKw+Pj7w9PRETk4O1q9fj88++wynT5/G/Pnz4e7uDolEgtatW2Pjxo1VOgf0gkSQZ98rUSWFh4fDxsYGw5fvg75V05oOh16DQ4M+TrxOqTz8TKA3VVhYCCMjI/z888/o0aNHjcRgZWUFX19f2Nvbv7N9rl+/HgkJCfjmm2/e2T6pZnG4IBERERFVm5MnTyI7Oxu5ublYvnw56tati06dOtV0WO+UpaUlPD09azoMeoc4XJBqliWAxjUdBBFViNcpEb2F06dPY9y4cSgqKkKrVq3w888/Q0VFpabDeqdGjhxZ0yHQO8Yki4iIiIiqzfbt27F9+/aaDkP06ox9RNWBwwWJiIiIiIjkiEnWe8rX1xe2trbllo8ePRre3t5y2dfChQvh7u4O4H8PGnz5gYBERERERPQ/TLJqmaFDh0JFRQUpKSk1HUqZSh40WNHUq0REREREHzPek1WLPH78GKdOnYKmpiYOHDiAmTNn1nRI1c6rTXO0b9++psMgogrwOiUiIqoa9mTVIvv370fDhg0xb948+Pn5yZTFxsaiX79+0NDQQPfu3RETEyNTfu7cObRo0QIaGhoYN24c8vPzy93Pw4cP0a9fP+jq6kJPTw/Ozs4yT1+/efMmOnbsCA0NDTg4OMiURUdHQyKRIDc3V05HTURERET0YWGSVYv4+fnB1dUVLi4uiIiIQHh4uFjm7OyMJk2aICUlBVu2bMHu3bvFsrS0NAwZMgQLFy5Eeno6+vXrhxMnTpS7H0EQsGDBAiQkJCAyMhKJiYlYunQpAKCgoABDhgzBsGHDkJaWhhkzZmDfvn1vfEyJiYkIDw8v9bp79+4bt0lE8sXrlIjeRyEhITAyMpJ7u6tXrxbvRSd6UxwuWEuEhobi9u3bcHFxgbm5OXr16gU/Pz+0b98esbGxuHTpEk6fPo06derAxsYGLi4uuHbtGgAgKCgIjRo1wtixYwEAbm5u2LRpU7n7ql+/PurXrw8AUFVVxaxZs7BkyRIAwKVLl/Ds2TMsXLgQCgoKsLOzw4ABA974uL7//nv4+Pi88fZEVP14nRLVXqG/TqrW9jsM8K10XVtbW4wePRqTJv0vpujoaFhbWyMnJwd16tSpjhDf6X4AYPHixdXa/vukU6dO2LRpE7p164aoqCgsXboUZ8+eRW5uLkxNTTFq1CjMmzcP9erVq9Y4JBIJ7t69i6ZNm1brfuSJPVm1hJ+fH3r06AErKysAwNixYxEYGIi8vDwkJCRAS0sLWlpaYn1LS0vx3wkJCbCwsJBp7+XyVyUnJ2P06NEwNTWFpqYmXFxckJqaKrZlamoKBYX//WlU1NbrfP755wgLCyv12r9//xu3SUTyxeuUiKh2EwQBRUVF73SfycnJiImJQZcuXfDgwQN06tQJurq6CA8PR2ZmJoKDg5Gamop///33ncb1JgoLC9/5PtmTVQvk5eXh4MGDyMvLE7u9CwsLkZaWhhMnTqBz5854+vQpMjMzoampCeDFPVolTExMZJZLytu2bVvm/hYvXozi4mLcvHkTurq6OH78OD7//HOxrfj4eBQXF4uJVmxsrLjfqjI2NoaxsXG55XfWfYliA/1yyztsqfwvbET0Zt72OqUX+HlFH7uwsDD0798fSUlJ4izEly9fxsCBA5GYmIjAwED4+vqiW7du2LNnD3R1dbFmzRqMHDkSAJCZmYlZs2bh1KlTUFJSwsiRI7FmzRqoqqqiW7duAAA9PT0AwI8//ij2aG3duhWrV69GUVER5s2bh3nz5gF4kZh888038PX1RWpqKrp06YIdO3bAzMwMgiBg/vz5CAgIQE5ODkxMTMTH43h7eyMyMhKHDh1Cbm4uJk2ahJMnT6KwsBBWVlY4ePAgmjdvXur4bW1t0bNnT1y4cAGhoaFo3bo1AgMDy/2xOjc3F1OnTsXRo0ehq6uLmTNnYurUqRAEQWyvW7duuHjxIq5cuYLff/8dGRkZ8PLywr///gstLS24u7tj5cqVAP7X2+fv74/ly5cjLS0N8+bNg6urK8aOHYubN2+iR48eOHToEDQ1NV97bKdOncKAAQOgoKAAb29vdOrUCVu3bhXjt7KywrfffisuX7lyBTNmzMCdO3dgYWGBNWvWYNCgQQAAd3d3GBkZ4auvvgIAREZGolmzZjLHWt65K3nvbWxsIJFIsHHjRkycOBFXr17FnDlzEBERASMjI3z55ZcYPny4uL86deogOTkZZ86cwc6dO+Hs7FzZP2W5YE9WLfDzzz8jPz8fERERuH79Oq5fv45bt27BxcUFfn5+sLCwQJcuXbB48WLk5eXh2rVrOHDggLi9g4MD7t27h8DAQBQWFmLfvn2IiIgod39ZWVmoV68etLS0kJCQgK+//los69q1K+rWrYt169ahoKAAZ8+exS+//FKtx09ERETvPxsbG5ibm+PkyZPiuoCAADg7O0NFRQXAi9sj9PT0kJKSgu3bt8PNzQ33798HAEyfPh3x8fGIjIxEaGgo/vvf/2LFihUAgIsXLwIAUlNTkZ2dDXt7e3E5Li4OMTExCAoKEhMQAPj2229x4MAB/Pbbb0hOTkb79u0xevRoAMBvv/2GQ4cO4fr162KvzKujggBg7969iIiIQFRUFDIyMnDo0CHo6OiUew4CAgKwdetWPHnyBMbGxuI972VZuXIlbt++jX/++QcXL17E4cOHS9Xx9/fHpk2bkJ2djfbt26NevXrw9/dHRkYGTp8+jZ07d+LHH3+U2eb8+fO4c+cOzp8/jxUrVmDChAnw8/NDYmIinjx5IiZGrzu2oKAgMUn67bff4OTkVO6xpKenw97eHuPHj8eTJ0+wYcMGjBo1qkr39pZ37kre+7CwMGRnZ2PixIlITEyEvb09Zs+ejdTUVPj7+8PT01Nmf/v378f06dORlZWFoUOHVjoOeWGSVQv4+flh3LhxqF+/PoyMjMTXrFmz8NtvvyEhIQGBgYG4ffs29PT0MHXqVIwfP17cXldXF8eOHcOqVaugra2NM2fOYPDgweXub/ny5bh58yakUikGDhwo84enrKyM48eP48cff4S2tja++eYb8V6vskyaNElmbDYRERF9uGbPng2pVCq+WrduLVPu4eGBgIAAAEB+fj4OHz4MNzc3sVxHRwcLFy6EsrIy7OzsYG9vj8OHD6OoqAgHDx7E2rVrIZVKYWRkBB8fH7Gt8igoKGDVqlVQUVFBp06d0LRpU1y/fh0AsH37dqxatQpWVlZQVlaGt7c3rl69itjYWKioqCA3Nxe3b99GQUEBrK2txfvVX6aiooKsrCxERkZCEAQ0a9aswsk2PDw80LJlS6ioqGDMmDEyk5i96uDBg/Dy8oK+vj709fUxf/78UnXGjRuHtm3bQkFBAaqqqujVqxdat24NBQUFtGrVCs7Ozjh//rzMNsuWLYOamhratWuHZs2awc7ODo0aNUK9evUwZMgQMaaKji0/Px8hISHifflPnjyBiYlJucdy6tQpWFpa4rPPPoOSkhLs7OwwePBgBAYGlrvN25y7ffv2oX///hg6dCgUFRXRuXNnDBs2DD/88INYZ9CgQejTpw8kEgnq1q1b6TjkhcMFa4HyeopsbGxkxpCeO3eu3Db69++PO3fuVGp/LVq0QFhYmMy6OXPmiP9u27YtQkNDy9y2qKgIEokESkov/nR8fTk8hoiI6GOxcePGMie+KOHq6oolS5YgJSUFFy5cgIGBATp27CiWl3Xfd3x8PFJTU5Gfny/emw68GI6WmJgoDikri46OjthLBgBqamrIzs4WYxs1apTM/hQUFPDo0SP06dMHPj4+WLx4Me7duwd7e3ts3LixVCIxduxYJCQkYOLEiUhISMDw4cPx9ddfl3sbxcsJ2MuxrF69GqtXrwYA9OzZE8HBwUhISIC5ublY/+V/v3x+Xnb58mUsXLgQt27dQn5+PvLy8jBs2LAKYygvpoqO7fz582jbtq04H4Curi4SEhLKPGYAiI+Pl3nvgBfvX3x8fLnbvKq8OMsSHR2N48ePQyqViusKCwtlOgbeZk4BeWBPFlVJREQEzM3NxSSLiIiIqIS+vj7s7Oxw8OBBBAQEyPRiARDv+y4RGxsLU1NT6OnpQUVFBdHR0WJZdHQ0jI2NIZFIIJFIqhyLhYUFTpw4gYyMDPGVk5Mj3uMzefJkXLlyBQ8fPkRhYSEWLFhQqg0lJSV4eXnh5s2biIiIwJ07d7B+/foqx7J48WJkZ2cjOzsbwcHBAF7cBx8XFyfWefnfJV497jFjxsDBwQGxsbF4+vQpvvjiiwqT0IpUdGxBQUFwcHAQ69rZ2ZUalvgyU1NTmfcOePH+mZqaAgDU1dXx/PlzsSwpKemNYi5hYWGB0aNHy7y32dnZ2L59u1jnTf5m5InflKnSVq9eja1bt1Y4PXyVWQIo/cONKPRbDkX8kHWYyp7Q98JrrlN64UP+vOK1SlXh4eGBBQsWICYmBt99951MWVpaGtatW4c5c+bg/PnzCA4Oxpo1a6CoqIjRo0dj0aJF4uzKPj4+Ys+Evr4+FBQUEBUVhRYtWlQqji+++AJeXl4ICAhAgwYNkJ6ejjNnzmDkyJG4evUqCgsLYWNjAzU1Naipqcn0eJU4d+4cdHR00LJlS6irq0NVVRWKiopvf5IAjBo1CmvWrEGXLl0AQOYe+fJkZWVBW1sbdevWRWhoKAIDA9GnT5832n9Fx3bq1CmcOnVKrOvt7Y2OHTti5syZmD9/vjjp2oYNGzBhwgQMHDgQ06ZNw549ezBu3DiEhITg5MmT4siodu3aYc2aNVi6dCkUFRWxdu3aKsVqaGiIqKgocQp3V1dX2NjY4OTJk/jkk09QXFyMa9euQVNTE82aNXuj8yFvTLKo0hYvXsxnRxAREb1DVXmOVW0xcOBATJw4Eb169So1/K5Dhw54/Pgx9PX1oaOjgz179qBJkyYAgC1btmDmzJlo0qQJFBUV4eTkhGXLlgF4MXxsyZIl6NOnj3ivV8kMhuWZNm0aFBQUMGjQIMTHx0MqleI///kPRo4ciczMTMyePRsPHjyAiooKevToge+//75UG0lJSfjiiy/w6NEjqKmpwd7eXpy98G0tW7YMkydPRqNGjaCnp4cpU6bg0qVLFW6zbds2zJkzBzNnzkTv3r3h5OQkPoanqso7trt370IikYjvC/DiGauXL1/G0qVL0aZNG+Tl5cHU1BTOzs5o2LAh1NTUcPr0acycOROzZs2Cubk5AgMDxYTH1dUVf/zxBxo2bAgTExPMnj27ShOr+fj4wNPTEzk5OVi/fj0+++wznD59GvPnz4e7uzskEglat26NjRs3vtG5qA4S4U37GIneQnh4OGxsbLBv/nA0NefU0B8r/jpeu/E6pRK8Vqmq2rdvj3nz5slMm+3v7w9fX1/8/fffNRhZ7XXs2DHMmzevxp87tX79eiQkJOCbb76p0Tjed7wni4iIiIjk5vfff0d8fLz4zCIqW1JSEv78808UFxcjJiYGK1eurBXnzNLSEp6enjUdxnuPwwWJiIiISC769++Pa9eu4fvvv3/tcL6PXWFhIaZOnYoHDx5AXV0djo6O4vDImlTycGh6O0yyqGaZA2hU00EQUYV4nRJRJZ09e7bcMnd3d7i7u7+7YGo5MzMz3Lx5s6bDoGrC4YJERERERERyxCSLiIiIiIhIjphkERERERERyRGTLCIiIiIiIjnixBdUo5p380L79u1rOgwiqgCvUyIioqphTxYREREREZEcMckiIiIiIiKSIyZZREREREREcsQki4iIiIiISI6YZBEREREREckRkywiIiIiIiI5YpJFREREREQkR0yyiIiIiIiI5IgPI6Ya9eV3d6BvVPzG2/t+2UGO0RBRWd72OiWqLvw/gIhqK/ZkERERERERyRGTLCIiIiIiIjlikkVERERERCRHvCeLapY5AKs333ySf6i8IhH5unOMP5GMt7xOiSqDn71E9CFhTxYREREREZEcMckiIiIiIiKSIyZZREREREREcsQki4iIiIiISI448QXVLEsAjWs6CCKqEK9TIiKiKmFPFhERERERkRwxySIiIiIiIpIjJlmv4evrC1tb23LLR48eDW9v73cWT3kkEgkiIyNrOgwiIiIioo/eR5tkDR06FCoqKkhJSam2fYSEhEBBQQHq6urQ0NBA/fr1sX79erHcysoKdevWhbq6OgwMDODk5ISkpKRqi4eIiIiIiKrfRznxxePHj3Hq1CloamriwIEDmDlzZrXty8DAQEycLly4ADs7O7Ru3RoDBgwAABw7dgz29vZITU2Fk5MTZs2ahYMHD1ZbPABQWFgIJaXa8dZ7tWmO9u3b13QYRFQBXqdERERV81H2ZO3fvx8NGzbEvHnz4OfnJ1MWGxuLfv36QUNDA927d0dMTIxM+blz59CiRQtoaGhg3LhxyM/Pr/R+e/TogRYtWiAiIqJUmZ6eHkaMGFFmGQDY2tpi0aJF6NatGzQ0NNCvXz/ExcXJ1Pnzzz/RtGlTaGlpwdXVVYwtJCQERkZG2LRpE0xNTTFo0CAAgLOzM4yNjaGlpYWePXvK7PuXX35Bq1atoKGhASMjI8ybN08su3r1Knr16gVtbW00a9YMR48erfQ5ICIiIiL60H2USZafnx9cXV3h4uKCiIgIhIeHi2XOzs5o0qQJUlJSsGXLFuzevVssS0tLw5AhQ7Bw4UKkp6ejX79+OHHiRKX2KQgC/vzzT9y6davMX4QfP36MH374ocJfi/fs2YNvv/0WKSkpaNiwIVxdXWXKf/zxR/z3v//F/fv3cenSJezfv18sS01NxcOHDxEVFYVjx44BAOzs7PDPP//g8ePH6NSpE5ydncX6Hh4emD9/PrKysvDvv/9ixIgRAIDExETY29tj9uzZSE1Nhb+/Pzw9PXH37t0yY05MTER4eHipV3n1iejd43VKREQkZ8JH5urVq4JEIhEePnwoCIIg9O7dW5g6daogCIIQExMjSCQSISMjQ6w/c+ZMoXfv3oIgCMLevXuF9u3by7TXtm1bYfny5WXu69y5c4JEIhG0tLQEbW1toXnz5sK3334rlltaWgr16tUTpFKpYGZmJowbN05ITU0ts63evXsLs2fPFpfT09MFiUQixMXFCYIgCACEc+fOieXTpk0TpkyZIsahoKAgPHv2rNzzkpGRIQAQ929hYSEsW7ZMSElJkam3du1aYeTIkTLrxo8fL/j4+JTZ7vLlywUA5b7CwsLKjYmI3g1ep0RERPJVO27MeYf8/PzQo0cPWFlZAQDGjh2L+fPn4+uvv0ZCQgK0tLSgpaUl1re0tMS1a9cAAAkJCbCwsJBpz9LSssL9vXxPVll+/PFH2NvbVyr2l/ctlUqhqamJ+Ph4mJmZAQCMjIzEcjU1NZn96unpQU1NTVwuKiqCl5cXfvjhB6SkpEBB4UWnZmpqKnR1dXHs2DGsWrUKjRo1QsOGDbF8+XIMGjQI0dHROH78OKRSqdhWYWEhxo4dW2bMn3/+ORwdHUutv3v3bqmeOCKqGbxOiYiI5OujSrLy8vJw8OBB5OXliQlJYWEh0tLScOLECXTu3BlPnz5FZmYmNDU1Aby4R6uEiYmJzHJJedu2bd9J/C/vOyMjA5mZmTA1Na3UthKJRGY5MDAQR48exZkzZ2BtbY3MzExIpVIIggAAaN++PY4ePYqioiIcPnwYI0aMwJMnT2BhYYHRo0fD39+/Uvs1NjaGsbFxueV31n2JYgP9Uus7bPGtVPtE9Pbe9Dql2oOfmUREtctHdU/Wzz//jPz8fEREROD69eu4fv06bt26BRcXF/j5+cHCwgJdunTB4sWLkZeXh2vXruHAgQPi9g4ODrh37x4CAwNRWFiIffv2lTtRRXU4cOAArl+/jtzcXCxatAjdu3cXe7GqKisrC6qqqtDV1UVOTg6WLFkiluXn52Pfvn1IT0+HoqIipFIpJBIJFBUV4erqiuDgYJw8eRKFhYXIz8/H5cuXee8GEREREdH/+6iSLD8/P4wbNw7169eHkZGR+Jo1axZ+++03JCQkIDAwELdv34aenh6mTp2K8ePHi9u/PIxOW1sbZ86cweDBg99Z/O7u7pg8eTL09fXxzz//yCSAVVVyHkxNTdGsWTN07NhRpjwwMBANGjSAhoYGFixYgCNHjqBOnTowMzPD6dOnsWnTJhgaGsLExASLFi1CXl7e2x4eEREREdEHQSKUjA+jWs3W1hajR4/GpEmTajoUuQgPD4eNjQ32jRqOphwuSFQrve46pdqDn5lERLXLR9WTRUREREREVN0+qokvqBayBGBeenXot/Lrseswlb/wEr2Vcq5Tqj1KPjP5eUdEVDswyXpPhISE1HQIRERERERUCRwuSEREREREJEdMsoiIiIiIiOSISRYREREREZEc8Z4sqlnmABrVdBBEVCFep0RERFXCniwiIiIiIiI5YpJFREREREQkR0yyiIiIiIiI5IhJFhERERERkRxx4guqUc27eaF9+/Y1HQYRVYDXKRERUdWwJ4uIiIiIiEiOmGQRERERERHJEYcLUo3IyckBANy9e7eGIyH6cDRt2hRqampya4/XKVH1kPe1SkS1D5MsqhHXrl0DALi6utZwJEQfjrCwMLneOxUdHQ2A1ymRvMn7WiWi2odJFtWIZs2aAQB2796Ntm3b1mwwlXD37l24urpi//79Yuy12fsWL/D+xVwb423atKlc2xswYAD2798PKysr1K1bV65tv63aeP7fBzxvb06e507e1yoR1T5MsqhGaGtrAwDatm37Xv2a16xZM8Zbzd63mN+3eKtCT08PLi4uNR1GhT7k81+deN7eHM8dEVUGJ74gIiIiIiKSIyZZREREREREcsQki4iIiIiISI6YZBEREREREckRkyyqEcbGxli+fDmMjY1rOpRKYbzV732L+X2L90PD8/9meN7eHM8dEVWFRBAEoaaDICIiIiIi+lCwJ4uIiIiIiEiOmGQRERERERHJEZMsIiIiIiIiOWKSRUREREREJEdMsoiIiIiIiOSISRa9UxkZGRg5ciQ0NDRgYmKCTZs2vZP95uXlwdPTE9bW1tDQ0ECLFi0QGBhYbn2JRIJ69epBXV0d6urq+OSTT2TKf/zxRzRo0ABqamro168fYmJiZMqXLl0KfX19aGlpwdPTE3l5eVWO2d3dHSoqKmIM6urqiI2NLbf+rVu30KVLF6ipqaF58+b4448/3mnML8eprq4OJSUlODo6llu/Js7xt99+iw4dOkBVVRWjR4+WKXvd+SurLVNTU6irq+PTTz9Fenq6WJafn49JkyZBKpVCT08PixYtwssTucbFxcHOzg716tWDtbU1Dh069NrYPxY//fQTWrZsiXr16sHS0hJHjx4FUPH7c+PGDTRv3hz6+vrYuHGjTHuffPIJwsLC3ukx1ITo6Gg4ODhAR0cHBgYGGDduHLKysgBU/PcWFxeHLl26QFtbG3PmzJFp87PPPsPPP//8Lg+j2r3NZ0BFn0mBgYEwNjaGtbU1zp07J65PT0+HjY2N+F4Q0UdEIHqHXFxcBEdHR+Hp06fCzZs3BX19feH06dPVvt/s7Gxh6dKlQlRUlFBcXCz89ddfgqampnDx4sUy6wMQ7t69W2bZnTt3hHr16gm//vqr8Pz5c2HmzJlCp06dxPKdO3cK1tbWQlRUlJCamip0795dmD9/fpVjdnNzExYsWFCpuvn5+YKVlZXw5ZdfCrm5ucKhQ4cETU1NITk5+Z3GXKKwsFAwMTER9u3bV26dmjjHP/30k3Ds2DFhypQpwqhRo8T1rzt/r/rtt98EHR0dISwsTMjMzBScnJyEkSNHiuVeXl5Cx44dheTkZCEmJkZo3LixsG3bNrG8e/fuwhdffCE8f/5cOHfunKCuri5ERES8Nv4P3e+//y6YmZkJFy5cEIqKioTk5GQhKirqte/PJ598IuzYsUNITEwUdHV1hcTEREEQBOHAgQPC9OnTa/KQ3pn//Oc/gqurq/D8+XMhLS1N6NOnjzBjxgxBECr+e/viiy+ExYsXC5mZmULDhg2Fq1evCoIgCBcuXBCGDBlSQ0dTfd70M6Ciz6SCggJBW1tbuHPnjhAcHCy0aNFCbPezzz4Tjh49+m4PkohqBSZZ9M5kZ2cLKioqMl8mFy9eLIwYMaJG4vnkk0+Er7/+usyyihKAxYsXC59++qm4nJmZKaiqqgq3bt0SBEEQunXrJmzdulUs/+233wR9ff0qx1eVJOu3334TDAwMhKKiInFdt27dhG+//fadxlwiKChI0NTUFJ4/f15unZo8x8uXL5f5gvW68/eqMWPGCHPmzBGX7927JygpKQkZGRmCIAiCiYmJcPLkSbF8x44dQocOHWTqpqWlybQ3d+7cSsf/oerevbuwY8eOUutf9/40/b927j2mqSuOA/gX0iBIgbbYIM6tU3zgwAgGFU1QwY3pmBrjY2q2TAlRGcrEJRpkCTFxcSa+wlxiXDKy+BoSM+P7QYhMY6YRU6Xb1KAgIyq+oKWMYqHf/WG4o45RO+GWye+TNOntOfT8fr/eHnqa2xMdzZs3b5IkExMTeenSJTY0NHDs2LG02WzqBO9jw4YN4/Hjx5XjnTt3MiUlxeP5Nn36dJ4+fZokuXDhQhYVFdHpdDIxMZF3795VNwkVeTsHdDUnPXjwgJGRkSRJh8PBoKAgkuTFixc5a9YsNdIRQvRCcrmgUM2tW7fgcrkQGxurPBYXFweLxaJ6LE1NTbhy5YpbLC9KSUlBREQE0tLS8OuvvyqPWywWxMXFKcchISGIiopS8nixPS4uDo8ePUJdXZ3Xce7evRsGgwFjxozB999//6/9LBYLRo8eDX//v9/SHWurZswAUFhYiIULFyIoKKjLfr2hxu3P11X9Ouvfcfzhw4cjICAAN27cQH19Pe7du/eP+DrGbjKZoNfrX2qsvqKtrQ2XL1/G06dPMWLECAwaNAhLly6F1Wr1+PrExsaitLQU9+/fR3V1NaKiopCbm4vc3FyEhIT4KiVVrV69Gvv374fdbsfjx49RXFyMGTNmeDzf2mtns9lQXl6OmJgYbNu2DXPnzsVbb73lq3RU9ypzqNFoVPqUlpYiJiYGra2t+OKLL1BQUKBqHkKI3kMWWUI1drsdYWFhbo/pdDrVr1V3uVxYsmQJxo0bh9TU1E77nDt3DtXV1aisrER8fDxSU1Nhs9kAPM9Dp9O59e+Yx4vt7fe9zTM7Oxu3bt3Cw4cPsWPHDqxduxaHDh3qtK+3MfVUzADw+PFjHD16FOnp6V326w01budpPG/62+12t5ja7zscDrS2tno9Vl9RV1cHp9OJH3/8EaWlpfjtt99QV1eH1atXe6zZli1bcOTIEaSlpWHbtm24ffs2/vjjD0ybNg2LFy/G5MmTsXPnTh9kpZ6pU6fixo0bCAsLg9FoRL9+/ZCdne2xdrm5uaiqqkJSUhIyMzOh1Wpx+PBhZGZmYsWKFZg8eTK+/PJLH2SkrleZQ/39/bFnzx5kZGTg66+/xnfffYcdO3Zgzpw5sFqtSE1NRXJyMsrKylTKRgjRG8giS6hGq9UqH6LbWa1WVb9pJokVK1bg3r17KCoqgp+fX6f9pkyZgoCAAISEhGDjxo3QaDS4ePEigOd5WK1Wt/4d83ixvf2+t3mOHTsWAwYMgEajQXJyMrKyslBcXNxpX29j6qmYAWDfvn0YNmwYJkyY0GW/3lDjdp7G86a/Vqt1i6n9fmBgIDQajddj9RX9+/cHAKxcuRKDBw+GTqdDXl4ejh075rFmJpMJJ06cwNWrV7FgwQLk5OSgoKAAmzZtQkxMDEpKSrBr1y78/vvvquelhra2NkyfPh1paWloamqC1WrFG2+8gY8//thj7QwGA4qKinDt2jXk5OQgOzsbW7duxQ8//IC2tjaUlZXh0qVLOHXqlC9SU82rzqHTpk3DL7/8grKyMhgMBhw6dAg5OTnIyMhAfn4+CgsL8cknn7htgCOEeL3JIkuoZsSIEfDz83O7LMxsNnd5yV53IomsrCyYzWacPHlS+TD8Mvz9/ZV/jrGxsTCbzUqb3W7H7du3lTxebDebzTAajYiIiHil+DvG8KLY2FhUVFTA5XK5jftvMfVkzIWFhVi6dKnXf+fLGnuqX2f9O45fWVmJlpYWREdHQ6/XY9CgQf+Ir2Psd+/eRUNDw0uN1VfodDq8+eabnX7x4c3rU1BQgJkzZ2LIkCGoqKjAuHHjEBAQgNGjR6OioqJHc/CV+vp61NbWYtWqVQgMDERoaCgyMzNx4sQJr863n376CZGRkZg4caJSOz8/PyQkJOD69esqZqS+V51DO/r888+xdetWaDQapY5vv/02nE4nHj161OO5CCF6CV/+IEz0PYsXL+bs2bNps9lYUVHBiIgIVXYXJMnPPvuM8fHxbj8Ab2cymVhYWEiStFgsLC8vp9PpZFNTE/Pz8xkREcH6+nqSf+8ydfbsWTY3N3PNmjVuO9/t3r2bUVFRvHPnDp88ecKkpKT/tFNfUVERbTYb29raeP78eQ4YMIAHDhxQ2qdMmcL8/HySf++MtWnTJjocDh48eLDTnbF6Ouby8nJqNBo+ePDgH229ocZOp5PNzc3My8vj/Pnz2dzczGfPnnmsX1VVFQGwqqqK5PMfyYeHh/Pq1atsbGzkRx995La74Pr16zlhwgQ+fPiQNTU1jI6OdttdcNKkSczKyuKff/7JsrIyhoSEyO6CJDds2MD4+Hjev3+fNpuNH374IdPT0z2+Pu1qa2uZmJjIZ8+ekXy+c966devY2NjotnPe62jo0KHcsGEDW1paaLfbmZGRwUmTJpF8ufOtsbGRCQkJyntw8+bNXLRoER0OB5OSklhcXKx2Sj3iv84BnuakdocPH+ayZcuU41GjRvHkyZO0WCzU6/VsbW1VLVchhG/JIkuoqr6+nvPmzWNwcDAHDhzI7du3qzJudXU1AbBfv34MDg5Wbl999RVbWlqo1WqVne5KS0s5cuRI9u/fn+Hh4Xz//fdpNpvdnu/gwYMcMmQIg4KCmJyczOrqaqXN5XIxLy+P4eHhDA0NZXp6Oh0Oh9cxJyUlMSwsjFqtlu+88w537drl1j506FCeOXNGOb5+/TrHjx/PwMBARkdHs6SkRPWYV65c2em2z72lxvn5+QTgdvv0009Jdl2/n3/+mSaTSfnwTpLffPMNIyMjGRwczDlz5rgt3ltaWrhs2TKGhYXRYDBw3bp1dLlcSntNTQ3fffddBgUF0WQycf/+/R5j7wucTidXrVpFvV5Po9HIJUuW0Gq1kvR8fpPkvHnzeP78eeW4pqaG48ePp06nY05Ojmp5+MK1a9eYkpJCvV5Pg8HADz74gJWVlSRf7nxbs2YN9+7dqxw3NDTwvffeY2hoKBctWvTaLA7+6xxAdj0nkc930E1ISHCbC0pKSmgymThw4EC3L8mEEK8/P1IuEBZ924ULF/Dtt9/iwIEDvg7lpdXW1mLBggXKb5h6u/9jjTvauHEjjEYjli9f7utQhBBCCPE/IIssIYQQQgghhOhGsvGFEEIIIYQQQnQjWWQJIYQQQgghRDeSRZYQQgghhBBCdCNZZAkhhBBCCCFEN5JFlhBCCCGEEEJ0I1lkCSGEEEIIIUQ3kkWWEEIIIYQQQnQjWWQJIYQQQgghRDeSRZYQQgghhBBCdCNZZAkhhBBCCCFEN/oLGNcVYdvv6T4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 938.625x350 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with sns.plotting_context(\"paper\"):\n",
    "    g = sns.catplot(\n",
    "        data=df,\n",
    "        y=\"Shortcut\",\n",
    "        x=\"value\",\n",
    "        hue=\"Feature/Subset\",\n",
    "        hue_order=[\"SCFG/Supporting\",\n",
    "                   \"N-gram pairs/Supporting\",\n",
    "                   \"Hypothesis n-grams/Supporting\",\n",
    "                   \"\",\n",
    "                   \"SCFG/Counter\",\n",
    "                   \"N-gram pairs/Counter\",\n",
    "                   \"Hypothesis n-grams/Counter\"],\n",
    "        palette=[hls[10], hls[9], hls[8], hls[-1],\n",
    "                 hls[0], hls[1], hls[2]], \n",
    "        col=\"Metric\",\n",
    "        kind=\"bar\",\n",
    "        sharex=False,\n",
    "        height=3.5,\n",
    "    )\n",
    "    g.axes[0, 1].set_xlim(0.5, None)\n",
    "    g.set(ylabel=\"\")\n",
    "    g.axes[0, 0].set_title(\"Number of examples\")\n",
    "    g.axes[0, 1].set_title(\"BERT accuracy\")\n",
    "    g.axes[0, 0].set_xlabel(\"\")\n",
    "    g.axes[0, 1].set_xlabel(\"\")\n",
    "    g.axes[0, 0].xaxis.set_major_formatter(mtick.FuncFormatter(lambda x, p: format(int(x), ',')))\n",
    "    g.axes[0, 1].xaxis.set_major_formatter(mtick.PercentFormatter(1, decimals=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "489befed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}