2_LeNet-5_MNISTNet_on_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial 2 : Playing with MNIST dataset using LeNet-5 and MNISTNet\n",
    "\n",
    "## A PyTorch implementation to: [Rotation equivariant vector field networks](https://github.com/COGMAR/RotEqNet), ICCV 2017\n",
    "\n",
    "### Go to your IPython working directory in bash and copy this command:\n",
    "### git clone https://github.com/COGMAR/RotEqNet.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "from torchvision import transforms,datasets\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torchvision\n",
    "import sys\n",
    "sys.path.insert(0, '/home/sprakashdash/Documents/DLFA/RotEqNet/') # change path accordingly\n",
    "from layers_2D import RotConv, VectorMaxPool, VectorBatchNorm, Vector2Magnitude, VectorUpsampling\n",
    "import matplotlib.pyplot as plt\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.Resize(32),transforms.ToTensor()])\n",
    "apply_transform_rot = transforms.Compose([transforms.Resize(32),transforms.RandomRotation(90),transforms.ToTensor()])\n",
    "BatchSize = 256 # change according to system specs\n",
    "\n",
    "trainset = datasets.MNIST(root='./MNIST', train=True, download=True, transform=apply_transform)\n",
    "trainsetRot = datasets.MNIST(root='./MNIST', train=True, download=True, transform=apply_transform_rot)\n",
    "trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "trainLoaderRot = torch.utils.data.DataLoader(trainsetRot, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "\n",
    "# Validation set with random rotations in the range [-90,90]\n",
    "testsetRot = datasets.MNIST(root='./MNIST', train=False, download=True, transform=apply_transform_rot)\n",
    "testLoaderRot = torch.utils.data.DataLoader(testsetRot, batch_size=BatchSize,\n",
    "                                         shuffle=False, num_workers=4) # Creating dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of samples in train set: 60000\n",
      "No. of samples in test set: 10000\n"
     ]
    }
   ],
   "source": [
    "# Size of train and test datasets\n",
    "print('No. of samples in train set: '+str(len(trainLoader.dataset)))\n",
    "print('No. of samples in test set: '+str(len(testLoaderRot.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define network architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LeNet-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(LeNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 6, kernel_size=5)\n",
    "        self.pool1 = nn.MaxPool2d(kernel_size=2,stride=2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, kernel_size=5)\n",
    "        self.pool2 = nn.MaxPool2d(kernel_size=2,stride=2)        \n",
    "        self.fc1 = nn.Linear(400, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = self.pool1(x)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool2(x)\n",
    "        x = x.view(-1, 400)\n",
    "        x = F.relu(self.fc1(x)) \n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return F.log_softmax(x,dim=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNISTNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MnistNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MnistNet, self).__init__()\n",
    "\n",
    "        self.main = nn.Sequential(           \n",
    "            RotConv(1, 6, [9, 9], 1, 9 // 2, n_angles=17, mode=1), #The first RotConv must have mode=1 \n",
    "            VectorMaxPool(2),\n",
    "            VectorBatchNorm(6),\n",
    "            \n",
    "            RotConv(6, 16, [9, 9], 1, 9 // 2, n_angles=17, mode=2), #The next RotConv has mode=2 (since the input is vector field)\n",
    "            VectorMaxPool(2),\n",
    "            VectorBatchNorm(16),\n",
    "            \n",
    "            RotConv(16, 32, [9, 9], 2, 1, n_angles=17, mode=2), # changed stride 1->2\n",
    "            Vector2Magnitude(), #This call converts the vector field to a conventional multichannel image/feature image\n",
    "            \n",
    "        )\n",
    "        self.linear = nn.Sequential(\n",
    "            nn.Linear(32,128),\n",
    "            nn.Dropout2d(0.7),\n",
    "            nn.Linear(128,10)               \n",
    "        )\n",
    "\n",
    "    def forward(self,x):\n",
    "        x = self.main(x)\n",
    "        x = x.view(-1,32)\n",
    "        x = self.linear(x)\n",
    "        return  F.log_softmax(x,dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the networks on vanilla MNIST and validate on MNIST with random rotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LeNet(\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
      "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
      "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
      ")\n",
      "MnistNet(\n",
      "  (main): Sequential(\n",
      "    (0): RotConv()\n",
      "    (1): VectorMaxPool()\n",
      "    (2): VectorBatchNorm()\n",
      "    (3): RotConv()\n",
      "    (4): VectorMaxPool()\n",
      "    (5): VectorBatchNorm()\n",
      "    (6): RotConv()\n",
      "    (7): Vector2Magnitude()\n",
      "  )\n",
      "  (linear): Sequential(\n",
      "    (0): Linear(in_features=32, out_features=128, bias=True)\n",
      "    (1): Dropout2d(p=0.7)\n",
      "    (2): Linear(in_features=128, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net1 = LeNet() \n",
    "net2 = MnistNet()\n",
    "print(net1)\n",
    "print(net2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU is available!\n"
     ]
    }
   ],
   "source": [
    "# Check availability of GPU\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')   \n",
    "    net1 = net1.cuda()\n",
    "    net2 = net2.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer1 = optim.SGD(net1.parameters(), lr=1e-2, momentum=0.9) # SGD\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr=1e-2, momentum=0.9) # SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training and validation routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1 /10  ; LeNet:  Training Loss: 0.005049 ; Testing Acc: 46.760 ; Time consumed: 7m 51s \n",
      "Iteration: 1 /10  ; MNISTNet:  Training Loss: 0.002655 ; Testing Acc: 67.110 ; Time consumed: 7m 51s \n",
      "Iteration: 2 /10  ; LeNet:  Training Loss: 0.000775 ; Testing Acc: 51.650 ; Time consumed: 8m 12s \n",
      "Iteration: 2 /10  ; MNISTNet:  Training Loss: 0.000614 ; Testing Acc: 68.790 ; Time consumed: 8m 12s \n",
      "Iteration: 3 /10  ; LeNet:  Training Loss: 0.000437 ; Testing Acc: 53.070 ; Time consumed: 8m 8s \n",
      "Iteration: 3 /10  ; MNISTNet:  Training Loss: 0.000459 ; Testing Acc: 72.370 ; Time consumed: 8m 8s \n",
      "Iteration: 4 /10  ; LeNet:  Training Loss: 0.000334 ; Testing Acc: 53.080 ; Time consumed: 8m 7s \n",
      "Iteration: 4 /10  ; MNISTNet:  Training Loss: 0.000402 ; Testing Acc: 73.680 ; Time consumed: 8m 7s \n",
      "Iteration: 5 /10  ; LeNet:  Training Loss: 0.000278 ; Testing Acc: 53.210 ; Time consumed: 7m 38s \n",
      "Iteration: 5 /10  ; MNISTNet:  Training Loss: 0.000324 ; Testing Acc: 74.260 ; Time consumed: 7m 38s \n",
      "Iteration: 6 /10  ; LeNet:  Training Loss: 0.000240 ; Testing Acc: 53.160 ; Time consumed: 7m 38s \n",
      "Iteration: 6 /10  ; MNISTNet:  Training Loss: 0.000290 ; Testing Acc: 72.750 ; Time consumed: 7m 38s \n",
      "Iteration: 7 /10  ; LeNet:  Training Loss: 0.000214 ; Testing Acc: 53.840 ; Time consumed: 9m 3s \n",
      "Iteration: 7 /10  ; MNISTNet:  Training Loss: 0.000272 ; Testing Acc: 71.970 ; Time consumed: 9m 3s \n",
      "Iteration: 8 /10  ; LeNet:  Training Loss: 0.000195 ; Testing Acc: 54.280 ; Time consumed: 9m 26s \n",
      "Iteration: 8 /10  ; MNISTNet:  Training Loss: 0.000239 ; Testing Acc: 74.840 ; Time consumed: 9m 26s \n",
      "Iteration: 9 /10  ; LeNet:  Training Loss: 0.000175 ; Testing Acc: 54.810 ; Time consumed: 8m 57s \n",
      "Iteration: 9 /10  ; MNISTNet:  Training Loss: 0.000213 ; Testing Acc: 74.590 ; Time consumed: 8m 57s \n",
      "Iteration: 10 /10  ; LeNet:  Training Loss: 0.000155 ; Testing Acc: 53.540 ; Time consumed: 9m 31s \n",
      "Iteration: 10 /10  ; MNISTNet:  Training Loss: 0.000210 ; Testing Acc: 74.090 ; Time consumed: 9m 31s \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de3wV9Z3/8dcnJzlJSMItpIqEGirYioKo0dZb66UVtVZltSvYql3og+riT2prre6ubGW1W3pRa6tWV1ytbb3Uri3requKW20VDUqtwCIRqAJS7nIJJCT5/P6YyYVwkpxDzmRyeT/3MY/vnJnvzPnMccsn3/nOfL/m7oiIiKQrJ+4ARESkd1HiEBGRjChxiIhIRpQ4REQkI0ocIiKSkdy4A+gOw4YN84qKirjDEBHpNRYuXLjR3ctS7esXiaOiooKqqqq4wxAR6TXM7K/t7dOtKhERyYgSh4iIZESJQ0REMtIv+jhS2bNnD6tXr2b37t1xh9LrFRQUUF5eTl5eXtyhiEg36LeJY/Xq1ZSUlFBRUYGZxR1Or+XubNq0idWrVzNq1Ki4wxGRbtBvb1Xt3r2b0tJSJY0uMjNKS0vVchPpR/pt4gCUNLJEv6NI/xJp4jCzM81smZlVm9l1Kfbnm9kj4f4FZlbRat/14fZlZjax1fZVZvYXM1tkZtG9nFFTA1VV8MYbkX2FiEhvFFniMLMEcAdwFjAWmGJmY9tUmwZscffRwK3AnPDYscBk4HDgTODO8HxNTnX3Ce5eGVX8FBQEZWNjZF9RXFycdt3777+fnJwc3nrrreZtRxxxBKtWrerwuNtuu42ampqU+1atWkVhYSETJkxgwoQJXH755WnHIyL9V5QtjuOAandf4e51wMPAeW3qnAc8EK4/BpxuwX2P84CH3b3W3VcC1eH5uk9Oz7uLV15ezs0335zRMR0lDoBDDjmERYsWsWjRIn72s591NUQR6Qei/NdxBPB+q8+rw20p67h7PfAhUNrJsQ48a2YLzWx6e19uZtPNrMrMqjZs2NClC2H79q4dn4ENGzZwwQUXcOyxx3Lsscfyxz/+sXnfOeecw+LFi1m2bNk+xz377LMcf/zxHH300Xzxi19kx44d3H777axdu5ZTTz2VU089tduuQUT6tigfx03VY9p2ntr26nR07InuvtbMPgL83sz+z93/sE9l93uAewAqKys7nh/361+HRYv23d6UMHJyoKiow1PsY8IEuO22zI4BZs6cydVXX81JJ53Ee++9x8SJE1m6dGkYRg7XXnst3/3ud3nggQeaj9m4cSM33XQTzz33HEVFRcyZM4dbbrmFWbNmccsttzB//nyGDRuW8vtWrlzJUUcdxcCBA7nppps4+eSTM45ZRPqXKBPHamBkq8/lwNp26qw2s1xgELC5o2Pdvalcb2aPE9zC2idxZEUiAQ0NkfZztPXcc8+xZMmS5s/btm1je6sWz8UXX8zNN9/MypUrm7e9+uqrLFmyhBNPPBGAuro6jj/++E6/a/jw4bz33nuUlpaycOFCzj//fBYvXszAgQOzeEUi0tdEmTheB8aY2ShgDUFn98Vt6swDLgNeAS4EXnB3N7N5wK/M7BbgIGAM8JqZFQE57r49XD8DmN3lSNtrGTQ2tjxVVRldP/zeX9nIK6+8QmFhYcr9ubm5fPOb32TOnDnN29ydz33uczz00EMdnvvxxx/nxhtvBODee++lsrKS/Px8AI455hgOOeQQ3nnnHSq76VpFpHeKrI8j7LO4EngGWAo86u6LzWy2mZ0bVpsLlJpZNfAN4Lrw2MXAo8AS4Glghrs3AAcAL5vZn4HXgP9x96ejuoY4OsjPOOMMfvrTnzZ/XpTiFtpXvvIVnnvuOZr6bj71qU/xxz/+kerqagBqamp45513ACgpKWlusUyaNKm5I7yyspINGzbQ0NAAwIoVK1i+fDkf+9jHIr0+Een9Ih1yxN2fBJ5ss21Wq/XdwBfbOfZm4OY221YAR2Y/0jTs3t3yiG6W1NTUUF5e3vz5G9/4BrfffjszZsxg/Pjx1NfX8+lPf3qfp52SySRXXXUVM2fOBKCsrIz777+fKVOmUFtbC8BNN93EoYceyvTp0znrrLMYPnw48+fP3+s8f/jDH5g1axa5ubkkEgl+9rOfMXTo0Kxeo4j0Pebecb9xX1BZWeltJ3JaunQphx12WOcHNx1XUABHHBFBdH1D2r+niPQKZrawvXflet7LCj3NgAFBqbGYREQAJY7OjR4ddwQiIj2KEkdnksm4IxAR6VGUODJRXx93BCIisVPiyMTy5XFHICISOyWOdDQ9htvBYIEiIv2FEkc6ml6Ky/Kjy2bGJZdc0vy5vr6esrIyzjnnHKDzodQrKirYuHEjADfffDOHH34448ePZ8KECSxYsIBJkyYxYcIERo8ezaBBg5qHT//Tn/7EKaecstcb4lVVVZxyyikdxrtq1Sp+9atfZenqRaS3UuJIR9MjuVlWVFTE22+/za5duwD4/e9/z4gRew8gnM5Q6q+88gpPPPEEb7zxBm+99RbPPfccI0eO5PHHH2fRokXce++9nHzyyc1vjZ9wwgkArF+/nqeeeirteJU4RASUODKX5QEPzzrrLP7nf/4HgIceeogpU6bstb+jodSbfPDBBwwbNqx53Klhw4Zx0EEHdfrd3/rWt7jpppv22d7Q0MC3vvUtjj32WMaPH8/dd98NwHXXXcdLL73EhAkTuPXWW9O+RhHpWyIdcqS3+PrTX2fRuhTDqrfWNELta4m0WiATDpzAbWd2Pqz65MmTmT17Nueccw5vvfUWU6dO5aWXXmre395Q6q2dccYZzJ49m0MPPZTPfvazXHTRRXzmM5/p9LuPP/54Hn/8cebPn09JSUnz9rlz5zJo0CBef/11amtrOfHEEznjjDP43ve+xw9/+EOeeOKJTs8tIn2XWhzpahrwMBwUMFvGjx/PqlWreOihhzj77LNT1rn44ot59dVX9xpKvbXi4mIWLlzIPffcQ1lZGRdddBH3339/Wt//L//yL/u0Op599ll+/vOfM2HCBD75yU+yadMmluuJMhEJqcUBabUM2L4dmm4XZXnY8XPPPZdrrrmGF198kU2bNu2zP9VQ6m0lEglOOeUUTjnlFMaNG8cDDzzAV77ylU6/+7TTTuOGG27g1Vdfbd7m7vzkJz9h4sSJe9V98cUX074mEem71OJIV6tbOdk2depUZs2axbhx49qt03Yo9daWLVu2V4tg0aJFHHzwwWl//z//8z/z/e9/v/nzxIkTueuuu9izZw8A77zzDjt37txriHYR6b+UOPZHljvIy8vLm4dIb0/TUOrr16/fZ9+OHTu47LLLGDt2LOPHj2fJkiV85zvfSfv7zz77bMrKypo/f/WrX2Xs2LEcffTRHHHEEXzta1+jvr6e8ePHk5uby5FHHqnOcZF+TMOqZ6LpHEOGwCGHZCm6vkHDqov0LRpWPVvy8oJy69Z44xARiZESRyY++tGg7AetNBGR9vTrxJHxbbohQ6IJpJfrD7c7RaRFv00cBQUFbNq0Sf/odZG7s2nTJgqyPB+7iPRc/fY9jvLyclavXp3y8dYOhYMK8vLLUFqa/cB6oYKCAsrLy+MOQ0S6Sb9NHHl5eYwaNSrzA086CTZvDoYd2bkz+4GJiPRw/fZW1X5rGqlWc3OISD+lxJGpyy+POwIRkVgpcYiISEaUOLpi1qy4IxAR6XZKHPtj4MCgvC2NUXVFRPoYJY798c1vBqVGihWRfkiJY3/oFpWI9GNKHCIikhEljq768Y/jjkBEpFtFmjjM7EwzW2Zm1WZ2XYr9+Wb2SLh/gZlVtNp3fbh9mZlNbHNcwszeNLMnooy/Q8XFQTl7dmwhiIjEIbLEYWYJ4A7gLGAsMMXMxrapNg3Y4u6jgVuBOeGxY4HJwOHAmcCd4fmazASWRhV7Wi69NCg3b441DBGR7hZli+M4oNrdV7h7HfAwcF6bOucBD4TrjwGnm5mF2x9291p3XwlUh+fDzMqBzwP3Rhh75+64I9avFxGJS5SJYwTwfqvPq8NtKeu4ez3wIVDaybG3AdcC2Z34W0RE0hJl4rAU29pOftFenZTbzewcYL27L+z0y82mm1mVmVVlPHR6ph55JNrzi4j0IFEmjtXAyFafy4G17dUxs1xgELC5g2NPBM41s1UEt75OM7NfpPpyd7/H3SvdvbKsrKzrV5NKYWFQzpwZzflFRHqgKBPH68AYMxtlZkmCzu55berMAy4L1y8EXvBgSr55wOTwqatRwBjgNXe/3t3L3b0iPN8L7v7lCK+hY5MmBeX69bGFICLS3SJLHGGfxZXAMwRPQD3q7ovNbLaZnRtWmwuUmlk18A3guvDYxcCjwBLgaWCGuzdEFet+uzfsn9f0syLSj1h/mHO7srLSq6qqojm5hd0x/eB3FJH+w8wWuntlqn16czxbnn8+7ghERLqFEkdXJZNBOXVqvHGIiHQTJY6uOvXUoHz//Y7riYj0EUocXfX440GpPg4R6SeUOLqq6V0OEZF+Qokjm15/Pe4IREQip8SRDbm5Qfnl+N5FFBHpLkoc2XDssUFZXR1vHCIi3UCJIxueCOeTatSAvSLS9ylxZMPQoXFHICLSbZQ4sm3NmrgjEBGJlBJHtiTCmW0///l44xARiZgSR7YccURQvv12vHGIiERMiSNbHnssKBt63ujvIiLZpMSRLaNHxx2BiEi3UOKIwubNcUcgIhIZJY5sygl/znPP7bieiEgvpsSRTU23qxYsiDcOEZEIKXFk0y9+EZT19fHGISISISWObGoas0pEpA9T4ojKrl1xRyAiEomMEocFiqIKpk8wC8oLLog3DhGRiHSaOMzs52Y20MwGAIuBlWb2jehD66VGjgzK55+PNw4RkYik0+IY5+7bgPOBZ4Fy4CtRBtWr3XdfUNbVxRuHiEhE0kkcSTPLBc4DfuvudYAmnmjP6afHHYGISKTSSRz3Au8BQ4D/NbOPAjsijaqvUAe5iPRBnSYOd7/V3Q9y9zPc3YH3gdOiD60Xa+og/+pX441DRCQC6XSOX2lmA8P1u4EFwMlRB9arlZUF5eOPxxuHiEgE0rlVNd3dt5nZGcAI4Arg+9GG1cv96EdBqVtVItIHpZM4PCzPAv7T3RemeVz/9eUvxx2BiEhk0kkAfzazJ4EvAE+ZWTEtyURERPqZdBLHPwDfAY5z9xqgAJgWZVB9ysyZcUcgIpJV6TxV1QAMA641s+8Bx7r7m+mc3MzONLNlZlZtZtel2J9vZo+E+xeYWUWrfdeH25eZ2cRwW4GZvWZmfzazxWZ2Y5rX2f2GDAnKphcCRUT6iHSeqroZuBZYES7fMrOb0jguAdxB0DcyFphiZmPbVJsGbHH30cCtwJzw2LHAZOBw4EzgzvB8tcBp7n4kMAE408w+lc6Fdrt/+qeg3KFXXkSkb0nnVtUXgM+6+z3ufg9wBpDOFHfHAdXuviJ82/xhgrfPWzsPeCBcfww43cws3P6wu9e6+0qgmuBWmbt707/EeeHSM/tbrrkm7ghERCKR7tNRJe2sd2QEwcuCTVaH21LWcfd64EOgtKNjzSxhZouA9cDv3T3ldHtmNt3MqsysasOGDWmGLCIinUkncXwfeMPM7jWzuUAV4S2lTliKbW1bB+3VafdYd29w9wkEgy0eZ2ZHpPrysIVU6e6VZU0v5MXle9+L9/tFRLIonc7xXwAnAU+Gy6fd/ZdpnHs1MLLV53JgbXt1woEUBwGb0znW3bcCLxL0gfRMJWHjbE46eVZEpHdoN3GY2fimheD2UTWwHCgNt3XmdWCMmY0ysyRBZ/e8NnXmAZeF6xcCL4TjYc0DJodPXY0CxgCvmVmZmQ0O4ysEPgv8X7oX2+2uuCIot26NNw4RkSzK7WDfHR3sc+DTHZ3Y3evN7ErgGSAB3Ofui81sNlDl7vOAucCDZlZN0NKYHB672MweBZYA9cAMd28ws+HAA+ETVjnAo+7+RFpXGoc5c+D7Gp1FRPoWC/7A79sqKyu9qqoqni9vGim3H/zOItJ3mNlCd69MtU9jTnWXu+6KOwIRkaxQ4ojagAFBecMN8cYhIpIlShxRaxopd9OmeOMQEcmSdIYcGZ9iOdjMlHTScffdcUcgIpJVHT1V1WQuwbhQiwlezDsMeBsYZGbT3f35COMTEZEeJp1Ww3LgGHefEA4ueAywCJgI/CjK4PocTSUrIn1AOonjMHd/q+mDu/8FONrdq6MLq+fYvGtz10+Snx+UM2Z0/VwiIjFLJ3G8a2Y/MbMTw+V2oNrM8glezuuT5lbNxW40Sr9f2vWTnX12UK5b1/VziYjELJ3EcSnB2FHXAdcTjBl1GUHSOD260OI1rbJlksNdu3Z17WS/DIf20kuAItIHdNo5Hk4XO4fUI+J+mPWIepAECRpoYMSPR7D5ui7csioszF5QIiIxS+dx3E+Z2VNmtsTM3mlauiO4uM2bHIzJuKV2S/ZO+vLL2TuXiEgM0rlV9Z/AnQQj0Z7caunzzv742c3r0x6f1kHNNCSTQXnJJV07j4hIzNJJHNvc/b/dfa27/61piTyyHuLEkScCcN9b93XtRCecEJR//WsXIxIRiVc6ieMFM/t3Mzu2zRwd/cLLU1tuLa3ZvGb/T/Tkk0GpDnIR6eXSeXP8pDYlpDEfR1+SzElS11jHmLvGUPPPNft3EnWQi0gfkc5TVf2iP6Mji2csZsxPxrCrvouP5Tb5y19g3LjsnEtEpJu1mzjMbIq7P2RmV6Xa7+63RxdWzzJ66Ojm9YkPTuSZS57ZvxPl5kJ9PUyeDIsXZyk6EZHu1VEfx5CwLGtn6VcuPuJiAJ5d8ez+n+Soo4Ly/3ruNOkiIp3R1LEZsBuDaWBfuuwlTqo4qZPaKaxZA+XlwXo/+N1FpPfqaOrYTvs4zGwYMBWoaF3f3adnK8DeoiRZwva67Zz681PZM2tP5icYMSL7QYmIdLN0Hsf9HXAA8DLwfKul31n19VUA1HsWxnZc04VHe0VEYpTO47hF7v7NyCPpBYYWDm1eP/ynh7P4yv3o4E4koKEBJk2C117LYnQiIt0jnRbHU2Z2RuSR9BI3nHQDAEs2Ldm/E3z840H55ptZikhEpHulkzguB542sx1mttnMtphZFmY36p1mnz67ef3BRQ9mfoLf/S4o6/vsVCYi0selkziGAXnAIILHcIfRDx/HbW148XAALvvdZZkfPHp053VERHqwdhOHmY0JVw9vZ+m33v3HdwFwvGuTPG3utw03EenFOuocvw6YBtyRYl+/GquqrcLCQgzDcT52x8f44JoPMjuBWfAexwUXwPz50QQpIhKRdlsc7j4tLE9OsfTbpNHkgfMeAGDdzv2YR/zgg4PyT3/KYkQiIt0jnT4OzOwTZvZ3ZnZx0xJ1YD3dJRNaJmSa9fyszA5+MOxUr6vLYkQiIt0jnalj/wW4B/gZcBZwG3BhxHH1Ckd+5EgA/u3lf8vswJP2Y7gSEZEeIp0Wx0XAqcAH7n4JcCTpvTjY5y26YlHz+uZd+9nR3ZXOdRGRGKSTOHa5ewNQb2YlwDrgY+mc3MzONLNlZlZtZtel2J9vZo+E+xeYWUWrfdeH25eZ2cRw20gzm29mS81ssZnNTCeOKOVakEMrbq3I7EALBkzkS1/KbkAiIhFLJ3G8aWaDgfuAKuA14I3ODjKzBMETWWcBY4EpZja2TbVpwBZ3Hw3cCswJjx0LTCZ47PdM4M7wfPXAN939MOBTwIwU5+xW8y8Nnoravmd7ZgceeGBQNk0pKyLSS3SYOMzMgO+4+1Z3vwP4PPA1d780jXMfB1S7+wp3rwMeBs5rU+c84IFw/THg9PA7zwMedvdad18JVAPHufsH7v4GgLtvB5YCsQ4523p49cm/npz+gXfeGZS1tVmOSEQkWh0mDg8m63ii1efqpn+40zACeL/V59Xs+498cx13rwc+BErTOTa8rXUUsCDVl5vZdDOrMrOqDRs2pBny/jnrkLMAeGTJI+kfdP75EUUjIhKtdG5VvWZmR+/HuS3FtrazF7VXp8NjzawY+A3wdXfflurL3f0ed69098qysmhHSHnyyy23m/6y5i+RfpeISNw6GnKk6cmpkwiSxzIze8PM3jSzdFodq4GRrT6XA2vbqxN+3yBgc0fHmlkeQdL4pbv/VxpxdIvC3EIAKu9LOWFWx6ZNy3I0IiLR6ajF0TRZxPnAx4GzgS8SvMPxxTTO/TowxsxGmVmSoLN7Xps684CmkQIvBF4Ib4/NAyaHT12NAsYQJC8D5gJL3f2WNGLoNsuvWA5AXWMGL/WVlgblQw9FEJGISDQ6ShwG4O7vplo6O3HYZ3El8AxBJ/aj7r7YzGab2blhtblAqZlVA98gGB8Ld18MPAosAZ4GZoSPBJ8IXAKcZmaLwuXs/bnwbBsxtKUL5oS5J6R30Jw5Qal3OUSkF7HgD/wUO8xWA+3+Vd/T/uLvSGVlpVdVVUX+PTOemMGdC4OnpfxfU/+u+2h6n6Od/w4iInEws4XunvLee0ctjgRQDJS0s0gbd5zTMpDwk8v0foaI9E0dDR3ygbvP7mC/pDC0YCibd2/mCw9/gYZ/bUj/wG9/u+XWlYhID9ZpH4dkZvVVqwFopDG9AwYPDsq77oooIhGR7OoocZzebVH0IU2TPAF87MdpDOn17W8H5fYMhywREYlJRxM5aV7T/XTrGbcCsHLrys4rX7fP2I8iIj1aWhM5SWZmHt8yaO+PX/lxjJGIiGSfEkdERg0eBcDVz16d/kE//GFE0YiIZI8SR0RWzFwBgOPs6uwFv+LioPzudyOOSkSk65Q4IpQT/rzlt5d3XHHq1KDcsiXiiEREuk6JI0KP/f1jAGze3clzBj9WP4iI9B5KHBGadNik5vUZT8yIMRIRkexR4ojYJ0d8EqB5DKtOPfhghNGIiHSdEkfEXv3qq83razavab9iYTCfB9dcE3FEIiJdo8TRDZI5SQDG3DWm/UqTwttaEU9zKyLSVUoc3aBqajCk+676Dh7LvffeoNTw6iLSwylxdINxI8Y1r5/54JmpKzXdqhIR6eGUOLrJhZ+4EIBnVjzTeeUnNZeHiPRcShzd5NcX/bp5/eVVL6eulAz6Qvja17ohIhGR/aPE0Y2K84KhRU79+ampK5wejmS/poOnr0REYqbE0Y3+evVfAaj3+tQVfvOboFQHuYj0YEoc3Who4dDm9XF3jNu3gjrIRaQXUOLoZteecC0Ab298u+OKr7/eDdGIiGROiaObzfncnOb1BxelGF4kNzcov/SlbopIRCQzShwxOKDoAAAu+91l++78ZDC2Fe++240RiYikT4kjBitnBHORp5zkad68oGxs7OaoRETSo8QRg8LCQgwDYMydbcavGjo0xREiIj2HEkdM/uPz/wHAmh0dvLNRXd1N0YiIpE+JIybTKqc1r984/8a9dyYSQXnhhd0YkYhIepQ4YjS2dCwA3/nDd/beMS58x+PtTh7ZFRGJgRJHjBZfubh5ffOuVvOSP/FEUDY0dHNEIiKdU+KIWcKC21IVt1W0bBwxIp5gRETSoMQRsxcvfRGA7XXbU1fYvDn1dhGRmESaOMzsTDNbZmbVZnZdiv35ZvZIuH+BmVW02nd9uH2ZmU1stf0+M1tvZn2iA+CkipOa17/0m1Zvi+eE/2nOOaebIxIR6VhkicPMEsAdwFnAWGCKmY1tU20asMXdRwO3AnPCY8cCk4HDgTOBO8PzAdwfbuszTh8VDKf+q7d/1bJx9Oig1JhVItLDRNniOA6odvcV7l4HPAyc16bOecAD4fpjwOlmZuH2h9291t1XAtXh+XD3PwB96v7Nc5c+17y+fNPyYOWxx4Kyvp0h2EVEYhJl4hgBvN/q8+pwW8o67l4PfAiUpnlsh8xsuplVmVnVhg0bMgy9+xXkFgBwxJ1HBBvGpRh2XUSkB4gycViKbW1nKGqvTjrHdsjd73H3SnevLCsry+TQWFRfEbwlXtdYt+/OtuNZiYjEKMrEsRoY2epzObC2vTpmlgsMIrgNlc6xfcqIoS0NqpPuCzvMLcyfkybFEJGISGpRJo7XgTFmNsrMkgSd3fPa1JkHNI0tfiHwgrt7uH1y+NTVKGAM8FqEsfYI04+eDsAf3/9jsGFkmDvnz48pIhGRfUWWOMI+iyuBZ4ClwKPuvtjMZpvZuWG1uUCpmVUD3wCuC49dDDwKLAGeBma4ewOAmT0EvAJ83MxWm9k0+oi7v3B38/qTy56E++4LPtSluH0lIhITC/7A79sqKyu9qqoq7jDSMvR7Q9lSu4UECer/tb7ldlU/+O8kIj2HmS1098pU+/TmeA+zZmYwzHoDbcapUge5iPQQShw9TGFhYfP6obcf2tLi+OpXY4pIRGRvShw90A8++wMAlm9ZDh/5SLDxV7+Co46KMSoRkYASRw90zYnXNK//7O7LW3YsWhS0QPLy4KmnYohMRESJo8c6eNDBAPzjotlBx/gPftBy26q+Hs4+O/h8wgkxRiki/ZESRw+16uurAHCcXbt2wTXXQGNj0Ek+fHhLxVdeCRJIfj68/HI8wYpIv6LE0YPlhP95Rv6k1Uv0BQWwdm3QCrnhhpZWSF0dnHxy8Plzn4shWhHpL5Q4erDH/j4YIXfTrk2pK8yeHbRCtmyBYcNatj/3XJBACguDfhERkSxS4ujBJh3WMkbVgT88kN8t/V3qioMHw4YNQSvk6qtbWiG7dwdPYpnBeW1HtBcR2T96c7yHO2HuCbyy+pV9thtGSbKEC8deyB0T76CgoGDvCuvWwWGHwdate28fMADeeAM+/vEIoxaR3k5vjvdif5r2J6YcNoXiZDHWarR5x9lWt437Ft1H4ZxC7EbDbjSSs5NMuHMCi1gX3MJyh6lTW1ohNTXwiU8Eny+5JKarEpHeTC2OXur+N+9n1ouzWLN9DY3e2GFdwxhaOJSrKqYwa9r9sGPH3hUGDoRly+DAA6MLWER6lY5aHEocfci6reuY/NvJvLr6VWobajuse+A2+NpCuGAJjN0ICSdohVx+Odx5Z/cELCI9lhJHP0kc7Zn94mx+8vpP2FSzCU8xkWKiEQ7dCCe9D+PXQfk2OGRXIeP+vDboeBeRfjg3SwwAAAzySURBVEeJo58njlRee/81pv/3dJZsXMIe3wNATiM0tur1+uhWGLwbPiiGmiFFnFZxGl8a9yXOP/R88vPzY4pcRLqDEocSR1p2797NPz7zjzy18BGOWFmDAauGQPVQ8NazwLc3K3woYQnycvIYkBxAaWEpFYMqqBxeyaTDJnFs+bERX4WIZIMShxLH/jvlFNYv+F/eKYPlpbC5EDYOCFohH5TA+qLg87Z8qMmD+hw6TCodybEcEpYgmUhSkizhI0UfYczQMZxy8ClcNPYiygaWZfXSRKR9ShxKHF33/PNwwQXB47x79rRbrdFgaz5sGABrS+C9wfDuEFgxJCjfGwR/K4aGRHbCyiGHZG6SgcmBDC8ZzjEHHsPfHfZ3fG7U50gmk9n5EpF+SIlDiSNar78Oc+YEAy5u2gS1HT/RlVJeHhvLipl/zFB+e+Iw3ij+kE07N7Fzz05qG2pp9MaUHfuZyM3JJZmTZHDhYCoGVXD8yOO5dPyljD9wfJfOK9IXKXEoccTr3Xfh5pth/vzgjfa6umCMrUzk5gbvm4wdC1OmwOTJMHQoO+p28OCfH+Tp5U+zdNNSNuzcwM49O6lvrN/vRGMYuTm5FOQWUFpYyqFDD+UzB3+GqcdM5cBivesi/YMShxJHz7ZhA/z7vweTU73/fjB0fKaJpbWcHEgkIJmEoqLgkeLyctaMOZCHR+/m6UEbeWfHKjbv3szu+t00NDZ0KcnkWE7QP5OTINdyyUvkkUwkGZA3gKJkEUPyhzCkcAgHFB9AeUk5hww9hENLD+XwjxzOgLwB+3+dIhFS4lDi6N1qa+FHP4Jf/xpWrICdO4OhVJqWbAqTzupBOVSNMBYOd94qa2T54EbWFjfyYdL3u/N/fxiGmWEYCUuQyAmeWEsmkhTmFTIgbwCDCwZTNqCMAXkDKMwrpCRZwoC8AQzMH8jA5EAGFQ5iUHIQpUWlDCkYwgEDDmBwwWByc3O770Kk11HiUOLoX+rqYOFCeOYZ+POfYeXKoFWzY0eQhPbsyX7SMaM2x9k8wNg0wNhYFCzri2HjAONvxcamImdzgbFpgLM1H7YlnR15Tl3CafrfYVf7caLSepy0pkTWnNTMyCGHomQRJfklFOQWBK2tvCJKkiWU5JcwuGAwQwqHMKxwGGUDyigrKuOgkoMYUTKCIQVDSCSy9LSEZE1HiUN/ckjfk0zC8ccHS6bq6oJO/mefhbfegr/+Nejw3749SDr19alvo7mT3wDDtzvDt2c3IQGQk0NjDnxYmGDdwAQbSowPwmV7SZKagYXsKEiwrQB25Bvb82FH0tiRdHbmOXU0Ut9YT73X09DYQENjA43e2LLQiHuQwJr/r1VibZ3Q9vpjs9Xq7l272587JpNLbtXKaroFmLAEeYm85tZWfiK/ucVVlBckrIHJgQwuHMyYoWM4sPjAYFv+QAbmD2RQ/iAGFQxiYHIgOTka27WrlDhEWksm4TOfCZaueO+9IPEsXRrcXlu3Lmj1bNkStHx27Wpp/TQlo6aEtNc/zOF6QwM5DTBkTwNDtsFhXYtuX60SFGZgYT9R05KbC3l5we+Tnx/MRFlYGAzTX1ISLAcdxM6yIawvbGRdspYNyT2sz61lQ6KOjTm72JRTy9bGnWyv28GOPTuoqathd/1udjfspq6+jj2Ne9jTuIf6xnoaG4Nk1uAN1NeHDzq0/xT4fsmxnOZbgDk5wTtEuTm55Obk7pWkkokk+bn55CfyKcgtoCC3gMLcwubE1ZS8ipJFFCeLKU4WU5RXxCFDDyGZSDYnwqbva26xpVnmWE7GxzR9X47lUJwszu4PhxKHSDQ++tFgOeec7J73vfeC+VTeeSe4Bbd2bTBhF7QkpJqaICnV1QVLfX2wNDS0JKjGxr1v17VKUM06eF+nPUXAqHDJWFPyMmu1JFqSWU5LMtuVZ2wthK0FxtYCmpcP85360qHsLi5gVx7szA1aXDWJRmpyG6nJaWRXopFdOQ3sppHd1kCtNVBLA3XeQF1DA3UNu6mp20mDB4nL8U5HoO6pDig6gHXXrMv6eZU4RHqTpoTUnerrYePG4Im3tWvhb38LWk+bNwctqG3bgiWZDFomu3cHS11dSwLbs6dlaUpirZNZU+m+d1JrbAzqwV4tscJwGZ4y4C3Z/w3CRNaQY9TmGbuSOdTk51CTNGryjZ3JcL15gaFjxmMDBuC5uXheLo25iWC9dZlItHxuXm+1PREsjYmc8HNO87bmz2Y4NN9ebF0W5hZm/7dAiUNEOpObG8zV0lPna2loCFpbmzfDhx8GyWbPnuDpu5qaYGlqie3aFSxNya1pqa1tKdsmvFaJL1Ffz4A9exhQX09pQwPU1MP2hjDxtUqC7rB0eZD0mlp+tbXZfwoQgqSWTKZeDjgAXroi61+pxCEivVsiAYMGBUtP19DQkkgyWVrfesxkKSqK5DKUOEREuksiETxUUBjNLaTuoufSREQkI5EmDjM708yWmVm1mV2XYn++mT0S7l9gZhWt9l0fbl9mZhPTPaeIiEQrssRhZgngDuAsYCwwxczGtqk2Ddji7qOBW4E54bFjgcnA4cCZwJ1mlkjznCIiEqEoWxzHAdXuvsLd64CHgfPa1DkPeCBcfww43cws3P6wu9e6+0qgOjxfOucUEZEIRZk4RgDvt/q8OtyWso671wMfAqUdHJvOOQEws+lmVmVmVRs2bOjCZYiISGtRJo5UY4i2fYi5vTqZbt93o/s97l7p7pVlZZpyVEQkW6JMHKuBka0+lwNr26tjZrnAIGBzB8emc04REYlQlInjdWCMmY0ysyRBZ/e8NnXmAZeF6xcCL3gw9OY8YHL41NUoYAzwWprnFBGRCEX2AqC715vZlcAzQAK4z90Xm9lsoMrd5wFzgQfNrJqgpTE5PHaxmT0KLAHqgRnu3gCQ6pydxbJw4cKNZvbX/byUYcDG/Ty2r9FvsTf9HnvT79GiL/wWB7e3o19M5NQVZlbV3mQm/Y1+i73p99ibfo8Wff230JvjIiKSESUOERHJiBJH5+6JO4AeRL/F3vR77E2/R4s+/Vuoj0NERDKiFoeIiGREiUNERDKixNEODd/ewsxGmtl8M1tqZovNbGbcMcUtHK35TTN7Iu5Y4mZmg83sMTP7v/D/R46PO6Y4mdnV4f9O3jazh8ysIO6Ysk2JIwUN376PeuCb7n4Y8ClgRj//PQBmAkvjDqKH+DHwtLt/AjiSfvy7mNkI4Cqg0t2PIHhReXK8UWWfEkdqGr69FXf/wN3fCNe3E/zDkHJU4v7AzMqBzwP3xh1L3MxsIPBpglEgcPc6d98ab1SxywUKw/H3BtAHx9NT4kgt7eHb+5twlsajgAXxRhKr24Brgca4A+kBPgZsAP4zvHV3r5kVxR1UXNx9DfBD4D3gA+BDd3823qiyT4kjtbSHb+9PzKwY+A3wdXffFnc8cTCzc4D17r4w7lh6iFzgaOAudz8K2An02z5BMxtCcHdiFHAQUGRmX443quxT4khNw7e3YWZ5BEnjl+7+X3HHE6MTgXPNbBXBLczTzOwX8YYUq9XAandvaoE+RpBI+qvPAivdfYO77wH+Czgh5piyTokjNQ3f3ko4ne9cYKm73xJ3PHFy9+vdvdzdKwj+/+IFd+9zf1Gmy93XAe+b2cfDTacTjGrdX70HfMrMBoT/uzmdPviwQGTDqvdm7Q0JH3NYcToRuAT4i5ktCrf9k7s/GWNM0nP8P+CX4R9ZK4B/iDme2Lj7AjN7DHiD4GnEN+mDw49oyBEREcmIblWJiEhGlDhERCQjShwiIpIRJQ4REcmIEoeIiGREiUNkP5lZg5ktarVk7Y1pM6sws7ezdT6RbNJ7HCL7b5e7T4g7CJHuphaHSJaZ2Sozm2Nmr4XL6HD7wWb2vJm9FZYfDbcfYGaPm9mfw6VpiIqEmf1HOLfDs2ZWGNa/ysyWhOd5OKbLlH5MiUNk/xW2uVV1Uat929z9OOCnBKPpEq7/3N3HA78Ebg+33w78r7sfSTDOU9MoBWOAO9z9cGArcEG4/TrgqPA8l0d1cSLt0ZvjIvvJzHa4e3GK7auA09x9RTg45Dp3LzWzjcBwd98Tbv/A3YeZ2Qag3N1rW52jAvi9u48JP38byHP3m8zsaWAH8Fvgt+6+I+JLFdmLWhwi0fB21turk0ptq/UGWvokP08wQ+UxwMJwwiCRbqPEIRKNi1qVr4Trf6JlGtEvAS+H688DV0DzXOYD2zupmeUAI919PsFkUoOBfVo9IlHSXyoi+6+w1WjBEMy73fRIbr6ZLSD442xKuO0q4D4z+xbBrHlNo8jOBO4xs2kELYsrCGaPSyUB/MLMBhFMOHarpmqV7qY+DpEsC/s4Kt19Y9yxiERBt6pERCQjanGIiEhG1OIQEZGMKHGIiEhGlDhERCQjShwiIpIRJQ4REcnI/wckTwsUxYh5OgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU5dn/8c+VjSzsJAoSEGRRiyJocEMroiAoYnm0RaxWap9SrVal1do+tfqTQqu2Wmvro1Lq0se6VKsUlSqguCEqAREFiixSCIhAWEMSQsL1++NMkiEMYYBMTki+79drXpmzzVwzyvnOue9z7mPujoiISE1JYRcgIiINkwJCRERiUkCIiEhMCggREYlJASEiIjGlhF1AXcnOzvYuXbqEXYaIyGFl7ty5G909J9ayRhMQXbp0IT8/P+wyREQOK2b2n30tUxOTiIjEpIAQEZGYFBAiIhJTo+mDiGXXrl0UFBRQWloadimNQnp6Orm5uaSmpoZdiojUg4QGhJkNAf4AJAOT3P3uGst/D5wbmcwEjnD31pFlFcCnkWWr3H34gb5/QUEBLVq0oEuXLpjZwX4MAdydwsJCCgoK6Nq1a9jliEg9SFhAmFky8BAwCCgA5pjZFHdfVLmOu4+NWv9HQN+olyhx9z6HUkNpaanCoY6YGe3atWPDhg1hlyIi9SSRfRCnAsvcfYW7lwHPApfUsv4o4Jm6LkLhUHf0XYo0LYlsYuoIrI6aLgBOi7WimR0NdAXejJqdbmb5QDlwt7tPjrHdGGAMQOfOneuobBFpyJ7+9Gl+PuPnnHDkCbRJbxM8MtqQk5lDdmY2R2QdwZHNj+So5kfROr01SUk6F+dgJTIgYv3c3NfNJy4HXnD3iqh5nd19rZkdA7xpZp+6+/I9Xsx9IjARIC8vr0He2KJ58+YUFRXFte4TTzzBNddcw/z58+nduzcAJ5xwAq+88gq1XSX+wAMPMGbMGDIzM/datnLlSo4//niOPfZYAE4//XQeeeSRA/8gIiFbs3UN/R/vz3+2Btd1rdq2Kq7tDCPJkkhJSiE1OZVmyc1IT0knIzWDrNQsmqc1p0VaC1qlt6oKm3YZ7cjJyiEnM4cjmx9Jh+YdOCLrCJKTkhP5ERucRAZEAdApajoXWLuPdS8Hro+e4e5rI39XmNlbBP0Ty/fetHHJzc1lwoQJPPfcc3Fv88ADD3DllVfGDAiAbt26MX/+/LoqUaReVVRUcPmLl/PCohcAyEzN5LlLn+PkDiezbsc61hWtY8OODWws3sjG4o1sLtnMlp1b2FK6he07t1O0q4gdZTsoKS+htLyUneU7KSoronx3ORV7/CaNT1XYJKWSnpLOxcdeTMcWHTmqxVF0bNGRji2D50dmHXnYB0oiA2IO0MPMugJrCELgipormdmxQBtgdtS8NkCxu+80s2ygP3BvAmutVxs2bODaa69l1argF9ADDzxA//79ARg2bBjvvPMOS5YsqfrVX2natGnceeed7Ny5k27duvH444/z2GOPsXbtWs4991yys7OZOXNmvX8ekUT56/y/8v2Xv0/Z7jIM49q8a/nfi/63avlRLY865PfYvXs3hSWFrCsKwmb9jvVsKN5AYXEhm0o2saU0CJttO7dRtKuI4rJiisuLKd1VStnuMqYvn866onV7hU2yJdO+efsgOFp2jBkiHVt0pGWzlg22fy9hAeHu5WZ2A/A6wWmuj7n7QjMbB+S7+5TIqqOAZ33Pe58eDzxqZrsJOtLvjj776aDcfDPU9a/oPn3ggQcOeLObbrqJsWPHctZZZ7Fq1SouuOACFi9eDEBSUhI//elP+fWvf82TTz5Ztc3GjRsZP348M2bMICsri3vuuYf777+fO+64g/vvv5+ZM2eSnZ0d8/2++OIL+vbtS8uWLRk/fjxnn332wX3eJmJp4VLue/8+3ln1Dmu2raFtRltmXDmDbtndwi6tyVi1dRX9H+tPwbYCAHrl9GLWd2fRKqNVnb9XUlJS0JyUlcOJR554UK9RsbuC9TvWs2b7GtZuX8uabWuqn29fw9LCpby98m02l27ea9us1KyqEKkKkMowiQRLhxYdSEtOO9SPesASeh2Eu08FptaYd0eN6f8XY7v3gYP7L3UYmDFjBosWVefdtm3b2L59e9X0FVdcwYQJE/jiiy+q5n3wwQcsWrSo6kijrKyMM844Y7/v1aFDB1atWkW7du2YO3cu3/jGN1i4cCEtW7asw090eHl96es8kv8I89bNY0PxBnaW72Q3u/e5/raybXR/qDuG0allJyYNn8SgboPqseKmo6Kigsuev4zJS4JzUrJSs3hp5EsN/vtOTkqmQ4sOdGjRodb1incVs3b72pghsmbbGmavns2a7Wsoqyjba9uczJx9hkiX1l34Ws7X6vxzNeorqfdwEL/0E2X37t3Mnj2bjIyMmMtTUlL4yU9+wj333FM1z90ZNGgQzzxT+5nAL730EnfddRcAkyZNIi8vj2bNmgFwyimn0K1bNz7//HPy8vLq6NM0LGVlZTw671GeX/w8SzYuYUvpFnbt3oXv8/yIvaVYCpmpmXRq2YmzOp/FrIJZLFy/EMdZtW0Vg58aDEDb9LbcPehuvn/y9xP1cZqUv8z7C9e9eh27du/CMG487UYeGNJw/t3WhczUTLq37U73tt33uY67U1hSuHeIRD3PX5vP+h3rq7bpd1Q/Pvr+R3Veb9MJiAZk8ODB/OlPf+LWW28FYP78+fTps+c1gaNHj+bee++tOrI4/fTTuf7661m2bBndu3enuLiYgoICevbsSYsWLdi+fTvZ2dmMGDGCESNGVL3Ohg0baNu2LcnJyaxYsYKlS5dyzDHH1N+HrWPritbxu/d/x/Tl01m1dRVFu4LOxgORmpRKy2Yt6damG8N7DuemM26ieVrz/W63tXgrl75wKW//523Kd5ezqXQTY14ew5iXx5CVmsW1p1zL7y743cF+tCbri81fcNZjZ7G2KDiHpfcRvZn1vVlx/TdpjMyM7MxssjOz6X1k732uV1ZRxrqidazZtuaAfgAdCAVEghUXF5Obm1s1/eMf/5gHH3yQ66+/nt69e1NeXs7Xv/71vU49TUtL48Ybb+Smm24CICcnhyeeeIJRo0axc+dOAMaPH0/Pnj0ZM2YMQ4cOpUOHDnt1Ur/zzjvccccdpKSkkJyczCOPPELbtm0T/KkPzbad2zjvyfP4dP2n7KrYVWvzT02GkZacRrvMdvTO6c3oPqMZcewI0tIOvf22VWYrZnxnBhCM8/WDV3/AMwufobS8lB27dnDfB/dx3wf3kZaUxsU9L+aZ/3pG41bVoqKigkueu4RXl74KQIu0Fky5fAoDug4It7DDRFpyGp1bdaZzq8RdA2Z79g0fvvLy8rzmDYMWL17M8ccfH1JFjVOivtMfTf0Rk+ZNorSi9oEVk0giIzWDDs07cEbuGdx42o3kdQy/uWzCOxO4d9a9bCvbtsf8ZEum31H9mDJqCjlZMW/a1SQ9/NHD3Pj6jZTvLscwfnLGT/jt4N+GXVaTZGZz3T3mPyIFhByQuvpOX1v6GldPvpr1xetjLs9MzeTbJ3ybcQPH0b55+0N+v/r03KfPcdPrN/HVjq/2mG8Y3dt25++X/Z0+HQ5pmLHD1ucbP+ecJ85h3Y51APRt35dZ351FRlrs/jhJvNoCQk1MUi+27dzGuU+cy/yv5rPb924ySrZkzul8DlNHTa3qVD9cjTxxJCNPHAnAnDVzuOIfV7B883IcZ+mmpfSdGIxJeWTWkTw09CEu7XVpmOXWi4qKCi58+kKmrZgGQMu0lrz67Vc5q/NZIVcmtVFASMLsr9kot2UuL37zRfrl9qvnyupPv479WHrjUgC+3Pol//X8fzFn7RwqvIKvdnzFZS9cBi9Ay2Yt+Xn/n/Ozs38WcsV1748f/pEfT/txVXPSbWfdxm/O+03YZUkc1MQkB6S27/TVJa/y3X9+lw0lsYcEz0zN5LYzb+OOAXfEXN6U7Nq1i2/941tMXTqVst17nvOenpLOqF6jePSiRw/rTu7FGxYz4IkBVc2I/Y7qx9tXv63mpAZGTUySEFtLtzLwyYFNotmorqWmpvLS5S9VTY99bSwT502keFcxpeWlPP7J4zz+yeOkJKUwoMsA/nn5P8lMjT3WVkNTUVHB4KcG8+bKYHDm1umtee3br3FabszBnKUB0xGEHJBZ82Zx/tTzm3SzUaI9/NHD3D7zdjaVbtpjvmFkpWXRuWVnzulyDtf3u55eR/QKqcrY7nv/Pm6bcRsVXkGSJXH72bdz17l3hV2W1KK2IwgNlJ5gZsZVV11VNV1eXk5OTg7Dhg0DgiG+k5KSWLBgQdU6J5xwAitXrgSgS5cubNy4EYAJEybQq1cvevfuTZ8+ffjwww8ZMWIEffr0oXv37rRq1Yo+ffrQp08f3n//fQYMGLDHFdP5+fkMGDCg1npXrlzJ008/XTW9pXQL89fNJ39tPvlr89m+c/se4ZCVmsVd59yF3+n4nc7qsasVDofoulOvo/C2QvxOZ9qV0+jUshOG4ThFZUUs2riIh/Mf5oSHT8DuMpLuSiJrQhbdH+zOlS9eyXur3qv3mhd8tYCce3O4ZfotVHgFZ3Q8g5JflCgcDnNqYkqwrKwsPvvsM0pKSsjIyGD69Ol07Nhxj3XiGeJ79uzZvPLKK8ybN49mzZqxceNGysrKeOmloJnirbfe4ne/+x2vvPLKHtutX7+ef/3rXwwdOjSuelesWMGjjz9KzwE997nOwKMHqtmongzqNohVY6vve7CxaCMPfvQgry1/jWWblrFt5zYqvILi8mKWb17O8s3L+dunfwOCI45myc04IusITu5wMqNPGs1FPS4iJaXu/tmXVZQx6K+DeGfVOwC0SW/D9Kumc8pRp9TZe0h4dARRD4YOHcqrrwZXiz7zzDOMGjVqj+XDhg1j4cKFLFmyZJ+v8eWXX5KdnV21U87Ozuaoo/Y/1PGtt97K+PHj95pfUVHBrbfeSr9+/ejduzePPvooX27/kptuuYm5H87likFX8PTE4EgiLTmN47OPJ++oPI5ufTRvjH5D4RCS7ObZjBs4jo++/xGbbttE+R3l+J3O9tu28/sLfs/Znc8mJzOHlKQUHKe0opRV21YxeclkvvH3b5A6IRW7y2g2vhkdfteBC/7vAv4y7y+Ulx/YcCUA97x3DxkTMnhn1TskWRK/OvdXbLptk8KhEWkyRxA3v3Yz89fV7XDffdr3iWswscsvv5xx48YxbNgwFixYwDXXXMO7775btXxfQ3xHGzx4MOPGjaNnz56cf/75jBw5knPOOWe/733GGWfw0ksvMXPmTFq0aFE1/y9/+QutWrVizpw57Ny5k5NPPZn2J7Xnhv+5gaceeYrnXnyuTsbal/rRPL05N59+MzeffvMe88vLy3nqs6d49rNnWfDVAgpLCimrKAvG8dmxjnUr1jFtxTT+++X/BiAlKYXW6a05rt1xfOO4b/CDk39A8/Q9x0Sa/+V8zvu/89hUEvSRnNXpLN64+o1QhqOWxGoyARGm3r17s3LlSp555hkuvPDCmOvEGuI7WvPmzZk7dy7vvvsuM2fOZOTIkdx9992MHj16v+9/++23M378+D1Gh502bRoLFizg+Reep2RXCTu272D1F6vJSs+iVXorhUMjkZKSwug+oxndZ/Rey175/BUem/cYc9fNZX3RenZW7KR8dzkbizfyXvF7vLf6PW6ZfgsQnJHWolkLslKzWLN9DQDtMtrx5tVv1jqgnBzemkxAhD1s8PDhw7nlllt46623KCws3Gt5rCG+a0pOTmbAgAEMGDCAE088kSeffDKugBg4cCC//OUv+eCDD6rmuTt33H0Hx51+XNW8Lq278NlHnx3YB5PD1rCewxjWc9he899f/T6P5D/C7NWzWVu0lpJdJVR4RdWd1ZJI4jfn/4af9v9pCFVLfWoyARG2a665hlatWnHiiSfy1ltvxVyn5hDf0ZYsWUJSUhI9evQAgiHCjz766Ljf/xe/+AXXXntt1VDfXzv9a0yaOIm7T7mblNQUmm9rTkarjKqhw6XpOrPTmZzZ6cy95i9cv5DnFz3PDXk3kN089t0LpXFRQNST3NzcqqG796XmEN/RioqK+NGPfsSWLVtISUmhe/fuTJw4Me73v/DCC8nJycHdyV+bz8WjLqZgVQHfGfId0pLTyMnJYfLkyfTu3ZuUlBROOukkRo8ezdixYw/4s0rj1OuIXg3uugtJLF0o14Ss3rp6jxFGj2lzDG0zDuzeEPpORRoXDbUhzF83v+rOa4bRt31fkpJ0lrOI7JsCopHbWb6TT9d/WjWdlZrF8Tk6AhCR/Wv0AeHumFnYZYRi5ZaVbCzeWDXdvU13Wme0PujXayzNkSISn0YdEOnp6RQWFtKuXbsmFxIff/kxFV4BBE1Kh3p1q7tTWFhIenp6XZQnIoeBRh0Qubm5FBQUsGFD7PsTNEa7KnaxdvvaqulmKc1o37w9i7cuPuTXTk9PJzc395BfR0QOD406IFJTU+natWvYZdSb0S+N5skF1UN1zLhyBud2OzfEikTkcNaoA6IpafmblmwvCy5wa5bcjNLbY9+vQUQkXgk9z9HMhpjZEjNbZmZ73WzXzH5vZvMjj8/NbEvUsqvNbGnkcXUi6zyczV83H7vLqsJh8DGDFQ4iUicSdgRhZsnAQ8AgoACYY2ZT3H1R5TruPjZq/R8BfSPP2wJ3AnmAA3Mj225OVL2Ho2/+/Zu8sPiFqul3r36Xs7qcFWJFItKYJLKJ6VRgmbuvADCzZ4FLgEX7WH8UQSgAXABMd/dNkW2nA0OAZxJY72Ela0IWxeXFAGSkZFD8i+KQKxKRxiaRTUwdgdVR0wWReXsxs6OBrsCbB7KtmY0xs3wzy28qZyp9tPoj7C6rCofhxw5XOIhIQiTyCCLWhQf7utLqcuAF98iJ+3Fu6+4TgYkQjMV0MEUeToY9PYxXl75aNf3xDz6mT/s+IVYkIo1ZIgOiAOgUNZ0LrN3HupcD19fYdkCNbd+qw9oOO5kTMikpLwGC4TKK/qco5IpEpLFLZBPTHKCHmXU1szSCEJhScyUzOxZoA8yOmv06MNjM2phZG2BwZF6T897K97C7rCocLu91ucJBROpFwo4g3L3czG4g2LEnA4+5+0IzGwfku3tlWIwCnvWogX7cfZOZ/YogZADGVXZYNyWD/jqIGV/MqJpe/IPFHNf+uFq2EBGpO436fhCHs2bjm1FWUQZAy7SWbP351pArEpHGqLb7QeiGAA3Mv5b+C7vLqsLhmj7XKBxEJBQaaqMBOeuxs5i1elbV9Bc3fUGX1l3CK0hEmjQFRMiWbVrGyY+eXDVUBkDb9LYU3lYYYlUiIgqIUJSUlHDMQ8ewbse6vZaN6jWKpy97OoSqRET2pICoRyc+dCKfbfxsr/lpSWlMHjmZoT2HhlCViEhsCogEG/J/Q3h9xd6XcCSTzP2D7+fGM24MoSoRkf1TQCTA9a9cz8NzH8ZrjA5iGNedch0PDXsopMpEROKngKgjf5j9B3487cfsZvdey4Z2G8rUK6eGUJWIyMFTQByCqUumMuLvIyjbXbbXshOzT2TB9QtCqEpEpG4oIA7Qsk3L6PtIX4p27T0eUofmHVj+w+VkZGSEUJmISN1SQMShpKSErg915asdX+21rEVaC+b9YB7d23YPoTIRkcRRQNSi1596sahw7xvgpSWlMfWKqZzX7bwQqhIRqR8KiBrO/+v5vPHFG3vNT7Zk/jjkj1x36nUhVCUiUv8UEMAPXv4BE+dN3Gu+YYw9bSz3DbkvhKpERMLV5APi3CfO5a3/vLXHvIt7XMyUK/a6t5GISJPS5ANi6sipZN6bSZ8j+/DxtR+HXY6ISIPR5AMiIyMDv7Nx3DRJRKQu6YZBIiIS034DwsyeNbMLzMzqoyAREWkY4jmCeAK4BvjczMabma4IExFpAvYbEO7+mruPBE4F1gEzzewdM7vKzJp8H4aISGMVVx+EmbUBrgCuAhYAjwJnAq8lrjQREQnTfo8AzOzvwInA08Cl7l4QWfQ3M9N5oSIijVQ8TUSTgOnuvte5oO7et+5LEhGRhiCeJqZjgFaVE2bWxszGJK4kERFpCOIJiGvdfUvlhLtvBjRinYhIIxdPQCRHT5hZEpAaz4ub2RAzW2Jmy8zsZ/tY51tmtsjMFprZ01HzK8xsfuShgZFEROpZPH0Q083sGeARwAmOHmbsbyMzSwYeAgYBBcAcM5vi7oui1ukB/Bzo7+6bzeyIqJcocfc+8X8UERGpS/EExK3AD4GxgAHTCE5z3Z9TgWXuvgKCK7KBS4DoO/B8H3go0myFu6+Pv3QREUmk/QaEu1cAf4w8DkRHYHXUdAFwWo11egKY2SyCpqz/5+6V11akm1k+UA7c7e6TD/D9RUTkEMRzHUQ3YALwNSC9cr6799zfpjHm1TxVNgXoAQwAcoF3zeyESKd4Z3dfa2bHAG+a2afuvrxGbWOAMQCdO3fe30cREZEDEO9YTI8T7PCHAn8Hno1juwKgU9R0LrA2xjr/dPdd7v4FsIQgMHD3tZG/K4C3gL2uuXD3ie6e5+55OTk5cZQkIiLxiicgMt39dQB3X+7utwPnxrHdHKCHmXU1szTgcqDm2UiTK1/LzLIJmpxWRK61aBY1vz979l2IiEiCxdNJvTMy1PdyM7sWWAMcsZ9tcPdyM7sBeJ2gf+Exd19oZuOAfHefElk22MwWARXAre5eaGZnAo+a2W6CELs7+uwnERFJPIsxgsaeK5idRvDrvQ1BX0RL4F53n5X48uKXl5fn+fn5YZchInJYMbO57p4Xa1mtRxCRaxlGuPuHwHaC0VxFRKQJqLUPInKK66n1VIuIiDQg8fRBzDOzF4HngR2VMyN9CCIi0kjFExBHEgTDhVHznL3PSBIRkUYkniup1e8gItIExXMl9cRY891d94QQEWnE4mlieiPqeTowgj3HWBIRkUYoniam56Knzez/gOkJq0hERBqEeIbaqKkrcHRdFyIiIg1LPH0Qm6kehTUJ2ATEvDuciIg0HvH0QWRHPd/t+xubQ0REGoV4mpguApq7e4W7u5m1NrNhiS5MRETCFU9AjHP3rZUTkZv5/CpxJYmISEMQT0DEWieepikRETmMxRMQ88zsXjM72sw6m9lvgY8TXZiIiIQrnoC4IbLePwnGX3Lgh4ksSkREwhfPhXJFwC31UIuIiDQg+z2CMLPXzKx11HQbM3s1sWWJiEjY4mliOjJy5hIA7r4ZOCpxJYmISEMQT0DsNrPcygkz65zAekREpIGI53TVO4BZZvZmZPpc4LrElSQiIg1BPJ3Ur5rZqcAZgAG3ufv6hFcmIiKhinc011JgFfAV0N3MzkxcSSIi0hDEM5rrNcBPgI7Ap0A/4ANgQEIrExGRUMVzBDEWyANWuvvZwCnAlwmtSkREQhdPQJS6ewmAmaW5+0LguMSWJSIiYYvnLKYvIxfKvQy8bmabCPoiRESkEdvvEYS7D3f3Le7+S2A88Dfgknhe3MyGmNkSM1tmZjHvQmdm3zKzRWa20Myejpp/tZktjTyujvPziIhIHTmgYbvd/Y141zWzZOAhYBBQAMwxsynuvihqnR7Az4H+7r7ZzI6IzG8L3EnQ9+HA3Mi2mw+kXhEROXjxnuZ6ME4Flrn7CncvA55l7yOP7wMPVe74o66vuACY7u6bIsumA0MSWKuIiNSQyIDoCKyOmi6IzIvWE+hpZrPM7AMzG3IA22JmY8ws38zyN2zYUIeli4hIIgPCYszzGtMpQA+CaypGAZMiHeLxbIu7T3T3PHfPy8nJOcRyRUQkWjzDfW82s001Hl+Y2fNm1qWWTQuATlHTucDaGOv80913ufsXwBKCwIhnWxERSaB4jiD+CPwS6AZ0B24HngAmA4/Xst0coIeZdTWzNOBygjvSRZtMMPgfZpZN0OS0AngdGBy590QbYHBknoiI1JN4zmIa7O6nR03/r5l94O6nm9lP97WRu5eb2Q0EO/Zk4DF3X2hm44B8d59CdRAsAiqAW929EMDMfkUQMgDj3H3TgX88ERE5WHGd5mpm/+XuL1Y+p7qPYHdt27n7VGBqjXl3RD134MeRR81tHwMei6c+ERGpe/E0MV0JfD/S91BIcGrqVWaWCdyc0OpERCQ08dwPYhkwdB+L367bckREpKGIZ7jvbOAaoEv0+u4+JnFliYhI2OLpg/gnwf0f3iPoSBYRkSYgnoDIcvefJLwSERFpUOLppP6XmQ1OeCUiItKgxBMQ1wKvmVlR5EymzZF7QoiISCMWTxNTdsKrEBGRBmefAWFmPdx9KdBrH6ssSExJIiLSENR2BPEz4HsEN/2pyYGvJ6QiERFpEPYZEO7+vcjTge6+K3qZmaUmtCoREQldPJ3UH8Y5T0REGpHa+iCOADoAGWZ2ItUD9LUEMuuhNhERCVFtfRAXEQyxkUvQD1EZENsJ7g8hIiKNWG19EI8Dj5vZt9z97/VYk4iINADx9EEcYWYtAczsETP7yMzOS3BdIiISsngCYoy7b4sMt5ELXAfcm9iyREQkbPEEhEf+DgUed/e5cW4nIiKHsXh29J+Y2VTgYoKB+5pTHRoiItJIxTMW03eBU4Bl7l4cuYHQ9/azjYiIHOb2ewTh7hXAMQR9DwAZ8WwnIiKHt/3u6M3sT8C5wJWRWTuARxJZlIiIhC+eJqYz3f1kM/sYwN03mVlagusSEZGQxdNUtMvMkoh0TJtZO2B3QqsSEZHQ7TMgzKzy6OIh4B9AjpndBbwH3FMPtYmISIhqa2L6CDjZ3f9qZnOB8wnGY/qmu39WL9WJiEhoaguIysH5cPeFwMIDfXEzGwL8AUgGJrn73TWWjwZ+C6yJzPqTu0+KLKsAPo3MX+Xuww/0/UVE5ODVFhA5ZvbjfS109/tre2EzSyZonhoEFABzzGyKuy+qsepz7n5DjJcocfc+tb2HiIgkTm0BkQw0J+pI4gCdSnBx3QoAM3sWuASoGRAiItIA1RYQX7r7uEN47Y7A6qjpAuC0GOtdamZfBz4Hxrp75TbpZpYPlAN3u/vkmhua2RhgDEDnzp0PoVQRkf0oK4PPPoNPPoFFiw4bkysAAA7sSURBVGD5cigpgV//Gk46CZIa3/XDcfVBHKRY29ccw+ll4Bl332lm1wJPAgMjyzq7+1ozOwZ408w+dffle7yY+0RgIkBeXp7GhxKRg1NaCp9+Guz8Fy8Odv4FBbB+PWzZAsXFUFERe9vXXoPsbBg4EM47D84/H445pn7rT5DaAuJQ7/lQAHSKms4F1kav4O6FUZN/Jur0WXdfG/m7wszeAvoCewSEiMh+lZbCxx/DggXw73/DihWwenWw89+6NTgK2NfOPzUVsrLg6KOhfXvo3Bm6d4cTToA+fYJlM2fCG2/AjBnw98i91bp2rQ6LgQMhJ6f+Pm8dMvfE/PCOXEfxOUHQrAHmAFdEzoiqXKeDu38ZeT4CuM3dTzezNkBx5MgiG5gNXBKjg7tKXl6e5+fnJ+SziEgDVVQE8+fvufNfs2bPnf/ufVzXm5YW7ODbtave+ffoEez8+/YNdvIH0mzkDkuWBEHxxhtBcGzdGiw76aQgLM4/H84+O3jfBsLM5rp7XsxliQqIyBtfCDxA0OH9mLtPMLNxQL67TzGz3wDDCfoZNgHXufu/zexM4FGCK7aTgAfc/S+1vZcCQqSRqKiAzz+H2bODHX9lc8/GjbB9e7DTLy8P1q1t59+8ebDz79AhOALo3h169YK8POjUKfF9BuXlMHdu9dHFrFlBP0ZqKpxxRhAW550H/foF80ISWkDUJwWESAO3fHmwk1ywAJYtC5p5Nm6EbduCnf6uXfve4Uczg5QUyM0NdvxHH139y/+UU4L5DVFxcfD5Z8wIHh9/HBx1tGgBAwZUN0l97WvBZ6wntQVEPIP1iYjEtmpV8Et/3rxgp79qVbDT37o12CEeyE4/ORkyMoJf/m3aBL/8u3at/tXfr1+w/HCVmQmDBgUPgMLCoBmqsknq5ZeD+e3bV4fFeecFRzsh0RGEiMRWWgrPPQeTJwe/+r/6KpjnfmA7/crmnjZtgp1fly7Br+S8PDj11GCZwMqVQVBUNklt2BDM79mzOizOPTf4HuuQmphEJLbSUnjqKfjnP4Nz/DdsqL1jF4KdfrNmQUdr69bBTv/oo4Od/sknw+mnQ6tW9fcZGqPdu4P/HpVh8fbbsGNH0G9yyinVRxj9+0N6+iG9lQJCpCkrKoInn4RXX4WFC4MmoNLS2kMgKSkIgZycYMd/0UUwerR+7YelrAw++qi6/+KDD4LO/PT0ICSGDYObbz6ol1ZAiDR2W7bAE0/A1KnBhV6FhdXNQfuSlBS06efkBB28F18MV111eLfzNxXbt8M771QHxlFHweuvH9RLqZNaZF/WrYOlS2Ht2uCsmnXrgl/YmzcHHa1FRUGTSqtWwamIaWnBGTRpacEv7JSUYIeanh4sz8gIHs2aBX+zsqqbY1q0CNZr0SJ4vczM4BGvjRth0iSYNi04337TJti5M74QOPJI6N0bvvENGDnykJslJGQtWgRHdRddFEyXlSXkbRQQsqcHH4QJE4IdSGUThPveO6HK6cq/NZsropfXXLfm81ivV/maSUnBoXTlY/fu6teMVVdTVHn2T/v2wdW9l14Kl10WhJc0DWmJuQu0/g9q6kpLg86u999vOjvbynPMzYIAqvybnBw8UlKqjxbS06uPDipP2ayoCC6Cqny+e3f188ozfCoflSG2r2CLDuHKvzXDsvJvenpwyuPJJ8M3vwnDhysEJKH0f1dT9N57cMklQRNFLO3aVZ+FUrkzrbzqtHKHuq9ltc2P3rbyefQjen5lJ2mXLtVDIXTqFAyCdswx6iwVqQcKiKbihz+EP/+5eoiCaElJQefkE0/Ue1ki0nApIBqrLVvgtNOCMW1iadUK/vGP4HxqEZEYGt8dLpqyp58OzpYxC662jA4Hs+DK1ZKSoE17yxaFg4jUSgFxuLvwwup2+29/Oxj/plJaGvzmN9WdpHPm6PRGEYmbmpgON//+dzDy41dfxV7eoUMwYmTXrvValog0PjqCOByMGxec0WMGxx+/ZzgkJQWnO1aeHrl2rcJBROqEjiAaotLS4IYin3wS+9qErKzgjKPLLqv30kSk6VBANBQlJcGNz6P7ECqZwXHHBReztW5d/7WJSJOkJqaGoKQkGJMnOhxSUmDs2OoO5kWLFA4iUq90BBG2ynCo9MEHwfULIiIhU0CEqWY4FBdrqGURaTDUxBQWhYOINHAKiLBEh0NhocJBRBocBUQYKkcuhSAc2rYNrxYRkX1QQNQ3hYOIHCYUEPUpOhwKChQOItKgKSDqS3Q4LF0KHTuGV4uISBwSGhBmNsTMlpjZMjP7WYzlo81sg5nNjzz+O2rZ1Wa2NPK4OpF1JlxS1Ne8YAF07x5eLSIicUrYdRBmlgw8BAwCCoA5ZjbF3RfVWPU5d7+hxrZtgTuBPMCBuZFtNyeq3oRJSqoeT2nBAjjxxHDrERGJUyKPIE4Flrn7CncvA54FLolz2wuA6e6+KRIK04EhCaozcZKTq8Pho48UDiJyWElkQHQEVkdNF0Tm1XSpmS0wsxfMrNOBbGtmY8ws38zyN2zYUFd1142UlGAMJYB334V+/cKtR0TkACUyICzGvJpjV78MdHH33sAM4MkD2BZ3n+juee6el5OTc0jF1qmUFKioCJ6/+iqcdVa49YiIHIREBkQB0ClqOhdYG72Cuxe6+87I5J+BU+LdtsFKTa0OhxdfDG4JKiJyGEpkQMwBephZVzNLAy4HpkSvYGYdoiaHA4sjz18HBptZGzNrAwyOzGvYmjWD8vLg+bPPwogR4dYjInIIEnYWk7uXm9kNBDv2ZOAxd19oZuOAfHefAtxoZsOBcmATMDqy7SYz+xVByACMc/dNiaq1TqSnQ1lZ8Pyvf4WRI8OtR0TkEJnHuqXlYSgvL8/z8/PDefPMzGB0VoBJk+B73wunDhGRA2Rmc909L9YyXUl9qLKyqsPhgQcUDiLSaCggDkXLltW3Cf3tb+Gmm8KtR0SkDikgDlarVrB9e/D8rrvgllvCrUdEpI4pIA5Gu3awbVvw/Kc/hTvuCLceEZEEUEAcqCOOgE2RE6puvBHuuSfcekREEkQBcSA6dIDKIT3GjIE//CHcekREEkgBEa9OnWDduuD5d74Djz4abj0iIgmmgIhH167BHeAguADuySdrX19EpBFQQOxPjx6wcmXwfMSIYAgNEZEmQAFRm+OPh2XLgudDhwaD74mINBEKiH3p0wf+/e/g+YABMHVqqOWIiNQ3BUQseXnwySfB8/79YebMcOsREQmBAqKmM8+EuXOD56edBu+9F249IiIhUUBEO+ccmD07eN6nD3zwQbj1iIiESAFRacgQeOed4PkJJ8DHH4dbj4hIyBQQAMOHw+uRG9Ydeyx8+mm49YiINAAKiG9/G15+OXjetWv1mUsiIk2cAmLgwOBv586wYkW4tYiINCAKiO99D9zhP/8JuxIRkQZFASEiIjEpIEREJCYFhIiIxKSAEBGRmBQQIiISkwJCRERiUkCIiEhMCggREYnJ3D3sGuqEmW0ADuVqt2xgYx2Vc7jTd7EnfR970vdRrTF8F0e7e06sBY0mIA6VmeW7e17YdTQE+i72pO9jT/o+qjX270JNTCIiEpMCQkREYlJAVJsYdgENiL6LPen72JO+j2qN+rtQH4SIiMSkIwgREYlJASEiIjE1+YAwsyFmtsTMlpnZz8KuJ0xm1snMZprZYjNbaGY3hV1T2Mws2cw+NrNXwq4lbGbW2sxeMLN/R/4fOSPsmsJkZmMj/04+M7NnzCw97JrqWpMOCDNLBh4ChgJfA0aZ2dfCrSpU5cBP3P144HTg+ib+fQDcBCwOu4gG4g/Aa+5+HHASTfh7MbOOwI1AnrufACQDl4dbVd1r0gEBnAosc/cV7l4GPAtcEnJNoXH3L919XuT5doIdQMdwqwqPmeUCFwGTwq4lbGbWEvg68BcAdy9z9y3hVhW6FCDDzFKATGBtyPXUuaYeEB2B1VHTBTThHWI0M+sC9AU+DLeSUD0A/BTYHXYhDcAxwAbg8UiT2yQzywq7qLC4+xrgd8Aq4Etgq7tPC7equtfUA8JizGvy5/2aWXPgH8DN7r4t7HrCYGbDgPXuPjfsWhqIFOBk4GF37wvsAJpsn52ZtSFobegKHAVkmdmV4VZV95p6QBQAnaKmc2mEh4kHwsxSCcLhb+7+Ytj1hKg/MNzMVhI0PQ40s6fCLSlUBUCBu1ceUb5AEBhN1fnAF+6+wd13AS8CZ4ZcU51r6gExB+hhZl3NLI2gk2lKyDWFxsyMoI15sbvfH3Y9YXL3n7t7rrt3Ifj/4k13b3S/EOPl7uuA1WZ2bGTWecCiEEsK2yrgdDPLjPy7OY9G2GmfEnYBYXL3cjO7AXid4CyEx9x9Ychlhak/cBXwqZnNj8z7H3efGmJN0nD8CPhb5MfUCuC7IdcTGnf/0MxeAOYRnP33MY1w2A0NtSEiIjE19SYmERHZBwWEiIjEpIAQEZGYFBAiIhKTAkJERGJSQIjsh5lVmNn8qEedXUFsZl3M7LO6ej2RutSkr4MQiVOJu/cJuwiR+qYjCJGDZGYrzeweM/so8ugemX+0mb1hZgsifztH5h9pZi+Z2SeRR+XQDMlm9ufIvQWmmVlGZP0bzWxR5HWeDeljShOmgBDZv4waTUwjo5Ztc/dTgT8RjP5K5Plf3b038Dfgwcj8B4G33f0kgnGMKq/a7wE85O69gC3ApZH5PwP6Rl7n2kR9OJF90ZXUIvthZkXu3jzG/JXAQHdfERnkcJ27tzOzjUAHd98Vmf+lu2eb2QYg1913Rr1GF2C6u/eITN8GpLr7eDN7DSgCJgOT3b0owR9VZA86ghA5NL6P5/taJ5adUc8rqO4bvIjgjoenAHMjN6YRqTcKCJFDMzLq7+zI8/epvv3kt4H3Is/fAK6Dqntdt9zXi5pZEtDJ3WcS3LSoNbDXUYxIIukXicj+ZUSNbgvBfZkrT3VtZmYfEvzYGhWZdyPwmJndSnAXtspRT28CJprZ9wiOFK4juBtZLMnAU2bWiuDGVr/XLT6lvqkPQuQgRfog8tx9Y9i1iCSCmphERCQmHUGIiEhMOoIQEZGYFBAiIhKTAkJERGJSQIiISEwKCBERien/A1dn9Bo/kqlzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "iterations = 10\n",
    "trainLoss1 = [] # list for saving training loss of LeNet\n",
    "testAcc1 = [] # list for saving test accuracy of LeNet\n",
    "trainLoss2 = [] # list for saving training loss of MNISTNet\n",
    "testAcc2 = [] # list for saving test accuracy of MNISTNet\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss1 = 0 # accumulated loss for LeNet   \n",
    "    runningLoss2 = 0 # accumulated loss for MNISTNet      \n",
    "    net1.train(True) # For training\n",
    "    net2.train(True) \n",
    "    for data in trainLoader:\n",
    "        inputs,labels = data\n",
    "        # Wrap them in Variable\n",
    "        if use_gpu:\n",
    "            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n",
    "        else:\n",
    "            inputs, labels = Variable(inputs), Variable(labels)  \n",
    "       \n",
    "        # Initialize gradients to zero\n",
    "        optimizer1.zero_grad()\n",
    "        optimizer2.zero_grad()\n",
    "        # Feed-forward input data through the network        \n",
    "        outputs1 = net1(inputs)        \n",
    "        outputs2 = net2(inputs)        \n",
    "        # Compute loss/error\n",
    "        loss1 = criterion(outputs1, labels)\n",
    "        loss2 = criterion(outputs2, labels)\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss1.backward()\n",
    "        loss2.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer1.step()\n",
    "        optimizer2.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss1 += loss1.data.item()\n",
    "        runningLoss2 += loss2.data.item()\n",
    "    avgTrainLoss1 = runningLoss1/60000.0\n",
    "    avgTrainLoss2 = runningLoss2/60000.0\n",
    "    trainLoss1.append(avgTrainLoss1)\n",
    "    trainLoss2.append(avgTrainLoss2)\n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    with torch.no_grad(): # for evaluation\n",
    "        running_correct1 = 0\n",
    "        running_correct2 = 0\n",
    "        for data in testLoaderRot:\n",
    "            inputs,labels = data\n",
    "            # Wrap them in Variable\n",
    "            if use_gpu:\n",
    "                inputs = Variable(inputs.cuda())                \n",
    "                outputs1 = net1(inputs)\n",
    "                outputs2 = net2(inputs)\n",
    "                _, predicted1 = torch.max(outputs1.data, 1)\n",
    "                _, predicted2 = torch.max(outputs2.data, 1)\n",
    "                predicted1 = predicted1.cpu()\n",
    "                predicted2 = predicted2.cpu()\n",
    "            else:\n",
    "                inputs = Variable(inputs)\n",
    "                outputs1 = net1(inputs)\n",
    "                outputs2 = net2(inputs)\n",
    "                _, predicted1 = torch.max(outputs1.data, 1)\n",
    "                _, predicted2 = torch.max(outputs1.data, 1)\n",
    "            running_correct1 += (predicted1 == labels).sum()\n",
    "            running_correct2 += (predicted2 == labels).sum()\n",
    "        avgTestAcc1 = float(running_correct1)/10000.0\n",
    "        avgTestAcc2 = float(running_correct2)/10000.0\n",
    "        testAcc1.append(avgTestAcc1)\n",
    "        testAcc2.append(avgTestAcc2)\n",
    "        \n",
    "    # Plotting training loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss1,'r-',label='LeNet-5')        \n",
    "    plt.plot(range(epoch+1),trainLoss2,'g-',label='MNISTNet')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),testAcc1,'r-',label='LeNet-5')        \n",
    "    plt.plot(range(epoch+1),testAcc2,'g-',label='MNISTNet')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Testing accuracy')    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f}  ; LeNet:  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss1,avgTestAcc1*100,epochEnd//60,epochEnd%60))\n",
    "    print('Iteration: {:.0f} /{:.0f}  ; MNISTNet:  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss2,avgTestAcc2*100,epochEnd//60,epochEnd%60))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train and validate the networks on MNIST with random rotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU is available!\n"
     ]
    }
   ],
   "source": [
    "net1 = LeNet()\n",
    "net2 = MnistNet()\n",
    "# Check availability of GPU\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')   \n",
    "    net1 = net1.cuda()\n",
    "    net2 = net2.cuda()\n",
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer1 = optim.SGD(net1.parameters(), lr=1e-2, momentum=0.9) # SGD with momentum\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr=1e-2, momentum=0.9) # SGD with momentum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training and validation routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1 /10  ; LeNet:  Training Loss: 0.007286 ; Testing Acc: 68.880 ; Time consumed: 9m 34s \n",
      "Iteration: 1 /10  ; MNISTNet:  Training Loss: 0.005218 ; Testing Acc: 86.580 ; Time consumed: 9m 34s \n",
      "Iteration: 2 /10  ; LeNet:  Training Loss: 0.002363 ; Testing Acc: 89.010 ; Time consumed: 9m 36s \n",
      "Iteration: 2 /10  ; MNISTNet:  Training Loss: 0.001340 ; Testing Acc: 91.720 ; Time consumed: 9m 36s \n",
      "Iteration: 3 /10  ; LeNet:  Training Loss: 0.001216 ; Testing Acc: 92.450 ; Time consumed: 9m 33s \n",
      "Iteration: 3 /10  ; MNISTNet:  Training Loss: 0.000943 ; Testing Acc: 94.210 ; Time consumed: 9m 33s \n",
      "Iteration: 4 /10  ; LeNet:  Training Loss: 0.000892 ; Testing Acc: 93.470 ; Time consumed: 9m 33s \n",
      "Iteration: 4 /10  ; MNISTNet:  Training Loss: 0.000808 ; Testing Acc: 94.480 ; Time consumed: 9m 33s \n",
      "Iteration: 5 /10  ; LeNet:  Training Loss: 0.000758 ; Testing Acc: 94.660 ; Time consumed: 8m 14s \n",
      "Iteration: 5 /10  ; MNISTNet:  Training Loss: 0.000710 ; Testing Acc: 95.270 ; Time consumed: 8m 14s \n",
      "Iteration: 6 /10  ; LeNet:  Training Loss: 0.000652 ; Testing Acc: 95.310 ; Time consumed: 7m 38s \n",
      "Iteration: 6 /10  ; MNISTNet:  Training Loss: 0.000656 ; Testing Acc: 95.380 ; Time consumed: 7m 38s \n",
      "Iteration: 7 /10  ; LeNet:  Training Loss: 0.000585 ; Testing Acc: 95.780 ; Time consumed: 7m 38s \n",
      "Iteration: 7 /10  ; MNISTNet:  Training Loss: 0.000600 ; Testing Acc: 96.230 ; Time consumed: 7m 38s \n",
      "Iteration: 8 /10  ; LeNet:  Training Loss: 0.000543 ; Testing Acc: 96.420 ; Time consumed: 7m 39s \n",
      "Iteration: 8 /10  ; MNISTNet:  Training Loss: 0.000557 ; Testing Acc: 96.320 ; Time consumed: 7m 39s \n",
      "Iteration: 9 /10  ; LeNet:  Training Loss: 0.000509 ; Testing Acc: 96.240 ; Time consumed: 7m 37s \n",
      "Iteration: 9 /10  ; MNISTNet:  Training Loss: 0.000527 ; Testing Acc: 96.500 ; Time consumed: 7m 37s \n",
      "Iteration: 10 /10  ; LeNet:  Training Loss: 0.000482 ; Testing Acc: 96.460 ; Time consumed: 7m 35s \n",
      "Iteration: 10 /10  ; MNISTNet:  Training Loss: 0.000504 ; Testing Acc: 96.540 ; Time consumed: 7m 35s \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZwV1Zn/8c/T9/bCzgCtIq2CggsaaLVFjBoXIosSiaOO6MTRiKNx3JKoGZ2ZaPSniRqjRqMxDBg0MaIxMeHlLu4aRRtEIyDSAyitGBqQtffu5/dHVUPTdN++t7nVt5fv+/WqV9WtOqfqudekH6pOnXPM3REREUlWVqYDEBGRzkWJQ0REUqLEISIiKVHiEBGRlChxiIhISuKZDqA9DBo0yIcOHZrpMEREOo358+evdff85o51i8QxdOhQiouLMx2GiEinYWaftnRMj6pERCQlShwiIpISJQ4REUlJt2jjaE5NTQ2lpaVUVlZmOpROLy8vj4KCArKzszMdioi0g26bOEpLS+nTpw9Dhw7FzDIdTqfl7qxbt47S0lKGDRuW6XBEpB1020dVlZWVDBw4UEljF5kZAwcO1J2bSDfSbRMHoKSRJvodRbqXbvuoqlWbN8PSpZCVBYcdluloREQ6jG59x5FQr17Bur4+skv07t076bKzZs0iKyuLDz/8cNu+Qw45hJUrVyasd/fdd1NeXt7ssZUrV9KjRw8KCwspLCzke9/7XtLxiEj3pcTRkqyO99MUFBRwyy23pFQnUeIA2G+//Vi4cCELFy7kgQce2NUQRaQb6Hh/HTuiFSva7VJlZWWcfvrpHHHEERxxxBG89dZb245NnjyZRYsWsXTp0p3qvfDCCxx11FEcdthhnHnmmWzZsoV77rmHL774ghNOOIETTjih3b6DiHRtauMA+P73YeHCnfdv3br9UVWfPqmds7AQ7r475VCuvPJKfvCDH3DMMcfw2WefMWHCBJYsWQJAVlYWP/rRj/jpT3/KQw89tK3O2rVrufnmm5k7dy69evXitttu48477+T666/nzjvv5JVXXmHQoEHNXm/FihUceuih9O3bl5tvvpljjz025ZhFpHtR4kikR48gebSjuXPnsnjx4m2fN23axObNm7d9Puecc7jllltY0egu6J133mHx4sUcffTRAFRXV3PUUUe1eq3Bgwfz2WefMXDgQObPn8+3v/1tFi1aRN++fdP4jUSkq4k0cZjZROCXQAyY4e63NjmeCzwMHA6sA85y95XhseuAaUAdcIW7P29mBwCPNTrFvsD17p76P+0bS3Rn0DCqblHRLl0iWfX19bz99tv06NGj2ePxeJyrrrqK2267bds+d+ekk07i0UcfTXjuJ598khtvvBGAGTNmUFRURG5uLgCHH344++23H5988glF7fRdRaRziqyNw8xiwH3AJGAkcLaZjWxSbBrwlbsPB+4CbgvrjgSmAgcDE4H7zSzm7kvdvdDdCwmSTTnwZFTfYQf/+Ee7XGb8+PH86le/2vZ5YTOP0M4//3zmzp1LWVkZAGPHjuWtt96ipKQEgPLycj755BMA+vTps+2O5bTTTtvWEF5UVERZWRl1dXUALF++nGXLlrHvvvtG+v1EpPOLsnF8DFDi7svdvRqYDUxpUmYK0PCw/glgnAW9yaYAs929yt1XACXh+RobB/yfu7c4ZnxaNLxd9fnnaT91eXk5BQUF25Y777yTe+65h+LiYkaNGsXIkSObfdMpJyeHK664gjVr1gCQn5/PrFmzOPvssxk1ahRjx47l448/BuCiiy5i0qRJzTaOv/7664waNYrRo0dzxhln8MADDzBgwIC0f08R6VrM3aM5sdkZwER3vzD8fC5wpLtf1qjMR2GZ0vDz/wFHAj8B3nH334f7ZwLPuvsTjeo+CCxw9+3/PG9BUVGRN53IacmSJRx00EGtf5F167a/VaVHOC1K+vcUkU7BzOa7e7N/9KK842huHIqmWaqlMgnrmlkOcCrwxxYvbnaRmRWbWXHDI502GTiw7XVFRLqgKBNHKbBXo88FwBctlTGzONAPWJ9E3UkEdxstNjy4+3R3L3L3ovz8ZqfNTV2jt5tERLqrKBPHe8AIMxsW3iFMBeY0KTMHOC/cPgN42YNnZ3OAqWaWa2bDgBHAu43qnQ0kfoUonRoG8Vu+vN0uKSLSUUX2Oq6715rZZcDzBK/jPujui8zsJqDY3ecAM4HfmVkJwZ3G1LDuIjN7HFgM1AKXunsdgJn1BE4CLo4q9p3k58OaNVBT026XFBHpqCLtx+HuzwDPNNl3faPtSuDMFureAuw0MJO7lwPt2/Cw995B4hAREY1VlbLa2kxHICKSUUocqWpmgMG2MjPOPffcbZ9ra2vJz89n8uTJQOtDqQ8dOpS1a9cCcMstt3DwwQczatQoCgsLmTdvHqeddhqFhYUMHz6cfv36bRs+/W9/+xvHH3/8Dj3Ei4uLOf744xPGu3LlSv7whz+k6duLSGelxJGshkEOKyrSdspevXrx0UcfURGe88UXX2TIkCE7lElmKPW3336bp556igULFvDhhx8yd+5c9tprL5588kkWLlzIjBkzOPbYY7f1Gv/6178OwJo1a3j22WeTjleJQ0RAiSN5I0ZEctpJkybx9NNPA/Doo49y9tln73A80VDqDVavXs2gQYO2jTs1aNAg9txzz1avfc0113DzzTfvtL+uro5rrrmGI444glGjRvGb3/wGgGuvvZY33niDwsJC7rrrrqS/o4h0LRodF/j+c99n4ZfNDKveVEM/jo9aH2K9cI9C7p7Y+tiLU6dO5aabbmLy5Ml8+OGHXHDBBbzxxhvbjrc0lHpj48eP56abbmL//ffnm9/8JmeddRbHHXdcq9c+6qijePLJJ3nllVfo02jY+JkzZ9KvXz/ee+89qqqqOProoxk/fjy33nord9xxB0899VSr5xaRrkt3HG2RxsdVo0aNYuXKlTz66KOcfPLJzZY555xzeOedd3YYSr2x3r17M3/+fKZPn05+fj5nnXUWs2bNSur6//M//7PTXccLL7zAww8/TGFhIUceeSTr1q1j2bJlKX0vEem6dMcBSd0ZAPD3v0NVVbCdxnGrTj31VK6++mpeffVV1q1bt9Px5oZSbyoWi3H88cdz/PHH87WvfY2HHnqI888/v9Vrn3jiifz4xz/mnXfe2bbP3bn33nuZMGHCDmVfffXVpL+TiHRduuNIxQEHRHLaCy64gOuvv56vfe1rLZZpOpR6Y0uXLt3hjmDhwoXss88+SV//v//7v7n99tu3fZ4wYQK//vWvqQk7PH7yySds3bp1hyHaRaT7UuJIRU5OJKctKCjgyiuvbOXSOw6l3tiWLVs477zzGDlyJKNGjWLx4sX85Cc/Sfr6J598Mo3H87rwwgsZOXIkhx12GIcccggXX3wxtbW1jBo1ing8zujRo9U4LtKNRTasekeyS8OqN9Vwnj33DBYBNKy6SFeTqWHVu6Z42Cz05ZeZjUNEJEOUOFI1bFiwrq/PbBwiIhnSrRNHmx7T9euX/kA6ue7wuFNEtuu2iSMvL49169bt2h+9jRvTF1An5e6sW7eOvLy8TIciIu2k2/bjKCgooLS0tNnXW1u1bh24w/r1sNderZfv4vLy8igoKMh0GCLSTrpt4sjOzmZYQ3tFqv74R7jhhmBbj2lEpJvpto+qdsn117deRkSki1Li2FWff57pCERE2pUSR1s19OdoZfIjEZGuRomjrb71rWBdUpLZOERE2lmkicPMJprZUjMrMbNrmzmea2aPhcfnmdnQRseuC/cvNbMJjfb3N7MnzOxjM1tiZkdF+R1a9Oc/Z+SyIiKZFlniMLMYcB8wCRgJnG1mI5sUmwZ85e7DgbuA28K6I4GpwMHAROD+8HwAvwSec/cDgdHAkqi+Q9LSOD+HiEhHF+UdxxigxN2Xu3s1MBuY0qTMFKBhWrsngHFmZuH+2e5e5e4rgBJgjJn1Bb4BzARw92p33xDhd0jMLFi3MAGTiEhXFGXiGAKsavS5NNzXbBl3rwU2AgMT1N0XKAN+a2bvm9kMM+sVTfhJOPjgYP366xkLQUSkvUWZOKyZfU17y7VUpqX9ceAw4NfufiiwFdip7QTAzC4ys2IzK25T7/BkvPZasNaAhyLSjUSZOEqBxuNxFABftFTGzOJAP2B9grqlQKm7zwv3P0GQSHbi7tPdvcjdixpPUpRWAwZEc14RkQ4sysTxHjDCzIaZWQ5BY/ecJmXmAOeF22cAL3sw6uAcYGr41tUwYATwrrt/Cawys4Y5XMcBiyP8DslrZQY/EZGuIrLEEbZZXAY8T/Dm0+PuvsjMbjKzU8NiM4GBZlYC/JDwsZO7LwIeJ0gKzwGXuntdWOdy4BEz+xAoBH4a1XdIyh57BOsHHshoGCIi7aXbTh2bNu+9B2PGBNvd4LcUke5BU8dG6YgjMh2BiEi7UuJIp9/9LtMRiIhETokjHXr3DtaXXprZOERE2oESRzrcfnuw3rw5s3GIiLQDJY50uOSSTEcgItJulDjS7e9/z3QEIiKRUuJIl5ycYK0BD0Wki1PiSJfvfCdYl5ZmNg4RkYgpcaTLzJmZjkBEpF0ocURBEzuJSBemxJFOWeHPedxxmY1DRCRCShzpdOSRwTqqcbFERDoAJY50eumlYK3BDkWkC1PiSKcePTIdgYhI5JQ4ovKv/5rpCEREIqHEkW777BOsH388s3GIiEREiaMFb658k6wbs9jt9t1Sqzh3brCurU1/UCIiHYASRwuG9R2G45RVlKVWcfjwaAISEekglDhaMGTAkG3bn6//vG0nueOONEUjItJxKHEkkG3ZABw649DUKvbvH6xvuCHNEYmIZF6kicPMJprZUjMrMbNrmzmea2aPhcfnmdnQRseuC/cvNbMJjfavNLO/m9lCM4u0p93PT/o5QOqPqx54IFiXl6c5IhGRzIsscZhZDLgPmASMBM42s5FNik0DvnL34cBdwG1h3ZHAVOBgYCJwf3i+Bie4e6G7F0UVP8CVR13ZtopnnZXeQEREOpAo7zjGACXuvtzdq4HZwJQmZaYAD4XbTwDjzMzC/bPdvcrdVwAl4fky5oRZJ7St4ptvpjcQEZEMizJxDAFWNfpcGu5rtoy71wIbgYGt1HXgBTObb2YXRRD3DsYNGwfAq5++mlrF3Nxgffrp6Q1IRCTDokwc1sy+poM4tVQmUd2j3f0wgkdgl5rZN5q9uNlFZlZsZsVlZSm2UTQy99/mtq3ipZcG6zVr2nxtEZGOKMrEUQrs1ehzAfBFS2XMLA70A9YnquvuDes1wJO08AjL3ae7e5G7F+Xn5+/ylwH45du/TL7wL36RlmuKiHQ0KSUOC/RKsvh7wAgzG2ZmOQSN3XOalJkDnBdunwG87O4e7p8avnU1DBgBvGtmvcysTxhLL2A88FEq36Et8nsEieeaF69p2wnWr09jNCIimdVq4jCzh82sr5n1BBYBK8zsh63VC9ssLgOeB5YAj7v7IjO7ycxODYvNBAaaWQnwQ+DasO4i4HFgMfAccKm71wG7A2+a2QfAu8DT7v5cal85de9f+D4ANV6TWsVY+CLY0UenOSIRkcwxb2XuCDN7390PNbNzCB4L/QgodvdR7RFgOhQVFXnxLk6uZDcGzS7LLl/G8AFJDisycSI8/zyYQX39Ll1fRKQ9mdn8lro8JPOoKidsf5gC/CV8tbbb/RXMycoBYOyMsclXevLJYK2JnUSkC0kmccwAPgP+CXjNzPYGtkQaVQd094S7AVhXsS75So0ndqqoSHNEIiKZ0WricPe73H1Pdx8fNlyvAk6MPrSO5ZIxl7StooVvFqs/h4h0Eck0jl9mZn3D7d8A84Bjow6sI0upF/n++wfrF16IJhgRkXaWzKOqi9x9k5mNJ+i9fQlwe7RhdUxt6kX+0kvBuq4u/QGJiGRAMomjoWV3EvBbd5+fZL0up029yIc0HWVFRKRzSyYBfGBmzwDfAp41s97sPHRIt5NSL/IG11+f/kBERNpZMonju8BPgDHuXg7kEQyH3i3t1jOYgzylXuQDBwbrn/88gohERNpXMm9V1QGDgB+Z2a3AEe7+fuSRdVALpi0AUuxF/thjwbqyMoKIRETaVzJvVd1C0Ft8ebhcY2Y3Rx1YR9V4LvKS9SXJVRo3LqJoRETaXzKPqr4FfDMcbXY6wcCCp7ZSp0trUy/yBg29yUVEOqlk347q08J2t7RLvcinddvmIRHpIpJJHLcDC8xshpnNBIoJ5wbvrtrUi7zhjaqvvkpvMCIi7SyZxvHfA8cAz4TLN9z9kagD6yyS7kV+7bXRBiIi0k5aTBxmNqphIZgHvARYRjB/RqcZUj0qbZ6LHKAkyUZ1EZEOKJ7g2H0JjjnQ7Fzf3cXcf5u7bY6OpMXjUFsLJ50EK1ZEE5iISMRavONw92MTLN06aTSVdC/yhhFyV66MLBYRkah1yzGn0qWhF/nVL16dXIXZsyOMRkSkfShx7IKGXuS1Xpt6ZU3sJCKdlBLHLmhTL/KGiZ1OOimCiEREopfMkCOjmln2MTMlHdrQi3z06GD99tsRRSQiEq1k/vjPBOYDDwO/I+gA+CSwzMwSDsJkZhPNbKmZlZjZTh0ZzCzXzB4Lj88zs6GNjl0X7l9qZhOa1IuZ2ftm9lQS8Ucq5V7kf/tbsK6vjygiEZFoJZM4lgGHu3uhu48GDgcWAhOAX7RUycxiBK/0TgJGAmeb2cgmxaYBX7n7cOAuwh7pYbmpwMHAROD+8HwNrgSWJBF75FLuRd4w9IiISCeVTOI4yN0/bPjg7n8HDnP31h7qjwFK3H25u1cDs4EpTcpMAR4Kt58AxpmZhftnu3uVu68g6Hw4BsDMCoBTgBlJxN6ujvvtcalVuPjiaAIREYlQMonj/8zsXjM7OlzuAUrMLBdI9DrREGBVo8+l4b5my7h7LbCRoJd6orp3EwzznvBZj5ldZGbFZlZcVlaW8AvuqvH7jgfg9c9eT67C4MHBetasaAISEYlQMonj3wj+cF8LXAd8AZxHkDQStXE016266ZSzLZVpdr+ZTQbWhPOeJxQOA1/k7kX5+fmtFd8lz5/7fGoV/vrXYF1dnf5gREQilswgh+Xufpu7f8vdJ7v7re6+1d3r3H1jgqqlwF6NPhcQJJ1my5hZHOgHrE9Q92jgVDNbSfDo60Qz+31r36E9JdWL/Igjog9ERCQiybyOO9bMnjWzxWb2ScOSxLnfA0aY2TAzyyFo7J7TpMwcgrsXgDOAl93dw/1Tw7euhgEjgHfd/Tp3L3D3oeH5Xnb37yT1TSOWci/yBjNnRhCNiEh0knlU9VvgfuCbwLGNloTCNovLgOcJ3oB63N0XmdlNZtYwg+BMgtF2S4AfEjwOw90XAY8Di4HngEvDuc87rJR7kfcJ58P6wQ8iikhEJBoW/AM/QQGzee5+ZDvFE4mioiIvLi6O/DoNo+Uuu3wZwwcMT1x45ky48MJgu5X/BiIi7c3M5rt7UXPHkrnjeNnMfmZmRzSZo0OaSKkXuaaQFZFOKpnEcUy43EnQoe8+4FdRBtVZtWkucoD33osgGhGRaLT6qKoraK9HVbD9cZXfkMTvmpsbvJI7eDB80fSFMxGRzGnToyozOztcX9HcElWwXUVSvcgb2jhWr442GBGRNEr0qOqfwnV+C4s0Y8K+wXiMSfUivy/R7LwiIh1Ti3OOu/v94frH7RdO5/fcuc+lPhc5wPr1MGBA+gMSEUmzZDoADjKzH5nZ/WY2vWFpj+A6u6R6kWeF/wnGJRyhXkSkw0jmraq/ArsDbwIvNVqkBSn1Ij/qqGD9wQcRRiQikj4tPqpqpJe7XxV5JF3IgmkLKLi3ILle5C++CD17qhOgiHQaydxxPGtm4yOPpAtJaS5yTewkIp1MMonje8BzZrbFzNab2Vdmtj7qwDq7hl7kR85IYbSWM8+MKBoRkfRJJnEMArIJhjzPDz/rddxWNPQiX1+RRI4dOjRY/+Uv0QUkIpImiToAjgg3D25hkQRSmov8xReDdW2SI+uKiGRQosbxa4FpBGNTNeXANyKJqAs67rfH8dp3X2u5wPBWRtIVEelAWrzjcPdp4frYZhYljSSk1Iu8wa23RhSNiEh6JNPGgZkdaGb/bGbnNCxRB9YVPHfuc8kX/qdwhJebboomGBGRNEmm5/j/ANOBB4BJwN0E07xKCu54647EBRqmkK2oiD4YEZFdkMwdx1nACcBqdz8XGE1yHQcF2L3X7gBc99J1iQuedlo7RCMisuuSSRwV4XzftWbWB/gS2DfasLqO+RfMB1KYixzgJY3oIiIdVzKJ430z6w88CBQD7wILIo2qC0mpF3leXrA+66wIIxIR2TUJE4eZGfATd9/g7vcBpwAXu/u/JXNyM5toZkvNrMTMrm3meK6ZPRYen2dmQxsduy7cv9TMJoT78szsXTP7wMwWmdmNKXzXjEm6F/k11wTrdSlOPSsi0o4SJg4P5pV9qtHnEndP6m7DzGIEfUAmASOBs81sZJNi04Cv3H04cBdwW1h3JDCVoKPhROD+8HxVwInuPhooBCaa2dhk4smkpHuR640qEekEknlU9a6ZHdaGc48BStx9ubtXA7OBKU3KTAEeCrefAMaFdzlTgNnuXuXuK4ASYIwHtoTls8Olww8rm1Iv8gaff57+QERE0iDRkCMNb04dQ5A8lprZAjN738ySuesYAqxq9Lk03NdsGXevBTYCAxPVNbOYmS0E1gAvuvu8FuK/yMyKzay4rKwsiXCjZQSzArY6F3ksFqw1sZOIdFCJ7jjeDdffBg4ATgbOJOjDkcwwrs3Nn9r07qClMi3Wdfc6dy8ECoAxZnZIcxd39+nuXuTuRfn5mR+Tcfy+wcj0rfYiHx+OYP/JJxFHJCLSNokShwG4+/81tyRx7lJgr0afC4AvWioT3uH0A9YnU9fdNwCvErSBdHhJ9yL/05+CtSZ2EpEOKlFHvnwz+2FLB939zlbO/R4wwsyGAZ8TNHY3HapkDnAe8DbBnczL7u5mNgf4g5ndCewJjCB4XJYP1Lj7BjPrAXyTsEG9M7njrTu4+ugWppVtPLFTRYUmehKRDifRHUcM6A30aWFJKGyzuAx4HlgCPO7ui8zsJjM7NSw2ExhoZiXADwlG5MXdFwGPA4uB54BLw06Ig4FXzOxDgsT0ors/RSeRdC9yC5/UqTe5iHRA5i08EjGzBe7elrepOpyioiIvLi7OdBh8vv5zCu4tAMBvSPAo6qCD4OOPg4ZyzdEhIhlgZvPdvai5Y622cUj6JN2L/K23gnVdXcQRiYikLlHi0PugEUiqF/mAAe0UjYhI6hJN5JTEZNmSqpTmIge46qoIoxERSV1SEzlJ+iTdi3y33YL1fc3N3CsikjlKHBnQ0Iv8mAePablQQ3+Oqqp2iEhEJHlKHBkwcb+gz+Jbq95qudAxCZKKiEgGKXFkwDPfeSa1Co89Fk0gIiJtoMSRYQnnIu/ZM1h/73vtE4yISBKUODIkqV7kN4bzVG3Y0A4RiYgkR4kjQ5Kai/zqFsazEhHJICWODElpLnKAkiTKiIi0AyWODEqqF3k8HMD4hBPaISIRkdYpcWTQ/ZPuB1rpRd7QMF5aCldc0Q5RiYgkpsSRQdOKprVe6N57Yffdt2//x39EG5SISCuUODIsqV7kX34JBcFw7Pz61zAtiYQjIhIRJY4MS6oXOcCqVbDPPsH2gw/CuedGHJmISPOUODIspV7kK1fCfvsF27//PZx5ZiQxiYgkosTRgdzxZoJe5A1KSmD//YPtJ56A00+PNigRkSaUODqAPXrtAcB1L7cyF3mDpUth5Mhg+89/hsmTI4pMRGRnShwdQPEFwXzoCXuRN7VoEYweHWw//TRMmBBBZCIiO4s0cZjZRDNbamYlZnZtM8dzzeyx8Pg8Mxva6Nh14f6lZjYh3LeXmb1iZkvMbJGZXRll/O0l5V7kDRYuhMMOC7ZfeAHGabZfEYleZInDzGLAfcAkYCRwtpmNbFJsGvCVuw8H7gJuC+uOBKYCBwMTgfvD89UCV7n7QcBY4NJmztkpbetF/r8JepE3Z/58GDMm2H75ZfjGN9IcmYjIjqK84xgDlLj7cnevBmYDU5qUmQI8FG4/AYwzMwv3z3b3KndfAZQAY9x9tbsvAHD3zcASYAhdwLZe5JVtmOp93jw49thg+4034OtfT2NkIiI7ijJxDAFWNfpcys5/5LeVcfdaYCMwMJm64WOtQ4F5zV3czC4ys2IzKy4rK2vzl2gvSfUiT+T11+HEE4Ptt9+Gww/f9aBERJoRZeKwZvZ5kmUS1jWz3sCfgO+7+6bmLu7u0929yN2L8vPzkww5s5LqRZ7ISy/B+PHB9oIFUFiYpshERLaLMnGUAns1+lwAfNFSGTOLA/2A9Ynqmlk2QdJ4xN3/HEnkGZJ0L/JEnn8eTjkl2P7gAzj44DREJiKyXZSJ4z1ghJkNM7McgsbuOU3KzAHOC7fPAF52dw/3Tw3fuhoGjADeDds/ZgJL3P3OCGPPiJTnIm/JU0/BP/9zsL14MRx4YHrOKyJChIkjbLO4DHieoBH7cXdfZGY3mdmpYbGZwEAzKwF+CFwb1l0EPA4sBp4DLnX3OuBo4FzgRDNbGC4nR/UdMunW12/dtRP86U9wxhnB9tKlMHz4rgclIgJY8A/8rq2oqMiLi4szHUZSBt8xmC+3fknc4tRcX7PrJzz33GBcKwgGSVy5ctfPKSJdnpnNd/ei5o6p53gH06Ze5In87ndwwQXB9qefwl57JS4vItIKJY4OpnEv8mXrlqXnpDNnwiWXBNulpbDnnuk5r4h0S0ocHVBDL/KxM8am76T33w+XXx5sr14Ne+yRvnOLSLeixNEB7VIv8kTuuQeuuSbY/sc/oJP0bxGRjkWJowPa5V7kidx+O1wXDt++di0MHBjdtUSkS1Li6KAaepF/fWYE40799Kdw443B9vr10L9/+q8hIl2WEkcHNXlEMDnT26VvR3OB66+Hn/882N64Efr1i+Y6ItLlKHF0UHPOadrJPgJXXw2//GWwvWkT9OkT/TVFpNNT4ugEdrkXeSJXXAH33Rdsb9kCvXpFdy0R6RKUODqwhnFS/5MAABBPSURBVLnIf/zqj6O90H/8B0yfHmyXl0PPntFeT0Q6NSWODiztvcgT+fd/h0ceCbYrKiAvL/prikinpMTRgUXSizyRc86BP/4x2K6qUvIQkWYpcXRwDb3I9//V/nzjwXaYT/yMM+Avfwm2q6ogJyf6a4pIp6LE0cE9ccYT27bfWPUGdqORdWMWk/8wObqLTpkCz4Rzg9TUQHZ2dNcSkU5HiaOD+9ZB38JvcC4vupwsC/5zOc7Ty57elkTO/fO56b/wpEkwd26wXVur5CEi22g+jk7o/CfP5+EPH8abTOFuGJcVXcY9p9yTvou9+SYce2ywnZUVvHWVm5u+84tIh6T5OLqYWafNov6GevwG55QRp2wbnsRx7i2+F7vRiN0U4/qXr9/1ix1zDMybF2zX1wcN5mZBZ8Ff/WrXzy8inY7uOLqQ42cdz+ufvr7TnUjc4vzipF9wxVFXtP3kCxfCEUcEj62ac+CB8OKLUFDQ9muISIeR6I5DiaMLqqys5OhZR7PgHwt2OpaTlcNvJv+G8w89v+0XKCuDE06AJUuCu5CmcnODfiH33tv2a4hIRilxdLPE0VhlZSWHzTyMJWuX7HQsL5bH7NNnM+WgKbt2kRkz4KqrgvGumjN4MDz+ePDYS0Q6hYy1cZjZRDNbamYlZnZtM8dzzeyx8Pg8Mxva6Nh14f6lZjah0f4HzWyNmX0UZexdRV5eHosvXYzf4Hz1n1+xT799th2rrKvk249/G7vR6HVLL95c+WbbLnLhhcEIu+5QWQknnQSx2Pbjq1cHDexmEI/DyScHfUREpFOKLHGYWQy4D5gEjATONrORTYpNA75y9+HAXcBtYd2RwFTgYGAicH94PoBZ4T5JUf+8/qz8/kr8Bmf1lau3jYUFUF5bzrEPHYvdaPT7WT8WfrmwbRfJzYUXXgjaQtzh5Zdh9923H6+rg2ef3d7I3r8/zJq1a19MRNpVlHccY4ASd1/u7tXAbKDpM5EpwEPh9hPAODOzcP9sd69y9xVASXg+3P11IM1zqnY/e/Tfg9VXr8ZvcFZcuYIBPQZsO7apehOH/uZQ7EZj4O0DWblhZdsvdMIJ8OWXQRJxD9o+GvdG37gRvvvdIInEYjB6dNCGIiIdVpSJYwiwqtHn0nBfs2XcvRbYCAxMsm5CZnaRmRWbWXGZ/hAlNLT/UNb9aB1+g/P+xe/TN6fvtmPrK9Yz7JfDsBuN3X6+G1c8cwVlm3bh95w+PXhM5Q4lJTBixPZj9fXw4Yew225BIunRA/7rv3bhm4lIFKJMHNbMvqYt8S2VSaZuQu4+3d2L3L0oPz8/lardWuEehWy8biN+gzP3O3PpGd8+xHpZeRn3vncvu921G3aj7bDk/L8c8m/P57gHj2Pm/JlUJdOGsd9+8Mkn2+9GfvGLHecDqayEn/0sSCJmsPfewWvBIpJRkb1VZWZHAT9x9wnh5+sA3P1njco8H5Z528ziwJdAPnBt47KNy4WfhwJPufshycTSnd+qSpe/Lvkr5/31PMpryqmpr0m5fpZlkZuVy+69dmfsXmP5z6//J4V7FrZcoaoKxo2Dt99u/pVfCBrae/YM3toaOxYuuwyKmn0JRERSlJHXccNE8AkwDvgceA84x90XNSpzKfA1d/+emU0F/tnd/8XMDgb+QNCusSfwEjDC3evCekNR4ugwNlVt4tbXb+WpZU/x6cZP2VqzlbrgP1VKYhajV3YvhvYfypQDp3D116+mb2742Ozpp+H882Ht2uRP2PAWV9++MGwYjB8Pl18Oe+zRel2Rbi5j/TjM7GTgbiAGPOjut5jZTUCxu88xszzgd8ChBA3eU919eVj3v4ELgFrg++7+bLj/UeB4YBDwD+AGd5+ZKA4ljsx7r/Q97vjbHbzz+Tus2bqG6vpq6r2FO4kEcmI59M/tz4H5B3JM1jDOeW0jB79bAqtWwdatLfdsb0lWVtBYP2AAHHQQnH46TJum4eSl21MHQCWODq+qqooHP3iQR/7+CB+v+5hNVZva9EisQRZZxLPi7FEZ54jPYeyndRStqqPw81r6V6Z6sqzgkdjuuwePwi66CE48sc2xiXQGShxKHF1C6aZSbnvzNl5Z+QobKzeyoWoDVbVV1NbX7jQ+V9Ic+lTDnptg6AY4YC0csA6Gr4N9v4KCzZDX2lO3hleJc3KCx2KDBwd3L8cfD//yL9CvX9tiE8kgJQ4ljm6nurqaF1a8wJyP5/DBmg8o3VzKpspNVNZVUldfl1qicYjXQ+9qGFABg7dAwUYYshkGbYVB5cH+ftVBmV7humE7r7bR64sNSSY3N0gogwfDyJHBHcxppynJSIehxKHEIa2orq7mr8v+ynPLnuODNR+wevNqNlZtpKquKvVE04Q59KiBXjXQpwr6VW1PLL2bJJmcumCJ10EsTFhxssiO5ZDbqy+9Bw9l97HjOPCMi8nP36f1i4u0kRKHEoekWXV1Na999horN62kZG0Jqzat4sutX/JVxVdsrNpIeU05VbVVVNdVU1tfS53XbXsZwN2J10N2XXAn4kBdFtRkQX0KPavyaoK7mbxayK2F3IakUw/Z9UHiyXIjZjGy4tlYXi6xnn3JHpRPrz32ZnC/IQzpO4ThA4ZzwIADOGjQQcTj8Uh+L+l8EiUO/a9EpA1ycnI4afhJaT/vZxs/Y8EXC/h47ccs37Ccf2xcTc2KEnI//5L4lq3EqmvJqgv+secWJJzqLKiKQ0UcKrNhazZsyIPNubAlx6mK1xK8nFgBbIBNn8Gm+Ttd23zHBJQdJqEgAQUzTJKVhWfH8dxc6vv0Jq9nX/rm9mVAjwHs3W9vhvYfyoAeA9it127s1ms39uyzJ0P6DiEnprfUuhIlDpEOZO9+e7N3v7137SSVlfD++7B4MXzyCRs/Wcaqr1ZQtrWMjTWb2UQV5VZDRcypjDmVsSDxVDZaKrKhPLsh+cDmnCAJbc6tY0tOHW5VULEJKr5IKqR4ozuheD3EwoQUc4iRRRaGWRaxrBhZsRixeA5ZOXnEe/Qi1rM3PXN70TunN71zetMvtx/98voxoMcABuYNJL93Prv13I09eu/Bnn33pHdO7137/aRVShwiXU1eHhx1VLAA/cIlZaWlMH9+MGHXp5/CylIoK6Puq3Wsrd7IWt/KhlgNW2K1bIk7lXGoihtVsWC7OgaVMaiOB4mpKhYkpfLs7YmpIg7l2fXh5zq2xmuoyIJytlJTB2wJlxSYb09O8fBuKe4WPLajYckiZlnBY7ysGPGsOLFYNvF4NrF4DvHsXGI5eeTk9CCe14PceB65sVzy4nnkxoN1j+we5MXz6BnvSY/sHvTK7rVt3TO7J/sN2I8e8R5kx7LJieWQnZVNdiyb7KxsYlmx1r9IB6bEISLNKygIlik7DmodA3YPl5TU1QWjIX/xRTBHS1kZrFsXjAawYQNs2AibNwcTgpWXU1GxObhDqq8I7pK8mnKrozxWR2VWPRUxp7rhjikWJKqqeNBWVJsFteG+6hhUhWWr4g2fGx0L91U0+VwVJr0oZGFkEyPbYmRbnGyLkZOVHWxnZZOdFd8x4cRzyI4FS052Ltnx3GBfdm6wL5a7LSk1rHNiOfTL68dlYy5Le/xKHCLSPmKxoIf+gAFwSOujBfUIl10aIKa6OkhEmzcHSWvLlmB78+ZgpIEtW4J1eXmwrqjYYamvrKCqupyKmgrKayqoqquior6K6voaKr2Gaq+lijpqqKOGeqqsntosp9qcrD59qPU6aryWaq+lxuuoyYKaGNRkOdWxWmpitdRkVVETJq3tx9lpX2Uz+3Yo1/A5TJoAe1RlK3GIiKQkJwcGDQqWNshiewIb0ErZVrkHd13V1dEsldu3vbqKmupKanv3bD2uNlDiEBFpDw2DbjaM6hzlpYCccIlCpHOOi4hI16PEISIiKVHiEBGRlChxiIhISpQ4REQkJUocIiKSEiUOERFJiRKHiIikpFvMx2FmZcCnbaw+CFibxnA6M/0WO9LvsSP9Htt1hd9iH3fPb+5At0gcu8LMiluazKS70W+xI/0eO9LvsV1X/y30qEpERFKixCEiIilR4mjd9EwH0IHot9iRfo8d6ffYrkv/FmrjEBGRlOiOQ0REUqLEISIiKVHiaIGZTTSzpWZWYmbXZjqeTDKzvczsFTNbYmaLzOzKTMeUaWYWM7P3zeypTMeSaWbW38yeMLOPw/+NHJXpmDLJzH4Q/v/kIzN71MzyMh1TuilxNMPMYsB9wCRgJHC2mY3MbFQZVQtc5e4HAWOBS7v57wFwJbAk00F0EL8EnnP3A4HRdOPfxcyGAFcARe5+CBADpmY2qvRT4mjeGKDE3Ze7ezUwG5iS4Zgyxt1Xu/uCcHszwR+GIZmNKnPMrAA4BZiR6Vgyzcz6At8AZgK4e7W7b8hsVBkXB3qYWRzoCXyR4XjSTomjeUOAVY0+l9KN/1A2ZmZDgUOBeZmNJKPuBn4E1Gc6kA5gX6AM+G346G6GmfXKdFCZ4u6fA3cAnwGrgY3u/kJmo0o/JY7mWTP7uv17y2bWG/gT8H1335TpeDLBzCYDa9x9fqZj6SDiwGHAr939UGAr0G3bBM3snwieTgwD9gR6mdl3MhtV+ilxNK8U2KvR5wK64O1mKswsmyBpPOLuf850PBl0NHCqma0keIR5opn9PrMhZVQpUOruDXegTxAkku7qm8AKdy9z9xrgz8DXMxxT2ilxNO89YISZDTOzHILGrTkZjiljzMwInmEvcfc7Mx1PJrn7de5e4O5DCf538bK7d7l/USbL3b8EVpnZAeGuccDiDIaUaZ8BY82sZ/j/m3F0wZcF4pkOoCNy91ozuwx4nuCtiAfdfVGGw8qko4Fzgb+b2cJw33+5+zMZjEk6jsuBR8J/ZC0HvpvheDLG3eeZ2RPAAoK3Ed+nCw4/oiFHREQkJXpUJSIiKVHiEBGRlChxiIhISpQ4REQkJUocIiKSEiUOkTYyszozW9hoSVuPaTMbamYfpet8IumkfhwibVfh7oWZDkKkvemOQyTNzGylmd1mZu+Gy/Bw/z5m9pKZfRiu9w73725mT5rZB+HSMERFzMz+N5zb4QUz6xGWv8LMFofnmZ2hryndmBKHSNv1aPKo6qxGxza5+xjgVwSj6RJuP+zuo4BHgHvC/fcAr7n7aIJxnhpGKRgB3OfuBwMbgNPD/dcCh4bn+V5UX06kJeo5LtJGZrbF3Xs3s38lcKK7Lw8Hh/zS3Qea2VpgsLvXhPtXu/sgMysDCty9qtE5hgIvuvuI8PN/AtnufrOZPQdsAf4C/MXdt0T8VUV2oDsOkWh4C9stlWlOVaPtOra3SZ5CMEPl4cD8cMIgkXajxCESjbMard8Ot//G9mlE/xV4M9x+CbgEts1l3relk5pZFrCXu79CMJlUf2Cnux6RKOlfKiJt16PRaMEQzLvd8EpurpnNI/jH2dnhviuAB83sGoJZ8xpGkb0SmG5m0wjuLC4hmD2uOTHg92bWj2DCsbs0Vau0N7VxiKRZ2MZR5O5rMx2LSBT0qEpERFKiOw4REUmJ7jhERCQlShwiIpISJQ4REUmJEoeIiKREiUNERFLy/wE2wnRMMiD5QQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV9bnv8c+TBEjCIBCiIBEZREULOEQcWmfFsVq1rcCh93C05do6UCta7VWsFKtyrLUerdZjHY62OLUOl1oHLE63WAiKA1ABATECJRCmkDl57h9rJ2zCTrKT7J2V7Hzfr9d+rb3mJ1vZz17r91u/x9wdERGRhtLCDkBERDomJQgREYlJCUJERGJSghARkZiUIEREJKaMsANIlAEDBvjQoUPDDkNEpFNZvHjxZnfPjbUuZRLE0KFDKSgoCDsMEZFOxcy+aGydbjGJiEhMShAiIhKTEoSIiMSUMm0QsVRVVVFYWEh5eXnYoaSEzMxM8vLy6NatW9ihiEg7SOkEUVhYSO/evRk6dChmFnY4nZq7s2XLFgoLCxk2bFjY4YhIO0jpW0zl5eXk5OQoOSSAmZGTk6OrMZEuJKUTBKDkkED6LEW6lpS+xSQikghVNVUUlxVTXFbM1rKtbC3fyrbybZRWlXJY7mHUeA21XrvXq6a2keUJ3n5wn8FMPXpqwv9uJYgk69WrFyUlJXFt+/jjj3PZZZexZMkSxowZA8DXvvY15s6dS1NPid97771MnTqV7OzsvdatXbuWUaNGccghhwBw3HHH8dBDD7X8DxEJS0kJzJ0Lb74JH30EhYWwfTsMGkRlryy298xge690tmdnsDM7g21ZxvYsY0emsaOHs7077Ohey86MWnZm1FCSXkOJVVFq1ZTXVFBRU0FlTSWVNZVU11ZTXVtNTW0NNV6Du+N0/Jo5x+7oowTRFeTl5XH77bfzzDPPxL3Pvffey+TJk2MmCIARI0awZMmSRIUoXVB1dTVPffoUcz6ZQ0ZaBmlpaVRWV1JRU0FVTRWVtZVU1VRRXVsdTD34ko3+sq3/FVwb9SuYWtydtOoasipr6VFVS7dqyKiF9NrgHrgD1WlQ1RuqToKqdKhMh7KMz6lOjyN4ByoiL8AcsquCV2YVZFdDv2rIrIHuNXXTNLrXGpmeTg9Po4dn0MMyyLRuZKX1IDMjk6xuWWRnZDF85DGklZWTXlZBWln57ldpGWmlZaRHpmm7ykjbVUpaaSlp1bWkO6TFeKXXNljWI5O0nj1Jz+5FWnZP0nr22v3q1Zu0Xr3h4IOT8t9dCSIERUVFXHHFFaxbtw4IvuC//vWvA3D++efzzjvv8Nlnn9X/6q/z+uuvc+utt1JRUcGIESN47LHHePTRR1m/fj2nnnoqAwYMYP78+e3+90hqWbppKb9a8CveXvs263eup7wmAR0TIl92GbXQrRZ6VENm5JVdFUy71UK3mmBqHiSF6jSozIDybmmUZ6RTm9WNyp496JGeziF9h7JPj33IysiiV0YWfWq7s091Bn2q0uhbmUbfCtinzOlbbvQrrWGfXTX0Lamm565K0naVQmnkVV4evCoqoLIyeFVVQXU11NRATTXUVkKj1TcXBZP0dOjVq8GrD/TaH3J7Qc+eMdZHvWKt79kzOG5Iuk6C+PGPIdG/oo84Au69t8W7TZs2jWuvvZZvfOMbrFu3jrPOOovly5cDkJaWxg033MAvf/lLnnjiifp9Nm/ezKxZs5g3bx49e/bkrrvu4p577mHGjBncc889zJ8/nwEDBsQ835o1azjyyCPp06cPs2bN4sQTT2zd3ysppby6nIcKHuKZT5/hn1v+yY6KHdR6bcxtu6d3Z7+e+3HM/scwer/R9M3sS89uPenZvSe9uvUiu1s2vXaW0f/tAvb5x4f0Xr6GzMINpO3YGXzZNvblmpERfAnutx+MHAnHHQcXXACRW6wdTm0t7NgBW7ZAcTGUlcHhhwdf5t27Q4p15Og6CaIDmTdvHsuWLauf37FjBzt37qyfnzRpErfffjtr1qypX/b++++zbNmy+iuNyspKjj/++GbPNWjQINatW0dOTg6LFy/mW9/6FkuXLqVPnz4J/Iuko3tv3Xv85v3f8H7h+2wq3URlTWXM7dIsjT49+nDogEOZePhEpuZPJTMjM1hZXQ3z5sGrb8GSV2HVKigqgl27gl/asZgFX5x9+8LgwTB6NJx2Gpx/PvTvn5w/NpnS0oK/pW9fGDEi7GiSruskiFb80k+W2tpaFixYQFZWVsz1GRkZXHfdddx11131y9ydM888kzlz5jR57BdeeIHbbrsNgEceeYT8/Hx69OgBwNFHH82IESNYsWIF+fn5CfprpCPZVr6Ne/5+Dy+veJnPt37OrspdjTayZqZnsn+f/Tl18IlMrx7HoQs/DxqB162DLStg1w1QdW3wq7kpdbdWBgyA4cPh6KPh3HPhxBNDvT0ibdd1EkQHMn78eO6//36uv/56AJYsWcIRRxyxxzZTpkxh9uzZ9VcWxx13HFdeeSWrVq3ioIMOorS0lMLCQg4++GB69+7Nzp07GTBgABdddBEXXXRR/XGKioro378/6enprF69mpUrVzJ8+PD2+2MlaeaumMuDix7kgw0fsLlsM9W11XtuELnnn1tmjNqazslfpnP+qjSO+KqatKoaqC0HVkdeT8Q4Q4RZ8Ms5MxN694axY+Hss+HCC0FP1ac0JYgkKy0tJS8vr37+Jz/5Cffddx9XXnklY8aMobq6mpNOOmmvrqfdu3fnmmuuYdq0aQDk5uby+OOPM3HiRCoqgu4Ys2bN4uCDD2bq1Kmcc845DBo0aK9G6nfeeYcZM2aQkZFBeno6Dz30EP0746V9CiutKmXD9g2sL1nPpl2b2LRrE8XlQX/77RXbKaksYVvZNlZtXcX6kvWUVZZiDrVRj7n2qoDhW2HsRhj3FRxXCF/bFPTIifQDiryimAW/8DMzoU8fGDgw6A1z7LHBLaCDDmq/D0E6JPNGW+YTcHCzs4HfAOnAI+5+Z4P1BwKPArlAMTDZ3Qsj62qATyKbrnP3C5o6V35+vjcsGLR8+XJGjRqViD9FIrrKZzp/9XxufftWPtr4EZU1laRZGrUEXTQdp9aDafS/n/boL9+jGg7bBGP/BWMir9GbYN9dURtFf/Hvsw8MGgSHHALHHw/nnQeqvChRzGyxu8e855y0KwgzSwceAM4ECoFFZvayuy+L2uxu4H/c/QkzOw24A/heZF2Zu+9530Ukwd5Z+w4z3prBkg1L2Fm5k1qaud/eSsbu3i1mRmYVDNzpDCxxcksgtxRySqFfOfSu2N0nv3sNpNcEVwtDtsMRRen0yOy5+4v/pFFwwglBz5+BA5MSu3RdybzFNA5Y5e6rAczsaeBCIDpBHAZcG3k/H3gxifFIF1afCDYuYWdF84kgIy2DnKwcxg8fz6TRkxjWbxgDsweyT/Y+8Z2wqgqefRaeego++SToFllREenu2cSVhhl06wb9+sGhh8J3vws/+EGwTKSdJTNBDAa+jJovBI5tsM1HwCUEt6EuAnqbWY67bwEyzayA4Mbpne6+V/Iws6nAVIAhQ4Yk/i+QTmfBlwu4+W83s3jD4vgTQWYO40eMZ/b42Qzs1YJf4du3w4MPwksvwcqVQf/4qqrm9zMLbv/suy/k58Pll8M558R/XpF2kswEEeuJkYY/naYD95vZFOAd4Ct2t6QNcff1ZjYc+JuZfeLun+9xMPeHgYchaINIZPDSsdUlgg82fBA84BVnIjh9+OncPf5uBvUeFN+JVq6EO+6AhQvhyy+b7vMfLT0dsrPhgAPgpJNg2rTgikCkE0lmgigEDoiazwPWR2/g7uuBiwHMrBdwibtvj1qHu682s7eAI4E9EoSkvgVfLuCWv93C4g2L404E/TP7c8bwM7jj1DsY0r+FV5arVsGECcHzANXVTW/brVvQ+2fEiKDxd9q0oG1AJEUkM0EsAkaa2TCCK4MJwKToDcxsAFDs7rXATQQ9mjCzfkCpu1dEtvk6MDuJsUoSFe4oZO4/57Jww0JWbF7BhpIN7KjYQWl1KVU1VfWDuLVEhmXQP6s/pw49ldmnz255Iqizbh18+9vw4YexE4JZ0B6Qnw+TJgUvtQdIF5G0BOHu1WZ2FfAaQTfXR919qZnNBArc/WXgFOAOM3OCW0xXRnYfBfzOzGoJBnS8s0Hvp07DzJg8eTJPPvkkEIyKOWjQII499ljmzp3b7BDfQ4cOpaCggAEDBnD77bfzxz/+kfT0dNLS0vjd737HnXfeyZo1aygpKaGoqKi+HOhvf/tbfvazn1FSUkJd99+CggKmT5/OW2+91Wi8a9eu5b3/9x6T/20ytbW17Kzcyc6KnZRVl1FZU8n6Hes57o7jqKipoLq2ur67Z7LUJYKTDjyJX53xq9Yngjrr1sGll8LixbHbC8yCcYHuuAOmTGnbuUQ6uaQ+KOfurwCvNFg2I+r988DzMfb7OzA6mbG1l549e/Lpp59SVlZGVlYWb7zxBoMHD95jm3iG+F6wYAFz587lgw8+oEePHmzevJnKykpeeOEFAN566y3uvvtu5s6du8d+mzZt4q9//SvnxNEIunHnRl4reI2nHnuKQ0+Nfb+8praGHZU7mj1WtDRLwzC6pXWjR0YPenXvRW52LkP2GcLYgWMZP3w8xww8pn5IkITauBEuuQQWLWo8Iey7L/ziF0FvIRGppyep28E555zDX/7yF7797W8zZ84cJk6cyLvvvlu/vqkhvuts2LCBAQMG1H+JNjZya0PXX389s2bN2itB1NTUcOONN/LWW29RWl7KhZMv5OLvXcz9v7yfNavWMOnMSZz/nfOZNHUShmFmpFkaGWkZnDLkFEYOGMkJeSdw3ojzyO2T28pPJgk2bgy6hr7/fuM9ivbdF265Ba66qn1jE+lkukyC+PGrP2bJxsQO933EwCO49+zmBwGcMGECM2fO5Pzzz+fjjz/msssu2yNBNDbEd7Tx48czc+ZMDj74YM444wwuvfRSTj755GbPffzxx/PCCy8wf/58evfuXb/897//PVm9snjwpQeprKjk+9/6PseefCzTZ0znuUee2+tKpM7yrcuZ/x8dqOZEcTFcfDEsWBCM4x9Lbi7cdBNce23s9SISU1rzm0hbjRkzhrVr1zJnzhzOPffcmNtMmjSJ999/f48hvqP16tWLxYsX8/DDD5Obm8ull17K448/Htf5b775ZmbNmlU/X1VdxTMvPcMTTzzBpDMnMeX8KWzfup2sHVkc2PfAFv997aq4GM44A3r0CG4P5eTA22/vmRxycuA//zN4KM0dNm1SchBphS5zBRHPL/1kuuCCC+obiLds2bLX+lhDfDeUnp7OKaecwimnnMLo0aN54oknmBJHQ+ppp53GLbfcwoIFCyipLOGjTR/hONNnTefE005kzL5jSEsLfis01YAdipKSoA3h7beDJ5FjycmB6dPhxhvbNzaRFKcriHZy2WWXMWPGDEaPbrztfcqUKcybN4+ioqK91n322WesXLmyfn7JkiUceGB8v/Zra2uZ8KMJ3H7n7fU9jk445QTmPTOPw3MOJy0tjRUrVrBr1676ocNDM39+MIpoRkZwhdC7N7z++p7JoX9/uPXW3VcImzcrOYgkQZe5gghbXl5e/dDdjWk4xHe0kpISrr76arZt20ZGRgYHHXQQDz/8cLPnXbVlFWyE4089nn45/QAYu+9YjrruKG7eejNHHXUU7k5ubi4vvvgiY8aMISMjg7FjxzJlyhSuTeatmR07gofS5s8PagI3pm9fuPpqmDkzebGIyF6SOtx3e9Jw33taVrSM0qrSPZaNGjCKnt17tum4bfpMZ88O2gY2b256u5wc+MlP4Gc/a915RCRuoQz3LeFYuWUl2yu277FsRL8R9Mvq176BLFoUPGj22WdNj12UmQknnxyMfKo62SIdihJEivhi2xcUle7ZdpHXO4+BvduhRkBFBfzbv8Ff/wqlpY1vl54eFKt54AE466zkxyUibZLyCcLdMYs1sGxq2LhzI4U7C/dYtm/2vgzpm/jhz+tvRz74YNAesGlT0wXt+/aFqVOhiZ5ZItJxpXSCyMzMZMuWLeTk5KRckiguK2b11tV7LOub2ZeD+ie4jnBZGaxZg5eVsaWqisyFC4NRSxvq3h2OOw6efz54ME1EOr2UThB5eXkUFhbG7DbaWVVUV7CxZOMey7qnd2dQ70FUba9i+b+Wt+0E5eXBlUHDzgu1tWSuWkXez38OaWmQlxc0OH/3u207n4h0WCmdILp161Y/umlnt3bbWkbeN5Jq3z0k9Yi+I1g1bVXiTnL00fDBB3sv790bJk+GX/8arrkmcecTkQ4tpRNEKigvLyfnVzmUVu9u/M3JyqHo+qLE3TYrL4eePXe3J/TpEySKESMSc3wR6ZT0JHUHVV5eTs5dOWTdlVWfHLIzsin7aRmbb9icuORw112QlbU7OdxyS1BrWclBpMvTFUQHNPTeoXyx/Yv6+W7WjXXXrGNg3wR3WR04EP71r+C9WdBFNTMzsecQkU5LCaIDGf3b0Xxa9Gn9fJqlseA/FjDugHGJPdHatRDdNjN6NHz8cWLPISKdnm4xdRAZt2XUJwfDePG7L1IzoybxyWHixD2Tw3PPKTmISEy6gugAsm/PpoZgOIo7Tr2DG09K0sik2dnBcw0Q1FNoaoA8EenydAURspzZOZRVB1/aj5z3SHKSw0svBW0MdcnhgguUHESkWbqCCNEh9x1CcVkxADeccAOX51+e+JMceSQsiSq1unw5HHpo4s8jIilHCSIkJz92Miu2rgDg24d+m7vOTPB4RQ2fbRgwAFLoiXIRST7dYgrBZS9cxjvr3gHgmP2P4blLn0vsCWbO3PvZBiUHEWkhXUG0szvfuZPHPn4MgCF9hrDwBwsTewI92yAiCaIriHb0wvIXuGn+TQD07t6bL679opk9WmDNmiAh1CWHsWODKwglBxFpJSWIdvLJV59w8bMXA9A9rTs7btqRuIN/5zswfPju+Rdf3LNhWkSkFXSLqR0UlxUz5pExQPAQXMUtFYk7eFbW7i6rerZBRBIoqVcQZna2mX1mZqvMbK8O/mZ2oJm9aWYfm9lbZpYXte7fzWxl5PXvyYwzmcrKysiZnVM/X3trExXYWqLu2Ya6hHDxxUoOIpJQSbuCMLN04AHgTKAQWGRmL7v7sqjN7gb+x92fMLPTgDuA75lZf+BWIB9wYHFk363JijdZsmdn178vvaGJes0toWcbRKQdJPMKYhywyt1Xu3sl8DRwYYNtDgPejLyfH7X+LOANdy+OJIU3gLOTGGtSpN22++PdcsMWsrKy2nbA8nJIT9+dHHJzg8pvSg4ikgTJTBCDgS+j5gsjy6J9BFwSeX8R0NvMcuLcFzObamYFZlbQ0cqKdpvZDSco27niqhX0z+rftgM2fLZhxoygNKiISJIks5E6VkWbBoWOmQ7cb2ZTgHeAr4DqOPfF3R8GHgbIz8/fa31Yev+yd31p0L9M+Asjc0a27YD77bc7GaSlwa5d6r4qIkmXzCuIQuCAqPk8YH30Bu6+3t0vdvcjgf8TWbY9nn07qv1/tT8lVSUA3Dv+Xs495NzWH+yf/wwaouuSw9ixUFOj5CAi7SKZCWIRMNLMhplZd2AC8HL0BmY2wMzqYrgJeDTy/jVgvJn1M7N+wPjIsg7tiAePYEPJBgB+dPSPmHb8tNYf7JJLYNSo3fN6tkFE2lnSbjG5e7WZXUXwxZ4OPOruS81sJlDg7i8DpwB3mJkT3GK6MrJvsZn9giDJAMx09+JkxZoI5z51Lh9t+giAs4afxQPnP9D6g2VmQkXkWQk92yAiITH3DnPrvk3y8/O9oKAglHNP+8s07iu4D4DDcg5j6VVLW3egl16Cb31r9/zFF8Of/pSACEVEYjOzxe6eH2udnqRuowcXPlifHPbruV/rk8P06fCrX+2e17MNIhIyJYg2ePPzN/nRX38EQHZGNhunb2z9wX7962DasyeUlCQgOhGRttFgfa20qngVZzx1BgAZlsGu/7OrbQese76hgz3PISJdlxJEK5SVlTHyv4JnGwyjakZV4g7e1qetRUQSRAmiFaLHV0rI4HuXJ6EWtYhIGzWbIMzsaTM7y8xiPd3c5dhtuz+GhA2+99RTwTQvr+ntRETaUTxXEI8DlwErzGyWmR2U3JA6rvTb0uvfJ2TwvTqVlcH0lVcSczwRkQRoNkG4+6vufinB6Kwbgflm9o6Zfc/MukwvqMxZmdQS3E5a+P2FbR98L5bRoxN/TBGRVoqrDSIy3MUk4HvAx8DvgBOAV5MXWsfR/87+VNQETzY/ffHTHDP4mMQd/Pe/T9yxREQSqNkrADN7FhgN/BG4xN0LI6v+YGYfJjO4jmD4b4aztSKoU3TbSbdx6ehLE3uCa68Npr17J/a4IiJtFM8tokcIivfEGm77yMSH1HGc8PsTWLNtDQCTvjaJGafOSPxJdu4MpnUPyomIdBDx3GIaDuxTNxMZYXVq8kLqGCY8N4EFhQsAOD7veP5wyR+Se0J1dRWRDiaeBHGFu2+rm4mUAP1h8kIK34w3Z/DMsmcAGNZ3GH+//O/JOdEnnyTnuCIiCRBPgkiPnonUb+iWnHDC9+SSJ/nFe78AoF9mP1ZPW528k50bKSbUvXvyziEi0krxtEG8YWZzgIcIyn7+EJiX1KhCsuirRfyvl/4XAD3Se1D80ySXoCiMtPdPnpzc84iItEI8CeJ64EfAtQS1ol8n6OaaUr4q/opxj4wDII00ym9uxyI96uoqIh1QswnC3WuA/4q8UlJZWRl5/7V7mIuaW2va46TJP4eISBvE8xzECOB24DAgs265ux+cxLjaVfTge35rO1XYO/30YJqm8RJFpGOKdyymxwhuL50DPAs8ncSY2tW4/x5X/z5hg+/F4x//CKZHH91+5xQRaYF4EkS2u78G4O6fu/vNwKnJDav9LPzBQgb2HMjKq1cmbvC9eNQVCHr77fY7p4hIC8TTSF0RGer7czO7AvgK2De5YbWvDdM3hHdyFQgSkQ4qngRxLdALuIagLaIPwfDf0lr/+3+HHYGISLOaTBBmlg5c5O7/AHYSjOYqbfX448F08OBQwxARaUqTbRCRLq7jmtpGWqGuQNCLL4Ybh4hIE+K5xfSBmf0ZeA7YVbfQ3V9OWlRdRX5+2BGIiDQqngSxH0FiODdqmQNKEK2hp6ZFpJOI50lqtTskkgoEiUgnEc+T1A/HWu7uzdaEMLOzgd8QjAj7iLvf2WD9EOAJoG9kmxvd/RUzGwosBz6LbPq+u1/R3Pk6hboCQXfdFW4cIiLNiOcW05tR7zOBi4Avm9sp0gPqAeBMoBBYZGYvu/uyqM1uBp519wfN7DDgFWBoZN3n7n5EHPF1Tj9M6ZIaIpIC4rnF9Ez0vJk9CbwRx7HHAavcfXVkv6eBC4HoBOEEz1VAULVufRzH7bxUIEhEOpHWjBQ3DDgwju0Gs+eVRmFkWbSfA5PNrJDg6uHq6POY2Ydm9raZnRjrBGY21cwKzKygqKgo7j8gNOefH0xVIEhEOoFmE4SZbTWz4shrG8HVw8/iOLbFWNZwqNSJwOPunkfQS+rJSMW6DcAQdz8S+AnwRzPr02Bf3P1hd8939/zc3Nw4QgrZunXBdMKEcOMQEYlDPG0QA6Le17p7vONhFwIHRM3nsfctpMuBswHcfYGZZQID3H0TUBFZvtjMPgcOBgriPHfH9sQTYUcgItKseG4xnQf0cvcad3cz62tm58ex3yJgpJkNM7PuwAT2fnZiHXA6gJmNImgELzKz3EgjN2Y2HBgJJLE4dDtQgSAR6WTiSRAz3X173Yy7bwN+0dxO7l4NXAW8RtBl9Vl3X2pmM83sgshm1wE/MLOPgDnAlMgVyknAx5HlzwNXuHuSC0QnWV2BIIt1501EpOOJ5xZTrCQSz364+ysEjc/Ry2ZEvV8GfD3Gfn8C/hTPOTqNugJBGl5DRDqJeK4gPjCz2WZ2oJkNMbP/BD5MdmApRwWCRKSTiSdBXBXZ7iWCNgQHfpTMoFKaCgSJSCcRz4NyJcD0dogldV15ZdgRiIi0WDzPQbxqZn2j5vuZ2V+SG1aKeeSRYDpoULhxiIi0QDy3mPaL9FwCwN23AvsnL6QUVFcg6KWXwo1DRKQF4kkQtWaWVzcTGYFVWuOYY8KOQEQkbvF0V50B/D8z+1tk/lRAQ5HG68knw45ARKRV4mmk/ouZjQOOJxhf6aeRoTAkHlddFUx79Qo3DhGRFop3NNdygmEx/gUcZGYnJC+kFLNjRzCdPTvcOEREWiieinKXEQyJMRj4BDgGeB84JamRpRoVCBKRTiaeK4hrgXxgrbufCBxNMBy3NEcFgkSkE4snQZS7exmAmXV396XAockNK0V885vBtFu3cOMQEWmFeHoxbYg8KPd/gdfMrJigLUKa88UXwfQ73wk3DhGRVoinF1Pd0Ny3mNnpBLWj9SR1S/zhD2FHICLSYnEN213H3d9MViApRwWCRKSTi7ebq7TUmWcGUxUIEpFOSgkiWRYsCKZHHRVuHCIiraQEkSx1BYLefTfcOEREWimeB+W2EhQJirYdKACud/e1SYgrdahAkIh0UvE0Uv8XQbfWPxKMxTQByAVWAY8RDN4n0aZNCzsCEZE2iydBjHf346Lmf2tm77v7cWZ2Q7IC69QeeiiYDhwYbhwiIm0QVxuEmV3c4H1d15zaZATV6dUVCHr55XDjEBFpg3gSxGTgB2ZWbGZbgB8A3zOzbODHSY2us1OBIBHpxOJ5knoVcE4jq99ObDgp4Jlnwo5ARCQh4unFNAC4DBgavb27T01eWJ3YFVcEUxUIEpFOLp5G6pcI6j+8B9QkN5wUsG1bMJ01K9w4RETaKJ4E0dPdr2vNwc3sbOA3QDrwiLvf2WD9EOAJoG9kmxvd/ZXIupuAywmS0jXu/lprYgiNurqKSCcXTyP1X81sfEsPbGbpwAME7ReHARPN7LAGm90MPOvuRxI8X/HbyL6HReYPB84m6Fqb3tIY2t2qVWFHICKSMPEkiCuAV82sJNKTaWukJkRzxgGr3H21u1cCTwMXNtjGgT6R9/sA6yPvLwSedvjNTm4AAA11SURBVPcKd19D8FDeuDjOGa66AfoyWjRIrohIhxTPN9mAVh57MPBl1HwhcGyDbX4OvG5mVwM9gTOi9n2/wb6DG57AzKYCUwGGDBnSyjATaO3aYHrJJaGGISKSCI1eQZjZyMjbwxt5NSfWONcNx3SaCDzu7nnAucCTZpYW5764+8Punu/u+bm5uXGE1E6efjrsCERE2qypK4gbCRqJH4ixzoGTmjl2IXBA1Hweu28h1bmcoI0Bd19gZpkEVyzx7NuxqECQiKSYRhOEu18eeXuau1dFrzOzbnEcexEw0syGAV8RNDpParDNOuB04HEzGwVkAkXAy8AfzeweYH9gJLAwjnOG5+yzg6kKBIlIioinkfofcS7bg7tXA1cBrwHLCXorLTWzmWZWV+f6OoJhPD4C5gBTPLAUeBZYBrwKXOnuHfsZjPfeC6Zjx4Ybh4hIgpj7Xrf2gxVm+wKDCHoffZfd7QJ9CJ5pOLRdIoxTfn6+FxQUhBdA3ZXDli3Qv394cYiItICZLXb3/FjrmmqDOI9giI08gnaIugSxE7gloRGmEiUHEUkRTbVBPAY8Zmbfdfdn2zGmzkdPTYtICoqnDWJfM+sDYGYPmdlCMzs9yXF1LnUFgvbbL9w4REQSKJ4EMdXdd0SG28gDfgjMTm5YnUxdgaDnnw83DhGRBIonQdS1Yp8DPObui+Pcr+v5xjfCjkBEJGHi+aL/yMxeAb5JMHBfL2I81dxlqUCQiKSoeMZi+g/gaIKB90ojBYQub2afruOHPwym2dnhxiEikmDNXkFEHlAbTtD2AJAVz35dxtatwfS228KNQ0QkwZr9ojez+4FTgcmRRbuAh5IZVKc0fXrYEYiIJFQ8t5hOcPejzOxDAHcvNrPuSY6rc1CBIBFJYfHcKqqKDMHtAGaWA9QmNarOYnyk0J4KBIlICmqqHkTdt94DwJ+AXDO7DXgPuKsdYuv41qwJpioQJCIpqKmfvguBo9z9f8xsMUG1NwO+4+6ftkt0nYUKBIlICmoqQdQXNogMv700+eF0IioQJCIprqkEkWtmP2lspbvfk4R4Oo9zzw2mKhAkIimqqQSRDvQidn1oeffdYDpmTLhxiIgkSVMJYoO7z2y3SDqbmkiBu7/9Ldw4RESSpKlurrpyiIcKBIlIimoqQajmQ2Ouuy7sCEREkq7RBOHuxe0ZSKfywAPBdN99w41DRCSJNOhea1RUBNM//SncOEREkkgJoi1UIEhEUpgSREu98ELYEYiItAsliJa6PFIrSQWCRCTFKUG0lAoEiUgXoQTRWioQJCIpTgmiJVQgSES6kKQmCDM728w+M7NVZnZjjPW/NrMlkdcKM9sWta4mat3LyYwzbuecE0xVIEhEuoCkfdOZWTpBsaEzgUJgkZm97O7L6rZx92ujtr8aODLqEGXufkSy4muVuiuIb30r3DhERNpBMq8gxgGr3H21u1cCTwMXNrH9RGBOEuNJnOeeCzsCEZGkS2aCGAx8GTVfGFm2FzM7EBgGRA+NmmlmBWb2vpnF/MluZlMj2xQUFRUlKu7YVCBIRLqYZCaIWKPBeiPbTgCed/eaqGVD3D0fmATca2Yj9jqY+8Punu/u+bm5uW2PuCnf/GYwVYEgEekikpkgCoEDoubzgPWNbDuBBreX3H19ZLoaeIs92yfa31tvBdPDDw81DBGR9pLMBLEIGGlmw8ysO0ES2Ks3kpkdAvQDFkQt62dmPSLvBwBfB5Y13Ldd1RUIevvtUMMQEWkvSevF5O7VZnYV8BpB+dJH3X2pmc0ECty9LllMBJ529+jbT6OA35lZLUESuzO691OoVCBIRLqIpHbod/dXgFcaLJvRYP7nMfb7OzA6mbG1iAoEiUgXpCep46ECQSLSBSlBxEMFgkSkC1KCaAkVCBKRLkQJojkqECQiXZQSRHPqCgRlZYUbh4hIO1OCaE5dgaAZM5reTkQkxShBxOvGvUYrFxFJaUoQTfnqq7AjEBEJjRJEU047LZiqQJCIdEFKEE1ZuTKY1o3kKiLShShBNKVueKg//zncOEREQqAE0RgVCBKRLk4JojEXXRRMVSBIRLooJYjGzJsXTEeNCjcOEZGQKEE0pq5A0LvvhhuHiEhIlCCaowJBItJFKUHE8tOfhh2BiEjolCBiue++YJqbG24cIiIhUoKIpbw8mM6ZE24cIiIhUoJoyumnhx2BiEholCAaUoEgERFACWJv3/9+MFWBIBHp4pQgGiouDqY33RRuHCIiIVOCaMwtt4QdgYhIqJQgoqlAkIhIPSWIaHUFgtLTw41DRKQDUIKIVlcg6Nxzw41DRKQDSGqCMLOzzewzM1tlZjfGWP9rM1sSea0ws21R6/7dzFZGXv+ezDjr1RUIeuaZdjmdiEhHlrRiy2aWDjwAnAkUAovM7GV3X1a3jbtfG7X91cCRkff9gVuBfMCBxZF9tyYr3j0KBKmLq4hIUq8gxgGr3H21u1cCTwMXNrH9RKBubIuzgDfcvTiSFN4Azk5irHDJJcFUBYJERIDkJojBwJdR84WRZXsxswOBYcDfWrKvmU01swIzKygqKmpbtK+/HkwPOaRtxxERSRHJTBCxfop7I9tOAJ5395qW7OvuD7t7vrvn57Z15NW6AkF1leRERLq4ZCaIQuCAqPk8YH0j205g9+2llu6bWINjXuSIiHQ5yUwQi4CRZjbMzLoTJIGXG25kZocA/YAFUYtfA8abWT8z6weMjyxLjhkzknZoEZHOKmm9mNy92syuIvhiTwcedfelZjYTKHD3umQxEXja3T1q32Iz+wVBkgGY6e7FyYqVu+8Opjk5STuFiEhnY1Hfy51afn6+FxQUtG7nup5L8+apBoSIdClmttjd82Ot05PU0ZQcRETqKUG88krYEYiIdEhKEC+9FEz79g03DhGRDkYJ4ne/C8Zg2pq8UTxERDojJQgREYlJCUJERGJSghARkZiUIEREJCYlCBERiUkJQkREYlKCEBGRmJQgREQkppQZrM/MioAv2nCIAcDmBIXT2emz2JM+jz3p89gtFT6LA909ZsW1lEkQbWVmBY2NaNjV6LPYkz6PPenz2C3VPwvdYhIRkZiUIEREJCYliN0eDjuADkSfxZ70eexJn8duKf1ZqA1CRERi0hWEiIjEpAQhIiIxdfkEYWZnm9lnZrbKzG4MO54wmdkBZjbfzJab2VIzmxZ2TGEzs3Qz+9DM5oYdS9jMrK+ZPW9m/4z8P3J82DGFycyujfw7+dTM5phZZtgxJVqXThBmlg48AJwDHAZMNLPDwo0qVNXAde4+CjgOuLKLfx4A04DlYQfRQfwGeNXdDwXG0oU/FzMbDFwD5Lv714B0YEK4USVel04QwDhglbuvdvdK4GngwpBjCo27b3D3DyLvdxJ8AQwON6rwmFkecB7wSNixhM3M+gAnAb8HcPdKd98WblShywCyzCwDyAbWhxxPwnX1BDEY+DJqvpAu/IUYzcyGAkcC/wg3klDdC9wA1IYdSAcwHCgCHovccnvEzHqGHVRY3P0r4G5gHbAB2O7ur4cbVeJ19QRhMZZ1+X6/ZtYL+BPwY3ffEXY8YTCz84FN7r447Fg6iAzgKOBBdz8S2AV02TY7M+tHcLdhGLA/0NPMJocbVeJ19QRRCBwQNZ9HCl4mtoSZdSNIDn9w9z+HHU+Ivg5cYGZrCW49nmZmT4UbUqgKgUJ3r7uifJ4gYXRVZwBr3L3I3auAPwMnhBxTwnX1BLEIGGlmw8ysO0Ej08shxxQaMzOCe8zL3f2esOMJk7vf5O557j6U4P+Lv7l7yv1CjJe7bwS+NLNDIotOB5aFGFLY1gHHmVl25N/N6aRgo31G2AGEyd2rzewq4DWCXgiPuvvSkMMK09eB7wGfmNmSyLKfufsrIcYkHcfVwB8iP6ZWA/8Rcjyhcfd/mNnzwAcEvf8+JAWH3dBQGyIiElNXv8UkIiKNUIIQEZGYlCBERCQmJQgREYlJCUJERGJSghBphpnVmNmSqFfCniA2s6Fm9mmijieSSF36OQiROJW5+xFhByHS3nQFIdJKZrbWzO4ys4WR10GR5Qea2Ztm9nFkOiSyfD8ze8HMPoq86oZmSDez/47UFnjdzLIi219jZssix3k6pD9TujAlCJHmZTW4xXRp1Lod7j4OuJ9g9Fci7//H3ccAfwDuiyy/D3jb3ccSjGNU99T+SOABdz8c2AZcEll+I3Bk5DhXJOuPE2mMnqQWaYaZlbh7rxjL1wKnufvqyCCHG909x8w2A4PcvSqyfIO7DzCzIiDP3SuijjEUeMPdR0bmfwp0c/dZZvYqUAK8CLzo7iVJ/lNF9qArCJG28UbeN7ZNLBVR72vY3TZ4HkHFw6OBxZHCNCLtRglCpG0ujZouiLz/O7vLT/4b8F7k/ZvAD6G+1nWfxg5qZmnAAe4+n6BoUV9gr6sYkWTSLxKR5mVFjW4LQV3muq6uPczsHwQ/tiZGll0DPGpm1xNUYasb9XQa8LCZXU5wpfBDgmpksaQDT5nZPgSFrX6tEp/S3tQGIdJKkTaIfHffHHYsIsmgW0wiIhKTriBERCQmXUGIiEhMShAiIhKTEoSIiMSkBCEiIjEpQYiISEz/H6XG0znDIZXtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "iterations = 10\n",
    "trainLoss1 = []\n",
    "testAcc1 = []\n",
    "trainLoss2 = []\n",
    "testAcc2 = []\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss1 = 0    \n",
    "    runningLoss2 = 0    \n",
    "    net1.train(True) # For training\n",
    "    net2.train(True) \n",
    "    for data in trainLoaderRot:\n",
    "        inputs,labels = data\n",
    "        # Wrap them in Variable\n",
    "        if use_gpu:\n",
    "            inputs, labels = Variable(inputs.cuda()), \\\n",
    "                Variable(labels.cuda())\n",
    "        else:\n",
    "            inputs, labels = Variable(inputs), Variable(labels)  \n",
    "       \n",
    "        # Initialize gradients to zero\n",
    "        optimizer1.zero_grad()\n",
    "        optimizer2.zero_grad()\n",
    "        # Feed-forward input data through the network        \n",
    "        outputs1 = net1(inputs)        \n",
    "        outputs2 = net2(inputs)        \n",
    "        # Compute loss/error\n",
    "        loss1 = criterion(outputs1, labels)\n",
    "        loss2 = criterion(outputs2, labels)\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss1.backward()\n",
    "        loss2.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer1.step()\n",
    "        optimizer2.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss1 += loss1.data.item()\n",
    "        runningLoss2 += loss2.data.item()\n",
    "    avgTrainLoss1 = runningLoss1/60000.0\n",
    "    avgTrainLoss2 = runningLoss2/60000.0\n",
    "    trainLoss1.append(avgTrainLoss1)\n",
    "    trainLoss2.append(avgTrainLoss2)\n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    with torch.no_grad():\n",
    "        running_correct1 = 0\n",
    "        running_correct2 = 0\n",
    "        for data in testLoaderRot:\n",
    "            inputs,labels = data\n",
    "            # Wrap them in Variable\n",
    "            if use_gpu:\n",
    "                inputs = Variable(inputs.cuda())                \n",
    "                outputs1 = net1(inputs)\n",
    "                outputs2 = net2(inputs)\n",
    "                _, predicted1 = torch.max(outputs1.data, 1)\n",
    "                _, predicted2 = torch.max(outputs2.data, 1)\n",
    "                predicted1 = predicted1.cpu()\n",
    "                predicted2 = predicted2.cpu()\n",
    "            else:\n",
    "                inputs = Variable(inputs)\n",
    "                outputs1 = net1(inputs)\n",
    "                outputs2 = net2(inputs)\n",
    "                _, predicted1 = torch.max(outputs1.data, 1)\n",
    "                _, predicted2 = torch.max(outputs1.data, 1)\n",
    "            running_correct1 += (predicted1 == labels).sum()\n",
    "            running_correct2 += (predicted2 == labels).sum()\n",
    "        avgTestAcc1 = float(running_correct1)/10000.0\n",
    "        avgTestAcc2 = float(running_correct2)/10000.0\n",
    "        testAcc1.append(avgTestAcc1)\n",
    "        testAcc2.append(avgTestAcc2)\n",
    "        \n",
    "    # Plotting training loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss1,'r-',label='LeNet-5')        \n",
    "    plt.plot(range(epoch+1),trainLoss2,'g-',label='MNISTNet')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),testAcc1,'r-',label='LeNet-5')        \n",
    "    plt.plot(range(epoch+1),testAcc2,'g-',label='MNISTNet')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Testing accuracy')    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f}  ; LeNet:  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss1,avgTestAcc1*100,epochEnd//60,epochEnd%60))\n",
    "    print('Iteration: {:.0f} /{:.0f}  ; MNISTNet:  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss2,avgTestAcc2*100,epochEnd//60,epochEnd%60))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}