{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Для корректной работы ноутбука в той же директории должен находиться файл utils.py!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "from utils import mnist, plot_graphs\n",
    "import numpy as np\n",
    "\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Загрузка данных\n",
    "\n",
    "Функция `mnist` из файла `utils.py` подготавливает и загружает данные\n",
    "\n",
    "`none_lambda` - инициализация весов \"по умолчанию\", функция, которая всегда возвращает `None`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader, valid_loader, test_loader = mnist(valid=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "none_lambda = lambda _: None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Нейросеть\n",
    "\n",
    "В `__init__` создаются и инициализируются слои, в `forward` слои \"соединяются\" правильном порядке."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, batchnorm=False, dropout=False, lr=1e-4, l2=0., init_fn=none_lambda, optimizer='Adam'):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(28*28, 128)\n",
    "        init_fn(self.fc1.weight)\n",
    "        self.fc2 = nn.Linear(128, 128)\n",
    "        init_fn(self.fc2.weight)\n",
    "        self.fc3 = nn.Linear(128, 128)\n",
    "        init_fn(self.fc3.weight)\n",
    "        self.fc4 = nn.Linear(128, 10)\n",
    "        init_fn(self.fc4.weight)\n",
    "        if batchnorm:\n",
    "            self.bn = nn.BatchNorm1d(128)\n",
    "        self.batchnorm = batchnorm\n",
    "        \n",
    "        self.dropout = dropout\n",
    "        if optimizer == 'Adam':\n",
    "            self.optim = optim.Adam(self.parameters(), lr=lr, weight_decay=l2)\n",
    "        elif optimizer == 'Adadelta':\n",
    "            self.optim = optim.Adadelta(self.parameters(), lr=lr, weight_decay=l2)\n",
    "        else:\n",
    "            self.optim = optim.SGD(self.parameters(), lr=lr, weight_decay=l2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 28*28)\n",
    "        x = torch.tanh(self.fc1(x))\n",
    "        if self.batchnorm:\n",
    "            x = self.bn(x)\n",
    "        x = torch.tanh(self.fc2(x))\n",
    "        if self.dropout:\n",
    "            x = F.dropout(x, 0.5)\n",
    "        x = torch.tanh(self.fc3(x))\n",
    "        x = self.fc4(x)\n",
    "        x = F.log_softmax(x, dim=1)\n",
    "        return x\n",
    "    \n",
    "    def loss(self, output, target, **kwargs):\n",
    "        self._loss = F.nll_loss(output, target, **kwargs)\n",
    "        self._correct = output.data.max(1, keepdim=True)[1]\n",
    "        self._correct = self._correct.eq(target.data.view_as(self._correct)).to(torch.float).cpu().mean()\n",
    "        return self._loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Создание двух моделей с разными инициализаторами"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {'default': Net(False, False, optimizer='SGD'), \n",
    "          'xavier': Net(False, False, init_fn=nn.init.xavier_uniform_, optimizer='SGD')}\n",
    "train_log = {k: [] for k in models}\n",
    "test_log = {k: [] for k in models}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Функции для обучения и тестирования моделей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, models, log=None):\n",
    "    train_size = len(train_loader.sampler)\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        for model in models.values():\n",
    "            model.optim.zero_grad()\n",
    "            output = model(data)\n",
    "            loss = model.loss(output, target)\n",
    "            loss.backward()\n",
    "            model.optim.step()\n",
    "            \n",
    "        if batch_idx % 200 == 0:\n",
    "            line = 'Train Epoch: {} [{}/{} ({:.0f}%)]\\tLosses '.format(\n",
    "                epoch, batch_idx * len(data), train_size, 100. * batch_idx / len(train_loader))\n",
    "            losses = ' '.join(['{}: {:.6f}'.format(k, m._loss.item()) for k, m in models.items()])\n",
    "            print(line + losses)\n",
    "            \n",
    "    else:\n",
    "        batch_idx += 1\n",
    "        line = 'Train Epoch: {} [{}/{} ({:.0f}%)]\\tLosses '.format(\n",
    "            epoch, batch_idx * len(data), train_size, 100. * batch_idx / len(train_loader))\n",
    "        losses = ' '.join(['{}: {:.6f}'.format(k, m._loss.item()) for k, m in models.items()])\n",
    "        if log is not None:\n",
    "            for k in models:\n",
    "                log[k].append((models[k]._loss, models[k]._correct))\n",
    "        print(line + losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(models, loader, log=None):\n",
    "    test_size = len(loader.sampler)\n",
    "    avg_lambda = lambda l: 'Loss: {:.4f}'.format(l)\n",
    "    acc_lambda = lambda c, p: 'Accuracy: {}/{} ({:.0f}%)'.format(c, test_size, p)\n",
    "    line = lambda i, l, c, p: '{}: '.format(i) + avg_lambda(l) + '\\t' + acc_lambda(c, p)\n",
    "\n",
    "    test_loss = {k: 0. for k in models}\n",
    "    correct = {k: 0. for k in models}\n",
    "    with torch.no_grad():\n",
    "        for data, target in loader:\n",
    "            output = {k: m(data) for k, m in models.items()}\n",
    "            for k, m in models.items():\n",
    "                test_loss[k] += m.loss(output[k], target, reduction='sum').item() # sum up batch loss\n",
    "                pred = output[k].data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "                correct[k] += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "    \n",
    "    for k in models:\n",
    "        test_loss[k] /= test_size\n",
    "    correct_pct = {k: c.to(torch.float) / test_size for k, c in correct.items()}\n",
    "    lines = '\\n'.join([line(k, test_loss[k], correct[k], 100*correct_pct[k]) for k in models]) + '\\n'\n",
    "    report = 'Test set:\\n' + lines\n",
    "    if log is not None:\n",
    "        for k in models:\n",
    "            log[k].append((test_loss[k], correct_pct[k]))\n",
    "    print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Цикл обучения и валидации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/50000 (0%)]\tLosses default: 2.308198 xavier: 2.359339\n",
      "Train Epoch: 1 [10000/50000 (20%)]\tLosses default: 2.269763 xavier: 2.334126\n",
      "Train Epoch: 1 [20000/50000 (40%)]\tLosses default: 2.283448 xavier: 2.212579\n",
      "Train Epoch: 1 [30000/50000 (60%)]\tLosses default: 2.286848 xavier: 1.991649\n",
      "Train Epoch: 1 [40000/50000 (80%)]\tLosses default: 2.246888 xavier: 2.134698\n",
      "Train Epoch: 1 [50000/50000 (100%)]\tLosses default: 2.285596 xavier: 2.142307\n",
      "Test set:\n",
      "default: Loss: 2.2711\tAccuracy: 1475/10000 (15%)\n",
      "xavier: Loss: 2.0542\tAccuracy: 3078/10000 (31%)\n",
      "\n",
      "Train Epoch: 2 [0/50000 (0%)]\tLosses default: 2.272349 xavier: 2.004954\n",
      "Train Epoch: 2 [10000/50000 (20%)]\tLosses default: 2.276998 xavier: 2.036245\n",
      "Train Epoch: 2 [20000/50000 (40%)]\tLosses default: 2.282175 xavier: 1.953522\n",
      "Train Epoch: 2 [30000/50000 (60%)]\tLosses default: 2.267921 xavier: 1.955431\n",
      "Train Epoch: 2 [40000/50000 (80%)]\tLosses default: 2.232016 xavier: 1.881520\n",
      "Train Epoch: 2 [50000/50000 (100%)]\tLosses default: 2.217940 xavier: 1.618863\n",
      "Test set:\n",
      "default: Loss: 2.2370\tAccuracy: 2581/10000 (26%)\n",
      "xavier: Loss: 1.7947\tAccuracy: 4769/10000 (48%)\n",
      "\n",
      "Train Epoch: 3 [0/50000 (0%)]\tLosses default: 2.230115 xavier: 1.805197\n",
      "Train Epoch: 3 [10000/50000 (20%)]\tLosses default: 2.234115 xavier: 1.770892\n",
      "Train Epoch: 3 [20000/50000 (40%)]\tLosses default: 2.191661 xavier: 1.558681\n",
      "Train Epoch: 3 [30000/50000 (60%)]\tLosses default: 2.245537 xavier: 1.767478\n",
      "Train Epoch: 3 [40000/50000 (80%)]\tLosses default: 2.215444 xavier: 1.686975\n",
      "Train Epoch: 3 [50000/50000 (100%)]\tLosses default: 2.227696 xavier: 1.734049\n",
      "Test set:\n",
      "default: Loss: 2.2022\tAccuracy: 3668/10000 (37%)\n",
      "xavier: Loss: 1.5972\tAccuracy: 5737/10000 (57%)\n",
      "\n",
      "Train Epoch: 4 [0/50000 (0%)]\tLosses default: 2.220744 xavier: 1.758658\n",
      "Train Epoch: 4 [10000/50000 (20%)]\tLosses default: 2.184612 xavier: 1.525352\n",
      "Train Epoch: 4 [20000/50000 (40%)]\tLosses default: 2.181252 xavier: 1.571940\n",
      "Train Epoch: 4 [30000/50000 (60%)]\tLosses default: 2.203310 xavier: 1.608736\n",
      "Train Epoch: 4 [40000/50000 (80%)]\tLosses default: 2.148704 xavier: 1.339273\n",
      "Train Epoch: 4 [50000/50000 (100%)]\tLosses default: 2.208527 xavier: 1.630437\n",
      "Test set:\n",
      "default: Loss: 2.1663\tAccuracy: 4458/10000 (45%)\n",
      "xavier: Loss: 1.4400\tAccuracy: 6307/10000 (63%)\n",
      "\n",
      "Train Epoch: 5 [0/50000 (0%)]\tLosses default: 2.190596 xavier: 1.347486\n",
      "Train Epoch: 5 [10000/50000 (20%)]\tLosses default: 2.141085 xavier: 1.369738\n",
      "Train Epoch: 5 [20000/50000 (40%)]\tLosses default: 2.137255 xavier: 1.448626\n",
      "Train Epoch: 5 [30000/50000 (60%)]\tLosses default: 2.130921 xavier: 1.281674\n",
      "Train Epoch: 5 [40000/50000 (80%)]\tLosses default: 2.121513 xavier: 1.309246\n",
      "Train Epoch: 5 [50000/50000 (100%)]\tLosses default: 2.113038 xavier: 1.343189\n",
      "Test set:\n",
      "default: Loss: 2.1287\tAccuracy: 5054/10000 (51%)\n",
      "xavier: Loss: 1.3115\tAccuracy: 6741/10000 (67%)\n",
      "\n",
      "Train Epoch: 6 [0/50000 (0%)]\tLosses default: 2.109518 xavier: 1.287516\n",
      "Train Epoch: 6 [10000/50000 (20%)]\tLosses default: 2.099296 xavier: 1.190594\n",
      "Train Epoch: 6 [20000/50000 (40%)]\tLosses default: 2.119692 xavier: 1.413960\n",
      "Train Epoch: 6 [30000/50000 (60%)]\tLosses default: 2.084168 xavier: 1.216181\n",
      "Train Epoch: 6 [40000/50000 (80%)]\tLosses default: 2.056017 xavier: 1.065279\n",
      "Train Epoch: 6 [50000/50000 (100%)]\tLosses default: 2.077066 xavier: 1.247178\n",
      "Test set:\n",
      "default: Loss: 2.0891\tAccuracy: 5526/10000 (55%)\n",
      "xavier: Loss: 1.2049\tAccuracy: 7077/10000 (71%)\n",
      "\n",
      "Train Epoch: 7 [0/50000 (0%)]\tLosses default: 2.105590 xavier: 1.382847\n",
      "Train Epoch: 7 [10000/50000 (20%)]\tLosses default: 2.072440 xavier: 1.144631\n",
      "Train Epoch: 7 [20000/50000 (40%)]\tLosses default: 2.028756 xavier: 1.063888\n",
      "Train Epoch: 7 [30000/50000 (60%)]\tLosses default: 2.061166 xavier: 1.097110\n",
      "Train Epoch: 7 [40000/50000 (80%)]\tLosses default: 2.085145 xavier: 1.146988\n",
      "Train Epoch: 7 [50000/50000 (100%)]\tLosses default: 2.049276 xavier: 1.044615\n",
      "Test set:\n",
      "default: Loss: 2.0473\tAccuracy: 5869/10000 (59%)\n",
      "xavier: Loss: 1.1157\tAccuracy: 7348/10000 (73%)\n",
      "\n",
      "Train Epoch: 8 [0/50000 (0%)]\tLosses default: 2.024486 xavier: 1.021290\n",
      "Train Epoch: 8 [10000/50000 (20%)]\tLosses default: 2.051822 xavier: 1.207487\n",
      "Train Epoch: 8 [20000/50000 (40%)]\tLosses default: 2.057399 xavier: 1.065828\n",
      "Train Epoch: 8 [30000/50000 (60%)]\tLosses default: 1.977026 xavier: 0.918769\n",
      "Train Epoch: 8 [40000/50000 (80%)]\tLosses default: 1.954535 xavier: 0.855643\n",
      "Train Epoch: 8 [50000/50000 (100%)]\tLosses default: 1.983083 xavier: 0.984449\n",
      "Test set:\n",
      "default: Loss: 2.0031\tAccuracy: 6127/10000 (61%)\n",
      "xavier: Loss: 1.0404\tAccuracy: 7566/10000 (76%)\n",
      "\n",
      "Train Epoch: 9 [0/50000 (0%)]\tLosses default: 1.991289 xavier: 0.991182\n",
      "Train Epoch: 9 [10000/50000 (20%)]\tLosses default: 2.000386 xavier: 0.955845\n",
      "Train Epoch: 9 [20000/50000 (40%)]\tLosses default: 1.999967 xavier: 1.055188\n",
      "Train Epoch: 9 [30000/50000 (60%)]\tLosses default: 1.972834 xavier: 0.978958\n",
      "Train Epoch: 9 [40000/50000 (80%)]\tLosses default: 2.017722 xavier: 1.084175\n",
      "Train Epoch: 9 [50000/50000 (100%)]\tLosses default: 1.976354 xavier: 1.099072\n",
      "Test set:\n",
      "default: Loss: 1.9565\tAccuracy: 6275/10000 (63%)\n",
      "xavier: Loss: 0.9764\tAccuracy: 7728/10000 (77%)\n",
      "\n",
      "Train Epoch: 10 [0/50000 (0%)]\tLosses default: 1.953757 xavier: 1.036196\n",
      "Train Epoch: 10 [10000/50000 (20%)]\tLosses default: 1.989472 xavier: 0.969552\n",
      "Train Epoch: 10 [20000/50000 (40%)]\tLosses default: 1.927819 xavier: 0.919067\n",
      "Train Epoch: 10 [30000/50000 (60%)]\tLosses default: 1.890491 xavier: 0.909108\n",
      "Train Epoch: 10 [40000/50000 (80%)]\tLosses default: 1.929399 xavier: 0.932403\n",
      "Train Epoch: 10 [50000/50000 (100%)]\tLosses default: 1.981598 xavier: 1.143105\n",
      "Test set:\n",
      "default: Loss: 1.9079\tAccuracy: 6383/10000 (64%)\n",
      "xavier: Loss: 0.9217\tAccuracy: 7870/10000 (79%)\n",
      "\n",
      "Train Epoch: 11 [0/50000 (0%)]\tLosses default: 1.914196 xavier: 0.894735\n",
      "Train Epoch: 11 [10000/50000 (20%)]\tLosses default: 1.866627 xavier: 0.802822\n",
      "Train Epoch: 11 [20000/50000 (40%)]\tLosses default: 1.905914 xavier: 0.843604\n",
      "Train Epoch: 11 [30000/50000 (60%)]\tLosses default: 1.897716 xavier: 0.999853\n",
      "Train Epoch: 11 [40000/50000 (80%)]\tLosses default: 1.809325 xavier: 0.676988\n",
      "Train Epoch: 11 [50000/50000 (100%)]\tLosses default: 1.823944 xavier: 0.768598\n",
      "Test set:\n",
      "default: Loss: 1.8575\tAccuracy: 6449/10000 (64%)\n",
      "xavier: Loss: 0.8745\tAccuracy: 7968/10000 (80%)\n",
      "\n",
      "Train Epoch: 12 [0/50000 (0%)]\tLosses default: 1.883636 xavier: 0.944345\n",
      "Train Epoch: 12 [10000/50000 (20%)]\tLosses default: 1.814510 xavier: 0.697201\n",
      "Train Epoch: 12 [20000/50000 (40%)]\tLosses default: 1.915798 xavier: 0.858794\n",
      "Train Epoch: 12 [30000/50000 (60%)]\tLosses default: 1.890156 xavier: 1.112192\n",
      "Train Epoch: 12 [40000/50000 (80%)]\tLosses default: 1.756543 xavier: 0.797413\n",
      "Train Epoch: 12 [50000/50000 (100%)]\tLosses default: 1.805571 xavier: 0.881871\n",
      "Test set:\n",
      "default: Loss: 1.8059\tAccuracy: 6494/10000 (65%)\n",
      "xavier: Loss: 0.8336\tAccuracy: 8076/10000 (81%)\n",
      "\n",
      "Train Epoch: 13 [0/50000 (0%)]\tLosses default: 1.789205 xavier: 0.831697\n",
      "Train Epoch: 13 [10000/50000 (20%)]\tLosses default: 1.722010 xavier: 0.748478\n",
      "Train Epoch: 13 [20000/50000 (40%)]\tLosses default: 1.778913 xavier: 0.777628\n",
      "Train Epoch: 13 [30000/50000 (60%)]\tLosses default: 1.822951 xavier: 0.964368\n",
      "Train Epoch: 13 [40000/50000 (80%)]\tLosses default: 1.790506 xavier: 0.874978\n",
      "Train Epoch: 13 [50000/50000 (100%)]\tLosses default: 1.734046 xavier: 0.690588\n",
      "Test set:\n",
      "default: Loss: 1.7534\tAccuracy: 6541/10000 (65%)\n",
      "xavier: Loss: 0.7977\tAccuracy: 8138/10000 (81%)\n",
      "\n",
      "Train Epoch: 14 [0/50000 (0%)]\tLosses default: 1.786486 xavier: 0.866907\n",
      "Train Epoch: 14 [10000/50000 (20%)]\tLosses default: 1.783187 xavier: 0.836311\n",
      "Train Epoch: 14 [20000/50000 (40%)]\tLosses default: 1.718947 xavier: 0.800408\n",
      "Train Epoch: 14 [30000/50000 (60%)]\tLosses default: 1.745717 xavier: 0.878432\n",
      "Train Epoch: 14 [40000/50000 (80%)]\tLosses default: 1.742509 xavier: 0.881847\n",
      "Train Epoch: 14 [50000/50000 (100%)]\tLosses default: 1.713971 xavier: 0.815643\n",
      "Test set:\n",
      "default: Loss: 1.7008\tAccuracy: 6587/10000 (66%)\n",
      "xavier: Loss: 0.7661\tAccuracy: 8197/10000 (82%)\n",
      "\n",
      "Train Epoch: 15 [0/50000 (0%)]\tLosses default: 1.696162 xavier: 0.656596\n",
      "Train Epoch: 15 [10000/50000 (20%)]\tLosses default: 1.728340 xavier: 0.834279\n",
      "Train Epoch: 15 [20000/50000 (40%)]\tLosses default: 1.763821 xavier: 0.764397\n",
      "Train Epoch: 15 [30000/50000 (60%)]\tLosses default: 1.620346 xavier: 0.639710\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 15 [40000/50000 (80%)]\tLosses default: 1.621439 xavier: 0.713122\n",
      "Train Epoch: 15 [50000/50000 (100%)]\tLosses default: 1.664808 xavier: 0.801988\n",
      "Test set:\n",
      "default: Loss: 1.6485\tAccuracy: 6628/10000 (66%)\n",
      "xavier: Loss: 0.7380\tAccuracy: 8255/10000 (83%)\n",
      "\n",
      "Train Epoch: 16 [0/50000 (0%)]\tLosses default: 1.607511 xavier: 0.735784\n",
      "Train Epoch: 16 [10000/50000 (20%)]\tLosses default: 1.735733 xavier: 0.723881\n",
      "Train Epoch: 16 [20000/50000 (40%)]\tLosses default: 1.479133 xavier: 0.606842\n",
      "Train Epoch: 16 [30000/50000 (60%)]\tLosses default: 1.586823 xavier: 0.698413\n",
      "Train Epoch: 16 [40000/50000 (80%)]\tLosses default: 1.581913 xavier: 0.621877\n",
      "Train Epoch: 16 [50000/50000 (100%)]\tLosses default: 1.598461 xavier: 0.606158\n",
      "Test set:\n",
      "default: Loss: 1.5971\tAccuracy: 6692/10000 (67%)\n",
      "xavier: Loss: 0.7130\tAccuracy: 8311/10000 (83%)\n",
      "\n",
      "Train Epoch: 17 [0/50000 (0%)]\tLosses default: 1.588323 xavier: 0.677094\n",
      "Train Epoch: 17 [10000/50000 (20%)]\tLosses default: 1.640998 xavier: 0.706049\n",
      "Train Epoch: 17 [20000/50000 (40%)]\tLosses default: 1.618871 xavier: 0.742330\n",
      "Train Epoch: 17 [30000/50000 (60%)]\tLosses default: 1.504384 xavier: 0.495411\n",
      "Train Epoch: 17 [40000/50000 (80%)]\tLosses default: 1.501153 xavier: 0.678155\n",
      "Train Epoch: 17 [50000/50000 (100%)]\tLosses default: 1.490343 xavier: 0.542719\n",
      "Test set:\n",
      "default: Loss: 1.5469\tAccuracy: 6745/10000 (67%)\n",
      "xavier: Loss: 0.6904\tAccuracy: 8366/10000 (84%)\n",
      "\n",
      "Train Epoch: 18 [0/50000 (0%)]\tLosses default: 1.484296 xavier: 0.471357\n",
      "Train Epoch: 18 [10000/50000 (20%)]\tLosses default: 1.532313 xavier: 0.692453\n",
      "Train Epoch: 18 [20000/50000 (40%)]\tLosses default: 1.518887 xavier: 0.719480\n",
      "Train Epoch: 18 [30000/50000 (60%)]\tLosses default: 1.609391 xavier: 0.828791\n",
      "Train Epoch: 18 [40000/50000 (80%)]\tLosses default: 1.520348 xavier: 0.627923\n",
      "Train Epoch: 18 [50000/50000 (100%)]\tLosses default: 1.459725 xavier: 0.594671\n",
      "Test set:\n",
      "default: Loss: 1.4983\tAccuracy: 6794/10000 (68%)\n",
      "xavier: Loss: 0.6700\tAccuracy: 8405/10000 (84%)\n",
      "\n",
      "Train Epoch: 19 [0/50000 (0%)]\tLosses default: 1.513822 xavier: 0.787923\n",
      "Train Epoch: 19 [10000/50000 (20%)]\tLosses default: 1.425810 xavier: 0.607158\n",
      "Train Epoch: 19 [20000/50000 (40%)]\tLosses default: 1.509578 xavier: 0.697811\n",
      "Train Epoch: 19 [30000/50000 (60%)]\tLosses default: 1.354011 xavier: 0.560932\n",
      "Train Epoch: 19 [40000/50000 (80%)]\tLosses default: 1.494326 xavier: 0.855431\n",
      "Train Epoch: 19 [50000/50000 (100%)]\tLosses default: 1.271168 xavier: 0.488800\n",
      "Test set:\n",
      "default: Loss: 1.4514\tAccuracy: 6854/10000 (69%)\n",
      "xavier: Loss: 0.6515\tAccuracy: 8435/10000 (84%)\n",
      "\n",
      "Train Epoch: 20 [0/50000 (0%)]\tLosses default: 1.519320 xavier: 0.636167\n",
      "Train Epoch: 20 [10000/50000 (20%)]\tLosses default: 1.460136 xavier: 0.632479\n",
      "Train Epoch: 20 [20000/50000 (40%)]\tLosses default: 1.522743 xavier: 0.714946\n",
      "Train Epoch: 20 [30000/50000 (60%)]\tLosses default: 1.408419 xavier: 0.538581\n",
      "Train Epoch: 20 [40000/50000 (80%)]\tLosses default: 1.376678 xavier: 0.538532\n",
      "Train Epoch: 20 [50000/50000 (100%)]\tLosses default: 1.438367 xavier: 0.725316\n",
      "Test set:\n",
      "default: Loss: 1.4064\tAccuracy: 6922/10000 (69%)\n",
      "xavier: Loss: 0.6345\tAccuracy: 8473/10000 (85%)\n",
      "\n",
      "Train Epoch: 21 [0/50000 (0%)]\tLosses default: 1.470420 xavier: 0.719539\n",
      "Train Epoch: 21 [10000/50000 (20%)]\tLosses default: 1.337160 xavier: 0.643387\n",
      "Train Epoch: 21 [20000/50000 (40%)]\tLosses default: 1.385719 xavier: 0.550818\n",
      "Train Epoch: 21 [30000/50000 (60%)]\tLosses default: 1.443855 xavier: 0.650872\n",
      "Train Epoch: 21 [40000/50000 (80%)]\tLosses default: 1.278281 xavier: 0.572450\n",
      "Train Epoch: 21 [50000/50000 (100%)]\tLosses default: 1.361363 xavier: 0.570742\n",
      "Test set:\n",
      "default: Loss: 1.3633\tAccuracy: 6980/10000 (70%)\n",
      "xavier: Loss: 0.6190\tAccuracy: 8509/10000 (85%)\n",
      "\n",
      "Train Epoch: 22 [0/50000 (0%)]\tLosses default: 1.421486 xavier: 0.622498\n",
      "Train Epoch: 22 [10000/50000 (20%)]\tLosses default: 1.320608 xavier: 0.623551\n",
      "Train Epoch: 22 [20000/50000 (40%)]\tLosses default: 1.338634 xavier: 0.553882\n",
      "Train Epoch: 22 [30000/50000 (60%)]\tLosses default: 1.306868 xavier: 0.578369\n",
      "Train Epoch: 22 [40000/50000 (80%)]\tLosses default: 1.368944 xavier: 0.563601\n",
      "Train Epoch: 22 [50000/50000 (100%)]\tLosses default: 1.243787 xavier: 0.581105\n",
      "Test set:\n",
      "default: Loss: 1.3223\tAccuracy: 7042/10000 (70%)\n",
      "xavier: Loss: 0.6047\tAccuracy: 8541/10000 (85%)\n",
      "\n",
      "Train Epoch: 23 [0/50000 (0%)]\tLosses default: 1.360021 xavier: 0.662707\n",
      "Train Epoch: 23 [10000/50000 (20%)]\tLosses default: 1.359561 xavier: 0.522588\n",
      "Train Epoch: 23 [20000/50000 (40%)]\tLosses default: 1.316914 xavier: 0.677380\n",
      "Train Epoch: 23 [30000/50000 (60%)]\tLosses default: 1.173248 xavier: 0.492656\n",
      "Train Epoch: 23 [40000/50000 (80%)]\tLosses default: 1.305602 xavier: 0.632604\n",
      "Train Epoch: 23 [50000/50000 (100%)]\tLosses default: 1.300410 xavier: 0.613418\n",
      "Test set:\n",
      "default: Loss: 1.2832\tAccuracy: 7105/10000 (71%)\n",
      "xavier: Loss: 0.5915\tAccuracy: 8565/10000 (86%)\n",
      "\n",
      "Train Epoch: 24 [0/50000 (0%)]\tLosses default: 1.343706 xavier: 0.655591\n",
      "Train Epoch: 24 [10000/50000 (20%)]\tLosses default: 1.223992 xavier: 0.515607\n",
      "Train Epoch: 24 [20000/50000 (40%)]\tLosses default: 1.146517 xavier: 0.532852\n",
      "Train Epoch: 24 [30000/50000 (60%)]\tLosses default: 1.204984 xavier: 0.449934\n",
      "Train Epoch: 24 [40000/50000 (80%)]\tLosses default: 1.247812 xavier: 0.544455\n",
      "Train Epoch: 24 [50000/50000 (100%)]\tLosses default: 1.428120 xavier: 0.638490\n",
      "Test set:\n",
      "default: Loss: 1.2460\tAccuracy: 7154/10000 (72%)\n",
      "xavier: Loss: 0.5793\tAccuracy: 8579/10000 (86%)\n",
      "\n",
      "Train Epoch: 25 [0/50000 (0%)]\tLosses default: 1.226356 xavier: 0.512786\n",
      "Train Epoch: 25 [10000/50000 (20%)]\tLosses default: 1.229972 xavier: 0.558099\n",
      "Train Epoch: 25 [20000/50000 (40%)]\tLosses default: 1.366020 xavier: 0.682676\n",
      "Train Epoch: 25 [30000/50000 (60%)]\tLosses default: 1.332706 xavier: 0.578485\n",
      "Train Epoch: 25 [40000/50000 (80%)]\tLosses default: 1.101512 xavier: 0.395103\n",
      "Train Epoch: 25 [50000/50000 (100%)]\tLosses default: 1.276278 xavier: 0.589677\n",
      "Test set:\n",
      "default: Loss: 1.2107\tAccuracy: 7204/10000 (72%)\n",
      "xavier: Loss: 0.5679\tAccuracy: 8603/10000 (86%)\n",
      "\n",
      "Train Epoch: 26 [0/50000 (0%)]\tLosses default: 1.309619 xavier: 0.581796\n",
      "Train Epoch: 26 [10000/50000 (20%)]\tLosses default: 1.234185 xavier: 0.493477\n",
      "Train Epoch: 26 [20000/50000 (40%)]\tLosses default: 1.220752 xavier: 0.574325\n",
      "Train Epoch: 26 [30000/50000 (60%)]\tLosses default: 1.191243 xavier: 0.494055\n",
      "Train Epoch: 26 [40000/50000 (80%)]\tLosses default: 1.255639 xavier: 0.616562\n",
      "Train Epoch: 26 [50000/50000 (100%)]\tLosses default: 1.163613 xavier: 0.637998\n",
      "Test set:\n",
      "default: Loss: 1.1772\tAccuracy: 7250/10000 (72%)\n",
      "xavier: Loss: 0.5574\tAccuracy: 8617/10000 (86%)\n",
      "\n",
      "Train Epoch: 27 [0/50000 (0%)]\tLosses default: 1.169971 xavier: 0.503733\n",
      "Train Epoch: 27 [10000/50000 (20%)]\tLosses default: 1.159706 xavier: 0.518460\n",
      "Train Epoch: 27 [20000/50000 (40%)]\tLosses default: 1.045485 xavier: 0.453761\n",
      "Train Epoch: 27 [30000/50000 (60%)]\tLosses default: 0.925796 xavier: 0.364558\n",
      "Train Epoch: 27 [40000/50000 (80%)]\tLosses default: 1.092497 xavier: 0.529174\n",
      "Train Epoch: 27 [50000/50000 (100%)]\tLosses default: 1.116152 xavier: 0.510589\n",
      "Test set:\n",
      "default: Loss: 1.1454\tAccuracy: 7297/10000 (73%)\n",
      "xavier: Loss: 0.5474\tAccuracy: 8632/10000 (86%)\n",
      "\n",
      "Train Epoch: 28 [0/50000 (0%)]\tLosses default: 1.153111 xavier: 0.522359\n",
      "Train Epoch: 28 [10000/50000 (20%)]\tLosses default: 1.220624 xavier: 0.658619\n",
      "Train Epoch: 28 [20000/50000 (40%)]\tLosses default: 1.186273 xavier: 0.544084\n",
      "Train Epoch: 28 [30000/50000 (60%)]\tLosses default: 1.128566 xavier: 0.607573\n",
      "Train Epoch: 28 [40000/50000 (80%)]\tLosses default: 1.156897 xavier: 0.543418\n",
      "Train Epoch: 28 [50000/50000 (100%)]\tLosses default: 0.973109 xavier: 0.398582\n",
      "Test set:\n",
      "default: Loss: 1.1152\tAccuracy: 7350/10000 (74%)\n",
      "xavier: Loss: 0.5382\tAccuracy: 8654/10000 (87%)\n",
      "\n",
      "Train Epoch: 29 [0/50000 (0%)]\tLosses default: 1.133798 xavier: 0.536495\n",
      "Train Epoch: 29 [10000/50000 (20%)]\tLosses default: 1.148080 xavier: 0.592350\n",
      "Train Epoch: 29 [20000/50000 (40%)]\tLosses default: 1.040060 xavier: 0.520766\n",
      "Train Epoch: 29 [30000/50000 (60%)]\tLosses default: 1.126008 xavier: 0.555528\n",
      "Train Epoch: 29 [40000/50000 (80%)]\tLosses default: 1.075368 xavier: 0.662437\n",
      "Train Epoch: 29 [50000/50000 (100%)]\tLosses default: 0.945553 xavier: 0.407387\n",
      "Test set:\n",
      "default: Loss: 1.0865\tAccuracy: 7386/10000 (74%)\n",
      "xavier: Loss: 0.5294\tAccuracy: 8670/10000 (87%)\n",
      "\n",
      "Train Epoch: 30 [0/50000 (0%)]\tLosses default: 1.091842 xavier: 0.567043\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 30 [10000/50000 (20%)]\tLosses default: 1.058274 xavier: 0.535677\n",
      "Train Epoch: 30 [20000/50000 (40%)]\tLosses default: 1.216925 xavier: 0.564750\n",
      "Train Epoch: 30 [30000/50000 (60%)]\tLosses default: 1.112735 xavier: 0.575520\n",
      "Train Epoch: 30 [40000/50000 (80%)]\tLosses default: 1.219639 xavier: 0.763164\n",
      "Train Epoch: 30 [50000/50000 (100%)]\tLosses default: 0.860066 xavier: 0.359044\n",
      "Test set:\n",
      "default: Loss: 1.0593\tAccuracy: 7433/10000 (74%)\n",
      "xavier: Loss: 0.5212\tAccuracy: 8680/10000 (87%)\n",
      "\n",
      "Train Epoch: 31 [0/50000 (0%)]\tLosses default: 1.143005 xavier: 0.506766\n",
      "Train Epoch: 31 [10000/50000 (20%)]\tLosses default: 1.075650 xavier: 0.507864\n",
      "Train Epoch: 31 [20000/50000 (40%)]\tLosses default: 1.035199 xavier: 0.717848\n",
      "Train Epoch: 31 [30000/50000 (60%)]\tLosses default: 0.941410 xavier: 0.422022\n",
      "Train Epoch: 31 [40000/50000 (80%)]\tLosses default: 0.970144 xavier: 0.490640\n",
      "Train Epoch: 31 [50000/50000 (100%)]\tLosses default: 1.049497 xavier: 0.514602\n",
      "Test set:\n",
      "default: Loss: 1.0335\tAccuracy: 7480/10000 (75%)\n",
      "xavier: Loss: 0.5135\tAccuracy: 8695/10000 (87%)\n",
      "\n",
      "Train Epoch: 32 [0/50000 (0%)]\tLosses default: 1.049858 xavier: 0.422066\n",
      "Train Epoch: 32 [10000/50000 (20%)]\tLosses default: 1.042979 xavier: 0.478993\n",
      "Train Epoch: 32 [20000/50000 (40%)]\tLosses default: 0.973104 xavier: 0.518035\n",
      "Train Epoch: 32 [30000/50000 (60%)]\tLosses default: 1.051817 xavier: 0.515892\n",
      "Train Epoch: 32 [40000/50000 (80%)]\tLosses default: 1.204144 xavier: 0.530121\n",
      "Train Epoch: 32 [50000/50000 (100%)]\tLosses default: 0.958119 xavier: 0.550439\n",
      "Test set:\n",
      "default: Loss: 1.0089\tAccuracy: 7511/10000 (75%)\n",
      "xavier: Loss: 0.5062\tAccuracy: 8714/10000 (87%)\n",
      "\n",
      "Train Epoch: 33 [0/50000 (0%)]\tLosses default: 0.894240 xavier: 0.419226\n",
      "Train Epoch: 33 [10000/50000 (20%)]\tLosses default: 1.039316 xavier: 0.508564\n",
      "Train Epoch: 33 [20000/50000 (40%)]\tLosses default: 0.835896 xavier: 0.306168\n",
      "Train Epoch: 33 [30000/50000 (60%)]\tLosses default: 1.016217 xavier: 0.521053\n",
      "Train Epoch: 33 [40000/50000 (80%)]\tLosses default: 0.833286 xavier: 0.366204\n",
      "Train Epoch: 33 [50000/50000 (100%)]\tLosses default: 1.006885 xavier: 0.472203\n",
      "Test set:\n",
      "default: Loss: 0.9855\tAccuracy: 7559/10000 (76%)\n",
      "xavier: Loss: 0.4993\tAccuracy: 8721/10000 (87%)\n",
      "\n",
      "Train Epoch: 34 [0/50000 (0%)]\tLosses default: 0.942888 xavier: 0.511114\n",
      "Train Epoch: 34 [10000/50000 (20%)]\tLosses default: 0.802248 xavier: 0.285436\n",
      "Train Epoch: 34 [20000/50000 (40%)]\tLosses default: 0.949197 xavier: 0.442481\n",
      "Train Epoch: 34 [30000/50000 (60%)]\tLosses default: 1.091229 xavier: 0.608299\n",
      "Train Epoch: 34 [40000/50000 (80%)]\tLosses default: 1.058532 xavier: 0.594714\n",
      "Train Epoch: 34 [50000/50000 (100%)]\tLosses default: 0.787882 xavier: 0.411753\n",
      "Test set:\n",
      "default: Loss: 0.9632\tAccuracy: 7608/10000 (76%)\n",
      "xavier: Loss: 0.4927\tAccuracy: 8740/10000 (87%)\n",
      "\n",
      "Train Epoch: 35 [0/50000 (0%)]\tLosses default: 1.032136 xavier: 0.679917\n",
      "Train Epoch: 35 [10000/50000 (20%)]\tLosses default: 0.860907 xavier: 0.332089\n",
      "Train Epoch: 35 [20000/50000 (40%)]\tLosses default: 0.847220 xavier: 0.309955\n",
      "Train Epoch: 35 [30000/50000 (60%)]\tLosses default: 0.965925 xavier: 0.541766\n",
      "Train Epoch: 35 [40000/50000 (80%)]\tLosses default: 1.006724 xavier: 0.436503\n",
      "Train Epoch: 35 [50000/50000 (100%)]\tLosses default: 0.881142 xavier: 0.418278\n",
      "Test set:\n",
      "default: Loss: 0.9420\tAccuracy: 7656/10000 (77%)\n",
      "xavier: Loss: 0.4865\tAccuracy: 8755/10000 (88%)\n",
      "\n",
      "Train Epoch: 36 [0/50000 (0%)]\tLosses default: 0.999640 xavier: 0.552868\n",
      "Train Epoch: 36 [10000/50000 (20%)]\tLosses default: 0.829470 xavier: 0.414761\n",
      "Train Epoch: 36 [20000/50000 (40%)]\tLosses default: 0.809161 xavier: 0.307030\n",
      "Train Epoch: 36 [30000/50000 (60%)]\tLosses default: 0.971936 xavier: 0.573362\n",
      "Train Epoch: 36 [40000/50000 (80%)]\tLosses default: 0.887331 xavier: 0.374184\n",
      "Train Epoch: 36 [50000/50000 (100%)]\tLosses default: 0.790041 xavier: 0.367310\n",
      "Test set:\n",
      "default: Loss: 0.9218\tAccuracy: 7696/10000 (77%)\n",
      "xavier: Loss: 0.4806\tAccuracy: 8768/10000 (88%)\n",
      "\n",
      "Train Epoch: 37 [0/50000 (0%)]\tLosses default: 1.000053 xavier: 0.482821\n",
      "Train Epoch: 37 [10000/50000 (20%)]\tLosses default: 0.909960 xavier: 0.482907\n",
      "Train Epoch: 37 [20000/50000 (40%)]\tLosses default: 0.859873 xavier: 0.401655\n",
      "Train Epoch: 37 [30000/50000 (60%)]\tLosses default: 0.948488 xavier: 0.570636\n",
      "Train Epoch: 37 [40000/50000 (80%)]\tLosses default: 1.002955 xavier: 0.558936\n",
      "Train Epoch: 37 [50000/50000 (100%)]\tLosses default: 0.875414 xavier: 0.353646\n",
      "Test set:\n",
      "default: Loss: 0.9025\tAccuracy: 7740/10000 (77%)\n",
      "xavier: Loss: 0.4749\tAccuracy: 8778/10000 (88%)\n",
      "\n",
      "Train Epoch: 38 [0/50000 (0%)]\tLosses default: 0.809358 xavier: 0.401847\n",
      "Train Epoch: 38 [10000/50000 (20%)]\tLosses default: 0.920216 xavier: 0.502372\n",
      "Train Epoch: 38 [20000/50000 (40%)]\tLosses default: 0.892052 xavier: 0.528155\n",
      "Train Epoch: 38 [30000/50000 (60%)]\tLosses default: 0.998221 xavier: 0.645291\n",
      "Train Epoch: 38 [40000/50000 (80%)]\tLosses default: 0.875992 xavier: 0.535234\n",
      "Train Epoch: 38 [50000/50000 (100%)]\tLosses default: 0.900056 xavier: 0.542546\n",
      "Test set:\n",
      "default: Loss: 0.8840\tAccuracy: 7779/10000 (78%)\n",
      "xavier: Loss: 0.4695\tAccuracy: 8779/10000 (88%)\n",
      "\n",
      "Train Epoch: 39 [0/50000 (0%)]\tLosses default: 0.839767 xavier: 0.413869\n",
      "Train Epoch: 39 [10000/50000 (20%)]\tLosses default: 1.009794 xavier: 0.648740\n",
      "Train Epoch: 39 [20000/50000 (40%)]\tLosses default: 0.776866 xavier: 0.410297\n",
      "Train Epoch: 39 [30000/50000 (60%)]\tLosses default: 0.892549 xavier: 0.439017\n",
      "Train Epoch: 39 [40000/50000 (80%)]\tLosses default: 0.792735 xavier: 0.374195\n",
      "Train Epoch: 39 [50000/50000 (100%)]\tLosses default: 0.795997 xavier: 0.451552\n",
      "Test set:\n",
      "default: Loss: 0.8664\tAccuracy: 7817/10000 (78%)\n",
      "xavier: Loss: 0.4644\tAccuracy: 8791/10000 (88%)\n",
      "\n",
      "Train Epoch: 40 [0/50000 (0%)]\tLosses default: 0.802783 xavier: 0.381919\n",
      "Train Epoch: 40 [10000/50000 (20%)]\tLosses default: 0.776749 xavier: 0.402787\n",
      "Train Epoch: 40 [20000/50000 (40%)]\tLosses default: 0.928325 xavier: 0.619582\n",
      "Train Epoch: 40 [30000/50000 (60%)]\tLosses default: 1.002256 xavier: 0.582983\n",
      "Train Epoch: 40 [40000/50000 (80%)]\tLosses default: 0.775714 xavier: 0.356575\n",
      "Train Epoch: 40 [50000/50000 (100%)]\tLosses default: 0.700547 xavier: 0.351129\n",
      "Test set:\n",
      "default: Loss: 0.8496\tAccuracy: 7844/10000 (78%)\n",
      "xavier: Loss: 0.4595\tAccuracy: 8801/10000 (88%)\n",
      "\n",
      "Train Epoch: 41 [0/50000 (0%)]\tLosses default: 0.694316 xavier: 0.323144\n",
      "Train Epoch: 41 [10000/50000 (20%)]\tLosses default: 0.705427 xavier: 0.218846\n",
      "Train Epoch: 41 [20000/50000 (40%)]\tLosses default: 0.892259 xavier: 0.505690\n",
      "Train Epoch: 41 [30000/50000 (60%)]\tLosses default: 0.908549 xavier: 0.607913\n",
      "Train Epoch: 41 [40000/50000 (80%)]\tLosses default: 0.782990 xavier: 0.344345\n",
      "Train Epoch: 41 [50000/50000 (100%)]\tLosses default: 0.783862 xavier: 0.365211\n",
      "Test set:\n",
      "default: Loss: 0.8335\tAccuracy: 7889/10000 (79%)\n",
      "xavier: Loss: 0.4548\tAccuracy: 8808/10000 (88%)\n",
      "\n",
      "Train Epoch: 42 [0/50000 (0%)]\tLosses default: 0.722604 xavier: 0.347355\n",
      "Train Epoch: 42 [10000/50000 (20%)]\tLosses default: 0.922273 xavier: 0.597900\n",
      "Train Epoch: 42 [20000/50000 (40%)]\tLosses default: 0.792091 xavier: 0.540297\n",
      "Train Epoch: 42 [30000/50000 (60%)]\tLosses default: 0.798603 xavier: 0.432657\n",
      "Train Epoch: 42 [40000/50000 (80%)]\tLosses default: 0.777111 xavier: 0.382907\n",
      "Train Epoch: 42 [50000/50000 (100%)]\tLosses default: 0.979980 xavier: 0.500992\n",
      "Test set:\n",
      "default: Loss: 0.8181\tAccuracy: 7937/10000 (79%)\n",
      "xavier: Loss: 0.4502\tAccuracy: 8815/10000 (88%)\n",
      "\n",
      "Train Epoch: 43 [0/50000 (0%)]\tLosses default: 0.795834 xavier: 0.344492\n",
      "Train Epoch: 43 [10000/50000 (20%)]\tLosses default: 0.874953 xavier: 0.508322\n",
      "Train Epoch: 43 [20000/50000 (40%)]\tLosses default: 0.800450 xavier: 0.398120\n",
      "Train Epoch: 43 [30000/50000 (60%)]\tLosses default: 0.787101 xavier: 0.462767\n",
      "Train Epoch: 43 [40000/50000 (80%)]\tLosses default: 0.695647 xavier: 0.368684\n",
      "Train Epoch: 43 [50000/50000 (100%)]\tLosses default: 0.732787 xavier: 0.329114\n",
      "Test set:\n",
      "default: Loss: 0.8033\tAccuracy: 7974/10000 (80%)\n",
      "xavier: Loss: 0.4459\tAccuracy: 8822/10000 (88%)\n",
      "\n",
      "Train Epoch: 44 [0/50000 (0%)]\tLosses default: 0.814233 xavier: 0.474390\n",
      "Train Epoch: 44 [10000/50000 (20%)]\tLosses default: 0.673470 xavier: 0.312942\n",
      "Train Epoch: 44 [20000/50000 (40%)]\tLosses default: 1.094018 xavier: 0.705731\n",
      "Train Epoch: 44 [30000/50000 (60%)]\tLosses default: 1.010444 xavier: 0.708497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 44 [40000/50000 (80%)]\tLosses default: 0.688409 xavier: 0.401385\n",
      "Train Epoch: 44 [50000/50000 (100%)]\tLosses default: 0.703875 xavier: 0.343019\n",
      "Test set:\n",
      "default: Loss: 0.7891\tAccuracy: 8012/10000 (80%)\n",
      "xavier: Loss: 0.4418\tAccuracy: 8833/10000 (88%)\n",
      "\n",
      "Train Epoch: 45 [0/50000 (0%)]\tLosses default: 0.986332 xavier: 0.578763\n",
      "Train Epoch: 45 [10000/50000 (20%)]\tLosses default: 0.912409 xavier: 0.485543\n",
      "Train Epoch: 45 [20000/50000 (40%)]\tLosses default: 0.609329 xavier: 0.218129\n",
      "Train Epoch: 45 [30000/50000 (60%)]\tLosses default: 0.690555 xavier: 0.432169\n",
      "Train Epoch: 45 [40000/50000 (80%)]\tLosses default: 0.722822 xavier: 0.306025\n",
      "Train Epoch: 45 [50000/50000 (100%)]\tLosses default: 0.817873 xavier: 0.477394\n",
      "Test set:\n",
      "default: Loss: 0.7756\tAccuracy: 8050/10000 (80%)\n",
      "xavier: Loss: 0.4378\tAccuracy: 8843/10000 (88%)\n",
      "\n",
      "Train Epoch: 46 [0/50000 (0%)]\tLosses default: 0.722978 xavier: 0.491779\n",
      "Train Epoch: 46 [10000/50000 (20%)]\tLosses default: 0.969026 xavier: 0.745216\n",
      "Train Epoch: 46 [20000/50000 (40%)]\tLosses default: 0.708737 xavier: 0.421392\n",
      "Train Epoch: 46 [30000/50000 (60%)]\tLosses default: 0.545211 xavier: 0.269022\n",
      "Train Epoch: 46 [40000/50000 (80%)]\tLosses default: 0.748636 xavier: 0.408645\n",
      "Train Epoch: 46 [50000/50000 (100%)]\tLosses default: 0.731946 xavier: 0.365285\n",
      "Test set:\n",
      "default: Loss: 0.7625\tAccuracy: 8080/10000 (81%)\n",
      "xavier: Loss: 0.4339\tAccuracy: 8850/10000 (88%)\n",
      "\n",
      "Train Epoch: 47 [0/50000 (0%)]\tLosses default: 0.647747 xavier: 0.300241\n",
      "Train Epoch: 47 [10000/50000 (20%)]\tLosses default: 0.620145 xavier: 0.315600\n",
      "Train Epoch: 47 [20000/50000 (40%)]\tLosses default: 0.773230 xavier: 0.354486\n",
      "Train Epoch: 47 [30000/50000 (60%)]\tLosses default: 0.888035 xavier: 0.547301\n",
      "Train Epoch: 47 [40000/50000 (80%)]\tLosses default: 0.733804 xavier: 0.418992\n",
      "Train Epoch: 47 [50000/50000 (100%)]\tLosses default: 0.732666 xavier: 0.396462\n",
      "Test set:\n",
      "default: Loss: 0.7500\tAccuracy: 8125/10000 (81%)\n",
      "xavier: Loss: 0.4302\tAccuracy: 8855/10000 (89%)\n",
      "\n",
      "Train Epoch: 48 [0/50000 (0%)]\tLosses default: 0.644109 xavier: 0.436359\n",
      "Train Epoch: 48 [10000/50000 (20%)]\tLosses default: 0.664289 xavier: 0.307291\n",
      "Train Epoch: 48 [20000/50000 (40%)]\tLosses default: 0.765739 xavier: 0.583594\n",
      "Train Epoch: 48 [30000/50000 (60%)]\tLosses default: 0.790718 xavier: 0.545950\n",
      "Train Epoch: 48 [40000/50000 (80%)]\tLosses default: 0.778864 xavier: 0.488887\n",
      "Train Epoch: 48 [50000/50000 (100%)]\tLosses default: 0.749857 xavier: 0.580625\n",
      "Test set:\n",
      "default: Loss: 0.7379\tAccuracy: 8158/10000 (82%)\n",
      "xavier: Loss: 0.4266\tAccuracy: 8861/10000 (89%)\n",
      "\n",
      "Train Epoch: 49 [0/50000 (0%)]\tLosses default: 0.781844 xavier: 0.440016\n",
      "Train Epoch: 49 [10000/50000 (20%)]\tLosses default: 0.722647 xavier: 0.434830\n",
      "Train Epoch: 49 [20000/50000 (40%)]\tLosses default: 0.683906 xavier: 0.336037\n",
      "Train Epoch: 49 [30000/50000 (60%)]\tLosses default: 0.587263 xavier: 0.273123\n",
      "Train Epoch: 49 [40000/50000 (80%)]\tLosses default: 0.777871 xavier: 0.506558\n",
      "Train Epoch: 49 [50000/50000 (100%)]\tLosses default: 0.630984 xavier: 0.298362\n",
      "Test set:\n",
      "default: Loss: 0.7263\tAccuracy: 8190/10000 (82%)\n",
      "xavier: Loss: 0.4232\tAccuracy: 8868/10000 (89%)\n",
      "\n",
      "Train Epoch: 50 [0/50000 (0%)]\tLosses default: 0.648942 xavier: 0.349356\n",
      "Train Epoch: 50 [10000/50000 (20%)]\tLosses default: 0.756484 xavier: 0.446290\n",
      "Train Epoch: 50 [20000/50000 (40%)]\tLosses default: 0.668655 xavier: 0.364747\n",
      "Train Epoch: 50 [30000/50000 (60%)]\tLosses default: 0.715287 xavier: 0.427210\n",
      "Train Epoch: 50 [40000/50000 (80%)]\tLosses default: 0.750012 xavier: 0.446367\n",
      "Train Epoch: 50 [50000/50000 (100%)]\tLosses default: 0.756411 xavier: 0.393721\n",
      "Test set:\n",
      "default: Loss: 0.7152\tAccuracy: 8236/10000 (82%)\n",
      "xavier: Loss: 0.4198\tAccuracy: 8875/10000 (89%)\n",
      "\n",
      "Train Epoch: 51 [0/50000 (0%)]\tLosses default: 0.636301 xavier: 0.393093\n",
      "Train Epoch: 51 [10000/50000 (20%)]\tLosses default: 0.699475 xavier: 0.376958\n",
      "Train Epoch: 51 [20000/50000 (40%)]\tLosses default: 0.541265 xavier: 0.254243\n",
      "Train Epoch: 51 [30000/50000 (60%)]\tLosses default: 0.764341 xavier: 0.485302\n",
      "Train Epoch: 51 [40000/50000 (80%)]\tLosses default: 0.804542 xavier: 0.406825\n",
      "Train Epoch: 51 [50000/50000 (100%)]\tLosses default: 0.664775 xavier: 0.352066\n",
      "Test set:\n",
      "default: Loss: 0.7044\tAccuracy: 8260/10000 (83%)\n",
      "xavier: Loss: 0.4166\tAccuracy: 8882/10000 (89%)\n",
      "\n",
      "Train Epoch: 52 [0/50000 (0%)]\tLosses default: 0.556271 xavier: 0.281463\n",
      "Train Epoch: 52 [10000/50000 (20%)]\tLosses default: 0.563449 xavier: 0.299068\n",
      "Train Epoch: 52 [20000/50000 (40%)]\tLosses default: 0.682086 xavier: 0.415764\n",
      "Train Epoch: 52 [30000/50000 (60%)]\tLosses default: 0.625356 xavier: 0.340223\n",
      "Train Epoch: 52 [40000/50000 (80%)]\tLosses default: 0.906730 xavier: 0.466807\n",
      "Train Epoch: 52 [50000/50000 (100%)]\tLosses default: 0.672191 xavier: 0.381661\n",
      "Test set:\n",
      "default: Loss: 0.6940\tAccuracy: 8287/10000 (83%)\n",
      "xavier: Loss: 0.4135\tAccuracy: 8880/10000 (89%)\n",
      "\n",
      "Train Epoch: 53 [0/50000 (0%)]\tLosses default: 0.674395 xavier: 0.439327\n",
      "Train Epoch: 53 [10000/50000 (20%)]\tLosses default: 0.573235 xavier: 0.304058\n",
      "Train Epoch: 53 [20000/50000 (40%)]\tLosses default: 0.712863 xavier: 0.384901\n",
      "Train Epoch: 53 [30000/50000 (60%)]\tLosses default: 0.744865 xavier: 0.404167\n",
      "Train Epoch: 53 [40000/50000 (80%)]\tLosses default: 0.632195 xavier: 0.293796\n",
      "Train Epoch: 53 [50000/50000 (100%)]\tLosses default: 0.625379 xavier: 0.309752\n",
      "Test set:\n",
      "default: Loss: 0.6840\tAccuracy: 8318/10000 (83%)\n",
      "xavier: Loss: 0.4104\tAccuracy: 8884/10000 (89%)\n",
      "\n",
      "Train Epoch: 54 [0/50000 (0%)]\tLosses default: 0.673166 xavier: 0.421719\n",
      "Train Epoch: 54 [10000/50000 (20%)]\tLosses default: 0.859655 xavier: 0.508474\n",
      "Train Epoch: 54 [20000/50000 (40%)]\tLosses default: 0.758726 xavier: 0.369892\n",
      "Train Epoch: 54 [30000/50000 (60%)]\tLosses default: 0.557507 xavier: 0.323943\n",
      "Train Epoch: 54 [40000/50000 (80%)]\tLosses default: 0.615260 xavier: 0.455923\n",
      "Train Epoch: 54 [50000/50000 (100%)]\tLosses default: 0.740212 xavier: 0.467881\n",
      "Test set:\n",
      "default: Loss: 0.6743\tAccuracy: 8333/10000 (83%)\n",
      "xavier: Loss: 0.4075\tAccuracy: 8894/10000 (89%)\n",
      "\n",
      "Train Epoch: 55 [0/50000 (0%)]\tLosses default: 0.646079 xavier: 0.475704\n",
      "Train Epoch: 55 [10000/50000 (20%)]\tLosses default: 0.626212 xavier: 0.355620\n",
      "Train Epoch: 55 [20000/50000 (40%)]\tLosses default: 0.576564 xavier: 0.315466\n",
      "Train Epoch: 55 [30000/50000 (60%)]\tLosses default: 0.673034 xavier: 0.468892\n",
      "Train Epoch: 55 [40000/50000 (80%)]\tLosses default: 0.647002 xavier: 0.436002\n",
      "Train Epoch: 55 [50000/50000 (100%)]\tLosses default: 0.474939 xavier: 0.194787\n",
      "Test set:\n",
      "default: Loss: 0.6649\tAccuracy: 8360/10000 (84%)\n",
      "xavier: Loss: 0.4047\tAccuracy: 8902/10000 (89%)\n",
      "\n",
      "Train Epoch: 56 [0/50000 (0%)]\tLosses default: 0.728794 xavier: 0.518907\n",
      "Train Epoch: 56 [10000/50000 (20%)]\tLosses default: 0.676407 xavier: 0.381106\n",
      "Train Epoch: 56 [20000/50000 (40%)]\tLosses default: 0.480185 xavier: 0.219090\n",
      "Train Epoch: 56 [30000/50000 (60%)]\tLosses default: 0.753234 xavier: 0.442378\n",
      "Train Epoch: 56 [40000/50000 (80%)]\tLosses default: 0.781191 xavier: 0.435761\n",
      "Train Epoch: 56 [50000/50000 (100%)]\tLosses default: 0.662560 xavier: 0.423438\n",
      "Test set:\n",
      "default: Loss: 0.6559\tAccuracy: 8391/10000 (84%)\n",
      "xavier: Loss: 0.4019\tAccuracy: 8910/10000 (89%)\n",
      "\n",
      "Train Epoch: 57 [0/50000 (0%)]\tLosses default: 0.653973 xavier: 0.358463\n",
      "Train Epoch: 57 [10000/50000 (20%)]\tLosses default: 0.691647 xavier: 0.382304\n",
      "Train Epoch: 57 [20000/50000 (40%)]\tLosses default: 0.604129 xavier: 0.393126\n",
      "Train Epoch: 57 [30000/50000 (60%)]\tLosses default: 0.689838 xavier: 0.336405\n",
      "Train Epoch: 57 [40000/50000 (80%)]\tLosses default: 0.688953 xavier: 0.383425\n",
      "Train Epoch: 57 [50000/50000 (100%)]\tLosses default: 0.520908 xavier: 0.312901\n",
      "Test set:\n",
      "default: Loss: 0.6471\tAccuracy: 8419/10000 (84%)\n",
      "xavier: Loss: 0.3992\tAccuracy: 8913/10000 (89%)\n",
      "\n",
      "Train Epoch: 58 [0/50000 (0%)]\tLosses default: 0.569891 xavier: 0.308453\n",
      "Train Epoch: 58 [10000/50000 (20%)]\tLosses default: 0.527909 xavier: 0.280511\n",
      "Train Epoch: 58 [20000/50000 (40%)]\tLosses default: 0.711948 xavier: 0.413517\n",
      "Train Epoch: 58 [30000/50000 (60%)]\tLosses default: 0.451654 xavier: 0.174822\n",
      "Train Epoch: 58 [40000/50000 (80%)]\tLosses default: 0.679483 xavier: 0.357761\n",
      "Train Epoch: 58 [50000/50000 (100%)]\tLosses default: 0.638967 xavier: 0.314061\n",
      "Test set:\n",
      "default: Loss: 0.6386\tAccuracy: 8446/10000 (84%)\n",
      "xavier: Loss: 0.3966\tAccuracy: 8914/10000 (89%)\n",
      "\n",
      "Train Epoch: 59 [0/50000 (0%)]\tLosses default: 0.480726 xavier: 0.282028\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 59 [10000/50000 (20%)]\tLosses default: 0.521408 xavier: 0.269455\n",
      "Train Epoch: 59 [20000/50000 (40%)]\tLosses default: 0.658976 xavier: 0.477158\n",
      "Train Epoch: 59 [30000/50000 (60%)]\tLosses default: 0.707440 xavier: 0.538772\n",
      "Train Epoch: 59 [40000/50000 (80%)]\tLosses default: 0.635866 xavier: 0.442733\n",
      "Train Epoch: 59 [50000/50000 (100%)]\tLosses default: 0.585085 xavier: 0.337160\n",
      "Test set:\n",
      "default: Loss: 0.6304\tAccuracy: 8461/10000 (85%)\n",
      "xavier: Loss: 0.3941\tAccuracy: 8921/10000 (89%)\n",
      "\n",
      "Train Epoch: 60 [0/50000 (0%)]\tLosses default: 0.560429 xavier: 0.287075\n",
      "Train Epoch: 60 [10000/50000 (20%)]\tLosses default: 0.572374 xavier: 0.351510\n",
      "Train Epoch: 60 [20000/50000 (40%)]\tLosses default: 0.620886 xavier: 0.436384\n",
      "Train Epoch: 60 [30000/50000 (60%)]\tLosses default: 0.717288 xavier: 0.434678\n",
      "Train Epoch: 60 [40000/50000 (80%)]\tLosses default: 0.527004 xavier: 0.274431\n",
      "Train Epoch: 60 [50000/50000 (100%)]\tLosses default: 0.462567 xavier: 0.244820\n",
      "Test set:\n",
      "default: Loss: 0.6224\tAccuracy: 8468/10000 (85%)\n",
      "xavier: Loss: 0.3916\tAccuracy: 8927/10000 (89%)\n",
      "\n",
      "Train Epoch: 61 [0/50000 (0%)]\tLosses default: 0.511173 xavier: 0.269050\n",
      "Train Epoch: 61 [10000/50000 (20%)]\tLosses default: 0.572568 xavier: 0.303366\n",
      "Train Epoch: 61 [20000/50000 (40%)]\tLosses default: 0.675676 xavier: 0.471295\n",
      "Train Epoch: 61 [30000/50000 (60%)]\tLosses default: 0.609403 xavier: 0.402924\n",
      "Train Epoch: 61 [40000/50000 (80%)]\tLosses default: 0.618598 xavier: 0.316771\n",
      "Train Epoch: 61 [50000/50000 (100%)]\tLosses default: 0.571058 xavier: 0.388464\n",
      "Test set:\n",
      "default: Loss: 0.6146\tAccuracy: 8482/10000 (85%)\n",
      "xavier: Loss: 0.3892\tAccuracy: 8931/10000 (89%)\n",
      "\n",
      "Train Epoch: 62 [0/50000 (0%)]\tLosses default: 0.785071 xavier: 0.613863\n",
      "Train Epoch: 62 [10000/50000 (20%)]\tLosses default: 0.726593 xavier: 0.459829\n",
      "Train Epoch: 62 [20000/50000 (40%)]\tLosses default: 0.577583 xavier: 0.290599\n",
      "Train Epoch: 62 [30000/50000 (60%)]\tLosses default: 0.433105 xavier: 0.203432\n",
      "Train Epoch: 62 [40000/50000 (80%)]\tLosses default: 0.480354 xavier: 0.285608\n",
      "Train Epoch: 62 [50000/50000 (100%)]\tLosses default: 0.642273 xavier: 0.394936\n",
      "Test set:\n",
      "default: Loss: 0.6071\tAccuracy: 8503/10000 (85%)\n",
      "xavier: Loss: 0.3869\tAccuracy: 8937/10000 (89%)\n",
      "\n",
      "Train Epoch: 63 [0/50000 (0%)]\tLosses default: 0.471244 xavier: 0.219370\n",
      "Train Epoch: 63 [10000/50000 (20%)]\tLosses default: 0.558536 xavier: 0.305241\n",
      "Train Epoch: 63 [20000/50000 (40%)]\tLosses default: 0.549419 xavier: 0.312837\n",
      "Train Epoch: 63 [30000/50000 (60%)]\tLosses default: 0.559820 xavier: 0.321335\n",
      "Train Epoch: 63 [40000/50000 (80%)]\tLosses default: 0.568527 xavier: 0.398803\n",
      "Train Epoch: 63 [50000/50000 (100%)]\tLosses default: 0.612005 xavier: 0.350472\n",
      "Test set:\n",
      "default: Loss: 0.5998\tAccuracy: 8516/10000 (85%)\n",
      "xavier: Loss: 0.3846\tAccuracy: 8940/10000 (89%)\n",
      "\n",
      "Train Epoch: 64 [0/50000 (0%)]\tLosses default: 0.505011 xavier: 0.335526\n",
      "Train Epoch: 64 [10000/50000 (20%)]\tLosses default: 0.766929 xavier: 0.495954\n",
      "Train Epoch: 64 [20000/50000 (40%)]\tLosses default: 0.475054 xavier: 0.276092\n",
      "Train Epoch: 64 [30000/50000 (60%)]\tLosses default: 0.532231 xavier: 0.402173\n",
      "Train Epoch: 64 [40000/50000 (80%)]\tLosses default: 0.713001 xavier: 0.431915\n",
      "Train Epoch: 64 [50000/50000 (100%)]\tLosses default: 0.632006 xavier: 0.403551\n",
      "Test set:\n",
      "default: Loss: 0.5927\tAccuracy: 8535/10000 (85%)\n",
      "xavier: Loss: 0.3824\tAccuracy: 8944/10000 (89%)\n",
      "\n",
      "Train Epoch: 65 [0/50000 (0%)]\tLosses default: 0.492304 xavier: 0.285222\n",
      "Train Epoch: 65 [10000/50000 (20%)]\tLosses default: 0.613800 xavier: 0.313585\n",
      "Train Epoch: 65 [20000/50000 (40%)]\tLosses default: 0.654738 xavier: 0.411673\n",
      "Train Epoch: 65 [30000/50000 (60%)]\tLosses default: 0.528847 xavier: 0.264986\n",
      "Train Epoch: 65 [40000/50000 (80%)]\tLosses default: 0.604111 xavier: 0.429859\n",
      "Train Epoch: 65 [50000/50000 (100%)]\tLosses default: 0.622733 xavier: 0.480300\n",
      "Test set:\n",
      "default: Loss: 0.5858\tAccuracy: 8551/10000 (86%)\n",
      "xavier: Loss: 0.3802\tAccuracy: 8950/10000 (90%)\n",
      "\n",
      "Train Epoch: 66 [0/50000 (0%)]\tLosses default: 0.482336 xavier: 0.373972\n",
      "Train Epoch: 66 [10000/50000 (20%)]\tLosses default: 0.436011 xavier: 0.180555\n",
      "Train Epoch: 66 [20000/50000 (40%)]\tLosses default: 0.472445 xavier: 0.241920\n",
      "Train Epoch: 66 [30000/50000 (60%)]\tLosses default: 0.617590 xavier: 0.278266\n",
      "Train Epoch: 66 [40000/50000 (80%)]\tLosses default: 0.664807 xavier: 0.430029\n",
      "Train Epoch: 66 [50000/50000 (100%)]\tLosses default: 0.536836 xavier: 0.346305\n",
      "Test set:\n",
      "default: Loss: 0.5791\tAccuracy: 8575/10000 (86%)\n",
      "xavier: Loss: 0.3781\tAccuracy: 8954/10000 (90%)\n",
      "\n",
      "Train Epoch: 67 [0/50000 (0%)]\tLosses default: 0.600469 xavier: 0.317649\n",
      "Train Epoch: 67 [10000/50000 (20%)]\tLosses default: 0.639419 xavier: 0.421614\n",
      "Train Epoch: 67 [20000/50000 (40%)]\tLosses default: 0.511554 xavier: 0.381633\n",
      "Train Epoch: 67 [30000/50000 (60%)]\tLosses default: 0.643394 xavier: 0.432208\n",
      "Train Epoch: 67 [40000/50000 (80%)]\tLosses default: 0.668583 xavier: 0.486255\n",
      "Train Epoch: 67 [50000/50000 (100%)]\tLosses default: 0.527085 xavier: 0.317183\n",
      "Test set:\n",
      "default: Loss: 0.5725\tAccuracy: 8599/10000 (86%)\n",
      "xavier: Loss: 0.3761\tAccuracy: 8958/10000 (90%)\n",
      "\n",
      "Train Epoch: 68 [0/50000 (0%)]\tLosses default: 0.616445 xavier: 0.439861\n",
      "Train Epoch: 68 [10000/50000 (20%)]\tLosses default: 0.551876 xavier: 0.305632\n",
      "Train Epoch: 68 [20000/50000 (40%)]\tLosses default: 0.756084 xavier: 0.504709\n",
      "Train Epoch: 68 [30000/50000 (60%)]\tLosses default: 0.688302 xavier: 0.478535\n",
      "Train Epoch: 68 [40000/50000 (80%)]\tLosses default: 0.666554 xavier: 0.513600\n",
      "Train Epoch: 68 [50000/50000 (100%)]\tLosses default: 0.628421 xavier: 0.526535\n",
      "Test set:\n",
      "default: Loss: 0.5662\tAccuracy: 8614/10000 (86%)\n",
      "xavier: Loss: 0.3740\tAccuracy: 8964/10000 (90%)\n",
      "\n",
      "Train Epoch: 69 [0/50000 (0%)]\tLosses default: 0.490513 xavier: 0.240611\n",
      "Train Epoch: 69 [10000/50000 (20%)]\tLosses default: 0.690674 xavier: 0.422655\n",
      "Train Epoch: 69 [20000/50000 (40%)]\tLosses default: 0.412930 xavier: 0.276606\n",
      "Train Epoch: 69 [30000/50000 (60%)]\tLosses default: 0.637130 xavier: 0.387231\n",
      "Train Epoch: 69 [40000/50000 (80%)]\tLosses default: 0.507661 xavier: 0.352653\n",
      "Train Epoch: 69 [50000/50000 (100%)]\tLosses default: 0.820194 xavier: 0.631656\n",
      "Test set:\n",
      "default: Loss: 0.5600\tAccuracy: 8629/10000 (86%)\n",
      "xavier: Loss: 0.3721\tAccuracy: 8969/10000 (90%)\n",
      "\n",
      "Train Epoch: 70 [0/50000 (0%)]\tLosses default: 0.632074 xavier: 0.439526\n",
      "Train Epoch: 70 [10000/50000 (20%)]\tLosses default: 0.614232 xavier: 0.408454\n",
      "Train Epoch: 70 [20000/50000 (40%)]\tLosses default: 0.586142 xavier: 0.459919\n",
      "Train Epoch: 70 [30000/50000 (60%)]\tLosses default: 0.572394 xavier: 0.308169\n",
      "Train Epoch: 70 [40000/50000 (80%)]\tLosses default: 0.543238 xavier: 0.332171\n",
      "Train Epoch: 70 [50000/50000 (100%)]\tLosses default: 0.618713 xavier: 0.512672\n",
      "Test set:\n",
      "default: Loss: 0.5540\tAccuracy: 8631/10000 (86%)\n",
      "xavier: Loss: 0.3701\tAccuracy: 8971/10000 (90%)\n",
      "\n",
      "Train Epoch: 71 [0/50000 (0%)]\tLosses default: 0.503161 xavier: 0.272887\n",
      "Train Epoch: 71 [10000/50000 (20%)]\tLosses default: 0.379000 xavier: 0.173386\n",
      "Train Epoch: 71 [20000/50000 (40%)]\tLosses default: 0.469259 xavier: 0.294606\n",
      "Train Epoch: 71 [30000/50000 (60%)]\tLosses default: 0.489222 xavier: 0.283236\n",
      "Train Epoch: 71 [40000/50000 (80%)]\tLosses default: 0.434239 xavier: 0.249916\n",
      "Train Epoch: 71 [50000/50000 (100%)]\tLosses default: 0.529127 xavier: 0.288604\n",
      "Test set:\n",
      "default: Loss: 0.5481\tAccuracy: 8642/10000 (86%)\n",
      "xavier: Loss: 0.3682\tAccuracy: 8977/10000 (90%)\n",
      "\n",
      "Train Epoch: 72 [0/50000 (0%)]\tLosses default: 0.491145 xavier: 0.319747\n",
      "Train Epoch: 72 [10000/50000 (20%)]\tLosses default: 0.508425 xavier: 0.302689\n",
      "Train Epoch: 72 [20000/50000 (40%)]\tLosses default: 0.400328 xavier: 0.207871\n",
      "Train Epoch: 72 [30000/50000 (60%)]\tLosses default: 0.513238 xavier: 0.333587\n",
      "Train Epoch: 72 [40000/50000 (80%)]\tLosses default: 0.690357 xavier: 0.455441\n",
      "Train Epoch: 72 [50000/50000 (100%)]\tLosses default: 0.512396 xavier: 0.355271\n",
      "Test set:\n",
      "default: Loss: 0.5424\tAccuracy: 8654/10000 (87%)\n",
      "xavier: Loss: 0.3664\tAccuracy: 8980/10000 (90%)\n",
      "\n",
      "Train Epoch: 73 [0/50000 (0%)]\tLosses default: 0.707476 xavier: 0.460893\n",
      "Train Epoch: 73 [10000/50000 (20%)]\tLosses default: 0.714629 xavier: 0.571639\n",
      "Train Epoch: 73 [20000/50000 (40%)]\tLosses default: 0.470290 xavier: 0.306963\n",
      "Train Epoch: 73 [30000/50000 (60%)]\tLosses default: 0.619304 xavier: 0.432037\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 73 [40000/50000 (80%)]\tLosses default: 0.596874 xavier: 0.396585\n",
      "Train Epoch: 73 [50000/50000 (100%)]\tLosses default: 0.386507 xavier: 0.261124\n",
      "Test set:\n",
      "default: Loss: 0.5369\tAccuracy: 8666/10000 (87%)\n",
      "xavier: Loss: 0.3646\tAccuracy: 8984/10000 (90%)\n",
      "\n",
      "Train Epoch: 74 [0/50000 (0%)]\tLosses default: 0.511342 xavier: 0.417040\n",
      "Train Epoch: 74 [10000/50000 (20%)]\tLosses default: 0.707169 xavier: 0.545415\n",
      "Train Epoch: 74 [20000/50000 (40%)]\tLosses default: 0.495756 xavier: 0.271244\n",
      "Train Epoch: 74 [30000/50000 (60%)]\tLosses default: 0.616858 xavier: 0.470379\n",
      "Train Epoch: 74 [40000/50000 (80%)]\tLosses default: 0.615700 xavier: 0.451184\n",
      "Train Epoch: 74 [50000/50000 (100%)]\tLosses default: 0.560439 xavier: 0.432785\n",
      "Test set:\n",
      "default: Loss: 0.5314\tAccuracy: 8679/10000 (87%)\n",
      "xavier: Loss: 0.3628\tAccuracy: 8989/10000 (90%)\n",
      "\n",
      "Train Epoch: 75 [0/50000 (0%)]\tLosses default: 0.585480 xavier: 0.366144\n",
      "Train Epoch: 75 [10000/50000 (20%)]\tLosses default: 0.417958 xavier: 0.256468\n",
      "Train Epoch: 75 [20000/50000 (40%)]\tLosses default: 0.612340 xavier: 0.480436\n",
      "Train Epoch: 75 [30000/50000 (60%)]\tLosses default: 0.643935 xavier: 0.521938\n",
      "Train Epoch: 75 [40000/50000 (80%)]\tLosses default: 0.614372 xavier: 0.497520\n",
      "Train Epoch: 75 [50000/50000 (100%)]\tLosses default: 0.600164 xavier: 0.458205\n",
      "Test set:\n",
      "default: Loss: 0.5262\tAccuracy: 8688/10000 (87%)\n",
      "xavier: Loss: 0.3611\tAccuracy: 8992/10000 (90%)\n",
      "\n",
      "Train Epoch: 76 [0/50000 (0%)]\tLosses default: 0.478490 xavier: 0.367621\n",
      "Train Epoch: 76 [10000/50000 (20%)]\tLosses default: 0.569455 xavier: 0.376769\n",
      "Train Epoch: 76 [20000/50000 (40%)]\tLosses default: 0.421921 xavier: 0.260038\n",
      "Train Epoch: 76 [30000/50000 (60%)]\tLosses default: 0.559390 xavier: 0.451741\n",
      "Train Epoch: 76 [40000/50000 (80%)]\tLosses default: 0.649103 xavier: 0.408593\n",
      "Train Epoch: 76 [50000/50000 (100%)]\tLosses default: 0.551890 xavier: 0.372432\n",
      "Test set:\n",
      "default: Loss: 0.5211\tAccuracy: 8698/10000 (87%)\n",
      "xavier: Loss: 0.3593\tAccuracy: 8998/10000 (90%)\n",
      "\n",
      "Train Epoch: 77 [0/50000 (0%)]\tLosses default: 0.560550 xavier: 0.352065\n",
      "Train Epoch: 77 [10000/50000 (20%)]\tLosses default: 0.577492 xavier: 0.354125\n",
      "Train Epoch: 77 [20000/50000 (40%)]\tLosses default: 0.476674 xavier: 0.310274\n",
      "Train Epoch: 77 [30000/50000 (60%)]\tLosses default: 0.495559 xavier: 0.332029\n",
      "Train Epoch: 77 [40000/50000 (80%)]\tLosses default: 0.367617 xavier: 0.220958\n",
      "Train Epoch: 77 [50000/50000 (100%)]\tLosses default: 0.483897 xavier: 0.247574\n",
      "Test set:\n",
      "default: Loss: 0.5161\tAccuracy: 8701/10000 (87%)\n",
      "xavier: Loss: 0.3577\tAccuracy: 9002/10000 (90%)\n",
      "\n",
      "Train Epoch: 78 [0/50000 (0%)]\tLosses default: 0.580533 xavier: 0.399088\n",
      "Train Epoch: 78 [10000/50000 (20%)]\tLosses default: 0.719033 xavier: 0.560777\n",
      "Train Epoch: 78 [20000/50000 (40%)]\tLosses default: 0.450047 xavier: 0.248767\n",
      "Train Epoch: 78 [30000/50000 (60%)]\tLosses default: 0.584721 xavier: 0.333389\n",
      "Train Epoch: 78 [40000/50000 (80%)]\tLosses default: 0.460264 xavier: 0.294559\n",
      "Train Epoch: 78 [50000/50000 (100%)]\tLosses default: 0.486244 xavier: 0.277028\n",
      "Test set:\n",
      "default: Loss: 0.5112\tAccuracy: 8711/10000 (87%)\n",
      "xavier: Loss: 0.3560\tAccuracy: 9004/10000 (90%)\n",
      "\n",
      "Train Epoch: 79 [0/50000 (0%)]\tLosses default: 0.495310 xavier: 0.260953\n",
      "Train Epoch: 79 [10000/50000 (20%)]\tLosses default: 0.435127 xavier: 0.238641\n",
      "Train Epoch: 79 [20000/50000 (40%)]\tLosses default: 0.504380 xavier: 0.363635\n",
      "Train Epoch: 79 [30000/50000 (60%)]\tLosses default: 0.603471 xavier: 0.409565\n",
      "Train Epoch: 79 [40000/50000 (80%)]\tLosses default: 0.462817 xavier: 0.360231\n",
      "Train Epoch: 79 [50000/50000 (100%)]\tLosses default: 0.521472 xavier: 0.359436\n",
      "Test set:\n",
      "default: Loss: 0.5065\tAccuracy: 8720/10000 (87%)\n",
      "xavier: Loss: 0.3544\tAccuracy: 9009/10000 (90%)\n",
      "\n",
      "Train Epoch: 80 [0/50000 (0%)]\tLosses default: 0.556659 xavier: 0.465363\n",
      "Train Epoch: 80 [10000/50000 (20%)]\tLosses default: 0.563937 xavier: 0.389519\n",
      "Train Epoch: 80 [20000/50000 (40%)]\tLosses default: 0.526375 xavier: 0.307527\n",
      "Train Epoch: 80 [30000/50000 (60%)]\tLosses default: 0.433062 xavier: 0.271393\n",
      "Train Epoch: 80 [40000/50000 (80%)]\tLosses default: 0.391935 xavier: 0.247875\n",
      "Train Epoch: 80 [50000/50000 (100%)]\tLosses default: 0.464229 xavier: 0.223844\n",
      "Test set:\n",
      "default: Loss: 0.5018\tAccuracy: 8730/10000 (87%)\n",
      "xavier: Loss: 0.3529\tAccuracy: 9010/10000 (90%)\n",
      "\n",
      "Train Epoch: 81 [0/50000 (0%)]\tLosses default: 0.488014 xavier: 0.372477\n",
      "Train Epoch: 81 [10000/50000 (20%)]\tLosses default: 0.463314 xavier: 0.341512\n",
      "Train Epoch: 81 [20000/50000 (40%)]\tLosses default: 0.624781 xavier: 0.436568\n",
      "Train Epoch: 81 [30000/50000 (60%)]\tLosses default: 0.665754 xavier: 0.548054\n",
      "Train Epoch: 81 [40000/50000 (80%)]\tLosses default: 0.579333 xavier: 0.447052\n",
      "Train Epoch: 81 [50000/50000 (100%)]\tLosses default: 0.543810 xavier: 0.309509\n",
      "Test set:\n",
      "default: Loss: 0.4973\tAccuracy: 8740/10000 (87%)\n",
      "xavier: Loss: 0.3513\tAccuracy: 9012/10000 (90%)\n",
      "\n",
      "Train Epoch: 82 [0/50000 (0%)]\tLosses default: 0.515393 xavier: 0.374899\n",
      "Train Epoch: 82 [10000/50000 (20%)]\tLosses default: 0.320418 xavier: 0.175014\n",
      "Train Epoch: 82 [20000/50000 (40%)]\tLosses default: 0.540330 xavier: 0.374967\n",
      "Train Epoch: 82 [30000/50000 (60%)]\tLosses default: 0.515602 xavier: 0.346800\n",
      "Train Epoch: 82 [40000/50000 (80%)]\tLosses default: 0.391605 xavier: 0.219857\n",
      "Train Epoch: 82 [50000/50000 (100%)]\tLosses default: 0.517342 xavier: 0.360562\n",
      "Test set:\n",
      "default: Loss: 0.4930\tAccuracy: 8748/10000 (87%)\n",
      "xavier: Loss: 0.3498\tAccuracy: 9017/10000 (90%)\n",
      "\n",
      "Train Epoch: 83 [0/50000 (0%)]\tLosses default: 0.292431 xavier: 0.210141\n",
      "Train Epoch: 83 [10000/50000 (20%)]\tLosses default: 0.374837 xavier: 0.218653\n",
      "Train Epoch: 83 [20000/50000 (40%)]\tLosses default: 0.578454 xavier: 0.533233\n",
      "Train Epoch: 83 [30000/50000 (60%)]\tLosses default: 0.573769 xavier: 0.368580\n",
      "Train Epoch: 83 [40000/50000 (80%)]\tLosses default: 0.358846 xavier: 0.231825\n",
      "Train Epoch: 83 [50000/50000 (100%)]\tLosses default: 0.507004 xavier: 0.398615\n",
      "Test set:\n",
      "default: Loss: 0.4887\tAccuracy: 8752/10000 (88%)\n",
      "xavier: Loss: 0.3483\tAccuracy: 9022/10000 (90%)\n",
      "\n",
      "Train Epoch: 84 [0/50000 (0%)]\tLosses default: 0.321570 xavier: 0.176638\n",
      "Train Epoch: 84 [10000/50000 (20%)]\tLosses default: 0.503249 xavier: 0.372434\n",
      "Train Epoch: 84 [20000/50000 (40%)]\tLosses default: 0.363138 xavier: 0.226571\n",
      "Train Epoch: 84 [30000/50000 (60%)]\tLosses default: 0.363238 xavier: 0.223915\n",
      "Train Epoch: 84 [40000/50000 (80%)]\tLosses default: 0.692568 xavier: 0.548809\n",
      "Train Epoch: 84 [50000/50000 (100%)]\tLosses default: 0.388328 xavier: 0.248555\n",
      "Test set:\n",
      "default: Loss: 0.4845\tAccuracy: 8762/10000 (88%)\n",
      "xavier: Loss: 0.3468\tAccuracy: 9028/10000 (90%)\n",
      "\n",
      "Train Epoch: 85 [0/50000 (0%)]\tLosses default: 0.449285 xavier: 0.265058\n",
      "Train Epoch: 85 [10000/50000 (20%)]\tLosses default: 0.631666 xavier: 0.529744\n",
      "Train Epoch: 85 [20000/50000 (40%)]\tLosses default: 0.438438 xavier: 0.336408\n",
      "Train Epoch: 85 [30000/50000 (60%)]\tLosses default: 0.374296 xavier: 0.214714\n",
      "Train Epoch: 85 [40000/50000 (80%)]\tLosses default: 0.641735 xavier: 0.501272\n",
      "Train Epoch: 85 [50000/50000 (100%)]\tLosses default: 0.391757 xavier: 0.239578\n",
      "Test set:\n",
      "default: Loss: 0.4805\tAccuracy: 8764/10000 (88%)\n",
      "xavier: Loss: 0.3453\tAccuracy: 9030/10000 (90%)\n",
      "\n",
      "Train Epoch: 86 [0/50000 (0%)]\tLosses default: 0.479789 xavier: 0.403137\n",
      "Train Epoch: 86 [10000/50000 (20%)]\tLosses default: 0.422745 xavier: 0.286287\n",
      "Train Epoch: 86 [20000/50000 (40%)]\tLosses default: 0.348534 xavier: 0.202898\n",
      "Train Epoch: 86 [30000/50000 (60%)]\tLosses default: 0.405293 xavier: 0.262601\n",
      "Train Epoch: 86 [40000/50000 (80%)]\tLosses default: 0.463956 xavier: 0.369638\n",
      "Train Epoch: 86 [50000/50000 (100%)]\tLosses default: 0.650242 xavier: 0.454011\n",
      "Test set:\n",
      "default: Loss: 0.4765\tAccuracy: 8767/10000 (88%)\n",
      "xavier: Loss: 0.3439\tAccuracy: 9032/10000 (90%)\n",
      "\n",
      "Train Epoch: 87 [0/50000 (0%)]\tLosses default: 0.346620 xavier: 0.244976\n",
      "Train Epoch: 87 [10000/50000 (20%)]\tLosses default: 0.526231 xavier: 0.394127\n",
      "Train Epoch: 87 [20000/50000 (40%)]\tLosses default: 0.874320 xavier: 0.723814\n",
      "Train Epoch: 87 [30000/50000 (60%)]\tLosses default: 0.447758 xavier: 0.338996\n",
      "Train Epoch: 87 [40000/50000 (80%)]\tLosses default: 0.380451 xavier: 0.186263\n",
      "Train Epoch: 87 [50000/50000 (100%)]\tLosses default: 0.449307 xavier: 0.260715\n",
      "Test set:\n",
      "default: Loss: 0.4727\tAccuracy: 8768/10000 (88%)\n",
      "xavier: Loss: 0.3425\tAccuracy: 9035/10000 (90%)\n",
      "\n",
      "Train Epoch: 88 [0/50000 (0%)]\tLosses default: 0.509232 xavier: 0.325372\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 88 [10000/50000 (20%)]\tLosses default: 0.467251 xavier: 0.355873\n",
      "Train Epoch: 88 [20000/50000 (40%)]\tLosses default: 0.460809 xavier: 0.279423\n",
      "Train Epoch: 88 [30000/50000 (60%)]\tLosses default: 0.548375 xavier: 0.358832\n",
      "Train Epoch: 88 [40000/50000 (80%)]\tLosses default: 0.290295 xavier: 0.166534\n",
      "Train Epoch: 88 [50000/50000 (100%)]\tLosses default: 0.486316 xavier: 0.381829\n",
      "Test set:\n",
      "default: Loss: 0.4689\tAccuracy: 8780/10000 (88%)\n",
      "xavier: Loss: 0.3412\tAccuracy: 9038/10000 (90%)\n",
      "\n",
      "Train Epoch: 89 [0/50000 (0%)]\tLosses default: 0.627528 xavier: 0.554531\n",
      "Train Epoch: 89 [10000/50000 (20%)]\tLosses default: 0.488994 xavier: 0.326875\n",
      "Train Epoch: 89 [20000/50000 (40%)]\tLosses default: 0.668476 xavier: 0.505802\n",
      "Train Epoch: 89 [30000/50000 (60%)]\tLosses default: 0.441508 xavier: 0.359982\n",
      "Train Epoch: 89 [40000/50000 (80%)]\tLosses default: 0.474029 xavier: 0.393036\n",
      "Train Epoch: 89 [50000/50000 (100%)]\tLosses default: 0.598161 xavier: 0.488985\n",
      "Test set:\n",
      "default: Loss: 0.4653\tAccuracy: 8784/10000 (88%)\n",
      "xavier: Loss: 0.3398\tAccuracy: 9040/10000 (90%)\n",
      "\n",
      "Train Epoch: 90 [0/50000 (0%)]\tLosses default: 0.470143 xavier: 0.401835\n",
      "Train Epoch: 90 [10000/50000 (20%)]\tLosses default: 0.298758 xavier: 0.157124\n",
      "Train Epoch: 90 [20000/50000 (40%)]\tLosses default: 0.460857 xavier: 0.329219\n",
      "Train Epoch: 90 [30000/50000 (60%)]\tLosses default: 0.547428 xavier: 0.390063\n",
      "Train Epoch: 90 [40000/50000 (80%)]\tLosses default: 0.423928 xavier: 0.272981\n",
      "Train Epoch: 90 [50000/50000 (100%)]\tLosses default: 0.517548 xavier: 0.352515\n",
      "Test set:\n",
      "default: Loss: 0.4617\tAccuracy: 8793/10000 (88%)\n",
      "xavier: Loss: 0.3385\tAccuracy: 9044/10000 (90%)\n",
      "\n",
      "Train Epoch: 91 [0/50000 (0%)]\tLosses default: 0.460668 xavier: 0.321410\n",
      "Train Epoch: 91 [10000/50000 (20%)]\tLosses default: 0.548675 xavier: 0.411448\n",
      "Train Epoch: 91 [20000/50000 (40%)]\tLosses default: 0.452679 xavier: 0.374110\n",
      "Train Epoch: 91 [30000/50000 (60%)]\tLosses default: 0.415392 xavier: 0.269438\n",
      "Train Epoch: 91 [40000/50000 (80%)]\tLosses default: 0.331673 xavier: 0.262950\n",
      "Train Epoch: 91 [50000/50000 (100%)]\tLosses default: 0.508492 xavier: 0.375346\n",
      "Test set:\n",
      "default: Loss: 0.4582\tAccuracy: 8804/10000 (88%)\n",
      "xavier: Loss: 0.3371\tAccuracy: 9047/10000 (90%)\n",
      "\n",
      "Train Epoch: 92 [0/50000 (0%)]\tLosses default: 0.359480 xavier: 0.222053\n",
      "Train Epoch: 92 [10000/50000 (20%)]\tLosses default: 0.325362 xavier: 0.174746\n",
      "Train Epoch: 92 [20000/50000 (40%)]\tLosses default: 0.494801 xavier: 0.406864\n",
      "Train Epoch: 92 [30000/50000 (60%)]\tLosses default: 0.395599 xavier: 0.312215\n",
      "Train Epoch: 92 [40000/50000 (80%)]\tLosses default: 0.538455 xavier: 0.399423\n",
      "Train Epoch: 92 [50000/50000 (100%)]\tLosses default: 0.346703 xavier: 0.234565\n",
      "Test set:\n",
      "default: Loss: 0.4548\tAccuracy: 8811/10000 (88%)\n",
      "xavier: Loss: 0.3358\tAccuracy: 9051/10000 (91%)\n",
      "\n",
      "Train Epoch: 93 [0/50000 (0%)]\tLosses default: 0.400183 xavier: 0.258785\n",
      "Train Epoch: 93 [10000/50000 (20%)]\tLosses default: 0.567312 xavier: 0.467059\n",
      "Train Epoch: 93 [20000/50000 (40%)]\tLosses default: 0.569617 xavier: 0.491196\n",
      "Train Epoch: 93 [30000/50000 (60%)]\tLosses default: 0.514918 xavier: 0.388843\n",
      "Train Epoch: 93 [40000/50000 (80%)]\tLosses default: 0.392424 xavier: 0.261981\n",
      "Train Epoch: 93 [50000/50000 (100%)]\tLosses default: 0.432430 xavier: 0.326394\n",
      "Test set:\n",
      "default: Loss: 0.4515\tAccuracy: 8817/10000 (88%)\n",
      "xavier: Loss: 0.3346\tAccuracy: 9054/10000 (91%)\n",
      "\n",
      "Train Epoch: 94 [0/50000 (0%)]\tLosses default: 0.438599 xavier: 0.317476\n",
      "Train Epoch: 94 [10000/50000 (20%)]\tLosses default: 0.364435 xavier: 0.273169\n",
      "Train Epoch: 94 [20000/50000 (40%)]\tLosses default: 0.398433 xavier: 0.237268\n",
      "Train Epoch: 94 [30000/50000 (60%)]\tLosses default: 0.300382 xavier: 0.177889\n",
      "Train Epoch: 94 [40000/50000 (80%)]\tLosses default: 0.472081 xavier: 0.362493\n",
      "Train Epoch: 94 [50000/50000 (100%)]\tLosses default: 0.475782 xavier: 0.306362\n",
      "Test set:\n",
      "default: Loss: 0.4483\tAccuracy: 8820/10000 (88%)\n",
      "xavier: Loss: 0.3333\tAccuracy: 9060/10000 (91%)\n",
      "\n",
      "Train Epoch: 95 [0/50000 (0%)]\tLosses default: 0.291594 xavier: 0.194131\n",
      "Train Epoch: 95 [10000/50000 (20%)]\tLosses default: 0.523660 xavier: 0.417064\n",
      "Train Epoch: 95 [20000/50000 (40%)]\tLosses default: 0.300277 xavier: 0.168208\n",
      "Train Epoch: 95 [30000/50000 (60%)]\tLosses default: 0.296075 xavier: 0.209679\n",
      "Train Epoch: 95 [40000/50000 (80%)]\tLosses default: 0.395128 xavier: 0.276300\n",
      "Train Epoch: 95 [50000/50000 (100%)]\tLosses default: 0.400439 xavier: 0.337934\n",
      "Test set:\n",
      "default: Loss: 0.4452\tAccuracy: 8826/10000 (88%)\n",
      "xavier: Loss: 0.3321\tAccuracy: 9066/10000 (91%)\n",
      "\n",
      "Train Epoch: 96 [0/50000 (0%)]\tLosses default: 0.274074 xavier: 0.126447\n",
      "Train Epoch: 96 [10000/50000 (20%)]\tLosses default: 0.338060 xavier: 0.215711\n",
      "Train Epoch: 96 [20000/50000 (40%)]\tLosses default: 0.248983 xavier: 0.222088\n",
      "Train Epoch: 96 [30000/50000 (60%)]\tLosses default: 0.392456 xavier: 0.378062\n",
      "Train Epoch: 96 [40000/50000 (80%)]\tLosses default: 0.463710 xavier: 0.355359\n",
      "Train Epoch: 96 [50000/50000 (100%)]\tLosses default: 0.437699 xavier: 0.236782\n",
      "Test set:\n",
      "default: Loss: 0.4421\tAccuracy: 8833/10000 (88%)\n",
      "xavier: Loss: 0.3309\tAccuracy: 9067/10000 (91%)\n",
      "\n",
      "Train Epoch: 97 [0/50000 (0%)]\tLosses default: 0.455455 xavier: 0.407181\n",
      "Train Epoch: 97 [10000/50000 (20%)]\tLosses default: 0.454801 xavier: 0.327059\n",
      "Train Epoch: 97 [20000/50000 (40%)]\tLosses default: 0.302503 xavier: 0.267649\n",
      "Train Epoch: 97 [30000/50000 (60%)]\tLosses default: 0.308253 xavier: 0.193582\n",
      "Train Epoch: 97 [40000/50000 (80%)]\tLosses default: 0.447580 xavier: 0.287353\n",
      "Train Epoch: 97 [50000/50000 (100%)]\tLosses default: 0.372057 xavier: 0.297818\n",
      "Test set:\n",
      "default: Loss: 0.4391\tAccuracy: 8841/10000 (88%)\n",
      "xavier: Loss: 0.3297\tAccuracy: 9070/10000 (91%)\n",
      "\n",
      "Train Epoch: 98 [0/50000 (0%)]\tLosses default: 0.454210 xavier: 0.356766\n",
      "Train Epoch: 98 [10000/50000 (20%)]\tLosses default: 0.393195 xavier: 0.230455\n",
      "Train Epoch: 98 [20000/50000 (40%)]\tLosses default: 0.414772 xavier: 0.312615\n",
      "Train Epoch: 98 [30000/50000 (60%)]\tLosses default: 0.465139 xavier: 0.334015\n",
      "Train Epoch: 98 [40000/50000 (80%)]\tLosses default: 0.571759 xavier: 0.425703\n",
      "Train Epoch: 98 [50000/50000 (100%)]\tLosses default: 0.358410 xavier: 0.217246\n",
      "Test set:\n",
      "default: Loss: 0.4362\tAccuracy: 8851/10000 (89%)\n",
      "xavier: Loss: 0.3285\tAccuracy: 9074/10000 (91%)\n",
      "\n",
      "Train Epoch: 99 [0/50000 (0%)]\tLosses default: 0.399995 xavier: 0.312931\n",
      "Train Epoch: 99 [10000/50000 (20%)]\tLosses default: 0.356439 xavier: 0.201362\n",
      "Train Epoch: 99 [20000/50000 (40%)]\tLosses default: 0.454657 xavier: 0.318490\n",
      "Train Epoch: 99 [30000/50000 (60%)]\tLosses default: 0.252546 xavier: 0.167894\n",
      "Train Epoch: 99 [40000/50000 (80%)]\tLosses default: 0.398319 xavier: 0.340600\n",
      "Train Epoch: 99 [50000/50000 (100%)]\tLosses default: 0.345184 xavier: 0.294917\n",
      "Test set:\n",
      "default: Loss: 0.4333\tAccuracy: 8860/10000 (89%)\n",
      "xavier: Loss: 0.3273\tAccuracy: 9078/10000 (91%)\n",
      "\n",
      "Train Epoch: 100 [0/50000 (0%)]\tLosses default: 0.417328 xavier: 0.364584\n",
      "Train Epoch: 100 [10000/50000 (20%)]\tLosses default: 0.302743 xavier: 0.162857\n",
      "Train Epoch: 100 [20000/50000 (40%)]\tLosses default: 0.429189 xavier: 0.310933\n",
      "Train Epoch: 100 [30000/50000 (60%)]\tLosses default: 0.326075 xavier: 0.282482\n",
      "Train Epoch: 100 [40000/50000 (80%)]\tLosses default: 0.288693 xavier: 0.208722\n",
      "Train Epoch: 100 [50000/50000 (100%)]\tLosses default: 0.637181 xavier: 0.489839\n",
      "Test set:\n",
      "default: Loss: 0.4305\tAccuracy: 8868/10000 (89%)\n",
      "xavier: Loss: 0.3261\tAccuracy: 9080/10000 (91%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 101):\n",
    "    for model in models.values():\n",
    "        model.train()\n",
    "    train(epoch, models, train_log)\n",
    "    for model in models.values():\n",
    "        model.eval()\n",
    "    test(models, valid_loader, test_log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Графики loss и accuracy моделей с различными инициализациями"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xV9f348dc7m+zJSoCEvUMgIAq4cGERa9W6q1W/SNWfo8Pa4ajWTltbt9SBtlbrFvfEgaASkI3sFUYSEsjeef/+OCdwDUkIkJuT8X4+Hudxz/2cc895Hy7JO+d8lqgqxhhjTEMBXgdgjDGmfbIEYYwxplGWIIwxxjTKEoQxxphGWYIwxhjTKEsQxhhjGmUJwhhjTKMsQRhjjGmUJQhjWpGIBLWk7HCPYYwXLEEY0wIi0ltEXhaRPBHZLCI3uOV3ishLIvIfESkCrmiiLFRE/iEiO93lHyIS6h7jRBHJFpFfishu4CkRSRSRN0Vkn4gUiMjnImI/r6ZN2X84Yw7B/cX8BrAMSAamAjeJyOnuLmcDLwGxwLNNlP0GmAiMAdKBCcBvfU7TE4gH+gEzgZ8B2UAS0AP4NWDj4pg2ZQnCmEMbDySp6l2qWqWqm4B/ARe62xeq6muqWqeq5U2UXQLcpaq5qpoH/A64zOccdcAdqlrp7l8N9AL6qWq1qn6uNnCaaWOWIIw5tH5Ab/dxzz4R2YfzF30Pd/v2Rj7TsKw3sNXn/Va3rF6eqlb4vP8rsAF4X0Q2icitR3UFxhwBSxDGHNp2YLOqxvosUap6pru9sb/sG5btxEk09fq6ZY3ur6rFqvozVe0PzAB+KiJTj+4yjDk8liCMObSvgWK3ErmbiASKyEgRGX8Yx3gO+K2IJIlIInA78J+mdhaR6SIyUEQEKARqcR5DGdNmLEEYcwiqWgtMx6lg3gzsAR4HYg7jML8HsoDlwApgiVvWlEHAh0AJsBB4WFXnHXbwxhwFsXovY4wxjbE7CGOMMY2yBGGMMaZRliCMMcY0yhKEMcaYRnWqQcESExM1NTXV6zCMMabDWLx48R5VTWpsW6dKEKmpqWRlZXkdhjHGdBgisrWpbfaIyRhjTKMsQRhjjGmUJQhjjDGN6lR1EMYYU11dTXZ2NhUVFYfeuQsJCwsjJSWF4ODgFn/GEoQxplPJzs4mKiqK1NRUnLEOjaqSn59PdnY2aWlpLf6cPWIyxnQqFRUVJCQkWHLwISIkJCQc9l2VJQhjTKdjyeFgR/Jv4rcEISJ9RGSeiKwWkVUicmMj+1wiIstFZIWILBCRdJ9tW9zypSLit84NqsqDH69n5Y5Cf53CGGM6JH/eQdQAP1PV4TiTtV8nIsMb7LMZOEFVRwF3A7MbbD9JVceoaqa/giwsr+a/X23jx3MWsb2gzF+nMcZ0YXfeeSf33ntvk9vz8vI45phjyMjI4PPPPz/s48+ZM4frr78egNdee43Vq1cfcay+/JYgVHWXqi5x14uBNUByg30WqOpe9+2XQIq/4mlKbHgIc66cQGV1LVc89TX7yqraOgRjTBf30UcfMWrUKL755humTJlyVMfqEAnCl4ikAhnAV83sdhXwjs97xZmwfbGIzGzm2DNFJEtEsvLy8o4ovsE9opj9o0y2F5Tzf89kUVFde0THMcaYevfccw+DBw9m8uTJrF27FoCNGzdyxhlnMG7cOKZMmcK3337L0qVLueWWW3j99dcZM2YM5eXl/OQnPyEzM5MRI0Zwxx137D9mamoqe/bsASArK4sTTzzxO+dcsGABc+fO5Re/+AVjxoxh48aNR3UNfm/mKiKRwMvATapa1MQ+J+EkiMk+xZNVdYeIdAc+EJFvVfWzhp9V1dm4j6YyMzOPeHq8if0T+PsF6Vz/32/46QtLefCisQQEWEWXMR3Z795Yxeqdjf7aOWLDe0dzx1kjmt1n8eLFPP/88yxdupSamhrGjh3LuHHjmDlzJo8++iiDBg3iq6++4tprr+Xjjz/mrrvuIisriwcffBBwkkt8fDy1tbVMnTqV5cuXM3r06EPGdtxxxzFjxgymT5/Oeeedd9TX6tcEISLBOMnhWVV9pYl9RuPM7ztNVfPry1V1h/uaKyKvAhOAgxJEa5o+uje79lVwz9tr+GPsGn7zvYZVJsYYc2iff/4555xzDuHh4QDMmDGDiooKFixYwPnnn79/v8rKykY//8ILLzB79mxqamrYtWsXq1evblGCaG1+SxDitKl6Alijqn9vYp++wCvAZaq6zqc8AghQ1WJ3/TTgLn/F6uvqKWls31vGvz7fTJ/4cH50bGpbnNYY4weH+ku/LdXV1REbG8vSpUub3W/z5s3ce++9LFq0iLi4OK644or9/ReCgoKoq6sDaJOe4v6sg5gEXAac7DZVXSoiZ4rILBGZ5e5zO5AAPNygOWsPYL6ILAO+Bt5S1Xf9GOt+IsIdZ43glGHduXPuKj5cndMWpzXGdCLHH388r732GuXl5RQXF/PGG28QHh5OWloaL774IuA0sV+2bNlBny0qKiIiIoKYmBhycnJ4550DVbOpqaksXrwYgJdffrnRc0dFRVFcXNwq1+HPVkzzVVVUdbTbVHWMqr6tqo+q6qPuPlerapzP9ky3fJOqprvLCFW9x19xNiYwQLj/ogxGJsdww/PfsGqn9ZEwxrTc2LFjueCCC0hPT2fatGmMHz8egGeffZYnnniC9PR0RowYweuvv37QZ9PT08nIyGDo0KFcfPHFTJo0af+2O+64gxtvvJHMzEwCAwMbPfeFF17IX//6VzIyMo66klpUj7het93JzMzU1pwwKLeogrMf+gIBXrtuEt2jw1rt2MYY/1izZg3Dhg3zOox2qbF/GxFZ3FRfMxtqoxndo8N4/PJM9pVX83//XmzNX40xXYoliEMY0TuGf1wwhuXZ+/j5i8voTHdcxhjTHEsQLXDaiJ7ccvpQ3ly+i0c/3eR1OMYY0yYsQbTQrBP6M310L/7y3rd8sjbX63CMMcbvLEG0kIjwl/NGM7RnNDc89w2b95R6HZIxxviVJYjDEB4SxOzLxhEYIFzz7yzKqmq8DskYY/zGEsRh6hMfzgMXjWV9bgm/fXWlVVobY1rdzp07W2UspaNlCeIITB6UyE1TB/PKNzt4ftF2r8MxxnQyvXv35qWXXmrx/jU1/nmaYQniCP2/kwcyZVAid8xdZbPRGWP2W7RoEaNHj6aiooLS0lJGjBjBypUrmTp1KmPHjmXUqFH7e1DfeuutPPTQQ/s/Wz+x0JYtWxg5ciQAtbW1/OIXv2D8+PGMHj2axx57DIBPPvmEKVOmMGPGDIYP98/Aon4f7ruzCggQ/nHBGKY/MJ9rn13CWzdMJios2OuwjDG+3rkVdq9o3WP2HAXT/tTk5vHjxzNjxgx++9vfUl5ezqWXXsrQoUN59dVXiY6OZs+ePUycOJEZM2ZwwQUXcNNNN3HdddcBziiu7733HrW1BzrlPvHEE8TExLBo0SIqKyuZNGkSp512GgBLlixh5cqVpKWlte41uixBHIWEyFAeuCiDHz62kNtfX8V9F4zxOiRjTDtw++23M378eMLCwrj//vupq6vj17/+NZ999hkBAQHs2LGDnJwcMjIyyM3NZefOneTl5REXF0efPn3YsmXL/mO9//77LF++fP8jp8LCQtavX09ISAgTJkzwW3IASxBHLTM1nhunDua+D9dx/OBEzslo81lTjTFNaeYvfX/Kz8+npKSE6upqKioqePHFF8nLy2Px4sUEBweTmpq6f7ju888/n5deeondu3dzwQUXHHQsVeWBBx7g9NNP/075J598QkREhF+vw+og6urgy0dg85HPRXT9yQOZkBrPb19dydZ86x9hTFd3zTXXcPfdd3PJJZfwy1/+ksLCQrp3705wcDDz5s1j69at+/e94IILeP7553nppZe+M5lQvdNPP51HHnmE6upqANatW0dpadv8nrEEERAA8/4Aa9444kMEBgj3XTiGwADhhueXUl1b14oBGmM6kmeeeYbg4GAuvvhibr31VhYtWkSvXr3Iyspi1KhRPPPMMwwdOnT//iNGjKC4uJjk5GR69ep10PGuvvpqhg8fztixYxk5ciTXXHON31otNWTDfQM8fCzEpcFF/z2q87+1fBfX/XcJN58ymBtPGXRUxzLGHBkb7rtpNtz3kYhJgcKj78/wvdG9+P6Y3jzw8XpWZFvTV2NMx+a3BCEifURknoisFpFVInJjI/uIiNwvIhtEZLmIjPXZdrmIrHeXy/0VJ+AmiOxWOdTvZowkMTKUn76w1OaPMMZ0aP68g6gBfqaqw4GJwHUi0rA3xzRgkLvMBB4BEJF44A7gGGACcIeIxPkt0pgUKC+AqqOv+IkJD+bP541mfW4Jf3t/bSsEZ4w5XJ3p0XlrOZJ/E3/OSb1LVZe468XAGiC5wW5nA8+o40sgVkR6AacDH6hqgaruBT4AzvBXrMT0cV4Ld7TK4U4YnMSlE/vy+PzNLN5a0CrHNMa0TFhYGPn5+ZYkfKgq+fn5hIUd3rTJbdIPQkRSgQzgqwabkgHfh//ZbllT5Y0deybO3Qd9+/Y9sgCj3UMXboekwUd2jAZ+NW0Y877N45aXlvPWDVMIC258gnFjTOtKSUkhOzubvLw8r0NpV8LCwkhJObx+Wn5PECISCbwM3KSqRa19fFWdDcwGpxXTER0kxv1HK2qdOwiAiNAg/vCDUVz+5Nc8+PEGfn76kFY7tjGmacHBwX7tXdyV+LUVk4gE4ySHZ1X1lUZ22QH08Xmf4pY1Ve4f0b0BabWK6nonDE7i3LEpPPrpRlbttFZNxpiOxZ+tmAR4Alijqn9vYre5wI/c1kwTgUJV3QW8B5wmInFu5fRpbpl/BAZDVK9WTxAAt00fRmx4CL98eTk11oHOGNOB+PMOYhJwGXCyiCx1lzNFZJaIzHL3eRvYBGwA/gVcC6CqBcDdwCJ3ucst859W6gvRUGx4CHedPYKVO4qYs2BLqx/fGGP8xW91EKo6H5BD7KPAdU1sexJ40g+hNS4mBXYt9cuhp43sydSh3fnb++uYNqoXybHd/HIeY4xpTdaTul5MitPM1Q9N40SE3509AoA7Xl/V6sc3xhh/sARRLyYFaiuhdI9fDp8SF87Npw7iwzU5vLdqt1/OYYwxrckSRL36pq5+qIeo9+NJaQztGcUdr6+ipLJtRmM0xpgjZQmi3v4E0fotmeoFBwbwhx+MYndRBfd/tN5v5zHGmNZgCaLe/uE2/JcgAMb2jePC8X14cv5m1ucU+/VcxhhzNCxB1OsWB8Hhrdqbuim3nDGUiNAgbn99lY0XY4xptyxB1BPxW1+IhuIjQvjF6UNYuCmfN5bv8vv5jDHmSFiC8NWK80IcykUT+jIyOZrfv7naKqyNMe2SJQhf0cltliACA4S7zx5JbnElD3xsFdbGmPbHEoSvmD5QkgM1lW1yuoy+cZw3LoUn529mU15Jm5zTGGNayhKEr/3Dfu9ss1PecsYQQoMCufvN1W12TmOMaQlLEL7aoC9EQ92jwrjplEHMW5vHx9/mtNl5jTHmUCxB+PIgQQD86NhUBiRFcNcbq6msqW3TcxtjTFMsQfjaP/Vo2yaIkKAAbj9rBFvyy3hy/pY2PbcxxjTFEoSv4DCI6A6F29r81CcMTuKUYd158OP15BZXtPn5jTGmIUsQDcX3h/yNnpz6N98bTlVtHfe+t9aT8xtjjC9/Tjn6pIjkisjKJrb/wmemuZUiUisi8e62LSKywt2W5a8YG5U4EPZ40y8hLTGCH09K48XF2azItjmsjTHe8ucdxBzgjKY2qupfVXWMqo4BfgV82mBa0ZPc7Zl+jPFgCYOgNBcqvPkFff3JA0mICOF3b9g4TcYYb/ktQajqZ0BL55G+CHjOX7EclsTBzuueDZ6cPjosmJ+fNoSsrXttnCZjjKc8r4MQkXCcO42XfYoVeF9EFovIzEN8fqaIZIlIVl5e3tEHlDjIed2z7uiPdYTOz+zDiN7R/OntNZRXWbNXY4w3PE8QwFnAFw0eL01W1bHANOA6ETm+qQ+r6mxVzVTVzKSkpKOPJi4VAoIg37vxkQIDhDvOGsHOwgoe+8ybCnNjjGkPCeJCGjxeUtUd7msu8Cowoc2iCQx2koRHFdX1JqTF871RvXj0043s3FfuaSzGmK7J0wQhIjHACcDrPmURIhJVvw6cBjTaEspvEgZBvjd1EL5unTaUOoU/v/ut16EYY7ogfzZzfQ5YCAwRkWwRuUpEZonILJ/dzgHeV9VSn7IewHwRWQZ8Dbylqu/6K85GJQ5y+kLUefv8v098ODOn9Of1pTtZvHWvp7EYY7qeIH8dWFUvasE+c3Caw/qWbQLS/RNVCyUOgtpK2LcN4tM8DeUnJw7gxcXbueuNVbx67SQCAsTTeIwxXUd7qINofxLclkzt4DFTRGgQt04byrLsQl75xv/zZRtjTD1LEI3Z39S1fcz0dnZ6Mhl9Y/nzu9/a9KTGmDZjCaIx4QkQFutpU1dfAW6z17ziSh782Pu7GmNM12AJojEiTo/qdnIHATCmT+z+6Um37Ck99AeMMeYoWYJoSuKgdpUgAG45fQjBgcLv31rjdSjGmC7AEkRTEgZCyW6oKPI6kv26R4fx/6YO4sM1OXyyNtfrcIwxnZwliKYktp+WTL6unJRG/0RnetKqmjqvwzHGdGKWIJrSjpq6+goJCuC2s4azaU8pT32x2etwjDGdmCWIpsSngQR6OqprU04a0p1ThnXn/o/Wk1Nk05MaY/zDEkRTgkIhrl+7TBAAt00fTnWt8se3rcLaGOMfliCa03047G7bcQJbql9CBNec0J/Xlu7kq035XodjjOmELEE0p9cYKNjYrloy+br2xIEkx3bj9tdXUV1rFdbGmNZlCaI5vUY7rznt8y6iW0ggd5w1nLU5xTy9YIvX4RhjOhlLEM3p6SaIXcu9jaMZpw7vwUlDkvjHh1ZhbYxpXZYgmhPVEyKSYHf7TRAiwp0zRlBVW2c9rI0xrcoSRHNEoFd6u76DAKfC+toTB/DGsp18ti7P63CMMZ2EP2eUe1JEckWk0Qf4InKiiBSKyFJ3ud1n2xkislZENojIrf6KsUV6joa8NVBT6WkYh/KTEwfQPzGC215fSUW1tzPhGWM6B3/eQcwBzjjEPp+r6hh3uQtARAKBh4BpwHDgIhEZ7sc4m9drNNTVQO5qz0JoidCgQH5/zki25pfZkODGmFbhtwShqp8BBUfw0QnABlXdpKpVwPPA2a0a3OHoABXV9Y4bkMgPxibz2Gcb2ZBb7HU4xpgOzus6iGNFZJmIvCMiI9yyZGC7zz7Zbpk34tIgJKpdV1T7+s2Zw4gIDeLXr6ykrk69DscY04F5mSCWAP1UNR14AHjtSA4iIjNFJEtEsvLy/FBBGxDgPGbataz1j+0HCZGh/PrMYXy9pYD/fr3N63CMMR2YZwlCVYtUtcRdfxsIFpFEYAfQx2fXFLesqePMVtVMVc1MSkryT7A9R0POKqjrGJW/549LYfLARP70zrfsKiz3OhxjTAflWYIQkZ4iIu76BDeWfGARMEhE0kQkBLgQmOtVnIBzB1Fd1u6G/m6KiPCHc0ZRW6f89tWVqNqjJmPM4fNnM9fngIXAEBHJFpGrRGSWiMxydzkPWCkiy4D7gQvVUQNcD7wHrAFeUNVV/oqzRfZXVHeMx0wAfRPC+dlpg/no21zeWL7L63CMMR1QkL8OrKoXHWL7g8CDTWx7G3jbH3EdkaQhEBjqJIjRP/Q6mhb78aQ03li2kzvnruK4AQkkRoZ6HZIxpgPxuhVTxxAYDD2Gd6g7CIDAAOEv56VTUlHDba/ZoyZjzOGxBNFSKeNhx2KorfY6ksMypGcUN506iHdW7uZNe9RkjDkMliBaqu+xTkV1B+kP4WvmlP6k94nlttdXkltsI74aY1rGEkRL9Z3ovG770ts4jkBQYAB/Oz+dsqpafmOtmowxLWQJoqWie0NsP9i20OtIjsjA7pHccvoQPlidw4uLs70OxxjTARwyQYhIoIjc3BbBtHt9j4WtC6GD/gV+5aQ0ju2fwO/mrmJrfqnX4Rhj2rlDJghVrQWabbLaZfQ7Fsr2QP5GryM5IgEBwt9+mE5AgHDz/5ZSY/NYG2Oa0dJHTF+IyIMiMkVExtYvfo2sPep7rPPaQR8zAfSO7cY954xiybZ9PDSvYyY6Y0zbaGlHuTHu610+ZQqc3LrhtHOJg6FbvFNRPfYyr6M5YjPSe/Pxmhzu/3g9kwclMK5fvNchGWPaoRYlCFU9yd+BdAgiTmumDnwHUe+u749kybZ93PDcUt66YTKx4SFeh2SMaWda9IhJRGJE5O/1w2qLyN9EJMbfwbVLfY+Fgo1QnON1JEclOiyYBy7KILe4gl++vNyavhpjDtLSOogngWLgh+5SBDzlr6Datfp6iO0drz9EQ+l9YvnlGUN5b1UO//5yq9fhGGPamZYmiAGqeoc7DegmVf0d0N+fgbVbvdIhqFuH7DDXmKsmp3Hy0O78/s01rMgu9DocY0w70tIEUS4ik+vfiMgkoGvORBMUAimZsPULryNpFSLCveenkxgZwk+eXcy+siqvQzLGtBMtTRCzgIdEZIuIbMEZpvsav0XV3qUd74zsWuKHKU49EB8RwsOXjiOnqIKb/7fU5rI2xgAt60kdAAxx544eDYxW1QxV7Xij1rWWgac4rxs/9jaOVjSmTyy3nzWCeWvzeGhex5g5zxjjXy3pSV0H3OKuF6lqkd+jau96jYHwRNjwgdeRtKpLj+nLORnJ/P3Ddcxbm+t1OMYYj7X0EdOHIvJzEekjIvH1S3MfEJEnRSRXRFY2sf0SEVkuIitEZIGIpPts2+KWLxWRrMO4nrYREAADp8KGj6Cu1utoWk39XNbDekZzw3PfsDGvxOuQjDEeammCuAC4DvgMWOwuh/rFPQc4o5ntm4ETVHUUcDcwu8H2k1R1jKpmtjDGtjXwVCgvgJ1LvY6kVXULCWT2j8YREhjA/z2TRVFFx5ogyRjTelpaB3GpqqY1WJpt5qqqnwEFzWxfoKp73bdfAimHE7jnBpwMSKd7zASQEhfOw5eMZVt+GTc+9w21VmltTJfU0jqIB/0cx1XAO76nBd4XkcUiMrO5D4rIzPoe3nl5bdiqKCIBksfB+s6XIACO6Z/AnTOcSus/vr3G63CMMR5o6SOmj0TkXBGR1g5ARE7CSRC/9CmerKpjgWnAdSJyfFOfV9XZqpqpqplJSUmtHV7zBp7izFNd1uSNUod26cR+XHFcKo/P32w9rY3pglqaIK4BXgAqRaRIRIpF5KhbM4nIaOBx4GxVza8vV9Ud7msu8Cow4WjP5ReDTgW0UzV3bei26cOZOrQ7d85dxSfWssmYLqWlCSIGuAL4vapGAyOAU4/mxCLSF3gFuExV1/mUR4hIVP06cBrQaEsoz/XOcIb/7qSPmQACA4T7L8pgSI8orv/vN6zeaa2cjekqWpogHgImcmBmuWIOUS8hIs8BC4EhIpItIleJyCwRmeXucjuQADzcoDlrD2C+iCwDvgbeUtV3W35JbSgg0Kms3vAh1NZ4HY3fRIQG8eQV44kKC+Lyp75me0GZ1yEZY9qAtGSYZxFZoqpjReQbVc1wy5a5vavbjczMTM3KauNuE6tegxcvhx+9Dv1PbNtzt7F1OcWc98gCEiNDeXHWsSREhnodkjHmKInI4qa6E7T0DqJaRAJxWhchIkmATWgMMOg0CI6AVa96HYnfDe4RxZNXjGfHvnKufDqL0srOe9dkjGl5grgfp7K4u4jcA8wH/uC3qDqSkHAYMg1Wz4Xazt+pLDM1ngcvHsuK7H1c8+/FVFR3np7kxpjvalGCUNVnccZj+iOwC/i+qr7oz8A6lJE/cHpVb/7U60jaxKnDe/CX89KZv2EP1/93CdW1djNpTGfU0jsIVPVbVX1IVR9UVes55WvAVAiNhpWd/zFTvfPGpXD32SP4cE0uN/9vqfW2NqYTanGCMM0IDoMhZ8K3b0BN15lw57JjU/nVtKG8uXwXv3hpmSUJYzoZSxCtZeQPoKIQNs3zOpI2dc0JA/jpqYN5ZckOSxLGdDJBXgfQafQ/CcJiYeUrMPh0r6NpUzdMHYQq3Peh09/xr+elExjQ6qOyGGPamCWI1hIUAsOmw6rXoarMad3Uhdx4yiDASRK1dcq956cTHGg3qMZ0ZPYT3JrSL4aqYlj1iteReOLGUwbxi9OH8PrSnVz77BIqa6wJrDEdmSWI1tTvOEgcAovneB2JZ647aSB3njWcD1bncPXTWZRVWWc6YzoqSxCtSQTGXQHZi2B3+xxfsC1cMSmNv543mi827OGSx79ib2nXadllTGdiCaK1pV8IgaGw+CmvI/HU+Zl9ePiSsazaWcS5jy4ge68N8GdMR2MJorWFx8OI78PyF6Cq1OtoPHXGyF7856pj2FNcyQ8eXsCaXTZUuDEdiSUIfxj3Y6gsgpUvex2J5yakxfPirOMIEOH8RxfapEPGdCCWIPyh70RIGgpZXfsxU70hPaN49brj6BMfzlVPZ9n0pcZ0EJYg/EEEMq+EnUtg+9deR9Mu9IrpxouzjuWEwUnc9tpK7npjNTU2yJ8x7ZpfE4SIPCkiuSLSaJMecdwvIhtEZLmIjPXZdrmIrHeXy/0Zp1+MucSZjvTzv3sdSbsRGRrE7MvGccVxqTz5xWZ+PGcR+8qshZMx7ZW/7yDmAGc0s30aMMhdZgKPAIhIPHAHcAwwAbhDROL8GmlrC42EY2bBuncgZ5XX0bQbQYEB3DljBH8+dxRfbspnxoNfsHZ3sddhGWMa4dcEoaqfAQXN7HI28Iw6vgRiRaQXcDrwgaoWqOpe4AOaTzTt04T/g5BImH+f15G0OxeM78vzM4+lvLqWcx7+grnLdnodkjGmAa/rIJKB7T7vs92ypsoPIiIzRSRLRLLy8vL8FugRCY93Os6tfBkKNnsdTbszrl8cb/6/yQzvFc0Nz33DnXNXUVVj9RLGtBdeJ4ijpqqzVTVTVTOTkpK8Dudgx14PAUGw4H6vI2mXekSH8dzMiVw5KY05C7ZwweyF1qnOmHbC6wSxA+jj8z7FLWuqvOOJ7iwT03EAABwbSURBVAXpF8E3z0KRPUZpTHBgALefNZwHL85gfU4JZ/7zc95dudvrsIzp8rxOEHOBH7mtmSYChaq6C3gPOE1E4tzK6dPcso5pyk8BhXn3eB1JuzZ9dG/eumEyqYkRzPrPYm57bSUV1TYirDFe8Xcz1+eAhcAQEckWkatEZJaIzHJ3eRvYBGwA/gVcC6CqBcDdwCJ3ucst65jiUmHCTOcuogsP4tcS/RIieGnWcVw9OY1/f7mV6Q/MZ+WOQq/DMqZLEtXOM0VkZmamZmVleR1G48r3wj/HQPI4uKxrzhdxuD5fn8fPX1xGQWkVN586mGuOH2Az1RnTykRksapmNrbN60dMXUe3ODjhFtj4EWz4yOtoOoQpg5J476bjOW14T/7y7lrOfWQBG3JLvA7LmC7DEkRbGn81xPaDD26HOnu23hKx4SE8eHEG91+UwZb8Us68/3Me+3SjDdNhTBuwBNGWgkLhlDshZyUsetzraDoMEWFGem/ev/l4ThycxB/f+ZYfPLKA1Ttt+HBj/MkSRFsbcQ4MmAof3QX7th96f7Nf96gwHrtsHA9enMHOfeWc9eB8/vTOt5RX2d2YMf5gCaKticD0+0Dr4K2fQidqJNAWRITpo3vz4U9P4NyxyTz66UZO+funfLg6x+vQjOl0LEF4Ia4fnHwbrH8fVrzkdTQdUmx4CH85L53nZ04kPCSQq5/J4uqnF7Et33phG9NaLEF45ZhrnCav7/4SSvd4HU2HNbF/Am/fOIVfTRvKgo35nHLfp/z1vW8prazxOjRjOjxLEF4JCIQZD0BlMbx2rT1qOgrBgQFcc8IAPv7ZiZw5sicPzdvIyX/7hJcXZ1NXZ/+uxhwpSxBe6jECTvs9rH8PFj7kdTQdXs+YMP5xYQYv/+RYekSH8bMXlzHjofks3JjvdWjGdEiWILw2YSYMnQ4f3gk7FnsdTacwrl88r107iX9eOIa9pdVc9K8vuXLOItbssmaxxhwOG2qjPSgrgMeOBwmAWZ9DWIzXEXUaFdW1zFmwhYfnbaC4soaz03tz86mD6ZcQ4XVoxrQLNtRGexceD+c9CUU74KWroNYqWFtLWHAgs04YwOe3nMysEwbw7qrdnPy3T7n15eU274Qxh2B3EO3J4jnwxo3OXNbT/ux1NJ1SblEFD3+ykf9+tQ1FOT+zDz85YQB94sO9Ds0YTzR3B2EJor157zew8EE4815nTmvjF7sKy3l43kb+t2g7taqck5HMtScOoH9SpNehGdOmLEF0JHW18PzFsP4DuOh5GHya1xF1arsLK3j004089/U2qmrrmDayJ7NOGMDolFivQzOmTViC6Ggqi2HO9yBvLVzyEqRN8TqiTi+vuJI5CzbzzMKtFFfUMGlgAldP6c+Jg5MQsTkoTOdlCaIjKs13ksS+bfCj16DPBK8j6hKKK6r571fbeOqLLewuqmBQ90iunJzGORnJhAUHeh2eMa3OswQhImcA/wQCgcdV9U8Ntt8HnOS+DQe6q2qsu60WWOFu26aqMw51vk6VIACKd8NT05xkcfnr0DvD64i6jKqaOt5asZN/fbaZ1buKiA0P5qIJfbl0Yj+SY7t5HZ4xrcaTBCEigcA64FQgG2du6YtUdXUT+/8/IENVr3Tfl6jqYdUYdroEAc6Q4HPOhLK9cPHzkDrZ64i6FFXl680FPPXFFt5fvRuAU4b14LJj+zFpQCIBNgWq6eCaSxBBfjzvBGCDqm5yg3geOBtoNEEAFwF3+DGejim2D/z4Xfj3OfCfc+H8OTBkmtdRdRkiwjH9EzimfwLZe8t49qtt/G/Rdt5fnUNqQjgXTejLueNSSIwM9TpUY1qdP+8gzgPOUNWr3feXAceo6vWN7NsP+BJIUdVat6wGWArUAH9S1deaOM9MYCZA3759x23dutUfl+O90nx49jzYtcwZ5C/jEq8j6rIqa2p5Z8Vu/vvVNr7eUkBwoHDa8J78cHwfJg9MJNDuKkwH4tUdxOG4EHipPjm4+qnqDhHpD3wsIitUdWPDD6rqbGA2OI+Y2iZcD0QkwOVz4X+XwuvXQt63zvSlAVZx2tZCgwL5fkYy389IZkNuMc99vZ2Xl2Tz1opd9I4J49xxKZw7NoXURBvOw3Rs/ryDOBa4U1VPd9//CkBV/9jIvt8A16nqgiaONQd4U1WbnV2nU9ZBNFRbDe/+Chb9CwadDuc+DmHRXkfV5VXW1PLh6lxeyNrOZ+vzUIVx/eL4wdhkvjeqF7HhIV6HaEyjvKqkDsKppJ4K7MCppL5YVVc12G8o8C6Qpm4wIhIHlKlqpYgkAguBs5uq4K7XJRJEvUWPw9u3QHyaUy/Rc5TXERnX7sIKXlu6g5cXZ7M+t4TgQOHEId05JyOZk4d2t+aypl3xspnrmcA/cJq5Pqmq94jIXUCWqs5197kTCFPVW30+dxzwGFCHM6DgP1T1iUOdr0slCIAt853B/cr3wrQ/wbgfO3Nem3ZBVVm1s4jXvtnB3GU7yS2uJCIkkFOH92D66N5MGZxIaJAlC+Mt6yjXmZXkwaszYePHMPxs+N59Tn2FaVdq65QvN+Xz5vKdvL1iN4Xl1USFBXHqsB6cOaoXkwcl2p2F8YQliM6urg4W/BM+vge6xcL0+2DYWV5HZZpQVVPHFxv28NaKXby/ajdFFTVEhgZx0tDunDGiJycMSSIytL20HzGdnSWIriJnFbw6C3Yvh5Hnwul/hKgeXkdlmlFVU8cXG/fw/qrdvL8qh/zSKkICAzhuYAKnDu/BKcN60CM6zOswTSdmCaIrqa2Gz/8On98LQd1g6m2QeaU1h+0AauuUrC0FfLA6h/dX57CtwJnQaFRyDFOHdefkod0Z2TvGem+bVmUJoivaswHe+ils/hR6pcPpf7BhOjoQVWVdTgkfrsnhozU5fLN9H6qQGBnKiUOSOGlIdyYPTCQmPNjrUE0HZwmiq1KFlS/DB7c705kO+R6cehckDvQ6MnOY9pRU8tm6POatzeOzdXkUllcTIJDRN47jByUxZXAio5NjCAq0WYTN4bEE0dVVl8PCh2D+fc76mIvhhFsgtq/XkZkjUFNbx7LsQj5dl8ena3NZvqMQVYgOC+K4AYlMGpjApIGJpCVG2FwW5pAsQRhHSS58/jfIetK5uxj7I5h0I8T18zoycxT2llbxxcY9fL5uD/M37GHHvnIAesWEcWz/BI4d4CwpcTbvtjmYJQjzXYXZ8Nm98M1/QOtg1Hkw+WboPszryMxRUlW25pcxf8MeFm7MZ+GmfApKqwBIievGxP4JTOyfwDFp8aTEdbM7DGMJwjShcIfz6GnxU1BdBgOmwsSfOK8B9iy7M6irU9blFrNwYz5fbSrgq8357C2rBpw7jPGp8YxPi2d8ahyDu0dZC6kuyBKEaV5ZASx6whnfqWQ3JAxymsamXwjh8V5HZ1pRfcJYtLmArzYX8PXmAnKLKwGnDmNsvzjG9Y1jXGoc6SmxRFiHvU7PEoRpmZoqWP06fPUo7MiCoDAY/n0Yexn0Pc7uKjohVWV7QTmLthSQtbWAxVv3si6nBIAAgaE9oxnbL5YxfeIY0yeW/okRdpfRyViCMIdv9wrIegqWvwBVxRCXCukXw+jzIb6/19EZPyosq2bJ9r18s3UvS7btY+n2fZRU1gDOXUZ6n1jSU2IZnRJDep9Y6+ndwVmCMEeuqhTWvAlLn3U63QEkj4NR5zuDA0b39jY+43e1dcrGvBKWbtvHN9v3sWz7PtbmFFNb5/zu6BEdyqhkJ2GMSo5hZHIMSVE2BWtHYQnCtI5922HVK7DiRecOA6DPMU6iGDrdmst2IeVVtazeVcjybGdZlr2PzXtKqf910jM6jBG9oxmRHMOI3tEM7xVtrabaKUsQpvXlrYXVc506ixw3WfQYCUOmweAzoHeGjf/UxZRU1rBqRyErdhSyemcRK3cWsiG3BPdGg6iwIIb3imZYr2iG9YpiWK9oBveIsmHOPWYJwvhX/kZY+w6sfRu2LXT6VoQnwMBTnKX/iRDZ3esojQfKq2pZm1PM6p1FrNpZyJpdRXy7u5iyKmf6+QCB1MQIhvV0ksWQns7SNz6cQKsMbxNezih3BvBPnBnlHlfVPzXYfgXwV5wpSQEeVNXH3W2XA791y3+vqk8f6nyWINqBsgJn8qL178OGD6Es3ynvMQr6nwBpx0PfY20e7S6srk7ZWlC2P1ms3e28biso2/+IKjQogIHdIxncI4pBPSIZ1D2KQd0j6WOJo9V5NSd1IM6c1KcC2ThzUl/kO6+0myAyVfX6Bp+NB7KATECBxcA4Vd3b3DktQbQzdXWwe5mTMDbOg+1fQ20lSKAzwmzqJOg3yanHsP4WXV5ZVQ3rc0pYu7uYdTnFrMstYd3uYnYXVezfJyQogP6JEQzoHsnApEgGdI9kQFIE/RMj6RZij6qORHMJwp+9YCYAG1R1kxvE88DZwOpmP+U4HfhAVQvcz34AnAE856dYjT8EBDh1Eb0zYMrPnIECt38Nmz+DrQvgq8dgwQPOvolDoO8xTrJIGe901rN+F11KeIjbhLZP7HfKiyqq2ZBbwoacEjbklbAht4QV2YW8vWIXvn/f9o4Jo39SJP2TIkhLPLAkx3azUW6PkD8TRDKw3ed9NnBMI/udKyLH49xt3Kyq25v4bHJjJxGRmcBMgL59bXTSdi24m/OYqf8JzvvqCqdD3rYvYftXTqX3kmecbaExkJwBvcdC8ljnNbo3WCuYLic6LJixfeMY2zfuO+UV1bVsyS9lQ24Jm/JK2ZRXwqY9pby6ZAfFbr8NgOBAoU9cOKmJEfRLCCc1wXntlxBBSlw3gi15NMnrfvRvAM+paqWIXAM8DZx8OAdQ1dnAbHAeMbV+iMZvgsOcSYzqJzKqq4P89ZCd5SSO7EWw4H6oc3/YwxOdR1O9RkNPd4lPs9ZSXVRYcCBDe0YztOd367NUlfzSKjbllbJlTymb853XLfllfLkpf38FOTiV5L1ju9EvIZy+8eH0iQ+nT5zz2jc+nLjw4C7dNNefCWIH0MfnfQoHKqMBUNV8n7ePA3/x+eyJDT77SatHaNqXgABIGuIsGZc4ZdXlzlzbO5Y49Rk7lzmPpeqTRnC4MwptjxHQfYSz3n04RCZ5dx3GUyJCYmQoiZGhTEj7bt2WqpJXXMnWgjK25pexNb+UbQVlbCso2z8nuK+IkEBS4sJJievmLuEkx3UjObYbyXHdSIgI6dQJxJ+V1EE4j42m4vzCXwRcrKqrfPbppaq73PVzgF+q6kS3knoxMNbddQlOJXVBc+e0SuouoqYS8r51OuvtXuEkkJxVUO7z3yM8AZKGOskmcQgkDXbqNaKTrW7DNKm0sobte8vYXlDOtoIysveWkb23nO0FZezYW/6dR1cAYcEB9I51EkbvmG70ig2jt896r5gwwkO8flDTPE8qqVW1RkSuB97Daeb6pKquEpG7gCxVnQvcICIzgBqgALjC/WyBiNyNk1QA7jpUcjBdSFCo+6gp/UCZKpTkQO4aJ3nkroa8dc6UqxWFB/YLDoeEAZAw0EkYCQMgfoAzvlR4vNVxdHERoUGNPraqV1heTfZeJ1ns2FfOjr3l7CqsIHtfOd/uziXPHRnXV0y3YHrFhNEzxkkYPaO70TMmlB7RTlnP6DBiurXPR1nWUc50bqrOTHr562HPOidp5G9wln1bnU599UJjID4V4tKcuo24VGeJ7QcxKRAY7NFFmI6iqqaOnKIKduwrZ1dhOTv3VbCrsJzdhZXsLipn176Kgx5jgdPvo0d0GD2iQ+keHUb3qFC6R7mv0c56j+hQvyQS60ltTGNqKmHvVijYBAUboWCzs753M+zbdqCeA0ACnMdTsf0gto8zn3dMHydxxPSBmGSnlZYxh1BZU0tuUSU5RRXsLqpgd2EFOUUV5BY7ZTlFleQWVVDqU5leLyQwgKSoUBKjQkmKDCEpKpSkyFB6xnTj4mOOrBWnV/0gjGnfgkKduomkwQdvq62B4p1O0ti3zV22Ogll82dQtBOnD6eP8EQnUcT0cZrkRvd2kkpUL2c9qheE2LzQXV1oUKDTWiq++f8LJZU15LqJI6+4cv9rXnEleSWVZO8tZ+n2QgpKK+keFXbECaI5liCMaUxgkHOXENvED11NFRTtcJbCbGek26JsZxrXgk2w5fPv1n3UC4uBqN4Q1dNJGFE9nSWyx4HXyB6WSAyRoUFEJkXSPymy2f1q65Tiimq/xGAJwpgjERTi1FPEpzW9T2WJc6dRvBOKdjnJpHg3FO9ylj3rnSle62oO/mxIlDPAYWQPp8luZA+I6A4RiU55/XpEIoREWuV6FxYYIMSGh/jl2JYgjPGX0MimH2HVq6tzBjQsyXGSRXGOu57rvC/Jg5zVsOmTxu9IwJkaNjzxQMKoXw+Pd9bDE767dIu1zoWmRSxBGOOlgAD3DiEJGNn8vjVVUJoHpblQusdZL8mFsj3ue7csb51TVl3WxIHEedQVnuAkkW7x7mucux7nrvssYbEQGm19SLoYSxDGdBRBIW4leKPDkh2sqszpPFiW7ySP8r3Oelm+Myx7Wb6zvXiX03+kvACqSpo+ngQ4iSUs1rkLOeg15sD277y6S5B/HoMY/7EEYUxnFRLuLDEpLf9MTSWU73OSRfk+J6nUr1fs83nd66wXbncefZXvg7pDVJQGdXPmAQmNdpNG/br7+p31KHc9yumfEhrlLMHdrL6lDVmCMMYcEBQKUT2c5XCoOuNm1SeRyqIDiaOyyCmvcMvqt1UUOS3AKoqcsiYfifmQwAPJon4JiXTXI53K/dBIt6y59xFOmdXFNMsShDHm6IkcuGOJ7n1kx6ithspiJ3lUlRxIHJUlUOkmlKoS932Rs29ViZN8CrMPvK8s5qA+Kk0J6uYki/okElKfPBoukc4wLfXrIeHu+0bWg7p1mroaSxDGmPYhMNhteXWUswuqQlXpgWRSVey+lh5IIAet1y9ugira6WyvL689eIylZgXXJ41wCI44kEQOKuvmrAd3c5aQiAZl4Qe27V8Pd/rptAFLEMaYzkXEuSMIjYSoVjpmbQ1UlzZIJqXOY7GG642WlTvrxbsOrNeXH27yAQgI9kkYYU7nyyvfaaWLPcAShDHGHEpgEAS6rbFaW22NkzBqKg4kleoK97V+KXe21VQ02FbuLMFhrR8XliCMMcZbgUEQGA00PsS4lzpHTYoxxphWZwnCGGNMo/yaIETkDBFZKyIbROTWRrb/VERWi8hyEflIRPr5bKsVkaXuMtefcRpjjDmY3+ogRCQQeAg4FcgGFonIXFVd7bPbN0CmqpaJyE+AvwAXuNvKVXWMv+IzxhjTPH/eQUwANqjqJlWtAp4HzvbdQVXnqWp998kvgcMYE8AYY4w/+TNBJAPbfd5nu2VNuQrwbcgbJiJZIvKliHzfHwEaY4xpWrto5ioilwKZwAk+xf1UdYeI9Ac+FpEVqrqxkc/OBGYC9O3b+lPuGWNMV+XPO4gdQB+f9ylu2XeIyCnAb4AZqrq/S6Gq7nBfNwGfABmNnURVZ6tqpqpmJiUltV70xhjTxYlqCwe1OtwDiwQB64CpOIlhEXCxqq7y2ScDeAk4Q1XX+5THAWWqWikiicBC4OwGFdyNnTMP2HqEIScCe47wsx1VV7xm6JrX3RWvGbrmdR/uNfdT1Ub/uvbbIyZVrRGR64H3gEDgSVVdJSJ3AVmqOhf4KxAJvCjOGO/bVHUGMAx4TETqcO5y/nSo5OCe84hvIUQkS1Uzj/TzHVFXvGbomtfdFa8ZuuZ1t+Y1+7UOQlXfBt5uUHa7z/opTXxuATDKn7EZY4xpnvWkNsYY0yhLEAfM9joAD3TFa4aued1d8Zqha153q12z3yqpjTHGdGx2B2GMMaZRliCMMcY0qssniEONONtZiEgfEZnnjp67SkRudMvjReQDEVnvvsZ5HWtrE5FAEflGRN5036eJyFfud/4/EQnxOsbWJiKxIvKSiHwrImtE5NjO/l2LyM3u/+2VIvKciIR1xu9aRJ4UkVwRWelT1uh3K4773etfLiJjD+dcXTpB+Iw4Ow0YDlwkIsO9jcpvaoCfqepwYCJwnXuttwIfqeog4CP3fWdzI7DG5/2fgftUdSCwF2ccsM7mn8C7qjoUSMe5/k77XYtIMnADzujQI3H6Xl1I5/yu5wBnNChr6rudBgxyl5nAI4dzoi6dIGjBiLOdharuUtUl7noxzi+MZJzrfdrd7WmgUw2MKCIpwPeAx933ApyM04MfOuc1xwDHA08AqGqVqu6jk3/XOP26urmjOIQDu+iE37WqfgYUNChu6rs9G3hGHV8CsSLSq6Xn6uoJ4nBHnO0URCQVZ2yrr4AeqrrL3bQb6OFRWP7yD+AWoM59nwDsU9Ua931n/M7TgDzgKffR2uMiEkEn/q7dsdvuBbbhJIZCYDGd/7uu19R3e1S/47p6guhyRCQSeBm4SVWLfLep0+a507R7FpHpQK6qLvY6ljYWBIwFHlHVDKCUBo+TOuF3HYfz13Ia0BuI4ODHMF1Ca363XT1BtGjE2c5CRIJxksOzqvqKW5xTf8vpvuZ6FZ8fTAJmiMgWnMeHJ+M8m491H0NA5/zOs4FsVf3Kff8STsLozN/1KcBmVc1T1WrgFZzvv7N/1/Wa+m6P6ndcV08Qi4BBbkuHEJxKrU45/7X77P0JYI2q/t1n01zgcnf9cuD1to7NX1T1V6qaoqqpON/tx6p6CTAPOM/drVNdM4Cq7ga2i8gQt2gqsJpO/F3jPFqaKCLh7v/1+mvu1N+1j6a+27nAj9zWTBOBQp9HUYfU5XtSi8iZOM+p60ecvcfjkPxCRCYDnwMrOPA8/tc49RAvAH1xhkr/oao2rADr8ETkRODnqjrdnYTqeSAeZ170S33nIukMRGQMTsV8CLAJ+DHOH4Sd9rsWkd/hzGlfg/O9Xo3zvL1Tfdci8hxwIs6w3jnAHcBrNPLdusnyQZzHbWXAj1U1q8Xn6uoJwhhjTOO6+iMmY4wxTbAEYYwxplGWIIwxxjTKEoQxxphGWYIwxhjTKEsQxrQDInJi/WizxrQXliCMMcY0yhKEMYdBRC4Vka9FZKmIPObONVEiIve5cxF8JCJJ7r5jRORLdxz+V33G6B8oIh+KyDIRWSIiA9zDR/rM4fCs28nJGM9YgjCmhURkGE5P3UmqOgaoBS7BGRguS1VHAJ/i9GwFeAb4paqOxunBXl/+LPCQqqYDx+GMPgrOCLs34cxN0h9nLCFjPBN06F2MMa6pwDhgkfvHfTecQdHqgP+5+/wHeMWdkyFWVT91y58GXhSRKCBZVV8FUNUKAPd4X6tqtvt+KZAKzPf/ZRnTOEsQxrScAE+r6q++UyhyW4P9jnT8Gt8xgmqxn0/jMXvEZEzLfQScJyLdYf88wP1wfo7qRwy9GJivqoXAXhGZ4pZfBnzqzuaXLSLfd48RKiLhbXoVxrSQ/YViTAup6moR+S3wvogEANXAdTgT8kxwt+Xi1FOAM+zyo24CqB9RFZxk8ZiI3OUe4/w2vAxjWsxGczXmKIlIiapGeh2HMa3NHjEZY4xplN1BGGOMaZTdQRhjjGmUJQhjjDGNsgRhjDGmUZYgjDHGNMoShDHGmEb9f4sxh6vxwRjRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_graphs(test_log, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3gc5bn4/e+tXq1iSW6SLRkExgXHtjCdEKoJYEgCoYR0yjmBhBR+CWmEkOQNOeGknZBCgABpBgMBJyGY3kKTXHADY1suKpYly+rWrrbc7x8zkteyZEu2Vitp7s917bWamWd3nvHCc888VVQVY4wx3hUX6wwYY4yJLQsExhjjcRYIjDHG4ywQGGOMx1kgMMYYj7NAYIwxHmeBwBhjPM4CgTHGeJwFAmOiSBz2/5kZ0ew/UOMJInKriGwRkTYR2SAiH4k4dp2IvBtxbL67v0hEHheRBhFpFJFfu/tvF5E/R3y+WERURBLc7ZdE5Eci8h9gLzBdRD4bcY5KEbmhV/4uEZHVItLq5nORiFwuIit6pfuqiDwZvX8p40UJsc6AMcNkC3A6UAdcDvxZRI4GTgNuBy4FKoCjgICIxAP/BF4APgmEgLJBnO+TwAXARkCAY4GLgErgDODfIlKuqitFZCHwEHAZ8DwwCcgEtgK/F5HjVPXdiO/94eH8AxjTH3siMJ6gqktVtVZVw6r6MLAJWAhcC/yPqparY7OqbnePTQb+n6p2qKpPVV8bxCkfUNX1qhpU1YCq/ktVt7jneBl4BicwAXweuF9Vn3XzV6Oq76mqH3gYuAZARGYBxTgBypghY4HAeIKIfMqtemkWkWZgNpAHFOE8LfRWBGxX1eBhnrKq1/kvEJE3RWSPe/4Pu+fvPldfeQB4ELhaRATnaeARN0AYM2QsEJgxT0SmAX8AbgLGq2o2sA6nyqYKpzqotypgane9fy8dQFrE9sQ+0vRM6ysiycBjwF3ABPf8T7nn7z5XX3lAVd8EunCeHq4G/tT3VRpz+CwQGC9IxymYGwBE5LM4TwQA9wK3iMgCt4fP0W7geBvYCdwpIukikiIip7qfWQ2cISJTRSQL+OYhzp8EJLvnD4rIBcB5EcfvAz4rImeLSJyITBGRGRHHHwJ+DQQGWT1lzIBYIDBjnqpuAP4XeAPYBcwB/uMeWwr8CPgr0AY8AeSqagi4GDga2AFUA1e4n3kWp+5+DbCCQ9TZq2ob8CXgEaAJ585+WcTxt4HPAj8HWoCXgWkRX/EnnMD1Z4yJArGFaYwZ2UQkFagH5qvqpljnx4w99kRgzMj330C5BQETLTaOwJgRTES24TQqXxrjrJgxzKqGjDHG46xqyBhjPG7UVQ3l5eVpcXFxrLNhjDGjyooVK3aran5fx0ZdICguLqaioiLW2TDGmFFFRLb3d8yqhowxxuMsEBhjjMdZIDDGGI+zQGCMMR5ngcAYYzzOAoExxnicBQJjjPG4qI4jEJFFwC+BeOBeVb2z1/FpwP1APrAHuEZVq6OZJ2OMGVbhMHS1O69AJ2gYwiHQEIS6INgFIT8E/c7xoM95Bdz3cMBJHwrAsYtgyoIhz2LUAoG7+PfdwLk4c7mXi8gyd274bncBD6nqgyJyFvBjnOX4jDHmyIRDTuHrb4euDrdADTqvUDDi7y63QPY7hW3I7+6LSBP0Q7DTffftK8CDvohjboEeCjgFeve5Ax1Dd02ZE0dXIMBZ/HuzqlYCiMgS4BIgMhDMBL7q/v0izqIgxpixIhyCwF7o2uu8B/37Ct7IQjrQse+OOBTYvwAOdDifD/mdu2lV53vDQadwDwXc7+h0z9Xhnss3tNcSlwgJKZCQDPFJkJAE8cmQmAIJqc52cqZzPCEZkjKc7aQMSM5w3pPSQeL2vRKSIT7R/b5U97vcV2Kqsz8+yUkjcSBy6HwehmgGginsv4B3NXBirzTvAB/FqT76CJApIuNVtTEykYhcD1wPMHXq1Khl2BhP6KmqcAvMA6ojOp19/raIu1q3IA/49t1ZhwL7F+Yh/4HVHOHA4eczLtEpKBPTICnNKXTj4vcViHGJEJfgFJQZBU667rSJaU6hG1kIx7vpJR7iE9zPxzvf21MYJ+9f+MYlOK8E99xjVKznGroF+LWIfAZ4BagBQr0Tqeo9wD0AZWVlNm+2GZsCPvC37n/XrGG3TjnoFMy+VieNvx262pz3oG9fYd5TMLv7gxHVHsHOfQX9YHUXsomp+wrH+ESngE0ZB+MmOXex3QVo5N1tZOGckOwWvEnOvu675sTUiLvgZIizfizDKZqBoAYoitgudPf1UNVanCcCRCQD+JiqNkcxT8YMDVX3rrl1X+Hsa3HeQ0Hn7jEuAdB9jX7+NuhogL27Ye8e6GyCzmbn3dd8eFUZcQn7CtiElP3vgtPy3OoL9063u3BOTNuXJjE9omojef8COTKNFcwxp6qEFeLjhr56KJqBoBwoFZESnABwJc6i3T1EJA/Yo6ph4Js4PYiMib5wCPY2OgVzRwN07N5XkPta969r7r7D9re5d+VuunBw8OeNS4T0fEgbD6nZkHc0pGRDao6znTxu/zvrnqqQeKdQTsly7qCTxzkFdUJy1OqNTXSoKr5AmFZfgJbOAM17AzTv7WJPRxeNHV00tnc523u7aNoboGVvFy2dAVp9QX506WyuXDj01eNRCwSqGhSRm4DlON1H71fV9SJyB1ChqsuAM4Efi4jiVA3dGK38mDEq0OkU4h0N0Lkn4u484i49cp+vGdrrnbtyDff9nRLvNuy51RnJGZCUCeMm7yuEU8a571lu4RyxLz5pX2Mn7LvLTs5wjlvBPWp1BcPs6eiioc1PQ7uPpo4Abb4Abb4g7X7n1eEP0hkI0RUMEwgpvkDI2d8VpN0XpM0XJBjuv4Y7LSme3PQkctKSyE5LZGpuGlmpCWSnJnHcpHFRua5Rt1RlWVmZ2noEY1TAB6010FrrvNp3uVUpjc7deXd9ub8N2uqcl7+l/++TuIgCOmtfQZ1RABkTnPf0fEjPc6pRUtw0iWlWWI8BobD23Gk3dwboCoYJhpVgKEwgFMbvFtRhVVSVYFjZ097FrjYf9a1+2v1BfIEQvkCYNn+Apo4A7f7+nwKTE+LISE4gPTmBtKR4khLiSIyPIzkhjvTkBDLcV2ZKApkpiWSmJJCVmkh2WiJZqYmMz0hmfHoSKYnRaZQWkRWqWtbXsVg3FhuvUHWqV3ytTp34nkrYvRF2b4I9W6F5u1Pw9xaf5BTWSenOnbrEOX/nHwvTz4SMfEh3C/S03P3vzpMyrEAfA3yBEE17nSoTp/rET2N7dzWKnz0dAVo7AzR3dtHuc+7G/cEwnYEQh3Ofm5OWSEFmCpkpCaQlJZCTFsexqZlkpyWSk5ZEXkYy+ZnJ5GUkkZue1FOoJ8aP3nYUCwRm6AQ6ncK8ucop2Ju2we73oWEjNG7puyvhuELILYHScyFrKmQXOVUwmZOdwTPJmVaYjwGqSmtnkMYOP017u2jqCLCno4vdHX72tHfR0dVdleLUne/p2FfwdwYO6EgIQEKcMD5jXxVKSV46mSmJpCbGk5wQ11PFkpuRTE5aIskJ8cTHCQlxQmJ8nHvHLsTHOa84EbLddF5jgcAMnKpTdVO3Fuo3OAV90zZo3uHU03e1759e4iCnxLl7Lz3PqYLpvmPPKYa8Y5x6czOq+QIhGtr81Lf52NXqp67F11O9sqvVR12Lj50tvn4L9NTEeNKTE0h2C+aMlATGpydzdEEGuWlJ5Lj15bnp+6pPxqcnMy41AbGbhCFhgcDsL+BzCveWamjZ4d7d73Du8Bs3O9U63dLznYJ+SplT554+3tmXPdV5jSt0ui+aUafdH6S6aS9Vezr3e9/V5icQDBMKK/5giMb2Ltr6qDdPio+jYFwyBZnJHDdpHB+aUcDEcSnkZTqFulOwJzE+I4m0JCuGYs1+Aa/ytTh18y1VTqFf/y7UrnLu9CO7RcYlQFYhZE+DmZfAhNkwcQ4UzHTq4c2o17y3i7U1LaypbmFtdQtra1qoad5/0FlqYjxFualMGJdCckIc8XFCUkI849OTyM9MJj8jmQlZKUwYl0xBZgo5aYl2tz6KWCDwAlWnrr7yJagudwr8xs37p0nJhsnz4JQvwYRZkFXkBIDMiWN6aL1XqCq1LT4217ezbXcHO/bsZXvjXt7f1caOPXt70hWPT2P+tByuPnEqU3PTKMpNozAnlfHpSVawj2EWCMaqrg7Y8iK8/2/Y/AK01Tr7MyfDlPkw90rIO9ZpnM2a6vS4sf/RRzVVZWeLj411bWzc1UbVnr3UtfiobfGxvbGDvV376uhTEuOYlpvO7CnjuGrhVI4vzGL25Cyy0hJjeAUmViwQjCUdjbDxKXj3H87df8jv9J8/6kw46iyY/iHImRbrXJrD4AuEqG7qpKppLzVNne5I0wCtnUHqWjqpbfZR09y5Xz/3nLREJmalMikrhRNLcjm6IIOjCzKYnpdOfmay3eGbHhYIRrNgl1PNs+0VqHwZtr/uLHaRNRXKPgfHXgDTTnGmKjCjSnXTXiq2NVGxfQ8rtjezsa6V3oNRk+LjyExJYMK4FIpy0zhpei5HFWQwY+I4jp2QaXf3ZsAsEIw2XXth0zOw4UnnvbvL5oTZcNpX4LiLYdJcq+YZZXY07uU/W3bzxpZGKrbtobbFmYAuIzmBeVOzOfesUqbnpVOUm8rk7FRy0qI3AtV4jwWC0aJpO7zxa1j1F2ehjrQ8mHMZHHU2TDvV6bppRoVAKExlQwcrtjt3/G9v3UN1k9NLpyAzmYUludxQnEtZcQ4zJo6LymyTxkSyQDDS7VoPr/0C1j3mDNCaczl84CqYeoqzuIYZ0dr9Qd7e2sh/Njc63TKbOtnZ0tlTzZOXkcSCaTlcd/p0Tj06j6Py063u3gw7K0lGqh1vwWs/g/efdubMOem/4aQvQNaUWOfM9ENV2dXqZ9WOJiq2N7FiexPraloIhpWkhDhmTx7HwpJcinJSKc5LZ/7UHKaNT7OC38ScBYKRRBW2vQYv/wS2vQqpufChb8MJ1zrdO82I09Dm54lVNbxZ2ciamhYa2vyAMxPl3MJsrj/DudNfMC3H6vTNiGWBYCRQdbp7vvwT2PGGM13D+f8fLPiMM9OmGVHafAFe39LI4yuref7deoJhpbQggzNK8zm+MIvjC7OYNTmLpITROxul8RYLBLHUHQBeuhOq3nQGe13wU5j/SWcxEzMihMLKmupmXnl/N69uamBVVTOhsDI+PYnPnVbCx8uKOLrAJs8zo5cFgljZ/jo8f4fzBDCuEC78X5j3SWfpQRNzobDy2ubdPL6ympffb6B5bwARmDMlixvOmM5ppXmUTcu1u34zJkQ1EIjIIuCXOEtV3quqd/Y6PhV4EMh209yqqk9FM08x17ARln8LNj8HGRPhw3fB/E9ZABgBfIEQK3c08eqm3TyxqoadLT6y0xI5e8YEzjgmj9NL88lNt9lUzdgTtUAgIvHA3cC5QDVQLiLLVHVDRLLvAI+o6m9FZCbwFFAcrTzF3LbX4G9XOd1Az/k+LLzeWRfXxExNcyfL19Xx7IZdrNjRRFcwTJzAGcfk850LZ3LOzAJPLlRivCWaTwQLgc2qWgkgIkuAS4DIQKBA91zGWUBtFPMTW+/9C5Z+1pnr55rHncneTExUNrTz9Po6lq+r451qZ83jYyZk8MmTpnHy9PGcUJJLVqpNz2C8I5qBYApQFbFdDZzYK83twDMi8kUgHTinry8SkeuB6wGmTp065BmNuhUPwD+/4kzzfPVSGwU8zFSV9bWtPLO+juXrd7FxVxsAxxdm8fVFx7Jo1kSm51tjr/GuWDcWXwU8oKr/KyInA38SkdmqGo5MpKr3APcAlJWVHcZy1DESCsKz34U3f+NMBfHxh2xpxmFU3bSXJW9X8fjKampbfMQJlBXn8r2LZ3LerIlMybaeWcZAdANBDRBZ/1Ho7ov0eWARgKq+ISIpQB5QH8V8DQ9fCzz6OadR+MT/gvN+ZFNCDINAKMwL79XzcHkVL26sR4APHpPPV849hrOPm2CNvcb0IZolUzlQKiIlOAHgSuDqXml2AGcDD4jIcUAK0BDFPA0Pfzs8cJGz7ONFv4Cyz8Y6R2NaZ1eINdXNPLthF0+srmF3excFmcl88UNHc8XCqXbnb8whRC0QqGpQRG4CluN0Db1fVdeLyB1AhaouA74G/EFEvoLTcPwZVR09VT99CYfh7zfArnVw1RI45vxY52jMUVXW1bTy9PqdvLa5kfXufD6J8cLZMybw8RMKOaM0n4R46+NvzEBEta7CHRPwVK99t0X8vQE4NZp5GHYv/hDe+ycsutOCwBAKh5VVVc38e+1O/r2ujprmTuLjhPlTs7nujOmUTcthwbQcstOs6seYwbJK66G05hF49X9h/qeddgFzxDbUtvL4ymr+tXYnO1t8JMXHcVppHjefU8o5VudvzJCwQDBU6t+FZV+Caac5o4VtauHDVtvcyVNrd/L4yho27GwlMV744DEFfGPRDM46roBxKdbH35ihZIFgKAQ6ncFiyRlw2f2QYHepg9XqC/DkqhqeXF1LxfYmwOnn//3Fs1g8dzI5dudvTNRYIBgKy78FDe/CNY9B5oRY52ZUWVPdzF/e3MGyd2rpDISYMTGTW847hg/PmWSDvIwZJhYIjtSGJ6Hifjjli3B0nwOjTS++QIh/vFPLn9/czjvVLaQlxXPpvMlcvXAacwqzYp09YzzHAsGR6GyGf9zsTB1x1m2HTu9xm+vb+dvbO3h0RTUtnQFKCzK445JZXDpvitX7GxNDFgiOxGs/d4LBp//P2gX6EAor62tb+M/mRl7cWM/bW/eQGC+cN2sinzxpGieW5Np6vcaMABYIDldLNbz1Ozj+Cpg4J9a5GVHeq2vlb2/t4Ml3amneGwDg2AmZfH3RsVy+oIj8TFt7wZiRxALB4Xrxx6BhOOvbsc7JiLC3K8g/1+zkb2/vYNWOZpLi4zh/9kTOOa6Ak48aT0FmSqyzaIzphwWCw7FrA7zzVzjpC5A9CqfFHiLBUJiK7U38c00tT66qpc0f5Kj8dL5z4XF8dH6hDfYyZpSwQHA4nrsdkjLh9K/FOifDTlV5o7KRpRXVvPBePS2dAZIS4rhoziSuOnEqZdNyrN7fmFHGAsFg7VwDm5bD2bdBWm6sczNsuoJh/rW2lntf3cr62lZnLd/jCjj3uAmcfkw+Gcn2n5Ixo5X93ztYb/0OEtOh7HOxzsmw2Lq7gyXlO3i0oprGji6OLsjgzo/O4dJ5U0hJtLV8jRkLLBAMRns9rF0K8z8FqTmxzk3U+AIhnl5Xx8PlVbxR2Uh8nHDOcQVctXAqZ5TmExdnVT/GjCUWCAaj4n4IdY3ZmUW3NLTzpze28/jKalp9QabmpnHLecfw8bIiCsZZrx9jxioLBAMV9EP5fVB6HuSVxjo3QyYUVl58r54H39jGq5t2kxgvXDB7ElcuLOKkkvF292+MB1ggGKh1j0NHPZz037HOyZBobPezpLyKv761g5rmTiaMS+Zr5x7DlQun2oAvYzwmqoFARBYBv8RZqvJeVb2z1/GfAx9yN9OAAlXNjmaeDosqvPkbyJ8B0z906PQjWKsvwB9eqeS+17aytyvEKUeN5zsXHsc5MyeQaEs7GuNJUQsEIhIP3A2cC1QD5SKyzF2eEgBV/UpE+i8C86KVnyNSuwrq1sCFPxu1C874AiH+/OZ27n5xM017A1x0/CRuPruU0gmZsc6aMSbGovlEsBDYrKqVACKyBLgE2NBP+quA70UxP4dv7aMQlwizPxrrnAxaIBRmaUU1v3p+E3WtPk4vzeMbi2Ywe4pN92yMcUQzEEwBqiK2q4ET+0ooItOAEuCFfo5fD1wPMHXqME/pEA7BusecRuJR1GU0EArz95U13P3SZrY37mX+1Gx+fsUHOPmo8bHOmjFmhBkpjcVXAo+qaqivg6p6D3APQFlZmQ5nxtj2GrTXwZzLhvW0hysYCrN0RTV3v7iZ6qZOZk8Zx32fLuOsGQU29YMxpk/RDAQ1QFHEdqG7ry9XAjdGMS+Hb+1SSMqAYxbFOieH9OqmBn7wzw28v6udDxRl84NLZnPmsfkWAIwxBxXNQFAOlIpICU4AuBK4unciEZkB5ABvRDEvhyfohw3L4LiLISkt1rnp13t1rdy1fCPPvVvP1Nw0fnfNAs6fNcECgDFmQKIWCFQ1KCI3Actxuo/er6rrReQOoEJVl7lJrwSWqOrwVvkMxKZnwd8yYquFNu1q4xfPb+Jfa3aSmZzA1xcdy+dOLbE5gIwxgxLVNgJVfQp4qte+23pt3x7NPByRtUshLQ9Kzox1Tnp0jwT+05vbefn9BtKT4rnpQ0dz7eklZKfZ/P/GmMEbKY3FI4+/Dd5/2plgLj72/0ytvgAPv13FA69v6xkJ/OVzSvnUycW2AIwx5ojEvoQbqSpfhqAPZl4S02zUt/q455VKlpRX0e4PsrAk10YCG2OGlAWC/mx5wektVLgwJqdv8wW455VK7n11K12hMBfOmcR1p09nTqENBDPGDC0LBP3Z8gIUnw4Jw1ftoqq8u7ONZe/UsrSiisaOLi46fhK3nHcsxXnpw5YPY4y3WCDoy55KaNo6bDON1rf6eHRlNY+vrGFzfTvxccIHj8nn5rNLmVs08ubgM8aMLRYI+rLlRef9qLOidopgKMxLGxtYUl7FixvrCYWVhcW5/PDS2Xx4ziRrADbGDBsLBH2pfBGyimD80UP+1dVNe3mkoppHyquoa/WRl5HMdadP5+NlhUzPzxjy8xljzKFYIOgtFITKV2DWJUM25bQvEGL5+jqWVlTzny27ATijNJ/bF8/i7OMKrPePMSamLBD0VrvSGU08BNVC4bDy2Mpq7npmI7ta/RTmpHLz2aV8bH4hRbkjd8oKY4y3WCDobcsLgEDJB4/oayq27eG2J9ezYWcrc4uyuevyuZx6VJ6tAWyMGXEsEPS25QWYPA/Scg/7K55cXcMtS9+hIDOFX101j4uPn2QTwBljRiwLBJF8LVBdAad95dBp+/HH/2zl+//YwIklufzh02WMS0kcwgwaY8zQs0AQacdboCGYfnjVQr947n1+8dwmzps5gV9dNc9mATXGjAoWCCLVrADEqRoapL+vquYXz23iY/ML+cnH5pBgPYGMMaOElVaRaldC/rGQnDmoj62raeHWx9aysCSXOy0IGGNGGSuxuqlCzUqYsmBQH9vT0cUNf1pBbnoSv/nEfBsTYIwZdaxqqFtLFezdPahqIVXl5iWraGj38+h/nUxeRnIUM2iMMdER1dtXEVkkIhtFZLOI3NpPmo+LyAYRWS8if41mfg6qZqXzPmX+gD/y9tY9vLppN7cumsHxhTY5nDFmdIraE4GIxAN3A+cC1UC5iCxT1Q0RaUqBbwKnqmqTiBREKz+HVLsS4hJhwuwBf+T3r1QyPj2Jq0+cGsWMGWNMdA3oiUBEHheRC0VkME8QC4HNqlqpql3AEqD3cl/XAXerahOAqtYP4vuHVs1KmDgbEgZWvbNpVxsvvFfPp04utm6ixphRbaAF+2+Aq4FNInKniBw7gM9MAaoitqvdfZGOAY4Rkf+IyJsisqivLxKR60WkQkQqGhoaBpjlQQiHYec7MHng1UL3vFJJSmIcnzx52tDnxxhjhtGAAoGqPqeqnwDmA9uA50TkdRH5rIgcydDZBKAUOBO4CviDiBxQ2a6q96hqmaqW5efnH8Hp+tG4GfytA24fqGvx8cTqGq4oK7J1A4wxo96Aq3pEZDzwGeBaYBXwS5zA8Gw/H6kBiiK2C919kaqBZaoaUNWtwPs4gWF41boNxQN8Ivjj61sJhZVrT58exUwZY8zwGGgbwd+BV4E04GJVXayqD6vqF4H+VlMpB0pFpEREkoArgWW90jyB8zSAiOThVBVVDvoqjlTNSkhMdwaTHYIvEOKvb+3ggjmTbCppY8yYMNBeQ79S1Rf7OqCqZf3sD4rITcByIB64X1XXi8gdQIWqLnOPnSciG4AQ8P9UtXHQV3GkalbA5A9A3KEbfV94r542X5BPLLSeQsaYsWGggWCmiKxS1WYAEckBrlLV3xzsQ6r6FPBUr323RfytwFfdV2wEu6BuLSy8bkDJl62uJT8zmROnj49yxowxZngMtI3guu4gAOB29xxYyTnS1W+AkH9ADcVtvgAvbKznwjmTiLcFZowxY8RAA0G8RKys4g4WGxvdZRo2Ou8T5hwy6bMbdtEVDHPx3MlRzpQxxgyfgVYNPQ08LCK/d7dvcPeNfi07nPfsooOnA5a9U8uU7FTmT7XpJIwxY8dAA8E3cAr//3a3nwXujUqOhlvzDkjPh8TUgybb09HFa5t2c+3p023ZSWPMmDKgQKCqYeC37mtsaa6C7EP3AHp6XR3BsHLx3EnDkCljjBk+AwoE7uRwPwZmAind+1V19I+oaqka0ERzy96pYXp+OjMnjRuGTBljzPAZaGPxH3GeBoLAh4CHgD9HK1PDRhVaqg/ZPrC73c9bW/dw8fGTrVrIGDPmDDQQpKrq84Co6nZVvR24MHrZGibt9RD0QfbBJ457q3IPqnDmsVGY58gYY2JsoI3FfncK6k3uaOEa+p9aYvRocSdHzTr4E8GblY2kJ8Uze0rWMGTKGGOG10CfCG7GmWfoS8AC4Brg09HK1LBpHljX0TcrGykrzrX1iI0xY9IhnwjcwWNXqOotQDvw2ajnarh0B4KDPBHsbvezqb6dj84vHKZMGWPM8DrkLa6qhoDThiEvw6+lClKyIaX/nkBvVe4B4KTpucOVK2OMGVYDbSNYJSLLgKVAR/dOVX08KrkaLs1VA6oWsvYBY8xYNtBAkAI0AmdF7FNglAeCHZB78KEQb1Y2ckKJtQ8YY8augY4sHjvtAt1Unaqh6Wf2m6ShzWkf+NgCax8wxoxdAx1Z/EecJ4D9qOrnhjxHw6WzCbraD1o19NZWZ42ck2ztAWPMGDbQqqF/RvydAnwEqB367AyjAYwh6GkfmGzTShhjxq4BVXyr6mMRr78AHwf6XKIykogsEpGNIrJZRG7t4/hnRKRBRFa7r2sHfwmHqWcMQf8Tzr1ZuYcTSnJJsPYBY8wYNvXUh4kAABXvSURBVNAngt5KgYKDJXDHH9wNnAtUA+UiskxVN/RK+rCq3nSY+Th8ze4TQT+BoKHNz+b6di6z9gFjzBg30DaCNvZvI6jDWaPgYBYCm1W10v2OJcAlQO9AEBstVZCYDqk5fR5eU+2szLlgWt/HjTFmrBhor6HMw/juKUBVxHY1cGIf6T4mImcA7wNfUdWq3glE5HrgeoCpUw+9dsCANO9wGor7mU10bU0LIti008aYMW9Ald8i8hERyYrYzhaRS4fg/P8AilX1eJxVzx7sK5Gq3qOqZapalp8/RDOANu84aPvAupoWpuelk558uLVnxhgzOgy0FfR7qtrSvaGqzcD3DvGZGiCyS06hu6+Hqjaqqt/dvBdnQrvh0VJ10B5Da2tamGOjiY0xHjDQQNBXukPdKpcDpSJSIiJJwJXAssgEIhK57uNi4N0B5ufI+NuccQT9jCGob/Oxq9Vv00oYYzxhoPUeFSLyM5xeQAA3AisO9gFVDbprFywH4oH7VXW9iNwBVKjqMuBLIrIYZ+WzPcBnDuMaBu8QPYbW17QC2BOBMcYTBhoIvgh8F3gYp/fQszjB4KBU9SngqV77bov4+5vANwea2SHTM5is70DQ3VA8ywKBMcYDBtprqAM4YEDYqNXqDooeN7nPw2trWijJSyfDGoqNMR4w0F5Dz4pIdsR2jogsj162oqyzyXlP63uNgXXWUGyM8ZCBNhbnuT2FAFDVJg4xsnhE62yChFRITD3g0O52PztbfMyebIHAGOMNAw0EYRHpqVAXkWL6mI101Ohs6ndE8boap5es9RgyxnjFQCvBvw28JiIvAwKcjjvSd1TqbILU7D4PdQeCWVNsRLExxhsG2lj8tIiU4RT+q4AngM5oZiyqOpv7fSLobigel5I4zJkyxpjYGOikc9cCN+OMDl4NnAS8wf5LV44enU2QW9LnoXU1rcy3ieaMMR4y0DaCm4ETgO2q+iFgHtB88I+MYP1UDe3p6KKmuZM5Vi1kjPGQgQYCn6r6AEQkWVXfA46NXrairJ/G4nd3OiOKZ1mPIWOMhwy0sbjaHUfwBPCsiDQB26OXrSgKdEKws89AULm7A4Cj8jOGO1fGGBMzA20s/oj75+0i8iKQBTwdtVxFU6dbo9VXIGhoJy0pngnjkoc5U8YYEzuDnkNBVV+ORkaGja//QLB1dwcleelIP4vVGGPMWOS9Vdm7p5c4SCAwxhgvsUDg8gdDVO3Zy3QLBMYYj/FuIEjZv/to1Z69hBWmW0OxMcZjvBsIej0RVDY4PYasasgY4zXeDAQSD8mZ++3u7jpakm+BwBjjLVENBCKySEQ2ishmEel3YRsR+ZiIqDufUXR1Dybr1TNoa0MHeRnJNseQMcZzohYIRCQeZ43jC4CZwFUiMrOPdJk4U1i8Fa287KefUcVbd3dYQ7ExxpOi+USwENisqpWq2gUsAS7pI90PgJ8AvijmZZ9+AkHl7nZrHzDGeFI0A8EUoCpiu9rd10NE5gNFqvqvg32RiFwvIhUiUtHQ0HBkueojELR0Btjd3mXtA8YYT4pZY7GIxAE/A752qLSqeo+qlqlqWX5+/pGduI9AsM1tKLaqIWOMF0UzENQARRHbhe6+bpnAbOAlEdmGs8bBsqg3GPexKM3W7kBgTwTGGA+KZiAoB0pFpEREkoArgWXdB1W1RVXzVLVYVYuBN4HFqloRtRyFguBv7WMMQTtxAkW5aVE7tTHGjFRRCwSqGgRuApYD7wKPqOp6EblDRBZH67wH5XPWIz4gEOzuoCg3jeSE+BhkyhhjYmvQs48Ohqo+BTzVa99t/aQ9M5p5ASJGFe8/vYRNNmeM8TJvjSzuY3oJVbVAYIzxNM8Hgl2tfvZ2hazHkDHGszwfCCp3twM266gxxrs8HwiqmzoBKMxJjUWOjDEm5rwZCFKyenbtanFmtpgwLiUWOTLGmJjzXiBIyYK4fd1Ed7b6GJ+eREqidR01xniT9wJBrzEEdS0+JmbZ04Axxrs8Hwh2tviYZIHAGONh3goEvgPnGapr6bT2AWOMp3krEHQ27bdovS8QomlvwJ4IjDGe5r1AEPFEUOf2GJqYZV1HjTHe5Z1AEA4fEAh2uoHAngiMMV7mnUDQ1QYa7jW9RPcTgQUCY4x3eScQ9DGquPuJYKI1FhtjPMzTgaCupZNxKQmkJ0d1Nm5jjBnRPB0IdtpgMmOM8XYgqGv1WY8hY4znRTUQiMgiEdkoIptF5NY+jv+XiKwVkdUi8pqIzIxaZvp5Iphk7QPGGI+LWiAQkXjgbuACYCZwVR8F/V9VdY6qfgD4H+Bn0coPfmfdge5lKruCYXa3+61qyBjjedF8IlgIbFbVSlXtApYAl0QmUNXWiM10QKOWm9O+DN/dDQnJANS3+VC1MQTGGBPN7jJTgKqI7WrgxN6JRORG4KtAEnBWX18kItcD1wNMnTr18HMUn9jzp40hMMYYR8wbi1X1blU9CvgG8J1+0tyjqmWqWpafnz8k5903qtgai40x3hbNQFADFEVsF7r7+rMEuDSK+dnPvnmG7InAGONt0QwE5UCpiJSISBJwJbAsMoGIlEZsXghsimJ+9rOzxUdqYjzjUmwwmTHG26JWCqpqUERuApYD8cD9qrpeRO4AKlR1GXCTiJwDBIAm4NPRyk9vde6CNCIyXKc0xpgRKaq3w6r6FPBUr323Rfx9czTPfzA7WzqtWsgYYxgBjcWxYmsVG2OMw5OBIBRW6tv8NobAGGPwaCBobPcTDKvNM2SMMXg0EPSMIbB5howxxtuBwNoIjDHGo4GgrqUTsEBgjDEQ5e6jI1Vdq5/EeCE3LSnWWTHGHKZAIEB1dTU+ny/WWRlRUlJSKCwsJDEx8dCJXd4MBC2dTBiXQlycDSYzZrSqrq4mMzOT4uJiGxjqUlUaGxuprq6mpKRkwJ/zZtVQq88WrDdmlPP5fIwfP96CQAQRYfz48YN+SvJkINjV6meCtQ8YM+pZEDjQ4fybeC4QqCo7Wzqt66gxxrg8FwhaO4P4AmHrMWSMGVK33347d911V7/HGxoaOPHEE5k3bx6vvvrqoL//gQce4KabbgLgiSeeYMOGDYed1948Fwh2tlrXUWPM8Hv++eeZM2cOq1at4vTTTz+i7xrqQOC5XkM9C9JY1ZAxY8b3/7GeDbWth044CDMnj+N7F886aJof/ehHPPjggxQUFFBUVMSCBQvYsmULN954Iw0NDaSlpfGHP/wBn8/H17/+dTo7O6moqOCNN97gq1/9KuXl5XR2dnLZZZfx/e9/H4Di4mIqKirIy8ujoqKCW265hZdeeqnnnK+//jrLli3j5Zdf5oc//CGPPfYYRx111BFdq+cCQfdaxRMsEBhjjsCKFStYsmQJq1evJhgMMn/+fBYsWMD111/P7373O0pLS3nrrbf4whe+wAsvvMAdd9xBRUUFv/71rwEniOTm5hIKhTj77LNZs2YNxx9//CHPe8opp7B48WIuuugiLrvssiG5Fs8Fgu7pJSwQGDN2HOrOPRpeffVVPvKRj5CWlgbA4sWL8fl8vP7661x++eU96fx+f5+ff+SRR7jnnnsIBoPs3LmTDRs2DCgQRIPnAsGuVh95GUkkJXiuecQYE2XhcJjs7GxWr1590HRbt27lrrvuory8nJycHD7zmc/09P1PSEggHA4DDNuo6aiWhiKySEQ2ishmEbm1j+NfFZENIrJGRJ4XkWnRzA84bQT2NGCMOVJnnHEGTzzxBJ2dnbS1tfGPf/yDtLQ0SkpKWLp0KeB0V3/nnXcO+Gxrayvp6elkZWWxa9cu/v3vf/ccKy4uZsWKFQA89thjfZ47MzOTtra2IbuWqAUCEYkH7gYuAGYCV4nIzF7JVgFlqno88CjwP9HKT7ed7lrFxhhzJObPn88VV1zB3LlzueCCCzjhhBMA+Mtf/sJ9993H3LlzmTVrFk8++eQBn507dy7z5s1jxowZXH311Zx66qk9x773ve9x8803U1ZWRnx8fJ/nvvLKK/npT3/KvHnz2LJlyxFfi6jqEX9Jn18scjJwu6qe725/E0BVf9xP+nnAr1X11L6OdysrK9OKiorDzte8O57hw3Mm8aOPzDns7zDGxN67777LcccdF+tsjEh9/duIyApVLesrfTSrhqYAVRHb1e6+/nwe+HdfB0TkehGpEJGKhoaGw86QLxCiaW/Auo4aY0yEEdFiKiLXAGXAT/s6rqr3qGqZqpbl5+cf9nm6u47aYDJjjNknmr2GaoCiiO1Cd99+ROQc4NvAB1W1735WQ6TOViYzxpgDRPOJoBwoFZESEUkCrgSWRSZw2wV+DyxW1foo5gVwpp8GrLHYGGMiRC0QqGoQuAlYDrwLPKKq60XkDhFZ7Cb7KZABLBWR1SKyrJ+vGxJ1NpjMGGMOENUBZar6FPBUr323Rfx9TjTP31tdq4/0pHgyUwa+hJsxxox1I6KxeLjUtfisfcAYM2LU1tYO2XxBR8JbgaDVAoExZuSYPHkyjz766IDTB4PBqOTDU3MN7WrxcdJR42OdDWPMUPv3rVC3dmi/c+IcuODOfg+Xl5fz+c9/nrfffptQKMTChQt5+OGHufnmm2lqaiIQCPDDH/6QSy65hFtvvZWioiJuvPFGwFnEJiMjg8suu4yLLrqIdevWEQqFuPXWW3nppZfw+/3ceOON3HDDDbz00kt897vfJScnh/fee4/3339/aK8TDwWCUFjZ1ea3HkPGmCFxwgknsHjxYr7zne/Q2dnJNddcw4wZM/j73//OuHHj2L17NyeddBKLFy/miiuu4Mtf/nJPIHjkkUdYvnw5oVCo5/vuu+8+srKyKC8vx+/3c+qpp3LeeecBsHLlStatW0dJSUlUrsUzgaCx3U8orDaq2Jix6CB37tF02223ccIJJ5CSksKvfvUrwuEw3/rWt3jllVeIi4ujpqaGXbt2MW/ePOrr66mtraWhoYGcnByKiorYtm1bz3c988wzrFmzpqeqqKWlhU2bNpGUlMTChQujFgTAQ4GgzhakMcYMscbGRtrb2wkEAvh8PpYuXUpDQwMrVqwgMTGR4uLinqmkL7/8ch599FHq6uq44oorDvguVeX//u//OP/88/fb/9JLL5Genh7V6/BMIOhekGZSVmqMc2KMGStuuOEGfvCDH7B161a+8Y1vUFpaSkFBAYmJibz44ots3769J+0VV1zBddddx+7du3n55ZcP+K7zzz+f3/72t5x11lkkJiby/vvvM2XKwaZnGzqeCQQ9S1RmJcc4J8aYseChhx4iMTGRq6++mlAoxCmnnMIZZ5zBkiVLmDNnDmVlZcyYMaMn/axZs2hra2PKlClMmjTpgO+79tpr2bZtG/Pnz0dVyc/P54knnhiWa4naNNTRcrjTUD+zvo6lK6r5/TULiIuTKOTMGDOcbBrq/g12GmrPPBGcN2si582aGOtsGGPMiOOpAWXGGGMOZIHAGDNqjbaq7eFwOP8mFgiMMaNSSkoKjY2NFgwiqCqNjY2kpAyum7xn2giMMWNLYWEh1dXVHMnytWNRSkoKhYWFg/qMBQJjzKiUmJgY1dG2XmJVQ8YY43EWCIwxxuMsEBhjjMeNupHFItIAbD9kwr7lAbuHMDujhRev24vXDN68bi9eMwz+uqepan5fB0ZdIDgSIlLR3xDrscyL1+3FawZvXrcXrxmG9rqtasgYYzzOAoExxnic1wLBPbHOQIx48bq9eM3gzev24jXDEF63p9oIjDHGHMhrTwTGGGN6sUBgjDEe55lAICKLRGSjiGwWkVtjnZ9oEJEiEXlRRDaIyHoRudndnysiz4rIJvc9J9Z5HWoiEi8iq0Tkn+52iYi85f7eD4tIUqzzONREJFtEHhWR90TkXRE52SO/9Vfc/77XicjfRCRlrP3eInK/iNSLyLqIfX3+tuL4lXvta0Rk/mDP54lAICLxwN3ABcBM4CoRmRnbXEVFEPiaqs4ETgJudK/zVuB5VS0Fnne3x5qbgXcjtn8C/FxVjwaagM/HJFfR9UvgaVWdAczFuf4x/VuLyBTgS0CZqs4G4oErGXu/9wPAol77+vttLwBK3df1wG8HezJPBAJgIbBZVStVtQtYAlwS4zwNOVXdqaor3b/bcAqGKTjX+qCb7EHg0tjkMDpEpBC4ELjX3RbgLOBRN8lYvOYs4AzgPgBV7VLVZsb4b+1KAFJFJAFIA3Yyxn5vVX0F2NNrd3+/7SXAQ+p4E8gWkUmDOZ9XAsEUoCpiu9rdN2aJSDEwD3gLmKCqO91DdcCEGGUrWn4BfB0Iu9vjgWZVDbrbY/H3LgEagD+6VWL3ikg6Y/y3VtUa4C5gB04AaAFWMPZ/b+j/tz3i8s0rgcBTRCQDeAz4sqq2Rh5Tp7/wmOkzLCIXAfWquiLWeRlmCcB84LeqOg/ooFc10Fj7rQHcevFLcALhZCCdA6tQxryh/m29EghqgKKI7UJ335gjIok4QeAvqvq4u3tX96Oi+14fq/xFwanAYhHZhlPldxZO3Xm2W3UAY/P3rgaqVfUtd/tRnMAwln9rgHOAraraoKoB4HGc/wbG+u8N/f+2R1y+eSUQlAOlbs+CJJzGpWUxztOQc+vG7wPeVdWfRRxaBnza/fvTwJPDnbdoUdVvqmqhqhbj/K4vqOongBeBy9xkY+qaAVS1DqgSkWPdXWcDGxjDv7VrB3CSiKS5/713X/eY/r1d/f22y4BPub2HTgJaIqqQBkZVPfECPgy8D2wBvh3r/ETpGk/DeVxcA6x2Xx/GqTN/HtgEPAfkxjqvUbr+M4F/un9PB94GNgNLgeRY5y8K1/sBoML9vZ8AcrzwWwPfB94D1gF/ApLH2u8N/A2nDSSA8/T3+f5+W0BwekVuAdbi9Kga1PlsigljjPE4r1QNGWOM6YcFAmOM8TgLBMYY43EWCIwxxuMsEBhjjMdZIDBmGInImd0zpBozUlggMMYYj7NAYEwfROQaEXlbRFaLyO/d9Q7aReTn7lz4z4tIvpv2AyLypjsX/N8j5ok/WkSeE5F3RGSliBzlfn1GxDoCf3FHyBoTMxYIjOlFRI4DrgBOVdUPACHgEzgTnFWo6izgZeB77kceAr6hqsfjjOzs3v8X4G5VnQucgjNSFJxZYb+MszbGdJy5coyJmYRDJzHGc84GFgDl7s16Ks4EX2HgYTfNn4HH3XUBslX1ZXf/g8BSEckEpqjq3wFU1Qfgft/bqlrtbq8GioHXon9ZxvTNAoExBxLgQVX95n47Rb7bK93hzs/ij/g7hP1/aGLMqoaMOdDzwGUiUgA9a8VOw/n/pXuGy6uB11S1BWgSkdPd/Z8EXlZnhbhqEbnU/Y5kEUkb1qswZoDsTsSYXlR1g4h8B3hGROJwZoC8EWfxl4XusXqcdgRwpgT+nVvQVwKfdfd/Evi9iNzhfsflw3gZxgyYzT5qzACJSLuqZsQ6H8YMNasaMsYYj7MnAmOM8Th7IjDGGI+zQGCMMR5ngcAYYzzOAoExxnicBQJjjPG4/x8I7y8fXXuedQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_graphs(test_log, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}