hw/hw1hw1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:15.080622Z",
     "iopub.status.busy": "2021-03-21T14:04:15.079672Z",
     "iopub.status.idle": "2021-03-21T14:04:15.085505Z",
     "shell.execute_reply": "2021-03-21T14:04:15.086237Z"
    },
    "papermill": {
     "duration": 0.027335,
     "end_time": "2021-03-21T14:04:15.086616",
     "exception": false,
     "start_time": "2021-03-21T14:04:15.059281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/ml2021spring-hw1/covid.test.csv\n",
      "/kaggle/input/ml2021spring-hw1/sampleSubmission.csv\n",
      "/kaggle/input/ml2021spring-hw1/covid.train.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:15.119652Z",
     "iopub.status.busy": "2021-03-21T14:04:15.117998Z",
     "iopub.status.idle": "2021-03-21T14:04:16.327317Z",
     "shell.execute_reply": "2021-03-21T14:04:16.328221Z"
    },
    "papermill": {
     "duration": 1.228857,
     "end_time": "2021-03-21T14:04:16.328445",
     "exception": false,
     "start_time": "2021-03-21T14:04:15.099588",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from matplotlib.pyplot import figure\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from matplotlib import pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.357208Z",
     "iopub.status.busy": "2021-03-21T14:04:16.356078Z",
     "iopub.status.idle": "2021-03-21T14:04:16.361594Z",
     "shell.execute_reply": "2021-03-21T14:04:16.362193Z"
    },
    "papermill": {
     "duration": 0.021853,
     "end_time": "2021-03-21T14:04:16.362523",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.340670",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 超参数\n",
    "batch_size = 64\n",
    "learning_rate = 0.001\n",
    "epochs = 3000\n",
    "model_path = 'model.pth'\n",
    "early_stop = 300\n",
    "target_only = False\n",
    "weight_decay = 0.1\n",
    "momentum = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.392136Z",
     "iopub.status.busy": "2021-03-21T14:04:16.391002Z",
     "iopub.status.idle": "2021-03-21T14:04:16.405610Z",
     "shell.execute_reply": "2021-03-21T14:04:16.406212Z"
    },
    "papermill": {
     "duration": 0.031123,
     "end_time": "2021-03-21T14:04:16.406416",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.375293",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class covidDataset(Dataset):\n",
    "    # def __init__(self, filepath, type='train'):\n",
    "    #     data = np.loadtxt(filepath, skiprows=1, delimiter=',', dtype=np.float32)\n",
    "    #     self.len = data.shape[0]\n",
    "    #     if type == 'train':\n",
    "    #         self.x_data = torch.from_numpy(data[:, 1:-1])\n",
    "    #         self.y_data = torch.from_numpy(data[:, [-1]])\n",
    "    #     else:\n",
    "    #         self.x_data = torch.from_numpy(data[:, 1:])\n",
    "    #\n",
    "    # def __getitem__(self, index):\n",
    "    #     return self.x_data[index], self.y_data[index]\n",
    "    #\n",
    "    # def __len__(self):\n",
    "    #     return self.len\n",
    "\n",
    "    def __init__(self,\n",
    "                 filepath,\n",
    "                 mode='train',\n",
    "                 target_only=False):\n",
    "        self.mode = mode\n",
    "\n",
    "        # Read data into numpy arrays\n",
    "        data = np.loadtxt(filepath, skiprows=1, delimiter=',', dtype=np.float32)\n",
    "        data = data[:, 1:]\n",
    "\n",
    "        if not target_only:\n",
    "            feats = list(range(93))\n",
    "        else:\n",
    "            # TODO: Using 40 states & 2 tested_positive features (indices = 57 & 75)\n",
    "            feats = list(range(40))\n",
    "            feats.append(57)\n",
    "            feats.append(75)\n",
    "\n",
    "        if mode == 'test':\n",
    "            # Testing data\n",
    "            # data: 893 x 93 (40 states + day 1 (18) + day 2 (18) + day 3 (17))\n",
    "            data = data[:, feats]\n",
    "            self.data = torch.from_numpy(data)\n",
    "        else:\n",
    "            # Training data (train/dev sets)\n",
    "            # data: 2700 x 94 (40 states + day 1 (18) + day 2 (18) + day 3 (18))\n",
    "            target = data[:, -1]\n",
    "            data = data[:, feats]\n",
    "\n",
    "            # Splitting training data into train & dev sets\n",
    "            if mode == 'train':\n",
    "                indices = [i for i in range(len(data)) if i % 10 != 0]\n",
    "            elif mode == 'dev':\n",
    "                indices = [i for i in range(len(data)) if i % 10 == 0]\n",
    "\n",
    "            # Convert data into PyTorch tensors\n",
    "            self.data = torch.from_numpy(data[indices])\n",
    "            self.target = torch.from_numpy(target[indices])\n",
    "\n",
    "        # Normalize features (you may remove this part to see what will happen)\n",
    "        # self.data[:, 40:] = \\\n",
    "        #     (self.data[:, 40:] - self.data[:, 40:].mean(dim=0, keepdim=True)) \\\n",
    "        #     / self.data[:, 40:].std(dim=0, keepdim=True)\n",
    "\n",
    "        self.dim = self.data.shape[1]\n",
    "\n",
    "        print('Finished reading the {} set of COVID19 Dataset ({} samples found, each dim = {})'\n",
    "              .format(mode, len(self.data), self.dim))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        # Returns one sample at a time\n",
    "        if self.mode in ['train', 'dev']:\n",
    "            # For training\n",
    "            return self.data[index], self.target[index]\n",
    "        else:\n",
    "            # For testing (no target)\n",
    "            return self.data[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        # Returns the size of the dataset\n",
    "        return len(self.data)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.435596Z",
     "iopub.status.busy": "2021-03-21T14:04:16.434579Z",
     "iopub.status.idle": "2021-03-21T14:04:16.441453Z",
     "shell.execute_reply": "2021-03-21T14:04:16.441998Z"
    },
    "papermill": {
     "duration": 0.023349,
     "end_time": "2021-03-21T14:04:16.442236",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.418887",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def dataloader(path, mode, batch_size, n_jobs=0, target_only=False):\n",
    "    ''' Generates a dataset, then is put into a dataloader. '''\n",
    "    dataset = covidDataset(path, mode=mode, target_only=target_only)  # Construct dataset\n",
    "    dataloader = DataLoader(\n",
    "        dataset=dataset,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=(mode == 'train'),\n",
    "        num_workers=n_jobs,                         # how many subprocesses to use for data loading. 0 means that the data will be loaded in the main process\n",
    "        pin_memory=True)                            # Construct dataloader\n",
    "    return dataloader\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.473921Z",
     "iopub.status.busy": "2021-03-21T14:04:16.472627Z",
     "iopub.status.idle": "2021-03-21T14:04:16.478693Z",
     "shell.execute_reply": "2021-03-21T14:04:16.479283Z"
    },
    "papermill": {
     "duration": 0.02386,
     "end_time": "2021-03-21T14:04:16.479483",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.455623",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.512809Z",
     "iopub.status.busy": "2021-03-21T14:04:16.511763Z",
     "iopub.status.idle": "2021-03-21T14:04:16.521610Z",
     "shell.execute_reply": "2021-03-21T14:04:16.522184Z"
    },
    "papermill": {
     "duration": 0.028764,
     "end_time": "2021-03-21T14:04:16.522381",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.493617",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Model(torch.nn.Module):\n",
    "    def __init__(self, features):\n",
    "        super(Model, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(features, 128)\n",
    "\n",
    "        self.linear2 = torch.nn.Linear(128, 64)\n",
    "\n",
    "        self.linear3 = torch.nn.Linear(64, 32)\n",
    "\n",
    "        self.linear4 = torch.nn.Linear(32, 8)\n",
    "\n",
    "        self.linear5 = torch.nn.Linear(8, 1)\n",
    "        self.relu = torch.nn.ReLU()\n",
    "\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        in_size = x.size(0)\n",
    "        x = self.relu(self.linear1(x))\n",
    "        x = self.relu(self.linear2(x))\n",
    "        x = self.relu(self.linear3(x))\n",
    "        x = self.relu(self.linear4(x))\n",
    "        x = self.relu(self.linear5(x))\n",
    "        x = x.view(in_size)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:16.553186Z",
     "iopub.status.busy": "2021-03-21T14:04:16.552137Z",
     "iopub.status.idle": "2021-03-21T14:04:17.105389Z",
     "shell.execute_reply": "2021-03-21T14:04:17.104794Z"
    },
    "papermill": {
     "duration": 0.570374,
     "end_time": "2021-03-21T14:04:17.105555",
     "exception": false,
     "start_time": "2021-03-21T14:04:16.535181",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished reading the train set of COVID19 Dataset (2430 samples found, each dim = 93)\n",
      "Finished reading the dev set of COVID19 Dataset (270 samples found, each dim = 93)\n",
      "Finished reading the test set of COVID19 Dataset (893 samples found, each dim = 93)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# load data\n",
    "tr_path = '../input/ml2021spring-hw1/covid.train.csv'\n",
    "tt_path = '../input/ml2021spring-hw1/covid.test.csv'\n",
    "train_loader = dataloader(tr_path, 'train', batch_size, target_only=target_only)\n",
    "dev_loader = dataloader(tr_path, 'dev', batch_size, target_only=target_only)\n",
    "test_loader = dataloader(tt_path, 'test', batch_size, target_only=target_only)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.146010Z",
     "iopub.status.busy": "2021-03-21T14:04:17.145282Z",
     "iopub.status.idle": "2021-03-21T14:04:17.167847Z",
     "shell.execute_reply": "2021-03-21T14:04:17.165897Z"
    },
    "papermill": {
     "duration": 0.048915,
     "end_time": "2021-03-21T14:04:17.168042",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.119127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "model = Model(train_loader.dataset.dim).to(device)\n",
    "\n",
    "# construct loss and optimizer\n",
    "criterion = torch.nn.MSELoss(reduction='mean')\n",
    "optimizer = torch.optim.Adam(model.parameters(),\n",
    "                             lr=learning_rate,\n",
    "                             weight_decay=weight_decay)\n",
    "# optimizer = torch.optim.SGD(model.parameters(),\n",
    "#                             lr=learning_rate,\n",
    "#                             weight_decay=weight_decay,\n",
    "#                             momentum=momentum)\n",
    "\n",
    "\n",
    "# lossRecord = [0.0 for _ in range(epochs)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.201429Z",
     "iopub.status.busy": "2021-03-21T14:04:17.200670Z",
     "iopub.status.idle": "2021-03-21T14:04:17.203018Z",
     "shell.execute_reply": "2021-03-21T14:04:17.202411Z"
    },
    "papermill": {
     "duration": 0.021217,
     "end_time": "2021-03-21T14:04:17.203225",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.182008",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_model():\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.242837Z",
     "iopub.status.busy": "2021-03-21T14:04:17.242070Z",
     "iopub.status.idle": "2021-03-21T14:04:17.245890Z",
     "shell.execute_reply": "2021-03-21T14:04:17.245214Z"
    },
    "papermill": {
     "duration": 0.028965,
     "end_time": "2021-03-21T14:04:17.246049",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.217084",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def train(tr_set, dv_set):\n",
    "    # running_loss = 0.0\n",
    "    # for batch_idx, data in enumerate(train_loader, 0):\n",
    "    #     inputs, target = data\n",
    "    #     inputs, target = inputs.to(device), target.to(device)\n",
    "    #     optimizer.zero_grad()\n",
    "    #\n",
    "    #     outputs = model(inputs)\n",
    "    #     loss = criterion(outputs, target)\n",
    "    #     loss.backward()\n",
    "    #     optimizer.step()\n",
    "    #\n",
    "    #     running_loss += loss.item()\n",
    "    #     lossRecord[epoch] += loss.item()\n",
    "    #     if batch_idx % 10 == 0:\n",
    "    #         print('[%d, %5d] loss: %.3f' % (epoch, batch_idx, running_loss / 10))\n",
    "    #         running_loss = 0.0\n",
    "\n",
    "    min_mse = 10000.\n",
    "    loss_record = {'train': [], 'dev': []}  # for recording training loss\n",
    "    early_stop_cnt = 0\n",
    "    epoch = 0\n",
    "    while epoch < epochs:\n",
    "        model.train()  # set model to training mode\n",
    "        for x, y in tr_set:  # iterate through the dataloader\n",
    "            optimizer.zero_grad()  # set gradient to zero\n",
    "            x, y = x.to(device), y.to(device)  # move data to device (cpu/cuda)\n",
    "            pred = model(x)  # forward pass (compute output)\n",
    "            mse_loss = criterion(pred, y)  # compute loss\n",
    "            mse_loss.backward()  # compute gradient (backpropagation)\n",
    "            optimizer.step()  # update model with optimizer\n",
    "            loss_record['train'].append(mse_loss.detach().cpu().item())\n",
    "\n",
    "        # After each epoch, test your model on the validation (development) set.\n",
    "        dev_mse = dev(dv_set)\n",
    "        if dev_mse < min_mse:\n",
    "            # Save model if your model improved\n",
    "            min_mse = dev_mse\n",
    "            print('Saving model (epoch = {:4d}, loss = {:.4f})'\n",
    "                  .format(epoch + 1, min_mse))\n",
    "            torch.save(model.state_dict(), model_path)  # Save model to specified path\n",
    "            early_stop_cnt = 0\n",
    "        else:\n",
    "            early_stop_cnt += 1\n",
    "\n",
    "        epoch += 1\n",
    "        loss_record['dev'].append(dev_mse)\n",
    "        # 如果一个epoch有early_stop个epoch没有更新，则停止训练\n",
    "        if early_stop_cnt > early_stop:\n",
    "            # Stop training if your model stops improving for \"config['early_stop']\" epochs.\n",
    "            break\n",
    "\n",
    "    print('Finished training after {} epochs'.format(epoch))\n",
    "    return min_mse, loss_record\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.284213Z",
     "iopub.status.busy": "2021-03-21T14:04:17.283145Z",
     "iopub.status.idle": "2021-03-21T14:04:17.287041Z",
     "shell.execute_reply": "2021-03-21T14:04:17.286391Z"
    },
    "papermill": {
     "duration": 0.026859,
     "end_time": "2021-03-21T14:04:17.287216",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.260357",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def dev(dv_set):\n",
    "    model.eval()                                # set model to evalutation mode\n",
    "    total_loss = 0\n",
    "    for x, y in dv_set:                         # iterate through the dataloader\n",
    "        x, y = x.to(device), y.to(device)       # move data to device (cpu/cuda)\n",
    "        with torch.no_grad():                   # disable gradient calculation\n",
    "            pred = model(x)                     # forward pass (compute output)\n",
    "            mse_loss = criterion(pred, y)  # compute loss\n",
    "        total_loss += mse_loss.detach().cpu().item() * len(x)  # accumulate loss\n",
    "    total_loss = total_loss / len(dv_set.dataset)              # compute averaged loss\n",
    "\n",
    "    return total_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.322689Z",
     "iopub.status.busy": "2021-03-21T14:04:17.321980Z",
     "iopub.status.idle": "2021-03-21T14:04:17.325760Z",
     "shell.execute_reply": "2021-03-21T14:04:17.325216Z"
    },
    "papermill": {
     "duration": 0.024572,
     "end_time": "2021-03-21T14:04:17.325918",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.301346",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def test():\n",
    "    model.eval()  # set model to evalutation mode\n",
    "    preds = []\n",
    "    for x in test_loader:  # iterate through the dataloader\n",
    "        x = x.to(device)  # move data to device (cpu/cuda)\n",
    "        with torch.no_grad():  # disable gradient calculation\n",
    "            pred = model(x)  # forward pass (compute output)\n",
    "            preds.append(pred.detach().cpu())  # collect prediction\n",
    "    preds = torch.cat(preds, dim=0).numpy()  # concatenate all predictions and convert to a numpy array\n",
    "    return preds\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.361838Z",
     "iopub.status.busy": "2021-03-21T14:04:17.360866Z",
     "iopub.status.idle": "2021-03-21T14:04:17.364824Z",
     "shell.execute_reply": "2021-03-21T14:04:17.364161Z"
    },
    "papermill": {
     "duration": 0.024901,
     "end_time": "2021-03-21T14:04:17.364977",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.340076",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def save_pred(preds, file):\n",
    "    ''' Save predictions to specified file '''\n",
    "    with open(file, 'w', newline=\"\") as fp:\n",
    "        writer = csv.writer(fp)\n",
    "        writer.writerow(['id', 'tested_positive'])\n",
    "        for i, p in enumerate(preds):\n",
    "            writer.writerow([i, p])\n",
    "    print('Successful save results to {}'.format(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.402755Z",
     "iopub.status.busy": "2021-03-21T14:04:17.401811Z",
     "iopub.status.idle": "2021-03-21T14:04:17.404830Z",
     "shell.execute_reply": "2021-03-21T14:04:17.404175Z"
    },
    "papermill": {
     "duration": 0.025493,
     "end_time": "2021-03-21T14:04:17.404985",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.379492",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def plot_learning_curve(loss_record):\n",
    "    ''' Plot learning curve of your DNN (train & dev loss) '''\n",
    "    total_steps = len(loss_record['train'])\n",
    "    x_1 = range(total_steps)\n",
    "    x_2 = x_1[::len(loss_record['train']) // len(loss_record['dev'])]\n",
    "    figure(figsize=(6, 4))\n",
    "    plt.plot(x_1, loss_record['train'], c='tab:red', label='train')\n",
    "    plt.plot(x_2, loss_record['dev'], c='tab:cyan', label='dev')\n",
    "    plt.ylim(0.0, 5.)\n",
    "    plt.xlabel('Training steps')\n",
    "    plt.ylabel('MSE loss')\n",
    "    plt.title('Learning curve')\n",
    "    plt.legend()\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-21T14:04:17.442937Z",
     "iopub.status.busy": "2021-03-21T14:04:17.442256Z",
     "iopub.status.idle": "2021-03-21T14:10:44.252984Z",
     "shell.execute_reply": "2021-03-21T14:10:44.251994Z"
    },
    "papermill": {
     "duration": 386.833628,
     "end_time": "2021-03-21T14:10:44.253228",
     "exception": false,
     "start_time": "2021-03-21T14:04:17.419600",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving model (epoch =    1, loss = 44.2446)\n",
      "Saving model (epoch =    2, loss = 18.4360)\n",
      "Saving model (epoch =    3, loss = 6.2738)\n",
      "Saving model (epoch =    4, loss = 5.6395)\n",
      "Saving model (epoch =    5, loss = 4.0320)\n",
      "Saving model (epoch =    6, loss = 3.0655)\n",
      "Saving model (epoch =    7, loss = 2.2270)\n",
      "Saving model (epoch =    8, loss = 1.7633)\n",
      "Saving model (epoch =    9, loss = 1.5082)\n",
      "Saving model (epoch =   11, loss = 1.4643)\n",
      "Saving model (epoch =   12, loss = 1.3422)\n",
      "Saving model (epoch =   14, loss = 1.2304)\n",
      "Saving model (epoch =   16, loss = 1.1859)\n",
      "Saving model (epoch =   18, loss = 1.1647)\n",
      "Saving model (epoch =   19, loss = 1.1531)\n",
      "Saving model (epoch =   21, loss = 1.1395)\n",
      "Saving model (epoch =   24, loss = 1.1075)\n",
      "Saving model (epoch =   33, loss = 1.0888)\n",
      "Saving model (epoch =   41, loss = 1.0633)\n",
      "Saving model (epoch =   46, loss = 1.0276)\n",
      "Saving model (epoch =   51, loss = 1.0117)\n",
      "Saving model (epoch =   58, loss = 1.0074)\n",
      "Saving model (epoch =   66, loss = 1.0057)\n",
      "Saving model (epoch =   69, loss = 0.9834)\n",
      "Saving model (epoch =   79, loss = 0.9831)\n",
      "Saving model (epoch =   90, loss = 0.9679)\n",
      "Saving model (epoch =   96, loss = 0.9523)\n",
      "Saving model (epoch =   99, loss = 0.9442)\n",
      "Saving model (epoch =  113, loss = 0.9382)\n",
      "Saving model (epoch =  114, loss = 0.9336)\n",
      "Saving model (epoch =  121, loss = 0.9329)\n",
      "Saving model (epoch =  137, loss = 0.9239)\n",
      "Saving model (epoch =  144, loss = 0.9164)\n",
      "Saving model (epoch =  156, loss = 0.9146)\n",
      "Saving model (epoch =  180, loss = 0.9068)\n",
      "Saving model (epoch =  188, loss = 0.9013)\n",
      "Saving model (epoch =  206, loss = 0.8997)\n",
      "Saving model (epoch =  208, loss = 0.8911)\n",
      "Saving model (epoch =  212, loss = 0.8858)\n",
      "Saving model (epoch =  246, loss = 0.8844)\n",
      "Saving model (epoch =  253, loss = 0.8707)\n",
      "Saving model (epoch =  268, loss = 0.8687)\n",
      "Saving model (epoch =  362, loss = 0.8682)\n",
      "Saving model (epoch =  385, loss = 0.8634)\n",
      "Saving model (epoch =  425, loss = 0.8589)\n",
      "Saving model (epoch =  450, loss = 0.8560)\n",
      "Saving model (epoch =  456, loss = 0.8507)\n",
      "Saving model (epoch =  586, loss = 0.8497)\n",
      "Saving model (epoch =  857, loss = 0.8480)\n",
      "Saving model (epoch =  888, loss = 0.8462)\n",
      "Saving model (epoch =  893, loss = 0.8424)\n",
      "Saving model (epoch = 1068, loss = 0.8401)\n",
      "Saving model (epoch = 1092, loss = 0.8398)\n",
      "Saving model (epoch = 1348, loss = 0.8384)\n",
      "Saving model (epoch = 1368, loss = 0.8382)\n",
      "Saving model (epoch = 1389, loss = 0.8381)\n",
      "Saving model (epoch = 1393, loss = 0.8380)\n",
      "Saving model (epoch = 1414, loss = 0.8326)\n",
      "Saving model (epoch = 1570, loss = 0.8312)\n",
      "Saving model (epoch = 1601, loss = 0.8298)\n",
      "Saving model (epoch = 1776, loss = 0.8286)\n",
      "Saving model (epoch = 1813, loss = 0.8278)\n",
      "Saving model (epoch = 1821, loss = 0.8269)\n",
      "Saving model (epoch = 1864, loss = 0.8264)\n",
      "Finished training after 2165 epochs\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABSA0lEQVR4nO2dd5wTVdfHfyfbG+zSywK71EVQkKaIIiAqIiDYKGJDwUfsihSRR1CxF/RRsaHyqhRFEBFFRVAREelNll52YWkLu2xvue8fmSSTZJJMkpkkO3u+nw+azNy598zdzJl7zz33HBJCgGEYhjEeplALwDAMw+gDK3iGYRiDwgqeYRjGoLCCZxiGMSis4BmGYQwKK3iGYRiDwgqeqbEQ0RVEtCfUcjCMXhD7wTOhgIgOA7hXCLEy1LIwjFHhETxjWIgoItQyBIoR7oEJHazgmbCCiExENJmIDhBRLhF9RUR1ZOe/JqITRJRPRH8QUQfZuc+IaDYR/UBERQD6EtFhIppARNulaxYSUaxUvg8RZcuud1tWOj+RiHKI6DgR3UtEgohau7mPOkT0qVT2HBF9Kx2/i4j+dCprq0fhHqZI9xshKz+MiLar6S+mZsMKngk3HgYwFMCVAJoAOAfgXdn5HwG0AdAAwGYAXzpdPwrATABJAKyK9FYAAwCkA7gIwF0e2lcsS0QDADwOoD+A1pJ8nvgcQDyADpKsb3op7+4eXgNQBKCf0/l50mdv/cXUYFjBM+HGfQCmCiGyhRBlAKYDuJmIIgFACPGJEKJAdq4TEdWWXb9UCLFWCGEWQpRKx94WQhwXQpwFsAxAZw/tuyt7K4BPhRC7hBDFAGa4q4CIGgO4DsB/hBDnhBAVQojffegD53uYD2CkVHcSgIHSMcBLfzE1G1bwTLjRAsASIsojojwAuwFUAWhIRBFE9JJkjjgP4LB0TT3Z9VkKdZ6QfS4GkOihfXdlmzjVrdSOlWYAzgohznko4wnnuucBuJGIYgDcCGCzEOKIdM5tf/nZNmMgWMEz4UYWgOuEEMmyf7FCiGOwmCZugMVMUhtAmnQNya7Xyy0sB0Cq7HszD2WzANQhomSFc0WwmG4AAETUSKGMwz0IIf4FcASWWYHcPGNty11/MTUcVvBMKIkioljZv0gA7wOYSUQtAICI6hPRDVL5JABlAHJhUZIvBFHWrwDcTUTtiSgewH/dFRRC5MCyVvAeEaUQURQR9ZZObwPQgYg6Swu401W2Pw8We3tvAF/LjnvqL6aGwwqeCSU/ACiR/ZsO4C0A3wH4mYgKAPwN4BKp/P/BMpI9BuBf6VxQEEL8COBtAKsB7AewTjpV5uaS2wFUAMgEcArAo1I9ewE8C2AlgH2wLwR7Yz6APgBWCSHOyI576i+mhsMbnRjGD4ioPYCdAGKEEJWhlodhlOARPMOoRPI/jyaiFAAvA1jGyp0JZ3QdwUvb0QtgWdWvFEJ0060xhtEZIloBoCcsv+ffAYyX7O0ME5YEQ8F3c7IZMgzDMEGATTQMwzAGRe8R/CFYtk4LAB8IIT5UKDMOwDgASEhI6JqRkeFzO8ezsnE6uQ7aHj2MuA4XBCg1wzBM9WHTpk1nhBD1lc7preCbCCGOE1EDAL8AeEgI8Ye78t26dRMbN270uZ1pD0/CR8NGYsXDd6Dzju0BSMwwDFO9IKJN7tY3dTXRCCGOS/8/BWAJgB56tEO6bV5kGIapvuim4IkoQQqMBCJKAHANLH7DuiEcdqwzDMPUbPSMONcQliBI1nbmCSFW6NKSNIAXxAqeYRjGim4KXghxEEAnveqXwyYahqm5VFRUIDs7G6Wlpd4LV2NiY2ORmpqKqKgo1dcYK2Y0D+AZpsaRnZ2NpKQkpKWlgQw6ixdCIDc3F9nZ2UhPT1d9nSH84EnyBGIbPMPUPEpLS1G3bl3DKncAICLUrVvX51mKQRR8qCVgGCaUGFm5W/HnHg2h4K3wIivDMIwdgyh4IfsvwzBM8MjLy8N7773n83UDBw5EXl6e9gLJMISCJ45pzzBMiHCn4Kuqqjxe98MPPyA5OVknqSwYzIuGTTQMwwSXyZMn48CBA+jcuTOioqKQmJiIxo0bY+vWrfj3338xdOhQZGVlobS0FI888gjGjRsHAEhLS8PGjRtRWFiI6667Dpdffjn++usvNG3aFEuXLkVcXFzAshlCwVvVOtvgGaZmc+KFF1C2O1PTOmPaZ6DRU0+5Pf/SSy9h586d2Lp1K3777Tdcf/312Llzp82d8ZNPPkGdOnVQUlKC7t2746abbkLdunUd6ti3bx/mz5+Pjz76CLfeeiu++eYbjB49OmDZDaHgwSYahmHChB49ejj4qr/99ttYsmQJACArKwv79u1zUfDp6eno3LkzAKBr1644fPiwJrIYQ8FLsJpnmJqNp5F2sEhISLB9/u2337By5UqsW7cO8fHx6NOnj6Ive0xMjO1zREQESkpKNJHFGIusVi8aNtEwDBNkkpKSUFBQoHguPz8fKSkpiI+PR2ZmJv7++++gymaIETxvdGIYJlTUrVsXvXr1QseOHREXF4eGDRvazg0YMADvv/8+LrroIrRr1w6XXnppUGUzhIK3ozyCL88+huK/1yH55puDLA/DMDWBefPmKR6PiYnBjz/+qHjOamevV68edu60R1KfMGGCZnIZQsHbYtG4sdAcueN2VB7PQa1Bg2CKjQ2iZAzDMKHDEDZ4b8urVblngyQHwzBM+GAQBW+BF1kZhmHsGELB8yIrwzCMK4ZQ8FY4HjzDMIwdQyh4W8o+1u8MwzA2jKHgQ2SiKc/KwrkFC0PTOMMwYcv06dPx2muvhVoMY7hJWgm2iebIqNtQefo0at84DKbo6KC2zTAM4w1DjOAhQhOqoCo/36F9hmFqLjNnzkS7du3Qv39/7NmzBwBw4MABDBgwAF27dsUVV1yBzMxM5OfnIy0tDWazGQBQXFyMZs2aoaKiQnOZDDGCJw4zxjAMgGn7srGzUJtAXVY6JsbhuTapHsts2rQJCxYswJYtW1BZWYkuXbqga9euGDduHN5//320adMG69evx/jx47Fq1Sp06tQJv//+O/r27Ytly5bh2muvRVRUlKZyAwZR8FZYzTMMEwrWrFmDYcOGIT4+HgAwZMgQlJaW4q+//sItt9xiK1dWVgYAGD58OBYuXIi+fftiwYIFGD9+vC5yGULB2xZZeaMTw9RovI209YSc9I/ZbEZycjK2bt3qUnbIkCGYMmUKzp49i02bNqFfv366yGQIG7xXEw3byBmG0ZHevXtjyZIlKCkpQUFBAZYtW4b4+Hikp6fj66+/BgAIIbBt2zYAQGJiInr06IFHHnkEgwYNQkREhC5yGULBW3EXbMyGXiN8foEwTI2mS5cuGD58ODp37oybbroJV1xxBQDgyy+/xJw5c9CpUyd06NABS5cutV0zfPhwfPHFFxg+fLhuchnCRGPzogn2Tic2CTEMIzF16lRMnTrV5fiKFSsUy998880QOg8ODTGC51g0DMMwrhhCwVvhaJIMwzB2DKHg2Q+eYWo2eps6wgF/7tEYCr4G/HEZhlEmNjYWubm5hlbyQgjk5uYi1seMdMZYZJUImYnGwD8shgl3UlNTkZ2djdOnT4daFF2JjY1Faqpvfv41S8FrrYit7bGCZ5iQERUVhfT09FCLEZbUDBONXiN7XtRlGCaM0V3BE1EEEW0hou/1bovH0QzDMHaCYaJ5BMBuALX0asDdCL7o7/WIqJWkV7MMwzBhja4KnohSAVwPYCaAx/VsS2rQ4evRu+6yHI6J0b1phmGYcENvE80sABMBmN0VIKJxRLSRiDb6uwpuVetuQxXwIijDMDUQ3RQ8EQ0CcEoIsclTOSHEh0KIbkKIbvXr1/evMbUKnBdFGYapQeg5gu8FYAgRHQawAEA/IvpCx/a8R5PUrWGeITAME37opuCFEFOEEKlCiDQAIwCsEkKM1qMta6iCoG904hkBwzBhjEH84EMtAcMwTPgRlJ2sQojfAPymf0s8omYYhrFijBG8zUQTYkEYhmHCCEMo+FAvcvIaK8Mw4YgxFLyEXin7Svfswe6M9ijPynI4bm+NNTzDMOGHIRS8dZFVLy+a/MWLAQAFv/7q1DDbhBiGCV+MoeB5BM0wDOOCIRS8DR5QMwzD2DCEgrdGk+RYNAzDMHYMoeBVwzZzhmFqEIZS8CHLycowDBOGGELB2000IYJNQAzDhCGGUvBeCVQRO1/OMwaGYcIYQyh4G+4UrhdFnPvZZ9id0R7C7C4vCStyhmGqH4ZQ8F4zOnnh9OtvWK6vrNRIIoZhmNBjCAXPNnCGYRhXjKHgJTijE8MwjB1DKHj99bobBW617bOCZxgmDDGEgreiux+8c/XsRcMwTBhjCAVvd5OsfgpXmM04PnkKSnbtCrUoDMMYDIMpeJ3RoZnKnBzkf/stsh96SPvKGYap0RhCwVsRBIiqKoUTgWrm6jczYMKfqsJC5d8rw2iEsRQ8CJkdOqJ482blAmwzZ8IEUVmJvd2648T0GaEWhTEwhlDwziaa4vXrQySJncqzZ7E7oz3yly4NtShMGCIqKgAA+cuWhVgSxsgYQsFb8dUPXlRU4PyKnwI3rSuYgMoPHwYAnFuwMNDaGYZh/MIQCt6Wss9HE8yZ2bNx7NFHAWk05XvDlvZOzZqFilOn/KuDYRhGJ4yh4P0cglccz3E84OdibN6ChciZPNk/IRiGYXTCEAreir/BxrTAXF4esrYZhmGUMIaCtyb80Fu/c0gCRiv4t8QEAUMoeHJeJlVri1f7kLmpz+Gou6r4QVaNEAJl+/eHWozgwq67jI4YQsFb8d1Eo2cmKF9devxowmCcnTsXBwcNRsm2baEWhWEMgSEUvG2RVfdgYzrUX01HcMWbtyD/u+80rbN0+3YAQHl2tqb1hiU8s2OCQGSoBdACFxONytGzCPQhq6bKWQuOjBoFAKg9ZEiIJanm1ODfEKM/hhjBW/G6yKrjqKni6FHd6mYYhvEHYyh4qxdNCO3eladPa1dZDSXgGVU1ogbdKhNCDKHgXdQ6T3urNVSD/n41506ZUKCbgieiWCL6h4i2EdEuItI9bJ41o9Ppt99WeYGPwyhZ+cI1f6IqL0/7NhiGYTRCz0XWMgD9hBCFRBQF4E8i+lEI8bfWDbkk/NA6xrbCiPLUm294uUZbERiGYXxFtxG8sFAofY2S/ukynCXbTtYAtSqPtkMLdz+jM1Xnz4dahKCiqw2eiCKIaCuAUwB+EUK4BGononFEtJGINp72c6HSquDNvip4VujhSY2wwfNvL9iU7tmDvT0uQd6Sb0MtStDQVcELIaqEEJ0BpALoQUQdFcp8KIToJoToVr9+fb/a8TdcMCt4JuTUiJdZeFC2dy8AoGjt2hBLEjx8UvBEZCKiWr42IoTIA/AbgAG+XqsGMnt2kxQq472X7dmDc/PnayaXT/DLpmbBf28mCHhV8EQ0j4hqEVECgH8B7CGiJ1VcV5+IkqXPcQD6A8gMUF7ltuBqoilcs8bneg4PH4ETM551OHbqzVko3rTJb9mE01S8bP9+VJ49az8Q5BFcVWEhyg4cCGqbqglTpVe8ZQuE2axP5TyCZ3REzQj+AiHEeQBDAfwAoDmA21Vc1xjAaiLaDmADLDb47/0V1BNKsWiyxo5zW77y3Dlk3T9elZtj7gcfoHTHDtc2vbjJuPPlPjhoMA4OvN5ru94oO3QIx56YoHp2YuXoXXfj4PWDAm5fV8JI6RX9vR5HRo7C2U8+CbUoDOMzatwkoyQ3x6EA3hFCVBB5z6EkhNgO4OIA5VOFdQSvNh782c/monD1ah0l8ozDi8XPUWvO09NQsmkTUkaOQHy3bqqvK92506/2PFF28CAi69RBRHKy5nWHmoocS9avsn01LIwxYwjUjOA/AHAYQAKAP4ioBYCw8jXyZoPXEnNJCUr37PX5ukPDhyPv22/dFwijUauvHBx4PQ7deFOoxWAYxgmvI3ghxNsA5FtDjxBRX/1E8h37CF5/JXl84kQU/LIS0S1aqCovyspR9Pd6lG7bjpxt23WWLnRUHD8eeCW22Uz1fdkxYUyYrvHoiZpF1kekRVYiojlEtBlAvyDIphqrwSgYCr54w0YAgLnCcw5Wa+CsssxMHL3rLr3FMgbCT3fXoKCxcgjrezU4NajP1ZhoxkiLrNcAqA/gbgAv6SqVzwRvBG/F2yIrYxD0+jP7oOAPDrkBJ5591ms5hnFGjYK3/gIHAvhUCLENYTaHNtls8EHEy4MZSEREIYRD6FxRWYlTb7yJqvx8v+s0EubSUpQdPBhqMYJG2d69ODcvRPszmGqNGgW/iYh+hkXB/0RESQB0cgr2j2Da4IMxtc5sfwEOjxhh+17wyy/I/fBDnHz5Fd3a1JKc6dORdf943eo/NmECDg68HubSUt3aYBgjoMZN8h4AnQEcFEIUE1FdWMw04YNWwcbUNGX9oHNbpbIFWVFZafl/uWe7v68IsxmorARFR2tab96Chf5KpKpU8d+WkEaiogKIjfWzLYYxPl5H8EIIMyyxZJ4motcAXCb5uIcNVlXrTcFXnj4TWEMarsKX7d/v6DYZghX+4xOeROZFnYLerhLFmzej4JeVli8avDvNWr8Ma6AHBlP9UeNF8xKAR2AJU/AvgIeJ6EW9BfOFuIwMAN4V/IH+/S0fAlEgGj3oBwcNRs7kKb7NBORtayDH+R9+CLgOXxFVVTg+aTLK9jtuHCr49VfN2ihatw57LuoUUIgJK97WUirPncOZjz7y/QXAL4zgUwP7XI0NfiCAq4UQnwghPoElYFjge+01JOWmGwEEZ6OTDYUHv+LYMb0a80kOJYo3bEDluXMayeM/ZXv3In/pUhyb4DWckXfcPLBFf60DABRvDFzBeyNn2jScfv0N5H+71L94NTXIZS9sqEFdrjaaZLLsc20d5AgIiogAoD5UQUBISqXi6FGXU4V//SUTSgdhhED50aM+KxIhBI7cfgeOjrnHp+uqCotw+r33ILTOkBUoYaQUzYVFAICcKVNwZvbsEEvjSHlWFpuWdCR/+XKUbA8ra7ULahT8iwC2ENFnRDQXwCYAL+grlm/YbkLvB1/H+q3xaTwFQCs7dAgHrrkWuR9+6FcbZbt3+1T+9Buv48zb/8P5H1f41Z7fBNDPajL2mIuLfQ7Spmb9t3jjRvXV6ax4i/75Bweuvgb5i5fo2o6vVObmamqOCyXHn5iAw7cOD7UYHlGzyDofwKUAFkv/egohFugtmC9Y1YGZCFUmE/rOno/51wx2Wz73/Q/8b0zjB7MiO9tSreTyZy4qclu2UgoHULxhoy0Ilp6Yi4otspXp4I5o3el76BDK9u0LaN+AlfM//Yy9PS7xOqra06Urjo5zH23UAW9yBfh70GvIUC6FhC7ZEV4jzKP33IvsBx6EuaQk1KLUCNwqeCLqYv0HS+jfbABZAJpIx8IG6zMoQCiPigIA/N/AGxXL7r20Z7DEUoW5TL23h1yVVPqg4M0B5qHMmfp0QNc74KQwD143EAcHD1Esen7FTzCXlbmvy0m5Fq2zmMhK//3XqxjF6zTP/R4wVXl5yJ3ziaHNKuVW06Ze8fUZBzz5wb/u4ZxAGMWjsakMufJw84yoiQHvEQ8PnxajUMUmy52UnEoFULhmDbLGjkODSZN0kEpjnPqueMMGHHv0UaSMGoVG/53msaxRyJn2XxT88gviOl3kUwhohnGHWwUvhAiriJGesPnB692QEJqbaNS8cE69OQsAYPYxVMHZzz8HoG5EG2zK9uzBgWvlGRztSpuIbLb0ihMngiyZG6S/e2lmJg4NHYaWPyxHTMuWgdcre1lVFRRYmvJ1fcAJc0mJY9YwxicKVq3GuQXz0dzPta5wQs1O1rDHZoM36ZpD3DdUjjKPT5jgtUzVmQA3aPmNDq9MWb+UHzmiQgQFGdS+ZDV4GZ/76iuH7+e/tyQlK/j1V4uCD0NzyuFbb7UnKAmT2U7luXOoUuGme2ziRMRdeBHq3D46CFIpkz1evzAbShSuWYOqc+dQe4iyqTIQwkgj+k8wf8Lm4mJ9G1B6IF2OhY9SyV++3McrVP611CgmIuQt+Rbl0kK1FsrszPsf4OSrrwKwKKUSnX3pvc3gzv/0s891hmP2qUM3DFWVqvL8d8twcuZMt+er8vNxfkWQvbp0JmvsOByfqI8Z1VAKXhAFd7OTiyAatK1iRFiyy25yKT/i6o8fTEp3aWT+kfedPHn66tWoKixUvkYI5EyZgsPDRyif94PTs2bh7Bwp/6qP/v96rMHkfbNI8zr9QQiBwjVr/F4Arjx1ShM5jj3+BI49+phumwpLfXQlDnc8edGMln3u5XTuQT2F8hXrTciVO4VklKvtA747o73FncxJccht8TlPPeU+FaCfCZKEEH4FNqsqLFKMASPKy3HiuefV24WJIBf6zLvvKZyXtZubKzWk/988EA+XMx99hCN3322tSKlyv+t2h1YvnbxFi5A1dhzyl3wbWEUB3qNVsfvifXbyxRdx/pdfvJYzl5Xh0DBl77vqiqcR/OOyz/9zOjdGB1n8xxpN0kSaRJQ8OOQGVBUUIG9RcEdPSqOSytyzXmcGOZOnKB4vWrMGgGXa6wt5ixYh86JOqDju6oq5r28/HLnjTrt8spHZ3m7dHMIcWylYuRLnvvwSJ198SdXL5szs91GRI0sBGIYudTbF6YPCOv36G+rcMzWcCZQfOYrdGe1RJvnF+4s1JWPFCY32XwRxbeDs3P/DsYce9lpOVFQGQZrg4knBk5vPSt9Dit2LRjaCD2CkULZ3L4rWrUPO09O8F5YR6OaNU6+/EdD1WlHw448AgLLDh1zOVebkoPiff2zfSzZvdjhf9q/rFFdICVk8mjxkD3zpjh04+dzzvojsUoevHHtyIvZd0dvv630mgN9nVX6+fd3BC0Vr1wIA8n18yeuGh/uu0MiMw9jxpOCFm89K30OKzURD+gp2Slp8k7MrvQ36zp6P3FrJOPm8TCn5q2xULbJqS/bDj+hS76lZsywmCTXKzJ9b1NCkcX7ZMlSePu140EO/V57JRcnOXZq1746qc3kuxw5cPwgH+l/tW0Ua9pWoqkLWff9B8eYt/lei0LdqFmEDwVfzmi/lzeXlqsJkBBtPCj6DiLYT0Q7ZZ+v3dkGSTxX2RVaT7YcTrCnG4r4WX+6tbS/QpX6LOVrd3YjKSpx49lmfwxgU/KzsqVGlJn6+B9ly3//A0SRB5NYmnLfwK8XjSm1425lrlvzJtVyIO//995Ygb9Izf/azz3D45ps1q1+O3A++dMcO2+e8bxbj9HvvhdBt1kJFzgkU/v67Khdfd5x69TWXEbvZ3WK6HA1eVM6/waJ//nG/jqWSo3fcib09LgmoDj3w5AffPmhSBIhdwduPhWqKYS4uhik+3q9rz//wg0uM9qwHHwJB3f0Ub9iAc/Pmo+zQIbT49FO/ZHBHVWEhIhITXU9oNDJU4yPtKwU//4zGz1mSVYvKSuRM+y/q3acyBo0CuR9+6Pf9Ose/V8Jq4st55hm0WbXK5XzO1Kker6/MzYVJrwxXGi8An5s3D+XZWf5vJlIYJwghULj6NyT27ePT4vJRaU2p1iYfZ0UySrZu9ftaPXE7ghdCHJH/A1AIoAuAetL3sEFugw9KXlYZziGK86WNML4+EOeSaikeL9u9W72Jxoc2fQ05rEUMn/PLl6M0c4/vF6q8L+eH2lxRgbJ9+wAApTt3In/JEhxzCttQvHkLTv/vHXcVOnw9Pest5C9d6rb9on82uD13cJD74HdWhKTgKxUWt9Wwr9flODjkBjeV+/Z7PP/Tz8hbtAiVVg8lCUsfO9YlKipQfvgwKk76ZkP3dceuuazM4+a4/MWLkT1+PPK++tqnet1Rcey46rWOcMWTm+T3RNRR+twYwE5YvGc+J6JHgyOeOuQjeD89A10oDYJ91cqKS3vjxlc+wJ7mylvf1bos+mIzzJmi7HnjlkptPAzyv/tOk3oA1/t1tqGL4mIcHDwElW5MGkIIHBk1CmfefVd1m1VOrp4O/t0a9VEgeDJLmcvKVCcqP/bII8h5ehr29brcYa1ByMN1SC/A45Mm48CA67D/yit9E1aqRlRW4tTrnkJfWSj46SfXKoSw7ZOwhrWoPOk5vEXZocMoXPOn1/YO9O/v+1pHmOHJBp8uhNgpfb4bwC9CiMEALkGYuUnKbfBajeDVxlwnJ51aumOnzzbwLe06AAAONUn16TpnitdbvFvUTE/zl2qjaK3uc6rRaYJVsnOXPaerE9YYL87kLfQ3ObgdNeEWnCNiFv651uIy6oyPoTZ825QjsO+K3tjT+WKf2gCAoj//RNFaWTIbJwXvU+pHhUFIwc8/I/ejjxWLVxUW2V5KSrs9cz/6GHu7dXeYPXgb6JRu346ssWMtba9U/s0YBU+/KPn86SoAPwCAEKIAQFg5JltdIi3mEt/9kwPB2UST9/XX2N83NIE27S8l/7Vo2YEDbt3Vip1cItWj/9/Cnx2f5UezdJDEFeE0as66915bTBsHfByclB9ydWP1hHxxumD1apybP1/VdadnzXJY7LUR6GDK+twquM9aQzjs7dbNKSidIwXSBqbKkye8ynN+ueOLSJjNKNkuuy8V9yOEQN43iz3mbQgnPC2yZhHRQ7DEge8CYAUAEFEcgKggyKYaRxt8SEWxE8IgT2639ntACAEiwsHrB7ktc2TUbYGIZcHsv7IXQiCzvd1bST5TyZvvIQdNWDn1urI7oz1azJun72/GacCTfb8loFbKyJE+VUNEmg2eitevd3vu3Pz5qHf//QCAypMnleVwd8xdrl5p458Vf8JsFG/YgJypUzVJ6B4MPCn4ewA8C6A/gOFCiDzp+KUAtHXRCBAT7FNGYXOTDPOnWkdK/cgTubd7D8ReoI+rp5yiP73bPl2wPrhOI72TL7+iuoryrOCM1v0l+5GH1bmlyghFYpDTb72NhF5S5BIvL6SSnbtw+OabEVGvnpdafX+xCSczEYSQZ/6xoXVsGWuwwcrcwF1Vg/H38+RFc0oI8R8hxA1CiJ9lx1cLIV7TXTIfkKfsUxNsbFd6G3zbW9vFk1C/UAqdRie+Yi4sdNihGk6U7typeDx/ifp8o8efnOhTm1qklKvKz0f54cPqyvqo3B2uLSxCzjPTPZZxp0zK9u/3nDVLgTPvWZKLKyWel5O/+BuLfLJFbuHU1qk3Z8FcEkCEVtv03X5/5z7/3Ha/ASf40YEjd9+NA9cOwLkv5znMSPXA7QieiDyuwgkhtA9e7Cf2Hy+pGgw8ONHiGz30D+8BiFTLEOLoDeVeHrbqTPHGjSg/cgRRTZr4WYPvL989F3dBm7/W+t5SRQVISht5aNiNqDh+HG3X65seMHfOx34vGB8cNBixHTog3Yc1DC3tz7kffICopk01qs3yDJqLi1H0xx9I9NWrJ0hYN//lL16se1ueFll7AkgFsAbAa7Ck8JP/CxsipKlZ2NjfQ4HBLVIHrh2AonXr/LvY36mwHzbxA4MGIX/5clScPGX3MNLLti4sLoaq3Gg9dEHpLrsbpLm4GDkzZnisqniD3d9fVdIWb6Ip+MOXHz7i0+hbVFU59LO51PuspOzAflW/jbxvFqNCWgco9zNoW/7y5did0d41HIbOeFLwjQA8BaAjgLcAXA3gjBDidyHE78EQTi12E40pZCPpUJtoHOLghBE7WraFWSNbo4PHQ5hSceQojj8xwXefcB/IatDI5lK7v//V9vj1GnD2//7P84K1E84eLhWnTmF3Rnuc8xR6wgmlWPH5S5e637QF+4IqSc/7kdtG+/widY7C6s45IWfqVOy/sg8AS4gFAD4PGqybr+RRPc3OuZZ1wJMNvkoIsUIIcScsC6v7AfwmedZ4hYiaEdFqItpNRLuISJ+IVpAn3YZ9kTXkI9qaPJ2wsCmjIx5+cgbmRCdpUt+Zd9zsOPVCqUKESwA4+4l2itETOf99RtP67pjxJsZMk7JOqc1ZK1NItpj0MgrX/Im9PS/D6VlvBSTb2c/mAgBOPBP4PatKEiJT6hVZdjOlP3FyTr7wosfz/rsJK1O+P7AQzmrwmJOViGIAXA9gJIA0AG8DUGs4qgTwhBBiMxElAdhERL8IITTPAO3gJql15X5ydu7cUIsQck6l1AUAHDSF1qv2+JNPhrT9Ap1SzPm7OKkUkz5n2jTN4wEFFG1SLXJX2a/t6wg2s4/XUb1dY5jdbIizoombcJDxtMg6FxbzzI8AZsh2tapCCJEDIEf6XEBEuwE0BaC5grdtmCCyR5MM1kYnNyN1xY0sQeTM+++HtH05xRs2hloEQ3Ji2n9Vly1c84eOkihTlpmpWV0+75hWSZHsZWeNne8rpXt8iK8UZNdWTyP42wEUAWgL4GHZxgICIIQQytGxFCCiNAAXA3DZ2UBE4wCMA4DmzZurrdKxDun/gsJoo1OIsU61S6OiEVvhe/o9JvTsztAuoKtXc4BGisfvhXAvOJuVTr78CpJvvcXjCH13RnvEd+/usV53Lri+cOiGoS7HqgqLsLdbN9QZM8bBFHj6befkePriyQZvEkIkSf9qyf4l+ajcEwF8A+BRIYRLIG8hxIdCiG5CiG7169f36yYcFHyQbd/hYxRyZWub9rju7bnYJMW6YaoPJUqhAXREK9fHMp2SVlc4JZcvXL3ashvXiwlG7vGjKW4Cy+3OaI+i9f9gb7duAFzXeUq2BMFsJcO36EY+QkRRsCj3L4UQujl92m3w8BpsbMLDT2nadqj93z2xvXUGAGCbTslIqiOl29Tv8vXXJU4LrJuJgoWqZBthiPPGqWBR9Jf7mcrZz/8viJJ4RjcFTxabzhwAu4UQuiYbtYYqUBNJclP7CzVr92jDxsizxnHXwbZmJsL6Dp38niMEOza+0Thy+x0ha1uEQejh6oCRN/hpgZ4j+F6w2PH7EdFW6d9APRqSm2iCyZ3T38BWP80fJ1Pq4mhDzzszl/buj8kPTsav3S/zqw0bOi/sLO5zLXJrJevaRkgIQawXK9bAWFVEKI7RKUuTEfCUyD1UhJHVVjcFL4T4UwhBQoiLhBCdpX8+BI5WT0yLNEubpG1Gp+z6jZDVoJFm9ckZ8cI7uHO65w3Bp1IsQZpOS+6G4cix+g3xv+F34Zlxj7ktwzMJ/3lz1L24ftanMHMfKlJdwvaGCl1t8MEiOq0FyGx2sMFr4SZ5+7Nv4o4ZbwZcj79YF3DNFL5/psqICABAQXxCiCUxJj9e1gdAWA0KmWpE+GoOHyEAwmQy1INg3Y1b3V0/g7UnwdDoOII/XTsFP11yhW71M6HD407WaoUQFuVuoKksCWviLD/vyUB9wejHhEem4mjjprhi6wbEl6nL2cpUD4wxgicCQTj4wevln15lMuHNkWNwoo63JAaBI49z7w9aJSBntOO7y6/CpoyOPl+npztubnIKAP9/Z4wjpf9qv1nfXwwzgjeZLQp+8oO+JXbwlZ0t2+K73lfjcOPAEmSrgcyWEbzwMRmzSz06mkjCeR9AOPLmbfcCAFbf71uqvOpupqtJVObkhFoEG8YYwQOIMFfBbDLhWIPGAOz264qICOQlahPNMNiYhHWRNfyf7nDe0WsE2BOJ8QfDKHiT2YwqU4TL8Zl3P4hhr34IACiJjgm2WAEh36FrJT8hEQDwW5dLNN+V6w+seIKFfv3Mr2bjYggFT0QojovHoqtc91H93vVS2+d5A9wnENCT8shI9J09H9/36ufTdVYTjXWxdE/zlhj62kf4pXsvzBj7qNdduUFVvqwldEVPE409UT1jNAyh4NUgAJRHusYlP1avAXalt9G17YJ4y6j7k8G3uJzb07wlimLjFK9zXmTd36wFAGCLj7tn+cFl1MDurMbDsAr+fEICyqLsCt0sedo4M/q5t2xJuAPBUwYpW7sKI+r/TJmJtZ2VQ5pa3SSdR+JhZe9W8fYQRDiVUgfv3HIHqjSeVRyr18Dh72xUgrGYzeY242FYBV8VEYkHnpQpbiKQWT/FWBEZ6V7t+rlhyb7RSUoqHsKx+P7UFug7ez4O+ek99MJdD+CbftdhV6u2mslURYTRz72FGffqlg0ybAiG8g2jYQOjEYZV8ABwoFma7bOA8gheK168+wHMvf4mxXPWdn1V0NYps/U66wtCbb5ZLe92ddeeAIC/LuqqeN5T35IQMAfo6qmEtc5/OnTyWrbKZKoW3khuCYbo1bl/GEUMreDlmE3kUTEuu7wf9qe2sH2fd80Qn9tY0fNKxeMkTynoA7brTE7XyWylqpS4DrbVE3XqYUdLdaNxvUefal54/d/9EhMfmqKrHHpiFBPNiTr1dF/zYuwYZqOTdwjO6rAiwu5W+cZtYwFYNqBUmiLw0TDfNqIAgMkWWsCpZT/1q/PIX+khF0RBXRyzziJGzrSkHvvkudAltLaZrsiSmrA8Kgq1it1HF9QyF0CwMYqJxvq78XWjF+MfNWYEL8jVS+DV2+9zW9Yf3Clya31+j+Ct19nc2XwcwWuC/y3J+91MJqy9qKtGctv78z9TZuKG1z8OuMaTKXXdejWFBJuZTscm2M/KsNQYBW8mk8vPeI0b7xW/R0tuRtLWcL/+LrJ6PB5iu6mvyuHr/tfj6fsnuO17X7Da1EkARzQKHTHihXdw35SZmtSlCUH8+2o9SzhRp16121xoNAyj4EetWIpIT2nOFEbwHgv7gcld/dYRfIAjJaUXhMc6NYyN7w21bWS2aAkAqFDYk+ArepktrOEuwoqgKHpt2xg583+Y+NBkTetkfMMwCr4kJhaVke6XFARI9WKjGm8LJeVCEDgfn4C+s+fjty6XOLYN+P2QOkvtYKLxUGU47lC0hpOIrAo856iWyV3CHb1eZgeaNkdpbKzUhvb175QSv2vJ2aTaONRE/2B/RsAwCn5J32s9nt/Rup3HF4Ac54epNCratYzCdWQWONKoKQBgUb/rAACHmqQG4CIonL4pL7L6SnFMLJ6+73Gcrp2ieP5A0+aYfeNtutp9j9drGHAd9kXWcHqF6YNef4t7n37Z9vlA0+Z4cMKMsDer3PbsLIyZ9mqoxagWGEbBe2Pyg5Mx/1p1sWicFUZBQiJ+uuQKB68boZBGj2D39zYJgb3N0jBm2qv4UoqBE+hDahthOVTku3Jb3a0n1nbujs8G36x4/rHHpuGrqwe5ScPnub2p/3kCw155H31nz0eZ9GKU96f184c3jvIq58EmzbBKFkvIGeOP22FfZA3CS2z2TaOxq1Vb7GjVTve2AsE646gO5CUm4WxSbdv3/IQkzXdze6LGKHgl3D00zsffueUOvHTXeFzzzhc4JSXANjv7pgOAsF9rMpttZZdeeY3lGuml0Hf2/IDk/u7Kq2Wy+lGBLQyx8p/ftqFK4Vx5ZCReduN9BAB/deqGPOkHnZdUy3bcHzPKPdNewXMedqla4+R7qrvv7Pn4cOgIn9sONmVRUZgz5FbFeEkWtFEKJ+rUw//chIyoCTOhYDPs1Q9x0yvvAwDOxydg6GsfYs4Nw4PWfs1W8AoPTRWRyw89V2bKON6gkeVahYfBJMyoko3gXRZd/X1+bG0pPJQeKnWn9twtBm9q1wF9Z89HoRQczUEE6Zq/L7wYK6RE0IBnpTB30M0O13or7yveXhnW82pnbqHk66uuxxfXDcNiZ1Ojxkr3+TEPYXG/67BHWuwOJvkJ1TMvg1acl+7/Tw08yNRiGAV/2baNmtTT/715WHb5VQ7HSL6BycPoV74l32Q2u4wsS2Ni8c8F3rfVy+sDPCtFB/MHgDdHjsHT9z3uoPyc5bB+t8p6Pj4B+1Nb4A/ZwrDSdZb2lH8ynjZz6TUy9FavFqEJfujZB18EIcx0hbQ+VObG/q00U5s78EbbgEItnsvrO4Kf8kDoNsWFG1vbtMf+ps11b8cwCv6OHxb7fE15tOviKeBqH5aPeK1Kb6/CCIiEsClAJQUPAJN8cBtzftwUFRpZFk0/umE41nbqhu96X421nbuj1MNCGTltnnnwyRkYO/UlVTK5M+uoRQtPjW97X42Pbhgh6w/Hfi6PjMSOlm3dvoyUOFurNs4oLDq/esd9mHODZxNPdv1GGPbK+zgpmeTkTHj4Kbx0x3+8tm/9jclNJ8Uxdluz0t/+s8G3YFW3y7zWrRb53+ZowyY4nVxHs7oB4Fj9RprW5y97m6VhxaW9QyrDY4//F2NlC9x6YZhQBXraD9Xajwn2UaNJmN37xWvIrvS2+LX7ZQ5mE0AKruVmtGafGVjOZ0meP84o9anz2oOaO7S8+OybkvxBACiKi0diSTHeGjkGAHDzqh8Uy757yx34rvfV+Pj5SYrnT6bURVRVJeqcz7cdu+nl913K5dStr0q276/oh7yk2hjxwjsYtWIpxi5dYDtnDY8w+f9c65djkpK7WP9mv3TvhRfGPGg7784UV+pmkOIP8h3Td05/HYD/IQWc/8zlkZEe1heCy31PvQgAGPD3Hw7Hj9VrgOiKCtTPPxcKsXTBMCP4ygjXdH1asa3tBarKkTA7mGi0CvLlqZYnH3nKRbkDwOA3P3Fre7Yq+JKYGFQqpDm0Yn1ZZbZoibzEWpIsTopGKnMw1f10093L9+cel7u9psLp77nsiv4Y/MYcZDWwjwLd1WsNGpfvJhfviBfeUVTozjwvU7CeKI+0K1l/s4Y5K/j1HTs7FnAzfvHXBXdLu4740ykyqJYhC5zNY7e8+F7Ye7+Mfu4t3PrSe6rLL+p3HY42bKKjRIFjIAUfnMnIp0NudXuOhP2BI6VFVpUUx8Ti5dvvQ1FcPADg2z7X4vFHn/bbvOG8am+Va23n7nj6P094uNLS4P2TZ2L5FZZ1CdcRvG9CyUecL979gNtyG9tf5PB93YUXAwCyZbtM3bXt62xuX2qa4nG1ytPbHgw1OCt455mOu3tVykPsicz01gCAj4eOwLT7JzietFm87I0XxCf4547q9Dc4ryLxfZXJhL6z52Np7/7+tGivh8hhgPDPBZ00jy9UaYrAu7fcgQefnKH6GltMqiBuPTSMgm9+4nioRYBJmG0jlwOpLfxe5FvcdwBWXNYHC/oPAmB56Le06xDQD+PBCTMw6rlZABx3wq6XFKcSSq6gzkpP/tKxJgR3Rm7iqlDYNGZlV3obfOTF3i1X3lOdFRQsD3NOvQYAgK/6X++xLivP3vuwqnJKlMS4rnWYyffMAxGSgq9ys27g7mWjZZx9pd/XkNc/xrcyt9xA6vJGuZSZ6/1ht/l8rZynHpiIa975AoDFHDfpocl48c77A6rTGevvvtiHWYkeORG8YRgFX6cg33shnclMa40TdS3K5WTd+pr7u+5p0crva3e1aoscaffopoyOimWcH0qlkbDzTGlvc/ti80E/vAJWdr8MubWSAQAPTnwW8wbcoOijreRRtCfN0h/yl8akhybb/PDXd3T/8pKT3dC/2DP5CUkYOOszl+NXvTcPnw5S3kTmDmuoaXdK4Es3pp9AF73lFMZbZox7W6Q7HH/n1rt8rsvbbPPBCTOwtU17h2MHm1h+P1UR9nvald4GfWfPx86W6mPI/9Ohs+1zibRQnaWBKaUkJgZvjLwHRbFxbmeJO1u2cftyt83OgrhFzzAKPlyYffNo22clTxs12GPIOP4QVl7i3mbtCz/06qd4fJnT1FiAXPKdnnRaePxY9hJz96PfJ8us5czMMQ/ZEnFESPFpSsIpXK+Me6a+jNEz3rB9H/rah27LWje3WZl51wMOOxqdseffVX4k3fmtV0WYcKx+QxRK5rxAOCt5ETl7DZlNJqxTeFn+3ONy9J09X9H84c2DaVerti7huq0eZvIXtjVb1yYnkx0AzRZtv7pqoMsI/56pL+Ntpxfb4j4DsKx3fyy8epDib/2Pzt3x0JPPYrmb58vd7ExPWMEzbjETYWlvz9Nzq+3YEweapWGHh6BT1gXa2PJyAFBUGMoOkb5zok49h+/fuLGfH2qcisy01i5yWiNNKik8Oc4KYOUll9t2NALAi3fe77DAHVElmWjcjODdrTFVmUwY/ews3DdZ3xDHu9Na4a3hdznIvOCawQCA5+55COWRkagissmvZr3I2YRZpPCSsg12FH5n7958OwBg1oi78cWAoaruQ4nZN9+On53cJg+mNndZW7GOwM1E9hDgslnv8fqWGXJ2Q2V3UDbRBMi8p/23pYYD+5uloe/s+fhEWsgNdZTE+dcO8ToylC8kByqt9WVRGRnpeu8a9UVJjKPNdHGfAYrlxvzXczCrpx6YGJAcP1/aG3tatMSplDp4c8QYm0KscjONd7eeYx1tW3dYzxlyK/rOno8HfFj8U8Pn19+Eb/tci7Wd7J43VuW7vuPFWNGzDyY+PAX93/3Sck6FDd7TYrjVW8rudiyw4OpBGDflBVsZa8iOpVde49YcKp8NZzVo5BBgz52p0hukIHthbBz2yMyV/6a1clmTCoWCN4wfPAA0zj0dahEC4hspAqWVUEcGWdJXWfnJUTOCV4u1roqISJS58+8OcL/D306LylU6uteq4bXbxmFDh07os3EdALjm37Wh7r6/uG4YAOBflflyfeXvjhfjsu2bMO2+J3C4STPb8TlDbnXwlHF/H3Y8KfgvBwzFTz2vRMbh/QAsyXmUTJ4bMxzTMDor7Scffsr2+Y4Zbzqcm/DIVCya5HnxdVvrDDz6xDNYNOl+h1eu/IVbEhODpx6Y6DBLfWDS80g7nuVQl5KC/63LJZgx9lGsePgOj3L4i6FG8AAwd/oTaHzmZKjFqDE45KENQPnmJSbZFtfWXdgF08c95nDeefetv3w4zL5L2UzksqagJWo2SjmbMrwFgPPEEj+8XXxlxWV9cCC1hYv3lVy572mejr8u7Op8qQtyhee8H+MnKYG91Uzmbj1r3rVDHL5PeGSqw/fcZMuI/aSTac7KzS/Pdjl2TPLCAoCvrxoIAPh00M0O+Qesge7MEREYOOszB+Vunb3IX4CA3QYvX/CdMfZRAMBHQ/XJUaubgieiT4joFBHt1KsNJZqfPI7Pn3kcI39aGsxm9aEaJLLocHCfJvUMe/VDW5Az500+u1u0so1fD0qbmLTA2/pCIAgCxk96zmsZq4nrt249Abi3wavx7X97xBiH7x/dMBwVERFYdnk/LLh6kBqxVbGlXQeP5/8z5QU8f89DXus5IwuFcPW7X/gli9q9JqUx6t0ZRz/3lu2zNdDg8iuusv0NPh94o0cX6LOSV5icf9NbezTRbPbSp/6i5wj+MwDe5/g6EGE2496lC0PRdI1jVXd7LBTn0ZO/OC8ojp/8PP6WFjU/89H90BOZaf65nb5w13hV5fI8eM0Alk1KziYumxJwo7icF4k9MW/AUPx8yRV447ax+ODG23BOFr45EOSzIDV4WpA+Vr9hQLMyLU2ESsiTBMl3y6/p3MPtNUrebg9MfM5Yi6xCiD8AnNWrfm+YhMCT//dBqJoPW+ZLng/hzK5WrvZjtdm4fMHZc8IThTLPnl8uuUKT9vOSaiHLyQffnRI43KQZfujZByNn/s+nNr6+yr7Za85g97uw9cTTgvSJuvUDys/rYCLUAXnAty+l9Q0AeH30WJ/rMpSCVwsRjSOijUS08fRpbRdJB677DTM+eAMXZwbVSqQZRYoZlQLD19EXY8HZ+0YLpo97zCXB9+pul+Gwm+Bvr97hPtGKO47IcpdqsdlHDwLpW/lmNufZjRbhorUcWNRIP3ghxIdCiG5CiG7162u/4NV76wa88dZMzJ3+BFpnHda8fqZm8OgTz/hUXilpilrufuY1lHsI6eAv29u2914oBLgkOfET59mNFhFmT/lgEvOGJ48tLXckyzGUm6Qnmp88jo9esOyYLIhPwJDXPw6xREx1wrqJJVj87iEXrZHQat1GCV+ToejNAQ8OAhU6mCCBMBjBh4Kk4iKsHD8Kk+bOxldTHkDdPEv854SS4hBLxjCMVvxwWd9Qi+CAp9hUxxvokwxFtxE8Ec0H0AdAPSLKBvCMEGKOXu35SoQQtoD/i6bYvSKyGjTCLz0ux9laydjbIh19Nq3H4cZN8UuIM8AwDOMbvzuloKyJkAgjX+tu3bqJjRv9y626O0Nf+6KZCLm1klEeFYU6BfkOUQTr5J/D2dopaHP0EPY1t0TiSzmfj3O1PLvJAcC3E8Z5DFrFMEzN4ETfzn5dR0SbhBDdlM7VGBt8oJiEcEjl5S6V2b7UNGxp1wHd/92GzwcOwxVbNuBooybo/u82ZBw5iINNm+Nnyc3u3qULEF1ZiZfeeQmTH7Tnam1y+iRKo6NxtnYKYktLcf83X2Bt52442LQ5umTuxM+X9sbL/3tJMb/rlE/fdUmkMfS3n7C6a0/ka+QH7Y33X3wKrbKP+r15JRAG/7ESy3r3xyfPPYkx0zzHk2EYo2OYEfzxSZOQv/Q7jSUKPwQsqeiSCwsgABxt1AQre1yO6/9chUZnzwAACuPiURITi8L4eKSezEFUVRUA+3bw84mJSCwuRnRlha3e0qho/NSzNxqfOY0e/27DyZS6aHAu17aDdFvrDDQ7lYPIykrUKi4CYEk2/dT4CXjn1WdwrH4jJBUXoe75c4grKwNgCbiUWFKMvKRa2NcsDecTklAQn4jeW9ajNDoGCaUlaH9oP47Xb4jc2slocPYM1nbqjtPJKbhyy3o0zD2D+vnn8Gu3y/Br98vw5Bcf4tdul2FT+wvx94Vd0O7wAUz6/H1sa90edc7noTg2DgP+/gNmIpiEwPF6DfDabWPxyMJP0eLEcRTGxWP0jDcxftHnOJ1SFy1OHENWw8bov/5P1D2fhzdG3Ys2WYewr1k64spKsUjapi5nwVMPYsQL7wAAvpj2CDZlXIh5A25Aw7NnEFVZiednv4ZPB9+CW1cux9T7J9hi1mcc2o/6eWex5uIeuGvZ10goKca7t95pq/eSnVtsLn/XrV2NH3v1xRtvPocOB/ehNDoay67oj4+HuiZDeeajWbbt7nLefflpPDV+ostLPaGk2CVq46U7NuPvC7vYvl+0bzfaHTmIVd0uQ+uswx6TwjjL7omLM3dii4cAX5PmzsbLGifmkPPo/DmYNfIev6+PKS9DmYdk9r6gNMPXYwTPCp5hVCIQ+gBwcuTyWF9qVUQAESLMluximWmtcMGh/Q7XVZlMECCYhBkkLJFTzibVRn5SElrkHHPZ/l9pigBBIMJsRllUFIpj47AvNQ0XHtxje5mv63gxUk/loNmpEzherwFiy8uQUFKMKlMEzickIqdeA1y891+cqZ2CevnnUGUyoTAuAUnFhcitlYxjDRrhov2Z2NL2AoAIyQXnEVlViaanTiIvqRayGzTCvGtvwPhFn6PBuVysvagrGp47g+2t26MqIgJ9Nq1DWVQ0EkuKURkRgfyEJHw+8Ea0yj6CO35YjKjKSkSaqxz6zmwyAULgu979cf3a31AYF4/8xCS0OGHpg5MpdVEl1dX09AnbwOZUSh2s7toTt65cjrKoaBxIbYFzSbVQGRmJ3NrJqJOfj4rISHQ8uBeJxUWoVVwEAWBJn2vR7ORxZDdohIF//Ybj9RsisrISW9p1xOXbNqDXP3/79TtgBc8wDBPmtM/c7dd1nhS8gdwkw2lsxTAME3oMpOAZhmEYOQZS8OFjamIYhgkHDKPgyV0GIIZhmBqKYRR8yujRoRaBYRgmrDCMgicdou8xDMNUZwyj4BmGYRhHWMEzDMMYFMMo+KjG9nCbjaZPR+yFF4ZQGoZhmNBjGAVviotD+8zdaJ+5GykjhiP966/QYMITqq5tufx7naVjGIYJPoZR8ErUvfde1L3/Px7LtF3/N2JatdJdlsSrrkK7rVt0b4dhGMaKoRU8ANQZNQpx3bqi7rhxAIBG0x1za0bUdh+zPfXddxy+t5j3pWK5uIsdI+nVGz/epUzDSRNhitU+cXMoqHX99bbPSddcg+QR7jPVeKLBhCfQeObzWomlC55+H0zNxvm5l9NmzR9BlMQ9hlfwkfXrI+2LL1D/sUfRbvMmpIwYoT6oD5kQ1dSe4T6+SxdkbN+GqObNAQBxnTujxbx5bhW/gxwNLTk94zp18v0mACRc1lPxeHxPS+5O8vLyiEhO9tyAypyQkQ0aoOnrr6HVyl/Q8ocfkPr2W2g8fTpa//G7rUyjGTNU1VXn7ruRfNNNqsoqkTJqFNK/XYKIOnX8rsOZJq++irbr/0ZCr15o/cfviO3QweF80jXXaNaWO5JvuRkNpz2teC6ifj3UGjTIax2ps98LSAZ/f6da4yxH808/0byN+EsvRdP/ve3zdSmjlHNCABa94wsJl13mc/tqMLyCt0JEMMXHey8IIOHyy6WLgPhLLWm/6j/ysOVQdDTSv1qIFvPnIW3BfMR3uRgky97eauUvgFOy3/aZu2GKscSRTlu4AHXuGQMASLr2WqR/uwRtN/zjIFtsR3vM7Jj27YHISDR94w1FWVNGjET7zN2I79oVAND07beQsWO7y+wjeeQI1H/8cdv3xKuuQuwFF6Ddls3I+HcXWi791qG8VWHXuftu2wst9Z3/Ie2rhQCA6NRUxLRMt5WPatDALtPwWxVltVJ37L3I2LUT5CbLfPO5cxWPW19ySdcNQPvM3Wj032mIzcgAyH2gucS+fR36tuF/pwEAGkyciPRvl6D+o486lK89eBAiatdG8zkfI6pBA0Q1bWK5btrTaLtxA5q+8ToAFS9MNzR55WVbHc5YZ0a1hwxBndtuUyxTe9BgNHlhpsOxtn+vc/ieMmoUkvq6z0ea7vS3llPvwQfRPnM3kq4bYG/zhiHI2P2vS1nn2VdUi+Zo+eMPaJ+5G203/OO2DWc89aXzyyyhp/JAxxMZO3d4PN/is09R6+qrfa43sm5d+wvIzW9ZdV2y50dLaoyCV0v7zN1AhNQtRKgzejQoLg61b7SPNCOSkxHvZnoWnZqKunffheSRrskZnIm7sCNiMzIQkZSEdps32Y43fGoKUt+fjVYrV6LlksVov3OHy0OQ9tVCtFrxI2pdaxlRWpVDXIcOoKgoJF11lcODTJGRqDdurO176jv/Q/rib2CKiwOZTC7rEFENGqB95m40nDQR6Yu/QfqSxUjq3x9RjXxLDpy2YL7LsQZPPOFWuQNAwiU9ECm10+bPNfZ7iI0DANSWmYgAACaLgq87dizaZ+5G699/Q50xY5Dx7y40m/0e4qSXH2Ax2bXP3I26Y+5GbEYGkm+52aP8DadMQeOXXkTKqFGISEwERUYi7auFaLnMHpra2YwTkZJi+1z/icdBsbFou/5vtJg3D7WHDEGtgQPRZt1fLm0lXXsN2m7cgPju3QFYvMEaTJ7kWEgIl7AcEcnJaPGlJXtWxo7taCS9xNwR264d2m74B2kLFwAA0r7+Gil33A4AqDPa8mKRD1qavPyyw3fA8iJwnn21/uknxKRbXvoRSUn247+utA2QlGj922qYEhPR+KUXkdj/KodzycNvRWyniwCoG73LZ9xWSGF2mrFju9e6PBHXuTPiZS+bFl987rG8KSHB0uffLHI4nnTNNUi88ko0fFp5xhYoNTZlX/qSxTAXFzscsyoVyGLkx7Zvj4wtm32q25SQgMbPPIOqs+dQe8hgl/MpI0eh8NdVqDV4iGP7TRqj3v33I75LF5drAMvos2zPXsRddBHiLrrI4VzyjcNQ+4YhDooztl07tzI6P7CeiEhMRER733LeUkwMRFkZ4jp3RlynTijZtg2JV16Jwt9/Vywf3boVGk6ahIKffwEAtFr+Pczl5YiUKcvkW25G4apVLmaT5h/PQd7Chaj/+GMAgKiGDdFw4pO286nvvoODA65Di/muL5vIunXR/NNPcPTuMYpymeLjkTx0qMMxa983++hDnP9+Oeo9MB4HrrnWLs9nn+HQDTcAAOqNHYt6Yy0v1vgu9kGB833lfb0I8d27IyIx0XY8RVrbSL75Fhx77DEUrVnj8NuUE9+1q0/xxCOSkhDXqZPtmrgLO6LRU0/ZzsdK91hrsP33G52Whqpz51CVn297yaQtmI/y7GOoPcjppSsjqmlTy+/6kktxZNQoRDZqhHr3348Tz1jWw0yxsWi3cYPlXocOxbEnJuD88uVo/cfvMEVHI33hQlX31OT115B4ZR+IkmJE1q8Pc0kJzCUlDmVarVwJiowARUW5XN987lyUbN6E02+9jVqDB6N4/XpUnjoFkmaAQqYvrAOX5OHDUbJtG6JbtEBc164o2bTJpV7AMvOPSEpymOlm7NgOREb69Cz6So1V8LFOCiv92yU2O3lS//4o+mNNwN41qW/NUjwendoUrVb86HK8zapVHuurM2qUx/OeRsW+0OTllwKuo+Xy5Sg/eAAA0Pz/5sJcXAxTQgLMBQUuZdtt2ghERcEUHY3EKyz5ak0JCTAlJDiUS+rbV1GJxbZr63HUaoqORutVv7o9H3/pparuyZnEK66wySuHIi1/h+j0dJdzSjR+7jk0fu45t+cjEhNQe+gNKFqzBpHSfo+2GzdibzfFHA8utM/cbUtKH92ypapr4i++GG03bkREov1v0GrFjzj1+uvI/ehj27G4zp0R17mzujq7XIwmr76C+C5dENW0qU3BO9N45vOoc9edDspQTtqC+Sj86y/Ed+mCvMWLcf67ZQBkMztJZlNcHExxcQ7XRqe6jvCtJFzSA5WnTlq+CIHWq35F2YEDtoFS8ebNODLK0XSWfOMwJN84DIDld1iyaZPthVB72DDkL7GYAZOusZiA5LZ5pZeM1tRYBe9MbEaG7XPyLbeg9uDBLj8OT8R3747iDRv0EC0g6tx1F85+9hlSRrpfELLSYNIkFG/YgNrS6NNXmr7xus08EZ3a1PYwmWJibGsQprp1Xa5zVuQu9b71FiJSkv2SSQ1ajKDarPsLB4cMQdXpMx7XBBzalWY5aqg1cCBMcfFIvLI3ADgoXl9I//or1WX9bcMTtWUzgmYffwxTjGsMKVNsLOI8bFSUv1QSLr3UpuA9kXT11Sj45ReHY+nfLUXV2bPKFwgBiox0mAW7m1lbaTBhAmLaZSD51lsAAI1fmInGzz2raCIKFqzgFSAikA/KHQCaz/kY5vJynSTyn4aTJ6Ghkx3X3cih7t13oe7dd/ndVq2BrkmqtcC6zhDORKakoNXy5ag4dgzRLVqg9k03os4dd3q8pvWqX1GVf15V/USEpH6OC6eNZsxAZEP3i3NNZ82CqKwEALRc9h1MtWt7fZl6I6adZSAU09r77NZUuzbM+fluzyde3isgWXyh6VuzALPZ4Vhs27YKJf1/2Zvi4x0cDIhI0Tut2ZyPkf/NYr/b8QVW8BpB0dGIqAYx6dtt3qR6hMn4RkStWoioVQsA0GTmTC+lLfb/SIUZjVq8eSvVGmBfF4hp08bvduTUHnQ9Ytu1VVVfy6Xfomz/AU3a9UTaokUoXu85YTWZTC7ebUpYnRkiGzXUQjRFEnv1QmKv4LzcWMHXMNS6itY04rp0sSlnxjNqXxZRjRr57HXlD3EdOyCuYwfvBVWQcHkvNJ31JpL69VM8H9m4MaJbtNCkrWBAws2qfCjo1q2b2LhxY6jFYBiGqTYQ0SYhhOKKO/vBMwzDGBRW8AzDMAaFFTzDMIxBYQXPMAxjUFjBMwzDGBRW8AzDMAaFFTzDMIxBYQXPMAxjUFjBMwzDGBRdFTwRDSCiPUS0n4gm69kWwzAM44huCp6IIgC8C+A6ABcAGElEF+jVHsMwDOOIniP4HgD2CyEOCiHKASwA4F+gcYZhGMZn9Iwm2RRAlux7NoBLnAsR0TgA46SvhUS0x8/26gE44+e1NQnuJ3VwP6mD+0kdevaT2/CWeip4paDjLqErhRAfAvgw4MaINrqLqMbY4X5SB/eTOrif1BGqftLTRJMNoJnseyqA4zq2xzAMw8jQU8FvANCGiNKJKBrACADf6dgewzAMI0M3E40QopKIHgTwE4AIAJ8IIXbp1R40MPPUELif1MH9pA7uJ3WEpJ/CKqMTwzAMox28k5VhGMagsIJnGIYxKNVewdfEcAhE1IyIVhPRbiLaRUSPSMfrENEvRLRP+n+K7JopUh/tIaJrZce7EtEO6dzbRETS8RgiWigdX09EaUG/UQ0goggi2kJE30vfuY8UIKJkIlpERJnS76on95UjRPSY9LztJKL5RBQb9n0khKi2/2BZvD0AoCWAaADbAFwQarmCcN+NAXSRPicB2AtLOIhXAEyWjk8G8LL0+QKpb2IApEt9FiGd+wdAT1j2LfwI4Drp+HgA70ufRwBYGOr79rOvHgcwD8D30nfuI+V+mgvgXulzNIBk7iuH/mkK4BCAOOn7VwDuCvc+CnnHBdjpPQH8JPs+BcCUUMsVgn5YCuBqAHsANJaONQawR6lfYPFs6imVyZQdHwngA3kZ6XMkLLvwKNT36mO/pAL4FUA/mYLnPnLtp1qS8iKn49xX9nux7syvI8n/PYBrwr2PqruJRikcQtMQyRISpGncxQDWA2gohMgBAOn/DaRi7vqpqfTZ+bjDNUKISgD5AOrqchP6MQvARABm2THuI1daAjgN4FPJnPUxESWA+8qGEOIYgNcAHAWQAyBfCPEzwryPqruCVxUOwagQUSKAbwA8KoQ476mowjHh4bina6oFRDQIwCkhxCa1lygcM3QfyYgE0AXAbCHExQCKYDE3uKPG9ZVkW78BFnNLEwAJRDTa0yUKx4LeR9VdwdfYcAhEFAWLcv9SCLFYOnySiBpL5xsDOCUdd9dP2dJn5+MO1xBRJIDaAM5qfye60QvAECI6DEsk035E9AW4j5TIBpAthFgvfV8Ei8LnvrLTH8AhIcRpIUQFgMUALkOY91F1V/A1MhyCtOo+B8BuIcQbslPfAbhT+nwnLLZ56/ER0ip9OoA2AP6RppQFRHSpVOcdTtdY67oZwCohGQerA0KIKUKIVCFEGiy/i1VCiNHgPnJBCHECQBYRtZMOXQXgX3BfyTkK4FIiipfu7SoAuxHufRTqxQsNFj8GwuJFcgDA1FDLE6R7vhyWqdt2AFulfwNhsdf9CmCf9P86smumSn20B9KqvXS8G4Cd0rl3YN/dHAvgawD7YVn1bxnq+w6gv/rAvsjKfaTcR50BbJR+U98CSOG+cumjGQAypfv7HBYPmbDuIw5VwDAMY1Cqu4mGYRiGcQMreIZhGIPCCp5hGMagsIJnGIYxKKzgGYZhDAoreCZsIaK6RLRV+neCiI7Jvkd7ubYbEb2too2/tJPYpe5kIhqvV/0M4w12k2SqBUQ0HUChEOI12bFIYYnZEZZIcYK+F0J0DLUsTM2ER/BMtYKIPiOiN4hoNYCXiagHEf0lBcn6y7obk4j6kD0G/HQi+oSIfiOig0T0sKy+Qln538geE/1LWZzugdKxP6X43d8ryNWBiP6RZhfbiagNgJcAtJKOvSqVe5KINkhlZkjH0qT650rHFxFRvHTuJSL6Vzr+mnO7DOMJ3ZJuM4yOtAXQXwhRRUS1APQWliTv/QG8AOAmhWsyAPSFJX7+HiKaLSwxReRcDKADLLFB1gLoRUQbAXwgtXGIiOa7kek/AN4SQnwpmY8iYAnY1VEI0RkAiOgaWLas94AlsNR3RNQblm3w7QDcI4RYS0SfABgv/X8YgAwhhCCiZF87iqnZ8AieqY58LYSokj7XBvA1Ee0E8CYsClqJ5UKIMiHEGVgCQjVUKPOPECJbCGGGJfxDGiwvhoNCiENSGXcKfh2Ap4hoEoAWQogShTLXSP+2ANgs1d1GOpclhFgrff4ClnAU5wGUAviYiG4EUOymbYZRhBU8Ux0pkn1+DsBqyc49GJZ4HkqUyT5XQXn2qlRGKYSrC0KIeQCGACgB8BMR9VMoRgBeFEJ0lv61FkLMsVbhWqWohGW0/w2AoQBWqJGFYaywgmeqO7UBHJM+36VD/ZkAWpI9P+ZwpUJE1BKWkf7bsEQFvAhAASwmISs/ARhDljj+IKKmRGRNENGciHpKn0cC+FMqV1sI8QOAR2EJCMYwqmEbPFPdeQXAXCJ6HMAqrSsXQpRIro4riOgMLFH+lBgOYDQRVQA4AeBZIcRZIlormY9+FEI8SUTtAayT1m8LAYyGZbawG8CdRPQBLJEJZ8Py8lpKRLGwjP4f0/r+GGPDbpIM4wUiShRCFEpeNe8C2CeEeFPD+tPA7pSMDrCJhmG8M5aItgLYBcuo+oPQisMw6uARPMMwjEHhETzDMIxBYQXPMAxjUFjBMwzDGBRW8AzDMAaFFTzDMIxB+X8m6i0JUQqYawAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successful save results to pred.csv\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if __name__ == '__main__':\n",
    "    model_loss, model_loss_record = train(train_loader, dev_loader)\n",
    "    plot_learning_curve(model_loss_record)\n",
    "\n",
    "    preds = test()\n",
    "\n",
    "    # print(preds.shape)\n",
    "    # print(type(preds))\n",
    "\n",
    "    save_pred(preds, 'pred.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.033301,
     "end_time": "2021-03-21T14:10:44.321619",
     "exception": false,
     "start_time": "2021-03-21T14:10:44.288318",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 396.639686,
   "end_time": "2021-03-21T14:10:45.267687",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-03-21T14:04:08.628001",
   "version": "2.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}