train_fp16_apex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# setup dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "from utils.to_fp16 import network_to_half"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## meta settings\n",
    "\n",
    "# select from efficientnet backbone or resnet backbone\n",
    "backbone = \"efficientnet-b0\"\n",
    "scale = 2\n",
    "# scale==1: resolution 300\n",
    "# scale==2: resolution 600\n",
    "useBiFPN = True\n",
    "HALF = True # enable FP16\n",
    "DATASET = \"VOC\"\n",
    "retina = False # for trying retinanets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make data.Dataset for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainlist:  16551\n",
      "vallist:  4952\n"
     ]
    }
   ],
   "source": [
    "if not DATASET == \"COCO\":\n",
    "    # load files\n",
    "    # set your VOCdevkit path here.\n",
    "    vocpath = \"../VOCdevkit/VOC2007\"\n",
    "    train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath)\n",
    "\n",
    "    vocpath = \"../VOCdevkit/VOC2012\"\n",
    "    train_img_list2, train_anno_list2, _, _ = make_datapath_list(vocpath)\n",
    "\n",
    "    train_img_list.extend(train_img_list2)\n",
    "    train_anno_list.extend(train_anno_list2)\n",
    "\n",
    "    print(\"trainlist: \", len(train_img_list))\n",
    "    print(\"vallist: \", len(val_img_list))\n",
    "\n",
    "    # make Dataset\n",
    "    voc_classes = ['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "                   'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "                   'cow', 'diningtable', 'dog', 'horse',\n",
    "                   'motorbike', 'person', 'pottedplant',\n",
    "                   'sheep', 'sofa', 'train', 'tvmonitor']\n",
    "\n",
    "    color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "    if scale == 1:\n",
    "        input_size = 300  # 画像のinputサイズを300×300にする\n",
    "    else:\n",
    "        input_size = 512\n",
    "\n",
    "    ## DatasetTransformを適応\n",
    "    transform = DatasetTransform(input_size, color_mean)\n",
    "    transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "    # Dataloaderに入れるデータセットファイル。\n",
    "    # ゲットで叩くと画像とGTを前処理して出力してくれる。\n",
    "    train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "    val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "        input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "else:\n",
    "    from dataset.coco import COCODetection\n",
    "    import torch.utils.data as data\n",
    "    from utils.dataset import VOCDataset, COCODatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn\n",
    "\n",
    "    color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "    if scale == 1:\n",
    "        input_size = 300  # 画像のinputサイズを300×300にする\n",
    "    else:\n",
    "        input_size = 512\n",
    "\n",
    "    ## DatasetTransformを適応\n",
    "    transform = COCODatasetTransform(input_size, color_mean)\n",
    "    train_dataset = COCODetection(\"../data/coco/\", image_set=\"train2014\", phase=\"train\", transform=transform)\n",
    "    val_dataset = COCODetection(\"../data/coco/\", image_set=\"val2014\", phase=\"val\", transform=transform)\n",
    "    \n",
    "batch_size = 32\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "# 辞書型変数にまとめる\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 512, 512])\n",
      "32\n",
      "torch.Size([1, 5])\n"
     ]
    }
   ],
   "source": [
    "# 動作の確認\n",
    "batch_iterator = iter(dataloaders_dict[\"val\"])  # イタレータに変換\n",
    "images, targets = next(batch_iterator)  # 1番目の要素を取り出す\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # ミニバッチのサイズのリスト、各要素は[n, 5]、nは物体数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define EfficientDet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.efficientdet import EfficientDet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b0\n",
      "use BiFPN\n",
      "layerc3: torch.Size([1, 40, 37, 37])\n",
      "layerc4: torch.Size([1, 80, 18, 18])\n",
      "layerc5: torch.Size([1, 320, 9, 9])\n",
      "layer size: torch.Size([1, 256, 37, 37])\n",
      "layer size: torch.Size([1, 256, 18, 18])\n",
      "layer size: torch.Size([1, 256, 9, 9])\n",
      "layer size: torch.Size([1, 256, 5, 5])\n",
      "layer size: torch.Size([1, 256, 3, 3])\n",
      "layer size: torch.Size([1, 256, 1, 1])\n",
      "torch.Size([1, 8096, 4])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    }
   ],
   "source": [
    "if not DATASET == \"COCO\":\n",
    "    num_class = 21\n",
    "else:\n",
    "    num_class = 81\n",
    "\n",
    "if scale==1:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 300*scale,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [37, 18, 9, 5, 3, 1],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "elif scale==2:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 512,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [64, 32, 16, 8, 4, 2],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264]*scale,  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315]*scale,  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "\n",
    "# test if net works\n",
    "net = EfficientDet(phase=\"train\", cfg=ssd_cfg, verbose=True, backbone=backbone, useBiFPN=useBiFPN)\n",
    "out = net(torch.rand([1,3,300,300]))\n",
    "print(out[0].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b0\n",
      "use BiFPN\n",
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "net = EfficientDet(phase=\"train\", cfg=ssd_cfg, verbose=False, backbone=backbone, useBiFPN=useBiFPN)\n",
    "\n",
    "if retina:\n",
    "    from utils.retinanet import RetinaFPN\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 300*scale,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [38, 19, 10, 5, 3, 1],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "    net = RetinaFPN(\"train\", ssd_cfg)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "net = net.to(device)\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EfficientDet(\n",
      "  (layer0): Sequential(\n",
      "    (0): Conv2dStaticSamePadding(\n",
      "      3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False\n",
      "      (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
      "    )\n",
      "    (1): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (layer2): Sequential(\n",
      "    (0): MBConvBlock(\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        32, 32, kernel_size=(3, 3), stride=[1, 1], groups=32, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        32, 8, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        8, 32, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(16, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (1): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        16, 96, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(96, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        96, 96, kernel_size=(3, 3), stride=[2, 2], groups=96, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(96, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        96, 4, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        4, 96, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        96, 24, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(24, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (2): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        144, 144, kernel_size=(3, 3), stride=(1, 1), groups=144, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        144, 6, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        6, 144, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        144, 24, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(24, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (3): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        144, 144, kernel_size=(5, 5), stride=[2, 2], groups=144, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 2, 1, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        144, 6, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        6, 144, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        144, 40, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(40, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential(\n",
      "    (0): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(240, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        240, 240, kernel_size=(5, 5), stride=(1, 1), groups=240, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(240, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        240, 10, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        10, 240, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(40, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (1): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(240, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        240, 240, kernel_size=(3, 3), stride=[2, 2], groups=240, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(240, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        240, 10, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        10, 240, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        240, 80, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(80, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "  )\n",
      "  (layer4): Sequential(\n",
      "    (0): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        480, 480, kernel_size=(3, 3), stride=(1, 1), groups=480, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        480, 20, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        20, 480, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(80, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (1): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        480, 480, kernel_size=(3, 3), stride=(1, 1), groups=480, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        480, 20, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        20, 480, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(80, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (2): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        480, 480, kernel_size=(5, 5), stride=[1, 1], groups=480, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(480, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        480, 20, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        20, 480, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        480, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (3): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        672, 672, kernel_size=(5, 5), stride=(1, 1), groups=672, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (4): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        672, 672, kernel_size=(5, 5), stride=(1, 1), groups=672, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (5): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        672, 672, kernel_size=(5, 5), stride=[2, 2], groups=672, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 2, 1, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "  )\n",
      "  (layer5): Sequential(\n",
      "    (0): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        1152, 48, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        48, 1152, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (1): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        1152, 48, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        48, 1152, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (2): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        1152, 48, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        48, 1152, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "    (3): MBConvBlock(\n",
      "      (_expand_conv): Conv2dStaticSamePadding(\n",
      "        192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn0): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_depthwise_conv): Conv2dStaticSamePadding(\n",
      "        1152, 1152, kernel_size=(3, 3), stride=[1, 1], groups=1152, bias=False\n",
      "        (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
      "      )\n",
      "      (_bn1): BatchNorm2d(1152, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_se_reduce): Conv2dStaticSamePadding(\n",
      "        1152, 48, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_se_expand): Conv2dStaticSamePadding(\n",
      "        48, 1152, kernel_size=(1, 1), stride=(1, 1)\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_project_conv): Conv2dStaticSamePadding(\n",
      "        1152, 320, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "        (static_padding): Identity()\n",
      "      )\n",
      "      (_bn2): BatchNorm2d(320, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
      "      (_swish): MemoryEfficientSwish()\n",
      "    )\n",
      "  )\n",
      "  (conv6): Conv2d(320, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv7): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (toplayer): Conv2d(320, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "  (smooth1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (smooth2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (latlayer1): Conv2d(80, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "  (latlayer2): Conv2d(40, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "  (loc): ModuleList(\n",
      "    (0): Conv2d(256, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): Conv2d(256, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (2): Conv2d(256, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (3): Conv2d(256, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (4): Conv2d(256, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (5): Conv2d(256, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  )\n",
      "  (conf): ModuleList(\n",
      "    (0): Conv2d(256, 84, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): Conv2d(256, 126, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (2): Conv2d(256, 126, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (3): Conv2d(256, 126, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (4): Conv2d(256, 84, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (5): Conv2d(256, 84, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  )\n",
      "  (BiFPN1): BiFPN(\n",
      "    (conv7up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv6up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv5up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv4up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv3up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv4dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv5dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv6dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv7dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (BiFPN2): BiFPN(\n",
      "    (conv7up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv6up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv5up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv4up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv3up): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv4dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv5dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv6dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "    (conv7dw): Sequential(\n",
      "      (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), groups=256)\n",
      "      (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=3, device=device, half=HALF)\n",
    "\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.\n",
      "\n",
      "Defaults for this optimization level are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n",
      "Processing user overrides (additional kwargs that are not None)...\n",
      "After processing overrides, optimization options are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n"
     ]
    }
   ],
   "source": [
    "if HALF:\n",
    "    from apex import amp, optimizers\n",
    "    # Initialization\n",
    "    opt_level = 'O1'\n",
    "    net, optimizer = amp.initialize(net, optimizer, opt_level=opt_level)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_lr(epoch):\n",
    "    \n",
    "    if DATASET == \"COCO\":\n",
    "        reduce = [20, 40]\n",
    "        # warmup\n",
    "        if epoch < 1:\n",
    "            lr = 1e-4\n",
    "        else:\n",
    "            lr = 1e-3\n",
    "    else:\n",
    "        reduce = [120,180]\n",
    "        lr = 1e-3\n",
    "        \n",
    "    for i,lr_decay_epoch in enumerate(reduce):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "batcht=[]\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device:\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('(train)')\n",
    "            else:\n",
    "                if((epoch+1) % 10 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('(val)')\n",
    "                else:\n",
    "                    # 検証は10回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "                if HALF:\n",
    "                    images = images.half()\n",
    "                    targets = [ann.half() for ann in targets]\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    tick = time.time()\n",
    "                    outputs = net(images)\n",
    "                    tock = time.time()\n",
    "                    batcht.append(tock-tick)\n",
    "                    print(\"batch time:\", np.mean(batcht))\n",
    "                    #print(outputs[0].type())\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        if HALF:\n",
    "                            with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                                scaled_loss.backward()\n",
    "                        else:\n",
    "                            loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iter {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "\n",
    "                        epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 10 == 0):\n",
    "            if useBiFPN:\n",
    "                word=\"BiFPN\"\n",
    "            else:\n",
    "                word=\"FPN\"\n",
    "            torch.save(net.state_dict(), 'weights/'+DATASET+\"_\"+backbone+\"_\" + str(300*scale) + \"_\" + word + \"_\" + \n",
    "                       str(epoch+1) + '.pth')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device: cuda:0\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 1/200\n",
      "-------------\n",
      "(train)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch time: 1.4208438396453857\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 32768.0\n",
      "batch time: 0.749925971031189\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 16384.0\n",
      "batch time: 0.5173380374908447\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 8192.0\n",
      "batch time: 0.400715708732605\n",
      "batch time: 0.3310044288635254\n",
      "batch time: 0.288063923517863\n",
      "batch time: 0.2540460654667446\n",
      "batch time: 0.22861766815185547\n",
      "batch time: 0.2091119024488661\n",
      "batch time: 0.19361252784729005\n",
      "Iter 10 || Loss: 16.3804 || 10iter: 10.8001 sec.\n",
      "batch time: 0.18068920482288708\n",
      "batch time: 0.16980804999669394\n",
      "batch time: 0.16061062079209548\n",
      "batch time: 0.15274827820914133\n",
      "batch time: 0.1459245522816976\n",
      "batch time: 0.1399286687374115\n",
      "batch time: 0.13468425414141486\n",
      "batch time: 0.13013850318060982\n",
      "batch time: 0.12593797633522436\n",
      "batch time: 0.12223780155181885\n",
      "Iter 20 || Loss: 14.7553 || 10iter: 5.4631 sec.\n",
      "batch time: 0.11879262470063709\n",
      "batch time: 0.11564928835088556\n",
      "batch time: 0.11281104709791101\n",
      "batch time: 0.11022448539733887\n",
      "batch time: 0.10784938812255859\n",
      "batch time: 0.10563441423269418\n",
      "batch time: 0.10446588198343913\n",
      "batch time: 0.1025756938116891\n",
      "batch time: 0.10080654867764177\n",
      "batch time: 0.0991185983022054\n",
      "Iter 30 || Loss: 13.8155 || 10iter: 5.5794 sec.\n",
      "batch time: 0.09798166828770791\n",
      "batch time: 0.09649457037448883\n",
      "batch time: 0.09507901740796638\n",
      "batch time: 0.09375166893005371\n",
      "batch time: 0.09251454898289271\n",
      "batch time: 0.09133277999030219\n",
      "batch time: 0.09037358696396286\n",
      "batch time: 0.08930755288977373\n",
      "batch time: 0.08887559328323756\n",
      "batch time: 0.0884905755519867\n",
      "Iter 40 || Loss: 12.4186 || 10iter: 5.7278 sec.\n",
      "batch time: 0.08762475920886528\n",
      "batch time: 0.08675802321661086\n",
      "batch time: 0.0859188201815583\n",
      "batch time: 0.08509161255576393\n",
      "batch time: 0.08436999320983887\n",
      "batch time: 0.08364476328310759\n",
      "batch time: 0.08342444643061211\n",
      "batch time: 0.08273407816886902\n",
      "batch time: 0.0820651540950853\n",
      "batch time: 0.08142916679382324\n",
      "Iter 50 || Loss: 12.3549 || 10iter: 5.6079 sec.\n",
      "batch time: 0.08083164458181344\n",
      "batch time: 0.08024632930755615\n",
      "batch time: 0.07966609271067493\n",
      "batch time: 0.07954637209574382\n",
      "batch time: 0.07902218211780895\n",
      "batch time: 0.0784937356199537\n",
      "batch time: 0.0779898626762524\n",
      "batch time: 0.07751142156535182\n",
      "batch time: 0.07705127182653394\n",
      "batch time: 0.07659920454025268\n",
      "Iter 60 || Loss: 11.4274 || 10iter: 5.6148 sec.\n",
      "batch time: 0.0761757131482734\n",
      "batch time: 0.07574760913848877\n",
      "batch time: 0.07532939078315856\n",
      "batch time: 0.07494180649518967\n",
      "batch time: 0.07455210318932166\n",
      "batch time: 0.07417322650100246\n",
      "batch time: 0.07415402469350331\n",
      "batch time: 0.0737981165156645\n",
      "batch time: 0.0734901117241901\n",
      "batch time: 0.07318522930145263\n",
      "Iter 70 || Loss: 11.1522 || 10iter: 5.6541 sec.\n",
      "batch time: 0.07289124878359513\n",
      "batch time: 0.07256931066513062\n",
      "batch time: 0.07257019330377448\n",
      "batch time: 0.07226589563730601\n",
      "batch time: 0.07197989463806152\n",
      "batch time: 0.07171386794040077\n",
      "batch time: 0.0714576801696381\n",
      "batch time: 0.0712275199401073\n",
      "batch time: 0.0709799150877361\n",
      "batch time: 0.0707482635974884\n",
      "Iter 80 || Loss: 10.4597 || 10iter: 5.5704 sec.\n",
      "batch time: 0.07053080311527958\n",
      "batch time: 0.0703033703129466\n",
      "batch time: 0.07008306089654026\n",
      "batch time: 0.0699147071157183\n",
      "batch time: 0.06969512771157657\n",
      "batch time: 0.06946807683900345\n",
      "batch time: 0.06926029303978229\n",
      "batch time: 0.06914151527664879\n",
      "batch time: 0.06892639599489363\n",
      "batch time: 0.06898348066541883\n",
      "Iter 90 || Loss: 9.9863 || 10iter: 5.6744 sec.\n",
      "batch time: 0.06881661205501347\n",
      "batch time: 0.06861689038898634\n",
      "batch time: 0.06842262514175908\n",
      "batch time: 0.06822733168906354\n",
      "batch time: 0.06804877080415425\n",
      "batch time: 0.06787678847710292\n",
      "batch time: 0.06769012175884444\n",
      "batch time: 0.06750818904565305\n",
      "batch time: 0.06734067261821092\n",
      "batch time: 0.06740596532821655\n",
      "Iter 100 || Loss: 9.1614 || 10iter: 5.6141 sec.\n",
      "batch time: 0.06742441536176323\n",
      "batch time: 0.06751562333574482\n",
      "batch time: 0.0673784654117325\n",
      "batch time: 0.06721041523493253\n",
      "batch time: 0.06707943961733863\n",
      "batch time: 0.06692614195481786\n",
      "batch time: 0.06677444404530748\n",
      "batch time: 0.06663252689220288\n",
      "batch time: 0.06647817366713778\n",
      "batch time: 0.0663298476826061\n",
      "Iter 110 || Loss: 9.1399 || 10iter: 5.6615 sec.\n",
      "batch time: 0.06618345750344766\n",
      "batch time: 0.06605063591684614\n",
      "batch time: 0.065926631995007\n",
      "batch time: 0.06580085921705815\n",
      "batch time: 0.06569127414537512\n",
      "batch time: 0.06558253230719731\n",
      "batch time: 0.06546792617210975\n",
      "batch time: 0.06533877122200142\n",
      "batch time: 0.06523782265286486\n",
      "batch time: 0.0651070515314738\n",
      "Iter 120 || Loss: 8.7496 || 10iter: 5.6069 sec.\n",
      "batch time: 0.06498512748844368\n",
      "batch time: 0.06486767041878622\n",
      "batch time: 0.06474677527823099\n",
      "batch time: 0.0646277243091214\n",
      "batch time: 0.0645151195526123\n",
      "batch time: 0.06440833636692592\n",
      "batch time: 0.06430324419276921\n",
      "batch time: 0.06419537216424942\n",
      "batch time: 0.06410771007685698\n",
      "batch time: 0.0640063762664795\n",
      "Iter 130 || Loss: 8.4081 || 10iter: 5.5237 sec.\n",
      "batch time: 0.06389738585202749\n",
      "batch time: 0.06379228830337524\n",
      "batch time: 0.06368623102517952\n",
      "batch time: 0.06358459458422305\n",
      "batch time: 0.06348912804215043\n",
      "batch time: 0.0633924919016221\n",
      "batch time: 0.06329547053706037\n",
      "batch time: 0.06320468411929366\n",
      "batch time: 0.06312381106314899\n",
      "batch time: 0.06304847683225359\n",
      "Iter 140 || Loss: 8.4660 || 10iter: 5.5505 sec.\n",
      "batch time: 0.06295868521886515\n",
      "batch time: 0.06286315179206955\n",
      "batch time: 0.06277639215642755\n",
      "batch time: 0.06282255550225575\n",
      "batch time: 0.0627371097433156\n",
      "batch time: 0.06265100387677755\n",
      "batch time: 0.06256636308164013\n",
      "batch time: 0.06248215726903967\n",
      "batch time: 0.06240285002945253\n",
      "batch time: 0.062321132024129235\n",
      "Iter 150 || Loss: 8.1069 || 10iter: 5.5379 sec.\n",
      "batch time: 0.062241726363731535\n",
      "batch time: 0.0621589626136579\n",
      "batch time: 0.062074679954379215\n",
      "batch time: 0.0619990113493684\n",
      "batch time: 0.06192571424668835\n",
      "batch time: 0.06198828189800947\n",
      "batch time: 0.06192863822742632\n",
      "batch time: 0.0618744484985931\n",
      "batch time: 0.06191416506497365\n",
      "batch time: 0.06185644268989563\n",
      "Iter 160 || Loss: 8.4378 || 10iter: 5.6182 sec.\n",
      "batch time: 0.0617834514712695\n",
      "batch time: 0.06171338940844124\n",
      "batch time: 0.061659482359154826\n",
      "batch time: 0.061590488364056846\n",
      "batch time: 0.061523058920195606\n",
      "batch time: 0.061594081212239096\n",
      "batch time: 0.06152993048022607\n",
      "batch time: 0.06146144015448434\n",
      "batch time: 0.06139544622432551\n",
      "batch time: 0.06146418487324434\n",
      "Iter 170 || Loss: 8.3321 || 10iter: 5.6247 sec.\n",
      "batch time: 0.06141030439856457\n",
      "batch time: 0.06134352573128634\n",
      "batch time: 0.061289001751497305\n",
      "batch time: 0.06123678026528194\n",
      "batch time: 0.061178490774972095\n",
      "batch time: 0.061126745559952476\n",
      "batch time: 0.06108187551552293\n",
      "batch time: 0.06104277894738015\n",
      "batch time: 0.060980218748806575\n",
      "batch time: 0.060931126276652016\n",
      "Iter 180 || Loss: 8.5638 || 10iter: 5.5763 sec.\n",
      "batch time: 0.06087437123883495\n",
      "batch time: 0.060813310382130385\n",
      "batch time: 0.06076785384631548\n",
      "batch time: 0.06072536240453306\n",
      "batch time: 0.06068295401495856\n",
      "batch time: 0.06063347862612817\n",
      "batch time: 0.06057402284387599\n",
      "batch time: 0.0605190599218328\n",
      "batch time: 0.060466370254597335\n",
      "batch time: 0.06041745637592517\n",
      "Iter 190 || Loss: 8.2222 || 10iter: 5.5282 sec.\n",
      "batch time: 0.06037704357926134\n",
      "batch time: 0.060333751142024994\n",
      "batch time: 0.06029024642983866\n",
      "batch time: 0.060237734588151126\n",
      "batch time: 0.06019675915057843\n",
      "batch time: 0.06027836459023612\n",
      "batch time: 0.06023728908015992\n",
      "batch time: 0.060189170066756434\n",
      "batch time: 0.060140676833876416\n",
      "batch time: 0.0600959575176239\n",
      "Iter 200 || Loss: 7.9824 || 10iter: 5.6508 sec.\n",
      "batch time: 0.06004442385773161\n",
      "batch time: 0.06000755919088231\n",
      "batch time: 0.059963792415675274\n",
      "batch time: 0.05992546969769048\n",
      "batch time: 0.05988603684960342\n",
      "batch time: 0.05983674179003077\n",
      "batch time: 0.05979261997241329\n",
      "batch time: 0.0597435121352856\n",
      "batch time: 0.059695886082626416\n",
      "batch time: 0.05965495563688732\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "Iter 210 || Loss: 7.5960 || 10iter: 5.4994 sec.\n",
      "batch time: 0.05961109224653922\n",
      "batch time: 0.05956816335894027\n",
      "batch time: 0.059528297101947625\n",
      "batch time: 0.05948166980921665\n",
      "batch time: 0.059438094427419264\n",
      "batch time: 0.05940400229560004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch time: 0.0593738775648829\n",
      "batch time: 0.059330021569488245\n",
      "batch time: 0.05928614912512096\n",
      "batch time: 0.059240774674849075\n",
      "Iter 220 || Loss: 7.7707 || 10iter: 5.5319 sec.\n",
      "batch time: 0.05919642362119925\n",
      "batch time: 0.05915580461691092\n",
      "batch time: 0.05911224305362445\n",
      "batch time: 0.05906997301748821\n",
      "batch time: 0.05903566784328885\n",
      "batch time: 0.059006193042856404\n",
      "batch time: 0.0589738795410694\n",
      "batch time: 0.059020756629475375\n",
      "batch time: 0.05899643377445671\n",
      "batch time: 0.058968152170595915\n",
      "Iter 230 || Loss: 7.5936 || 10iter: 5.5751 sec.\n",
      "batch time: 0.05893709442832253\n",
      "batch time: 0.05889900491155427\n",
      "batch time: 0.05886560345923952\n",
      "batch time: 0.05882938091571514\n",
      "batch time: 0.058795064560910486\n",
      "batch time: 0.05876249074935913\n",
      "batch time: 0.05874526349804069\n",
      "batch time: 0.058712213981051406\n",
      "batch time: 0.05867828484858429\n",
      "batch time: 0.058654058972994486\n",
      "Iter 240 || Loss: 7.0318 || 10iter: 5.5524 sec.\n",
      "batch time: 0.05862698020776772\n",
      "batch time: 0.0585996602192398\n",
      "batch time: 0.05857280742974929\n",
      "batch time: 0.058538882458796265\n",
      "batch time: 0.05850229652560487\n",
      "batch time: 0.05846852887936724\n",
      "batch time: 0.05843458388015809\n",
      "batch time: 0.05840096454466543\n",
      "batch time: 0.05836960781051452\n",
      "batch time: 0.058336546897888183\n",
      "Iter 250 || Loss: 7.8670 || 10iter: 5.5420 sec.\n",
      "batch time: 0.05831037669542776\n",
      "batch time: 0.05828315114218091\n",
      "batch time: 0.058250105899313225\n",
      "batch time: 0.05821978576540008\n",
      "batch time: 0.05827303587221632\n",
      "batch time: 0.058250741101801395\n",
      "batch time: 0.058231843584706344\n",
      "batch time: 0.05820982770402302\n",
      "batch time: 0.05818702631475382\n",
      "batch time: 0.058164302202371454\n",
      "Iter 260 || Loss: 7.6694 || 10iter: 5.6340 sec.\n",
      "batch time: 0.05823535663414732\n",
      "batch time: 0.05820624427941009\n",
      "batch time: 0.058179832683316654\n",
      "batch time: 0.05814859451669635\n",
      "batch time: 0.05811747335038095\n",
      "batch time: 0.05808528383871667\n",
      "batch time: 0.05806217657939325\n",
      "batch time: 0.058034346174837936\n",
      "batch time: 0.05800523013430457\n",
      "batch time: 0.057974606973153577\n",
      "Iter 270 || Loss: 7.7616 || 10iter: 5.6440 sec.\n",
      "batch time: 0.05794336962963822\n",
      "batch time: 0.05791299045085907\n",
      "batch time: 0.05788200766175658\n",
      "batch time: 0.057857537791676766\n",
      "batch time: 0.057831342870538885\n",
      "batch time: 0.05780518573263417\n",
      "batch time: 0.05777949030218572\n",
      "batch time: 0.0577534394298526\n",
      "batch time: 0.057728828067847904\n",
      "batch time: 0.05770993317876543\n",
      "Iter 280 || Loss: 7.3697 || 10iter: 5.5369 sec.\n",
      "batch time: 0.057690880901024436\n",
      "batch time: 0.05766579952645809\n",
      "batch time: 0.05764622149113632\n",
      "batch time: 0.05762563037200713\n",
      "batch time: 0.057607045926545794\n",
      "batch time: 0.05758745186812394\n",
      "batch time: 0.057569736387671495\n",
      "batch time: 0.05755247837967343\n",
      "batch time: 0.057537082157333004\n",
      "batch time: 0.05751067769938502\n",
      "Iter 290 || Loss: 7.1903 || 10iter: 5.5623 sec.\n",
      "batch time: 0.057493171331399084\n",
      "batch time: 0.05746781009517304\n",
      "batch time: 0.057449964939937655\n",
      "batch time: 0.05742404736629149\n",
      "batch time: 0.05742414442159362\n",
      "batch time: 0.057402354639929695\n",
      "batch time: 0.057407654495753024\n",
      "batch time: 0.05739105867859501\n",
      "batch time: 0.05736899934086114\n",
      "batch time: 0.057342863877614336\n",
      "Iter 300 || Loss: 7.6110 || 10iter: 5.5918 sec.\n",
      "batch time: 0.057315739286302333\n",
      "batch time: 0.057292158240514084\n",
      "batch time: 0.05726917820795141\n",
      "batch time: 0.05724754694261049\n",
      "batch time: 0.057222901797685466\n",
      "batch time: 0.057197658844243465\n",
      "batch time: 0.05717326530804463\n",
      "batch time: 0.05715017736732186\n",
      "batch time: 0.05712563243112904\n",
      "batch time: 0.05719927972362888\n",
      "Iter 310 || Loss: 7.5151 || 10iter: 5.5303 sec.\n",
      "batch time: 0.057176701125608\n",
      "batch time: 0.05715702588741596\n",
      "batch time: 0.057135599489790946\n",
      "batch time: 0.05711423743302655\n",
      "batch time: 0.057089913837493414\n",
      "batch time: 0.057066217253479774\n",
      "batch time: 0.057119635001342005\n",
      "batch time: 0.05709969622534026\n",
      "batch time: 0.05708000129293125\n",
      "batch time: 0.05705708712339401\n",
      "Iter 320 || Loss: 7.3837 || 10iter: 5.6028 sec.\n",
      "batch time: 0.05703541645751192\n",
      "batch time: 0.05701383522578648\n",
      "batch time: 0.05699154836105488\n",
      "batch time: 0.05704471358546504\n",
      "batch time: 0.057026809545663686\n",
      "batch time: 0.05701068573934169\n",
      "batch time: 0.056987809843244176\n",
      "batch time: 0.05696860464607797\n",
      "batch time: 0.05694742405668218\n",
      "batch time: 0.05693285392992424\n",
      "Iter 330 || Loss: 6.7227 || 10iter: 5.6218 sec.\n",
      "batch time: 0.0569792867066997\n",
      "batch time: 0.05695758454770927\n",
      "batch time: 0.05693856015935674\n",
      "batch time: 0.05699036364069956\n",
      "batch time: 0.057034780730062455\n",
      "batch time: 0.057015229548726766\n",
      "batch time: 0.05699640707021651\n",
      "batch time: 0.056976474953826364\n",
      "batch time: 0.05695547888764238\n",
      "batch time: 0.05693705432555255\n",
      "Iter 340 || Loss: 7.1228 || 10iter: 5.6160 sec.\n",
      "batch time: 0.05691503779279871\n",
      "batch time: 0.05689809196873715\n",
      "batch time: 0.056879584365266404\n",
      "batch time: 0.056858924932258074\n",
      "batch time: 0.05683964577274046\n",
      "batch time: 0.05682087564744012\n",
      "batch time: 0.05681100496297611\n",
      "batch time: 0.05679803675618665\n",
      "batch time: 0.05678267943483369\n",
      "batch time: 0.056762805666242325\n",
      "Iter 350 || Loss: 7.5340 || 10iter: 5.5440 sec.\n",
      "batch time: 0.0567441000218405\n",
      "batch time: 0.05673036114736037\n",
      "batch time: 0.05671244505106896\n",
      "batch time: 0.05669374519822288\n",
      "batch time: 0.05667476586892571\n",
      "batch time: 0.05665779113769531\n",
      "batch time: 0.056640883453753815\n",
      "batch time: 0.056622513845646184\n",
      "batch time: 0.056666401435405765\n",
      "batch time: 0.056648172934850055\n",
      "Iter 360 || Loss: 7.0870 || 10iter: 5.5493 sec.\n",
      "batch time: 0.056629512448720325\n",
      "batch time: 0.05661250543857806\n",
      "batch time: 0.05662959009491051\n",
      "batch time: 0.05661344200700194\n",
      "batch time: 0.05660657490769478\n",
      "batch time: 0.0565917257402764\n",
      "batch time: 0.05657587324241201\n",
      "batch time: 0.05655823129674663\n",
      "batch time: 0.05654297676189805\n",
      "batch time: 0.056525673737397066\n",
      "Iter 370 || Loss: 7.2454 || 10iter: 5.5724 sec.\n",
      "batch time: 0.05650683747468933\n",
      "batch time: 0.05649091671871882\n",
      "batch time: 0.05647326858050063\n",
      "batch time: 0.05651592762075006\n",
      "batch time: 0.056543030420939125\n",
      "batch time: 0.0565917396799047\n",
      "batch time: 0.056577115855735556\n",
      "batch time: 0.05656399361040226\n",
      "batch time: 0.05654613072136147\n",
      "batch time: 0.0565313784699691\n",
      "Iter 380 || Loss: 7.2290 || 10iter: 5.6370 sec.\n",
      "batch time: 0.05651854217208902\n",
      "batch time: 0.0565006027671055\n",
      "batch time: 0.05648478141963949\n",
      "batch time: 0.05647024450202783\n",
      "batch time: 0.05645566296267819\n",
      "batch time: 0.05644767889704729\n",
      "batch time: 0.05649755044192923\n",
      "batch time: 0.05648079791019872\n",
      "batch time: 0.056471555582355414\n",
      "batch time: 0.056456521229866226\n",
      "Iter 390 || Loss: 7.0981 || 10iter: 5.6176 sec.\n",
      "batch time: 0.0564417089037883\n",
      "batch time: 0.056427725115600895\n",
      "batch time: 0.056411549638549184\n",
      "batch time: 0.05639500545366161\n",
      "batch time: 0.05637873999680145\n",
      "batch time: 0.05636266385666048\n",
      "batch time: 0.056347197789689454\n",
      "batch time: 0.05633219342734946\n",
      "batch time: 0.05631730132234425\n",
      "batch time: 0.05630188465118408\n",
      "Iter 400 || Loss: 7.5046 || 10iter: 5.5368 sec.\n",
      "batch time: 0.056286032360390835\n",
      "batch time: 0.056271336564970256\n",
      "batch time: 0.056259524437689015\n",
      "batch time: 0.05624959669490852\n",
      "batch time: 0.056243583302439\n",
      "batch time: 0.05623340959032181\n",
      "batch time: 0.056225559342405425\n",
      "batch time: 0.05621970108911103\n",
      "batch time: 0.05620811562666392\n",
      "batch time: 0.05619261032197534\n",
      "Iter 410 || Loss: 7.1489 || 10iter: 5.5729 sec.\n",
      "batch time: 0.056205311540849595\n",
      "batch time: 0.05620395790026026\n",
      "batch time: 0.056190296754998675\n",
      "batch time: 0.056181191822181\n",
      "batch time: 0.056172854641833934\n",
      "batch time: 0.05615854664490773\n",
      "batch time: 0.056144485656591914\n",
      "batch time: 0.05614781664889395\n",
      "batch time: 0.056190633546196474\n",
      "batch time: 0.0561766471181597\n",
      "Iter 420 || Loss: 7.4090 || 10iter: 5.6094 sec.\n",
      "batch time: 0.05616275261813275\n",
      "batch time: 0.05614828780929059\n",
      "batch time: 0.05613663236018332\n",
      "batch time: 0.05612212981817857\n",
      "batch time: 0.056145488514619715\n",
      "batch time: 0.05618183713563731\n",
      "batch time: 0.05616815642953198\n",
      "batch time: 0.056153373183491076\n",
      "batch time: 0.056140530637372066\n",
      "batch time: 0.056126627256703934\n",
      "Iter 430 || Loss: 7.1384 || 10iter: 5.6170 sec.\n",
      "batch time: 0.05613189285703159\n",
      "batch time: 0.05612289905548096\n",
      "batch time: 0.056110824778779274\n",
      "batch time: 0.056098774281514954\n",
      "batch time: 0.05608511793202367\n",
      "batch time: 0.05607584200867819\n",
      "batch time: 0.056062876223426664\n",
      "batch time: 0.056049180357423546\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch time: 0.056036209462715446\n",
      "batch time: 0.056060953573747116\n",
      "Iter 440 || Loss: 6.4678 || 10iter: 5.6182 sec.\n",
      "batch time: 0.05606276810574694\n",
      "batch time: 0.05605098294996028\n",
      "batch time: 0.05604398869499396\n",
      "batch time: 0.05603116804415041\n",
      "batch time: 0.056020092160514234\n",
      "batch time: 0.056007610308215224\n",
      "batch time: 0.05599432290267091\n",
      "batch time: 0.05598111982856478\n",
      "batch time: 0.05596917934035405\n",
      "batch time: 0.05596178160773383\n",
      "Iter 450 || Loss: 6.9035 || 10iter: 5.5495 sec.\n",
      "batch time: 0.0559555400501598\n",
      "batch time: 0.05594354234965502\n",
      "batch time: 0.055934589693351563\n",
      "batch time: 0.05592159132600356\n",
      "batch time: 0.05591596718672868\n",
      "batch time: 0.05590790301038508\n",
      "batch time: 0.055900924837041475\n",
      "batch time: 0.05589237015320224\n",
      "batch time: 0.055939920068046885\n",
      "batch time: 0.055930867402449895\n",
      "Iter 460 || Loss: 6.8679 || 10iter: 5.6673 sec.\n",
      "batch time: 0.05592306989392593\n",
      "batch time: 0.055913625857530735\n",
      "batch time: 0.05590536218482526\n",
      "batch time: 0.055895702078424654\n",
      "batch time: 0.055932801769625756\n",
      "batch time: 0.055922792704831886\n",
      "batch time: 0.055912001485232364\n",
      "batch time: 0.05595050663010687\n",
      "batch time: 0.05593857785531961\n",
      "batch time: 0.0559292493982518\n",
      "Iter 470 || Loss: 7.3142 || 10iter: 5.6519 sec.\n",
      "batch time: 0.0559166414216319\n",
      "batch time: 0.055906072006387225\n",
      "batch time: 0.055894612257878826\n",
      "batch time: 0.055915206796509305\n",
      "batch time: 0.0559032204276637\n",
      "batch time: 0.05589399067293696\n",
      "batch time: 0.05588278230631127\n",
      "batch time: 0.05587082178522852\n",
      "batch time: 0.05586032728063786\n",
      "batch time: 0.0558477466305097\n",
      "Iter 480 || Loss: 7.3180 || 10iter: 5.5717 sec.\n",
      "batch time: 0.0558359617998595\n",
      "batch time: 0.05582475167587091\n",
      "batch time: 0.055817638618358666\n",
      "batch time: 0.05580704862421209\n",
      "batch time: 0.055799706940798416\n",
      "batch time: 0.05578891632488235\n",
      "batch time: 0.05577875162786527\n",
      "batch time: 0.055767294813375\n",
      "batch time: 0.05575540801986351\n",
      "batch time: 0.05579631279925911\n",
      "Iter 490 || Loss: 7.1327 || 10iter: 5.6494 sec.\n",
      "batch time: 0.05578406619442942\n",
      "batch time: 0.055774813260489366\n",
      "batch time: 0.05576442704964845\n",
      "batch time: 0.05580167200883873\n",
      "batch time: 0.05579142570495606\n",
      "batch time: 0.055781120734830054\n",
      "batch time: 0.05576880117297412\n",
      "batch time: 0.05580736977987021\n",
      "batch time: 0.05579704225421669\n",
      "batch time: 0.055788076877593996\n",
      "Iter 500 || Loss: 7.0576 || 10iter: 5.5983 sec.\n",
      "batch time: 0.0557783006907937\n",
      "batch time: 0.055785151591813895\n",
      "batch time: 0.055772387957715136\n",
      "batch time: 0.055759380734156046\n",
      "batch time: 0.05574606716042698\n",
      "batch time: 0.05573348094352149\n",
      "batch time: 0.05572049481393788\n",
      "batch time: 0.05570811549509604\n",
      "batch time: 0.055695152001671334\n",
      "batch time: 0.05568217249477611\n",
      "Iter 510 || Loss: 7.0183 || 10iter: 5.4891 sec.\n",
      "batch time: 0.05566876741771362\n",
      "batch time: 0.05565570434555411\n",
      "batch time: 0.05564272938183641\n",
      "batch time: 0.05563017691155816\n",
      "batch time: 0.05561865093638596\n",
      "batch time: 0.05560619101043819\n",
      "batch time: 0.05559334118546093\n",
      "batch time: 0.056772855257895924\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:4433.5437 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  295.8544 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 2/200\n",
      "-------------\n",
      "(train)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch time: 0.057033233789579946\n",
      "batch time: 0.05709133881788987\n",
      "Iter 520 || Loss: 6.7143 || 10iter: 4.0647 sec.\n",
      "batch time: 0.057128293958140425\n",
      "batch time: 0.057145640767853834\n",
      "batch time: 0.05713414416486401\n",
      "batch time: 0.057121746867667626\n",
      "batch time: 0.05711540040515718\n",
      "batch time: 0.05710609829471139\n",
      "batch time: 0.057096617045393476\n",
      "batch time: 0.05709120134512583\n",
      "batch time: 0.057081042255480934\n",
      "batch time: 0.05707931248646862\n",
      "Iter 530 || Loss: 6.9429 || 10iter: 5.8179 sec.\n",
      "batch time: 0.05707111825601754\n",
      "batch time: 0.05705815643296206\n",
      "batch time: 0.057050528714178206\n",
      "batch time: 0.0570403721448634\n",
      "batch time: 0.05705347061157227\n",
      "batch time: 0.05704433482084701\n",
      "batch time: 0.05703371374744721\n",
      "batch time: 0.05702323541322162\n",
      "batch time: 0.057029415813580515\n",
      "batch time: 0.057016751059779415\n",
      "Iter 540 || Loss: 7.1472 || 10iter: 5.5745 sec.\n",
      "batch time: 0.05700360164184006\n",
      "batch time: 0.05699129122209725\n",
      "batch time: 0.056980832505621304\n",
      "batch time: 0.05701239187927807\n",
      "batch time: 0.05699955126561156\n",
      "batch time: 0.05698636119618957\n",
      "batch time: 0.05697329432262581\n",
      "batch time: 0.05696182355393459\n",
      "batch time: 0.05694852934943305\n",
      "batch time: 0.05693699836730957\n",
      "Iter 550 || Loss: 6.5566 || 10iter: 5.5586 sec.\n",
      "batch time: 0.056926325315139685\n",
      "batch time: 0.056915280611618706\n",
      "batch time: 0.05690341875307383\n",
      "batch time: 0.0568917877837639\n",
      "batch time: 0.05692139230332933\n",
      "batch time: 0.056911339862741155\n",
      "batch time: 0.05690049226236943\n",
      "batch time: 0.05689049749818754\n",
      "batch time: 0.05688206070437628\n",
      "batch time: 0.05687615871429443\n",
      "Iter 560 || Loss: 6.6824 || 10iter: 5.5716 sec.\n",
      "batch time: 0.056906573487688086\n",
      "batch time: 0.056893719048686724\n",
      "batch time: 0.05688418376509292\n",
      "batch time: 0.05687751584019221\n",
      "batch time: 0.05686995286857132\n",
      "batch time: 0.05686156488560114\n",
      "batch time: 0.05685303324744815\n",
      "batch time: 0.05684188134233716\n",
      "batch time: 0.056833956907838844\n",
      "batch time: 0.056825002452783414\n",
      "Iter 570 || Loss: 6.3188 || 10iter: 5.5748 sec.\n",
      "batch time: 0.05681242708984482\n",
      "batch time: 0.05682939291000366\n",
      "batch time: 0.056817798298274866\n",
      "batch time: 0.05680571034394905\n",
      "batch time: 0.05679304786350416\n",
      "batch time: 0.05678515508770943\n",
      "batch time: 0.05677436170792869\n",
      "batch time: 0.05676621054283063\n",
      "batch time: 0.05675394670950934\n",
      "batch time: 0.05674306935277478\n",
      "Iter 580 || Loss: 7.5331 || 10iter: 5.5319 sec.\n",
      "batch time: 0.05673092203583446\n",
      "batch time: 0.056724608968623316\n",
      "batch time: 0.05671562297879934\n",
      "batch time: 0.05670874616871142\n",
      "batch time: 0.05670012327340933\n",
      "batch time: 0.05668909598536052\n",
      "batch time: 0.05667684187880976\n",
      "batch time: 0.05670426167598387\n",
      "batch time: 0.05669699743364784\n",
      "batch time: 0.0566899542081154\n",
      "Iter 590 || Loss: 7.1742 || 10iter: 5.6381 sec.\n",
      "batch time: 0.05667978010806941\n",
      "batch time: 0.05666822678334004\n",
      "batch time: 0.05665908173405099\n",
      "batch time: 0.05664675966256395\n",
      "batch time: 0.05663903540923816\n",
      "batch time: 0.05663091864361859\n",
      "batch time: 0.05661981589031379\n",
      "batch time: 0.056609202786831556\n",
      "batch time: 0.056600902235766685\n",
      "batch time: 0.056590958436330156\n",
      "Iter 600 || Loss: 6.7491 || 10iter: 5.5343 sec.\n",
      "batch time: 0.056580007968845464\n",
      "batch time: 0.0565680792165357\n",
      "batch time: 0.05655883951962093\n",
      "batch time: 0.05654739544091635\n",
      "batch time: 0.0565372423692183\n",
      "batch time: 0.056526110904051526\n",
      "batch time: 0.056516078983539414\n",
      "batch time: 0.05652722951612974\n",
      "batch time: 0.056516154059048355\n",
      "batch time: 0.056507519815788894\n",
      "Iter 610 || Loss: 6.7304 || 10iter: 5.5826 sec.\n",
      "batch time: 0.05649644068142006\n",
      "batch time: 0.05648420605005002\n",
      "batch time: 0.05647669219659553\n",
      "batch time: 0.05646856522327137\n",
      "batch time: 0.056458415830038425\n",
      "batch time: 0.05645122350036324\n",
      "batch time: 0.05644309926458078\n",
      "batch time: 0.05643511204272026\n",
      "batch time: 0.05645962293005529\n",
      "batch time: 0.05645079728095762\n",
      "Iter 620 || Loss: 6.9430 || 10iter: 5.5852 sec.\n",
      "batch time: 0.05644333266597631\n",
      "batch time: 0.056433816814729254\n",
      "batch time: 0.05642382444194959\n",
      "batch time: 0.05645409188209436\n",
      "batch time: 0.0564469181060791\n",
      "batch time: 0.05644021399866659\n",
      "batch time: 0.05643079687723893\n",
      "batch time: 0.05642267017607476\n",
      "batch time: 0.056412749525473496\n",
      "batch time: 0.056404321155850846\n",
      "Iter 630 || Loss: 6.5889 || 10iter: 5.6172 sec.\n",
      "batch time: 0.056396934007502585\n",
      "batch time: 0.05638871087303644\n",
      "batch time: 0.0563795521361003\n",
      "batch time: 0.05639351580045201\n",
      "batch time: 0.05638706177238404\n",
      "batch time: 0.056379973513525236\n",
      "batch time: 0.0563721046717238\n",
      "batch time: 0.05636470519636866\n",
      "batch time: 0.05635491260713628\n",
      "batch time: 0.056345709785819056\n",
      "Iter 640 || Loss: 6.7310 || 10iter: 5.5789 sec.\n",
      "batch time: 0.056336713097582744\n",
      "batch time: 0.05632603539856052\n",
      "batch time: 0.05631544727188988\n",
      "batch time: 0.056308578630411846\n",
      "batch time: 0.05629904954008354\n",
      "batch time: 0.05628942705160324\n",
      "batch time: 0.05628255014257416\n",
      "batch time: 0.05627365759861322\n",
      "batch time: 0.05626390197060325\n",
      "batch time: 0.05625587353339562\n",
      "Iter 650 || Loss: 7.0232 || 10iter: 5.5130 sec.\n",
      "batch time: 0.05624671172802716\n",
      "batch time: 0.0562368240093161\n",
      "batch time: 0.056234136657364334\n",
      "batch time: 0.056240732516717476\n",
      "batch time: 0.056231471054426586\n",
      "batch time: 0.05622279971111112\n",
      "batch time: 0.05621417182039815\n",
      "batch time: 0.05620470018010009\n",
      "batch time: 0.056199694620459505\n",
      "batch time: 0.05619253678755327\n",
      "Iter 660 || Loss: 6.9646 || 10iter: 5.5654 sec.\n",
      "batch time: 0.056187039244012645\n",
      "batch time: 0.05618055961643461\n",
      "batch time: 0.05621517837317281\n",
      "batch time: 0.05620654268437121\n",
      "batch time: 0.0561963206843326\n",
      "batch time: 0.05621724085764842\n",
      "batch time: 0.0562070627798741\n",
      "batch time: 0.05619742556246455\n",
      "batch time: 0.0561904582920274\n",
      "batch time: 0.05618180516940444\n",
      "Iter 670 || Loss: 6.5057 || 10iter: 5.5739 sec.\n",
      "batch time: 0.05618330654195453\n",
      "batch time: 0.05617814227229073\n",
      "batch time: 0.05617406003365368\n",
      "batch time: 0.05616546738395351\n",
      "batch time: 0.05615773695486563\n",
      "batch time: 0.05615208205386732\n",
      "batch time: 0.05614659317839269\n",
      "batch time: 0.05614607277872991\n",
      "batch time: 0.05614144601948307\n",
      "batch time: 0.0561348368139828\n",
      "Iter 680 || Loss: 6.4517 || 10iter: 5.6385 sec.\n",
      "batch time: 0.05612691698620498\n",
      "batch time: 0.056118708901391354\n",
      "batch time: 0.05611097725383708\n",
      "batch time: 0.05610589395489609\n",
      "batch time: 0.05609717160245798\n",
      "batch time: 0.05609050019489433\n",
      "batch time: 0.05608210362374696\n",
      "batch time: 0.056072772588840754\n",
      "batch time: 0.05606357210433362\n",
      "batch time: 0.05605483780736509\n",
      "Iter 690 || Loss: 6.1203 || 10iter: 5.5578 sec.\n",
      "batch time: 0.05604837530773731\n",
      "batch time: 0.056041275145690565\n",
      "batch time: 0.05603608871779229\n",
      "batch time: 0.056026751095005004\n",
      "batch time: 0.0560181518252805\n",
      "batch time: 0.05600968517106155\n",
      "batch time: 0.056001625923036334\n",
      "batch time: 0.055994502794435165\n",
      "batch time: 0.055985696667083175\n",
      "batch time: 0.0559770461491176\n",
      "Iter 700 || Loss: 7.0602 || 10iter: 5.5554 sec.\n",
      "batch time: 0.055969723621210596\n",
      "batch time: 0.05596108823759943\n",
      "batch time: 0.055954075130934734\n",
      "batch time: 0.05594538727944547\n",
      "batch time: 0.05593722289335643\n",
      "batch time: 0.05596072748092349\n",
      "batch time: 0.05595269567389805\n",
      "batch time: 0.05594382380361611\n",
      "batch time: 0.05597108678185553\n",
      "batch time: 0.056000156469748055\n",
      "Iter 710 || Loss: 6.5331 || 10iter: 5.7286 sec.\n",
      "batch time: 0.05602189156576551\n",
      "batch time: 0.05601379978522826\n",
      "batch time: 0.0560047539554505\n",
      "batch time: 0.05601881331756335\n",
      "batch time: 0.05601268881684417\n",
      "batch time: 0.056004752992917704\n",
      "batch time: 0.05599535559011803\n",
      "batch time: 0.05598658738362092\n",
      "batch time: 0.055977903917867056\n",
      "batch time: 0.05596909787919786\n",
      "Iter 720 || Loss: 6.2941 || 10iter: 5.5942 sec.\n",
      "batch time: 0.055960875113033555\n",
      "batch time: 0.055953471614383264\n",
      "batch time: 0.05594486220743646\n",
      "batch time: 0.0559388626346272\n",
      "batch time: 0.05593008436005691\n",
      "batch time: 0.055921547012224015\n",
      "batch time: 0.055912314615682523\n",
      "batch time: 0.055905589690575234\n",
      "batch time: 0.055896426097519276\n",
      "batch time: 0.05588930730950342\n",
      "Iter 730 || Loss: 7.2483 || 10iter: 5.5579 sec.\n",
      "batch time: 0.05588408110079785\n",
      "batch time: 0.05587685889885074\n",
      "batch time: 0.05586899839351805\n",
      "batch time: 0.0558617914080295\n",
      "batch time: 0.055854628686191274\n",
      "batch time: 0.05584814107936362\n",
      "batch time: 0.055840795690363106\n",
      "batch time: 0.055834645178259874\n",
      "batch time: 0.055826173583612713\n",
      "batch time: 0.05581781413104083\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 740 || Loss: 6.8518 || 10iter: 5.5964 sec.\n",
      "batch time: 0.05581476524291251\n",
      "batch time: 0.05581074487166906\n",
      "batch time: 0.05580368850625861\n",
      "batch time: 0.05579592335608698\n",
      "batch time: 0.055789068081234926\n",
      "batch time: 0.05578140950394699\n",
      "batch time: 0.05577423805533004\n",
      "batch time: 0.05576580539744168\n",
      "batch time: 0.05575773267147538\n",
      "batch time: 0.0557602596282959\n",
      "Iter 750 || Loss: 6.2037 || 10iter: 5.6235 sec.\n",
      "batch time: 0.055751520530202894\n",
      "batch time: 0.05574588223974755\n",
      "batch time: 0.05573946324635945\n",
      "batch time: 0.05573354665417254\n",
      "batch time: 0.055744785662518434\n",
      "batch time: 0.05573679845799845\n",
      "batch time: 0.05572903455484811\n",
      "batch time: 0.055720491270905746\n",
      "batch time: 0.055715232655621956\n",
      "batch time: 0.05570763129937022\n",
      "Iter 760 || Loss: 6.4748 || 10iter: 5.6492 sec.\n",
      "batch time: 0.055700014517905676\n",
      "batch time: 0.055693958687970016\n"
     ]
    }
   ],
   "source": [
    "if DATASET == \"COCO\":\n",
    "    num_epochs = 50\n",
    "else:\n",
    "    num_epochs = 200\n",
    "    \n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}