nn_isl.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nn-isl.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/etherzhhb/isl-nn/blob/master/nn_isl.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "f_-udCqroE38",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        },
        "outputId": "19213f12-8157-494c-e50f-d39311aca528"
      },
      "cell_type": "code",
      "source": [
        "!pip install islpy"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting islpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/0a08461588ac80a3d1adb21f733f7bc6f93c9d1ed717959abd4e2dcd7709/islpy-2018.2.tar.gz (1.2MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.2MB 17.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytest>=2 in /usr/local/lib/python3.6/dist-packages (from islpy) (3.10.1)\n",
            "Requirement already satisfied: cffi>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from islpy) (1.11.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from islpy) (1.11.0)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (18.2.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (40.6.3)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (1.7.0)\n",
            "Requirement already satisfied: pluggy>=0.7 in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (0.8.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (1.2.1)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=2->islpy) (4.3.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.1.0->islpy) (2.19)\n",
            "Building wheels for collected packages: islpy\n",
            "  Running setup.py bdist_wheel for islpy ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/c0/6e/ed/eee64c8588c6a281e1ed02486572fdddc06478f0e276a2d128\n",
            "Successfully built islpy\n",
            "Installing collected packages: islpy\n",
            "Successfully installed islpy-2018.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "h5-gRDE-sZR4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OXhM3o2KsZSB",
        "colab_type": "code",
        "outputId": "a87be540-8a6a-437c-9e45-937dc028dee2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        }
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
        "print(mnist.train.num_examples)\n",
        "print(mnist.test.num_examples)\n",
        "\n",
        "# The number of pixels in each dimension of an image.# The n \n",
        "img_size = 28\n",
        "\n",
        "# The images are stored in one-dimensional arrays of this length.\n",
        "img_size_flat = img_size * img_size\n",
        "\n",
        "# Tuple with height and width of images used to reshape arrays.\n",
        "img_shape = (img_size, img_size)\n",
        "\n",
        "# Number of classes, one class for each of 10 digits.\n",
        "num_classes = 10\n",
        "\n",
        "# Number of colour channels for the images: 1 channel for gray-scale.\n",
        "num_channels = 1\n",
        "\n",
        "# Tuple with height, width and depth used to reshape arrays.\n",
        "# This is used for reshaping in Keras.\n",
        "img_shape_full = (img_size, img_size, num_channels)\n",
        "\n",
        "def plot_images(images, cls_true, cls_pred=None):\n",
        "    assert len(images) == len(cls_true) == 9\n",
        "\n",
        "    # Create figure with 3x3 sub-plots.\n",
        "    fig, axes = plt.subplots(3, 3)\n",
        "    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n",
        "\n",
        "    for i, ax in enumerate(axes.flat):\n",
        "        # Plot image.\n",
        "        ax.imshow(images[i].reshape(img_shape), cmap='binary')\n",
        "        if cls_pred is None:\n",
        "            xlabel = \"True: {0}\".format(cls_true[i].argmax())\n",
        "        else:\n",
        "            xlabel = \"True: {0}, Pred: {1}\".format(cls_true[i].argmax(), cls_pred[i].argmax())\n",
        "\n",
        "        # Show the classes as the label on the x-axis.\n",
        "        ax.set_xlabel(xlabel)\n",
        "        \n",
        "        # Remove ticks from the plot.\n",
        "        ax.set_xticks([])\n",
        "        ax.set_yticks([])\n",
        "    \n",
        "    # Ensure the plot is shown correctly with multiple plots\n",
        "    # in a single Notebook cell.\n",
        "    plt.show()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-3-8706223a0a6a>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.one_hot on tensors.\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "55000\n",
            "10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9zAQADLdsZSJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def plot_example_errors(cls_pred):\n",
        "    # cls_pred is an array of the predicted class-number for\n",
        "    # all images in the test-set.\n",
        "\n",
        "    # Boolean array whether the predicted class is incorrect.\n",
        "    incorrect = (cls_pred.argmax(axis=1) != mnist.test.labels.argmax(axis=1))\n",
        "\n",
        "    # Get the images from the test-set that have been\n",
        "    # incorrectly classified.\n",
        "    images = mnist.test.images[incorrect]\n",
        "    \n",
        "    # Get the predicted classes for those images.\n",
        "    cls_pred = cls_pred[incorrect]\n",
        "\n",
        "    # Get the true classes for those images.\n",
        "    cls_true = mnist.test.labels[incorrect]\n",
        "    \n",
        "    # Plot the first 9 images.\n",
        "    plot_images(images=images[0:9],\n",
        "                cls_true=cls_true[0:9],\n",
        "                cls_pred=cls_pred[0:9])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7Z5qkN99sZSM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def plot_conv_weights(weights, input_channel=0):\n",
        "    # Get the lowest and highest values for the weights.\n",
        "    # This is used to correct the colour intensity across\n",
        "    # the images so they can be compared with each other.\n",
        "    w_min = np.min(weights)\n",
        "    w_max = np.max(weights)\n",
        "\n",
        "    # Number of filters used in the conv. layer.\n",
        "    num_filters = weights.shape[3]\n",
        "\n",
        "    # Number of grids to plot.\n",
        "    # Rounded-up, square-root of the number of filters.\n",
        "    num_grids = math.ceil(math.sqrt(num_filters))\n",
        "    \n",
        "    # Create figure with a grid of sub-plots.\n",
        "    fig, axes = plt.subplots(num_grids, num_grids)\n",
        "\n",
        "    # Plot all the filter-weights.\n",
        "    for i, ax in enumerate(axes.flat):\n",
        "        # Only plot the valid filter-weights.\n",
        "        if i<num_filters:\n",
        "            # Get the weights for the i'th filter of the input channel.\n",
        "            # See new_conv_layer() for details on the format\n",
        "            # of this 4-dim tensor.\n",
        "            img = weights[:, :, input_channel, i]\n",
        "\n",
        "            # Plot image.\n",
        "            ax.imshow(img, vmin=w_min, vmax=w_max,\n",
        "                      interpolation='nearest', cmap='seismic')\n",
        "        \n",
        "        # Remove ticks from the plot.\n",
        "        ax.set_xticks([])\n",
        "        ax.set_yticks([])\n",
        "    \n",
        "    # Ensure the plot is shown correctly with multiple plots\n",
        "    # in a single Notebook cell.\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Z4uQ2L1KsZSO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def plot_image(image):\n",
        "    plt.imshow(image.reshape(img_shape),\n",
        "               interpolation='nearest',\n",
        "               cmap='binary')\n",
        "\n",
        "    plt.show()\n",
        "\n",
        "def plot_conv_output(values):\n",
        "    # Number of filters used in the conv. layer.\n",
        "    num_filters = values.shape[3]\n",
        "\n",
        "    # Number of grids to plot.\n",
        "    # Rounded-up, square-root of the number of filters.\n",
        "    num_grids = math.ceil(math.sqrt(num_filters))\n",
        "    \n",
        "    # Create figure with a grid of sub-plots.\n",
        "    fig, axes = plt.subplots(num_grids, num_grids)\n",
        "\n",
        "    # Plot the output images of all the filters.\n",
        "    for i, ax in enumerate(axes.flat):\n",
        "        # Only plot the images for valid filters.\n",
        "        if i<num_filters:\n",
        "            # Get the output image of using the i'th filter.\n",
        "            img = values[0, :, :, i]\n",
        "\n",
        "            # Plot image.\n",
        "            ax.imshow(img, interpolation='nearest', cmap='binary')\n",
        "        \n",
        "        # Remove ticks from the plot.\n",
        "        ax.set_xticks([])\n",
        "        ax.set_yticks([])\n",
        "    \n",
        "    # Ensure the plot is shown correctly with multiple plots\n",
        "    # in a single Notebook cell.\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yuYyQ_oUsZSQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def plot_history(network_history):\n",
        "    plt.figure()\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.ylabel('Loss')\n",
        "    plt.plot(network_history.history['loss'])\n",
        "    plt.plot(network_history.history['val_loss'])\n",
        "    plt.legend(['Training', 'Validation'])\n",
        "\n",
        "    plt.figure()\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.plot(network_history.history['acc'])\n",
        "    plt.plot(network_history.history['val_acc'])\n",
        "    plt.legend(['Training', 'Validation'], loc='lower right')\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "D9ur6Ru_sZST",
        "colab_type": "code",
        "outputId": "7db9c28a-e689-4679-82c8-9fe19d402a0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        }
      },
      "cell_type": "code",
      "source": [
        "plot_images(images=mnist.test.images[0:9], cls_true=mnist.test.labels[0:9], cls_pred=mnist.test.labels[0:9])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUwAAAD1CAYAAAA73MxWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XvcV2O+//FXUkKEYTCkbPHBLnIW\n80sOxZBT2yHbNDGNU3sKezAeTRKT0C7aaLZhY2xJ5JAyaApFKockOfRBETPj3IwcksLvj7Wu73fd\nd/dhfe/v+e79fDx69L3Xd33X+tz3da/r/qzrutZ1tfjhhx8QEZHGrVfuAEREqoUqTBGRlFRhioik\npApTRCQlVZgiIimtn+fn18Uu9hblDqDEVMbNn8o4JWWYIiIpqcIUEUlJFaaISEqqMEVEUlKFKSKS\nkipMEZGU8h1WJNIko0ePBmDlypUAvPLKKwDcf//9a+173nnnAdCtWzcA+vXrV4oQRdaiDFNEJKUW\neU7vpgGvzV9By/jUU08FYNKkSTl/tlOnTgDMmDEDgB122KFwgdWkMi6TN998EwAzA+CGG24AYNCg\nQYU+lQaui4gUk9owpehCVgn1Z5a77rorAEcddRQAS5cuzbw3ZcoUAN5++20Axo8fD8CQIUMKH6yU\n1YIFCwBYb70ol9tuu+3KGc5alGGKiKSkDFOK5sUXXwTgoYceWuu9zp07A9nsccsttwSgbdu2AHz7\n7beZfQ844AAAFi5cCMBnn31WpIil3F5++WUg+3vQp0+fcoazFmWYIiIplTzDDOPsbr31VgB+8pOf\nZN5r06YNAKeffjoA22yzDZDtHZXq8sEHHwCQHIkRMstp06YBsO2229b52TBOE+CNN96o8V7v3r0L\nGqeU36JFiwC48cYbAfjFL35RznDqpQxTRCSlkmeYF198MQDvvvtuvfvcfPPNAGy66aYA7L777gU5\nd/v27QG45JJLANh3330Lclyp27HHHgtke7cBNtlkEwC22GKLBj977733Zl4n2zOleXJ3AL766iug\n5siKSqIMU0QkJVWYIiIplfyW/H//93+B7BCR5O3266+/DmQHr86cOROAefPmAdlH4d577716j9+q\nVSsgO0wldDwkjxNuzXVLXhodOnRIve9//dd/AdlH5JLC8KLwvzQfo0aNAqBjx45A5V6byjBFRFKq\n6Mk3/vGPfwDZjDP81XnhhRfq/cwGG2wAZB/eD4/cASxfvhyAcePGATBw4MCmhKWJGYrgkUceAeDk\nk08GYNWqVZn3tt56awAmTpwIwCGHHFLscFTGJZDs+N1xxx2B7HW7ePHiYp9ek2+IiBRTRT8aufnm\nmwNw2GGH1dh++OGHN/rZBx54AMhmqQB77LEHAH379i1UiFIg4THKZGYZhCEmJcgspYRmzZq11rat\nttqqDJGkpwxTRCSlis4wm+Ljjz8Gsu2TyTbaYcOGAY0PmpbSOeGEE4Dso5JB//79M69HjBhR0pik\nNMKyJEnhoZJKpQxTRCSlZpdhhh7wkGluttlmmfdCD5yUXxgfO2fOHCDbdhnasIYOHZrZN0z1Jc3D\n3LlzAbjjjjsy2/baay8AevbsWZaY0lKGKSKSUrPJMGfPng3ANddcU2P7ww8/nHkdphaT8gsTw376\n6ac1toep/XbaaaeSxySl8cQTTwA1R7CEpUnCFI+VShmmiEhKqjBFRFJqNrfkjz76KJCdO/GII44A\noFu3bmWLSdYW1vAJj7sGPXr0AODKK68sdUhSYmHinaTwSGylU4YpIpJS1WeYK1euBODxxx8HspNv\nXHHFFUB2ujcpn+QqjyNHjgTWnkW9a9eugIYQNWcffvghAM888wxQc2KcE088sSwx5UoZpohISlWf\nYYYJZ0Ob2M9+9jMADjrooLLFJDWNGTMm8/r555+v8V54NFJtl83fn/70JwA++ugjIHutVhNlmCIi\nKVVlhhkmmwX4/e9/D0C7du0AuOyyy8oSk9Tvuuuuq/e98Cir2i6bv2XLltX4OkzfWE2UYYqIpFRV\nGWbobR08eHBm25o1awA4+uijAY27rDahTNOMZgh3EWHf1atXA/D555+vtW947O7666+v81gtW7bM\nvL722msB2GijjdKGLU0wderUGl/37t27TJE0nTJMEZGUVGGKiKRUFbfk3333HZCd0eSdd97JvNep\nUycg2/kj1SWss5TGKaecAsC2224LZIenhNUkmyqsSpmcg1MKJwxUD+VVzZRhioikVBUZ5pIlS4Ds\nyoJJYciK5k+sXKFDDmDy5MlNPs59993X6D6hQ2i99WrmAscddxyQXds+6ac//WmTY5LGPfTQQ0C2\ngzbMrl6Nq4AqwxQRSamiM8ww0LVXr141to8ePTrzuhqHJqxrHnzwwczrUaNGAWtPvhG8/vrrQMPt\nkgMGDACgQ4cOa733b//2bwDstttuTQtWCubrr78G4LHHHquxPUzllhzaVS2UYYqIpNQiuW53E+T1\n4cYMGTIEgKuvvrrG9hdeeCHzuq42qSJrUeoTlllRy7hCqYwLIDxY0L17dyA7GmHChAlA2R8UaFIZ\nK8MUEUmpItsww7itm266qcyRiEhThRELYR3y5kAZpohIShWZYYY1xr/44osa28NTPZoKTETKQRmm\niEhKqjBFRFKqyFvy2sKKgk888QQAW2yxRTnDEZF1lDJMEZGUKnrgeoXSoObmT2Xc/GnguohIMeWb\nYYqIrDOUYYqIpKQKU0QkJVWYIiIpqcIUEUmpwYHrZjYK2B9oA+wFhGlHbnP3u4oRkJn9DuiZ2LQr\ncKG731PP/j2Ah4EF8aY2wEvA+e6+uokxjADWuPvwet7fDrg7sakNsLm7W1POV05lKuNNgTuArYC2\nwHXuPr6B/c8ArgXeiDdtCDzu7pfnEcN4YIa7/6mBfX4N9Ae+A5YAZ7p73VPFV7BylHF83v2Be4G7\n3b3BJTnLcR3H+7QGRgHnA63cfU1Dx2ywwnT3S+KDdgRmu3uP3ELOnbtfBVwVn3dLYDrQ2MpZi0Js\nZtYCmAicAxRlfjh3/xvQI3xtZr8HPijGuYqtHGUMXAa87u6XmdmPgTfNbJK7r2rgM9Pd/ecAZtYK\nmGVmL7j7I8UI0Mw6A4OBLu6+ysweAE4D7izG+YqpHGVsZjsBI4BpOXyspNdx7DqiijmVJj8aaWbD\ngR2BDsBvgDHACHefkSiY7c1sc+BmomyiHTDG3SeYWVdggLsPauA0I4Br3H1l2rjc/Qczm02UmWJm\nK4DbgJbuPtjMBgGnEH3vi4GB7r7SzK4CegPvA18RZzNmNha4y93n1/Nz6Aj8DDggbYzVoohlPAz4\nPn69HGhJlGk2VGFmuPtqM5sL7GpmrwJTgUXAq+4+0sxGAgcTZaKzgEuIBirfBnQBlgEbJ77PicBv\n4j+EwevAPolK/BNgyzTxVZMilvEHRNfFZTShninhdTzE3VeY2W1p4sr3WfIdgUPib66+fUYQ3T7d\nYWYbAwvNbLq7vwzUW1maWXuiX/qBuQRkZm2AY4l+uBBdiI+6+/T4FuFEoHsc8/XAr8xsGnA6YES3\nX88T/6Dd/YJGTnkpcL27f5dLnFWk4GVc6w/gIOBJd/8sbUBm1o6o2ebseNNuwEnu7mZ2MrCdux8S\n7/sQ0QW0iuji24+oIl1ClMHg7n3riPF74Iv4GDsCxwAHpY2xyhSjjL8GaOB4DSrVdezuK3KJK98K\nc567Nzby/VBgPzPrH3+9mqiAPmnkcwOBW+Nf3MZ0MbOZia+nuvu98esWwLPx6x5AJ+CpuCA3juPp\nAswP2YSZPZ3inKEt7hjg12n2r1JFK2MzGwz8HDgiRRw9E2X8PTDa3efFWdByd/dELN0S+7aLY2kF\nzIm/l6/N7LkU58TMdiNqWzvL3d9P85kqVMzrOBdluY5zkW+FmWwAT/7AWyderyJKl1/M8dgnAkem\n3DfT9lGPEOcqYIq716jgzOwksreIEN0ipnEMUdtagw3FVa4oZWxmvyWqKA9N+Vc+04bZSIyrgFvc\nfXRyBzO7iBzL2Mx2J6osz3D3Zxvbv4oV8zrORbmu49QKOaxoBdA+fn1YYvtsorYGzGxDM/uDmTXW\nO78lsJm7LytgfBD9hfqZmbWNzzPQzLoRpe17m1nruEPhkJTHO5go7V9XFKSMzexQotvk3rneEqUw\nG+gTzm9mw8xsZ6I2yQPNrIWZbUIjbc5x7+lEoG8zryxrK9h1XESFvo5TK2SFeRMw1Mymk2hQB4YD\nO8cNuE8DC9x9jZl1NbMb6zlWe+DD5AYz28bMJuUTYPzXcRwwM46nB7DQ3V8j6ol/DpgEvJw471gz\n2ydtnM1cocr4IqLOg2lmNjP+t08jvxNpPUh0Qc2JO4a2BpYS9da+R1TGt5MdWoOZTbRoqFjS8cAO\nwJhEjL/LM7ZqUJAyNrPj4tvrM4B+8c+vZ6Vdx2b2YKIZ4Akzu7v2PklVNfmGmd3i7mc3vqdUKzP7\no7ufU+44pHiq+Tqumid94lukqeWOQ4onboqZUO44pHiq/TquqgxTRKScqibDFBEpN1WYIiIp5Tss\nYF28n9d6L82fyrj505o+IiLFpApTRCQlVZgiIimpwhQRSUkVpohISqowRURSUoUpIpJSuaZnyslX\nX30FwMUXXwzAzTffnHlv3333BWDSpGgClA4dOpQ4OhFZVyjDFBFJKd/JN0ryhMBbb70FwO67777W\ne999Fy2lc8MNNwDw618XfbUIPQVSAC+9FC3U16dPHwDefffdJh/rL3/5S+b1brvtBkD79u3r2z0N\nlXGZTJ0aTWR03HHHAXDjjdFUm+edd15mn5YtCzKRup70EREppopuw/zkk2h9pf79+zeyp1SbadOi\n5apXrUq1sm6DpkyZknl9++23AzBx4sS8jyul89ln0aKhyUwSYNCgaEHKAQMGZLZtuOGGpQusFmWY\nIiIpVWSGGdojJ0+eDMALL7zQ6GeeeeYZAEKb7J577glA9+7dixGiNNGaNdECm48++mjBjhlGSgBc\nd911QHZkxcYbb1znZ6SyPP10tCLu3/72txrbTzvtNADatGlT8pjqogxTRCSliswwL7jgAiC33rAH\nH3ywxv877LADAPfdd19mn332qW/xRymVp556CoA5c+YA8Nvf/jbvYy5fvjzz+rXXXgPg66+/BpRh\nVrJk+/WIESPq3Kdfv34AtGhRGQMXlGGKiKSkClNEJKWKGrh+9NFHA/DYY48B6dLwLbfcEsjeei1b\ntqzefb///vt8QwQNas7ZokWLMq979OgBZMtt/vz5ALRt27bJxw/HhGzn34cffgjAVltt1ZRDqoxL\nINmZu//++9d4b/31o9bC1atXF+v0GrguIlJMZe/0mTVrVub14sWLgWxmWV+nz7nnnpt53atXLwDa\ntWsHwJNPPgnAVVddtdbn/ud//gdYe3CsFFeyLEJnzPjx44H8MsvQ2ZP8HaqUzgFpXOigrUvPnj1L\nGEl6yjBFRFIqW4YZJlvo27dvZtunn35a575hiNBJJ50EwOWXX555b6ONNqqxb5je7Y9//ONax7zk\nkksA+Oabb4DsRB2tWrVq2jchDbr//vuBmoPUO3XqBMB+++2X9/HDUJRkVhnaMzfbbLO8jy/Flbwz\nCFq3bg3AyJEjSx1OKsowRURSKluGGXq/6ssqIftY47333gtke1YbEjLMIUOGAPCf//mfmffC43Ih\n0wxTSO200045xS7phEmdw88dCtN+HO5OJkyYAGR7VAGGDh0K6K6hkoWHFubOnbvWe+GOsWvXriWN\nKS1lmCIiKZW9l7wuoX3rjjvuANJllrWF7PHuu+/ObHv++ecLEJ005vPPPwdg3rx5a703cODAvI9/\nyy23ANnp/5ITSx922GF5H1+Kq6HJdCp9BIsyTBGRlMqeYYYlJpKee+65vI8bnmBKPt0TtoVzht72\nMCZQCiNMqvDXv/4VyE7RVShLliyp8XXnzp0LenwprroyzDCqoRB3IMWkDFNEJCVVmCIiKZXtljys\nLV6gFeDWElafW7BgQWZb7Ucur7jiiqKce123ySabANmhIcnJN8LjjFtssUXOx/3444+B7HCl4OCD\nD25SnFJas2fPBrLDwZLCo83bb799SWPKlTJMEZGUypZhPvLIIwU9Xhhi8vrrrwMNP1oVhilpcHNx\nhFX9wmOQ4RFJgGOOOQao+UBBXV599dXM69DJE6buqz3Bxnrr6e9+NQgrQ9Y1pWSlTrZRm37TRERS\nKvuwokIJU4iNGzeu3n06duwIwJ133glkJ/WQ4hg+fDhQM6MIdxbJSVfqkpz4N2SU9T1Ge+aZZ+YT\nppRI7bbn5AQpZ599dqnDaRJlmCIiKZVtiQozA2Dp0qVrvZfLtPRhWYsw+fD7779f775HHnkkkHf7\n6bo2Q21Bly8IoxZqDz6vLUzll9S/f39g7QcN6nr4IU8q4wIKDzCEO7pQ5yQfOEiOpCgRLVEhIlJM\nZWvDrP2YYlJYBC0466yzAPj73/9e73HSLE1Q6J55yd1ee+1V4/9c/Mu//Eud25PZSZcuXZoWmBRN\nmM6t9t3s8ccfX45w8qIMU0QkJVWYIiIple2WPMx7F2Y/TwqDm2s/NlnXY5Thlj7NCpNS3cItXe1b\nO92GV7YwYD0ID45ccMEF5QgnL8owRURSKluG2adPHwBGjRqV2dbQ+j6NCX+1dtttNwBuvfVWALbd\ndtsmH1MqS+jY09rj1WXatGk1vm7fvj2QnXCjmijDFBFJqWwZZljdMawICTB58mQAxo4dm/Pxfve7\n3wHZtcal+QnryQdhkg+pTOEBlLfffrvG9jZt2gDVOfmNMkwRkZTKPvlGWHs8+bpXr15AdnXAMBnw\nscceC8A555yT+UzoMU2uHCjNU1hFNEzaMGzYsHKGI40I0+6FVWBfe+01AHbeeeeyxZQvZZgiIimV\nPcOsy1FHHVXjfxHIZioXXnghoDXIK10YGx2mXgyjG/bee++yxZQvZZgiIimVbXq3KrauDQJUGTd/\nKuOUlGGKiKSkClNEJCVVmCIiKanCFBFJSRWmiEhKqjBFRFLKd1iRiMg6QxmmiEhKqjBFRFJShSki\nkpIqTBGRlFRhioik1OD0bmY2CtgfaAPsBcyN37rN3e8qRkBmtiHwJ+AnwAbA7919agP79wAeBhbE\nm9oALwHnu/vqJsYwAljj7sMb2Kc3cBnwLfAB0N/dVzblfOVUjjJOnHt9YA7waCM/6x6UoYwT+x4P\nTHb3qpyUo1xlbGb7A/cCd7v70Eb27UF5ruPWwCjgfKCVu69p6JgNVpjufkl80I7AbHfvkVvITTIY\n+MzdTzWz9sBcM3vC3b9u4DOLQmxm1gKYCJwD3FSMAM2sDXAL0M3dl5nZDcCFwMhinK+YylTGwaVE\nf3DSKGkZB2b2I+C3RH8Uq1I5ytjMdgJGANMa2zehHGV8HVHFnEqTJxA2s+HAjkAH4DfAGGCEu89I\nFMz2ZrY5cDOwFdAOGOPuE8ysKzDA3QfVOvTPgOEA7v6+mS0GDgJmpInL3X8ws9nArnGcK4DbgJbu\nPtjMBgGnxN/7YmCgu680s6uA3sD7wFfAG/HnxwJ3ufv8xGkOBN5092Xx1/cB11CFFWZDiljGmNme\nwP8jKpsOucRVojIOxgFDgdtzibFaFLGMPyC6li+jCfVMCct4iLuvMLPb0sSVbxvmjsCh9fyiBSOA\nx939MKA7cKWZbeXuL9d1IRHdin+Y+PrDeFsqcfZ3LPBMvKkt0S3f4PgW4USgu7t3A/4J/MrMdgFO\nJ7ptOQHILDri7hfU8f3lFWOVKXgZx7dBfwDOowlzMZaojDGzU4Hl7v5krjFWmYKXsbt/7e7fNTWg\nUpWxu6/IJa58l6iY5+6N/cIfCuxnZv3jr1cTFdAnKc/RgsYvqi5mNjPx9VR3D+v3tgCejV/3ADoB\nT5kZwMZxPF2A+e6+CsDMnk4ZWy4xVqtilPEwYIK7LzWz7vXsU1tJy9jMtgYuir+35q4U13Ea5b6O\nG5VvhZlsf0r+wFsnXq8iSpdfTHnM94mytcXx1z8B/trIZzJtH43EuQqY4u41Fi83s5OA7xObWqaM\nMUgTY7UqRhmfAHxpZv2IbvE2MLN/untDC9KXuoyPIboQZ8QX5bZmNg84wt2/bOSz1aYYZdwUpS7j\nnBVyEbQVQPv4dXJ1qtlEbQ0vxj3gY4DBDfRGPQKcBjwZNxx3Iturl69ngcFm1tbdvzSzgUS9cm8A\ne8e3ij8AhwAPNnCc54AdzWwnd18C/ByYUqAYK1lBytjdO4fXZnYG0LGRyjIXBSljd7+dRLulmb3r\n7gcWKMZKVqjruJgKdR3nrJDjMG8ChprZdKK/zMFwYOe4AfdpYIG7rzGzrmZ2Yx3H+QPQxsyeBe4G\nfunu35jZNmY2KZ8A47+O44CZcTw9gIXu/howmaginAS8HD5jZmPNbJ9ax/kWGABMiOPcgCL31laI\nQpVxnSqpjNdhBSljMzsuvr0+A+hnZjPNrGellbGZPZhoBnjCzO5u6NxVNVuRmd3i7meXOw4pHpVx\n81fNZVw1T/rEaXa9A9il+qmMm79qL+OqyjBFRMqpajJMEZFyy7eXfF1MT6vyeeI8qIybP5VxSsow\nRURSUoUpIpKSKkwRkZRUYYqIpKQKU0QkJVWYIiIpqcIUEUlJFaaISEqFnN5NRKQg/vGPfwDw3nvv\n1btPhw7RyibXX389AJ07R7MG7rLLLgDsueeeBY9LGaaISEoVlWF+/PHHAJxyyikAHHTQQQCcfXY0\nE1THjh0Lcp7PP/8cgKefjmawP+qoowBo1apVQY4vIrl55JFHAJg6NZrIaObMmQC89dZb9X4mngmf\nd999F4BVq1bVeP/777+v/ZG8KcMUEUmp7BlmaKsA+Nd//VcgmwFuvfXWQOEzy7333huATz/9FIAX\nX4yWKdl5553r/qAUzYoV0aJ9l156KQCvvfYaADNmZFdVVubfPCxZsgSAcePGAXDLLbdk3lu5ciUA\nuUw36e4FjC4dZZgiIimVLcMM2V1orwT47LPPAPiP//gPAG68MfVyMKmMGDECgHfeeQfI/oVTZll6\n48ePB2Do0KHA2r2hIfME+NGPflS6wKRo/vrXaGHVsWPzW+9u1113BbK94qWkDFNEJKV8l6ho8of/\n8pe/ANke6qSPPvoIgK222qqph8949dVXM6+7dOkCwIknngjAnXfeCcAmm2ySyyE1uWweQpax1157\nAdk7jRYtav5Y+/btm3l9003RgpxbbLFFIUNpiMo4R6EcIZtB/vSnPwWy1/jcudFq2UcffTQAbdu2\nzXzmyy+jpd6PPPJIIJs9HnDAAUD29wVgww03BGDjjZOLWuZMEwiLiBSTKkwRkZRK3ukTBqc/8MAD\na713++23A4W9Fe/Zs+da7/Xp0wfI+VZcCmD06NFAtoOvPhMnTsy8fuyxx4BsB9GgQYMAaN26dTFC\nlBx89dVXQM3rbOHChQBMnjy5xr7dunUDYMGCBUDN4YKh02/77bcHYL31KjOXq8yoREQqUMk7ffr1\n6wdkh5WEQeSQfVQxz8ZcAG6++WYAzjvvvMy2M888E8hmsk2kDoEcLVu2LPN6jz32AOCLL76o8XV4\nSGH69On1HifsEzKUbbbZJt/Q6qMybsS3334LwMknnwxkH2kEGDJkSI3/N9poo7wDLAJ1+oiIFFPJ\n2zDD8JHw/3bbbZd5L582qfBo1ciRI4Hs41fJ4Sp5ZpbSRC+//HLmdRiQ3r17dwBmzZoFwDfffAPA\nhAkTALj66qszn3n77bcB+PDDDwE4/vjjgWzbZgmHG63zwvCfcJ2FzDLZ73DxxRcDFZtZ5kUZpohI\nSmWffCNM6wTQq1cvADbbbDOgZvtjfcI0UOH/efPm1Xg/tLFI+SSn3QoZ/4UXXlhjnzZt2gDwy1/+\nEoD7778/816YtCG0t4fMRb3kpRd6vq+55hogO4nvM888k9mnXbt2pQ+sRJRhioikVPIM8/zzzwfg\nySefBODvf/975r3QnhUyiYcffrjR44V9az9at9NOOwHZthYpn3vuuWetbX/+858BOOGEE+r8TJhy\nry4HHnggUPPROimNOXPm1Pg6PLIYxk82d8owRURSKtvkG2Hi4GQP6uOPPw7AqFGjgOy4u/79+9d7\nnDCuM4znq709TLBRQBqjl6P77rsv8zpMqhHKKzzRs2jRIgAeeughACZNmpT5THgiK/zOhF7x0G62\n++675xtibSrjevz4xz8GspNtbLDBBkB2AmiA4447Dqg5YUYF0jhMEZFiUoUpIpJS2W7JC2Xp0qVA\ntpOna9euQHa+zUJM5FGLbtdytHz58szrUE5hfaX6Ou2SkzmEhxB69+4NwJtvvglkVxMNj8EWkMq4\nHrUfPKlLy5YtATj33HOB7JyW77//PgCdOnUCsmt4JYU1ncJEHUXsTNItuYhIMVV9hnnGGWcA8H//\n938ATJs2Dah7WrcCUfaRh7Aa5EknnQRkM81g8ODBAFx77bWZbWFQe5jMITw2GaYHS64wGTLYPKmM\n6xEeexwzZkzRgoFs51KPHj2AmtP9FYgyTBGRYqrKDDM55CSsOrnpppsC8NRTTwE1p40rMGUfBRCy\nwjDZRngc9sorrwTqHpQeJlj593//dyD7YEMYQgYFG0amMq7Hd999B8BLL70EwOmnnw7A6tWrM/uE\ndZvCvvkIbaVXXHFFZluYSDrfQzflQ8owRURSKvvkG00RpvVKOuaYY4CiZpZSQEcccUSN/9MIqwWe\neuqpQDbDDHcVkO2R15RvxRF6wPfbbz8gO2Ih6YknngCyWefw4cMBeP7553M+X7gDnj9/fs6fLQZl\nmCIiKVV9hhmWs7jooovKFY6UWGi3njJlClCzBzWsYT5s2LDSByYAHH744TW+Do8/hwyzVatWQHbJ\nGICzzjoLgOuvvx7Itm1XGmWYIiIpqcIUEUmpqoYV1bUSZJjRKKz3UgIaclIhwq3eQQcdlNkW1gZa\nvHgxALvssktTDq0yLqAwBGnfffetd59DDz0UyK6cULteGjhwYOZ1aHbJk4YViYgUU1VlmGFijVde\neSWzLTwaGVaEDOtdh7kTd9hhh0KHoeyjwiQf0wudf3369AFg/PjxQHZIUkoq4wIKDxyE9Zruvffe\nRj+z/vpRf3QYLhjKEbIdvXlk0pfiAAAHDElEQVRShikiUkxVn2H+6le/ArLrXIdhCZ07dwY043oB\nVHyG+cknn2ReH3zwwQC89dZbACxcuBBYe0b+RqiMi+Cjjz4CYMCAAUDNwejhvTChyi9+8QsgO+i9\nCJRhiogUU9VnmLUnoA0Z52WXXQZA+/btCx2Gso8K9t577wHZ9bJPO+00IOeB0CrjErjrrrsyr+fO\nnQtkM8owvVsRKcMUESmmqsowwyqBl19+eWZbaLsMYzM333xzAFq3bl2sMJR9VIFevXoB2XW0w2N5\nKVeYVBk3f8owRUSKqaoyzAqh7KMKrFixAoA999wTgP/+7/8GsmtmN0Jl3PwpwxQRKSZVmCIiKemW\nPHe6XWv+VMbNn27JRUSKSRWmiEhKqjBFRFLKtw1TRGSdoQxTRCQlVZgiIimpwhQRSUkVpohISus3\n9KaZjQL2B9oAewFz47duc/e76v1gAZjZ+sAc4FF3H97Afj2Ah4EF8aY2wEvA+e6+uonnHgGsaei8\niX2PBya7e1UOdi5HGZvZlsBtwBZEg6YvcPeXGtj/DOBa4I1404bA4+5+eX2fSRHDeGCGu/+pgX0u\nAvoC3wB/dverm3q+cipTGQ8n+tmF5Vy/dPfeDezfgzJcx2b2DTAvsekmd7+/vv0brDDd/ZL4oB2B\n2e7eI4d483Up8G3KfReF2MysBTAROAcoyHqc9TGzHwG/BT4o5nmKqUxlPBR41d1/Z2YdgAeA+tdg\njUx3958DmFkrYJaZveDujxQjQDPrCpwLdCGqMKea2f7u/nwxzldMZbyOr2noD1IdynEdf5jLz6PB\nCrMh8V+QHYEOwG+AMcAId5+RKJjtzWxz4GZgK6AdMMbdJ8S/kAPcfVAdx94T+H9EWUiHXOJy9x/M\nbDawa3ysFfFxWrr7YDMbBJxC9L0vBga6+0ozuwroDbwPfEWczZjZWOAud5+/9tkYR3Tx355LjNWi\niGW8C/AHAHdfZmbfm9mO7v5OmrjcfbWZzQV2NbNXganAIqJKeKSZjQQOJspEZwGXED0KdxtRBbgM\nyCw9aGYTgd+4+99qxfiyu6+M93kUOAaougqzIcW8jvNR4us4tSZXmLEdgUPib66+fUYQ3T7dYWYb\nAwvNbLq7vwzUVVm2JrqY+gHdcw3IzNoAxxL9cAHaEt3WTzez/YETge5xzNcDvzKzacDpgAHfEV0U\nbwC4+wX1nOdUYLm7P9nA994cFLyMiW61jgMeMbOdgJ2BbYFUFaaZtQN6AmfHm3YDTnJ3N7OTge3c\n/ZB434eILqBVRBfffkQV6RKiDAZ371vHaRYCY+Lmg8+Bw4HP0sRXhYpRxgCnm1lfoj9ON7l74+vr\nxkp1HQObmtk9QHvgbeBid/+knn3zrjDnuXtjI98PBfYzs/7x16uJCqi+oIYBE9x9qZmlrTC7mNnM\nxNdTE4XTAng2ft0D6AQ8Ff9ibBzH0wWY7+6rAMzs6YZOZmZbAxfF31tzV4wyvhYYa2bPAq/E/75p\n5Bw9E2X8PTDa3efFWdByd/dELN0S+7aLY2kFzIm/l6/N7LmGThZXvpcTZa+fAg4UbRr/MitGGT8K\nPOnuT8fNLvPMbIG7v9nAOUp6HccuBe5x9y/M7FrgOqJkrU75VpjJNsbkDzz5i7WKKF1+MeUxTwC+\nNLN+ROn/Bmb2T3cf28BnMm0fjcS5Cpji7r9OvmlmJxFdhEHLRmI8hqiQZsQFtq2ZzQOOcPcvG/ls\ntSl4Gbv7F8CA8LWZvU10C9WQTBtmIzGuAm5x99HJHeIOnFzKGHe/nbi5xcyGxsdujopRxs8nXi+L\nr489gIYqzFJfx7j7LYkvJwB3N7R/vhVm0gqitBbgsMT22URtDS+a2YZEbSSD3X1NXQdx987hddw7\n2rGRyjIXzwKDzaytu39pZgOJeuXeAPaOmwN+AA4BHqzvIMkLKY7zXXc/sEAxVrKClHH8x7CTu19u\nZocTNbzXexuUo9nAb8xsrLuvMbNhwD3A68DxcWdCW+AAos6mOpnZZkRZ0mFEF95pQJ8CxVjJClXG\nNxH9kXs4bkLZm6iDtBAKch2b2e7AKODEuCf+CLK99HUq5DjMm4ChZjadRIM6MBzYOW7AfRpYEP8i\ndzWzG9Me3My2MbNJ+QQY/3UcB8yM4+kBLHT314DJwHPAJODlxHnHmtk++Zy3GSlUGU8BDo47bq4A\nzoSoZzqX34l6PEh0Qc2Jj781sBSYBrxHVMa3kx1ag5lNNLPtkgdx938SDXN5Lj7e9Ynb/uasUGX8\nB+AiM5sFzACGu/ublXQdu/vrwHzgufj2vQdwcUPnrqrJN8zsFnc/u/E9pVqZ2R/d/ZxyxyHFU83X\ncdU86ROn2VPLHYcUT9wjPaHccUjxVPt1XFUZpohIOVVNhikiUm6qMEVEUlKFKSKSkipMEZGUVGGK\niKT0/wFPU/TJiYZ3JAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f5c2a30a470>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "8bv9pNn9sZSX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d0548beb-774e-4948-cc48-6f533064dcf2"
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.python.keras.models import Sequential, Model\n",
        "from tensorflow.python.keras.layers import *\n",
        "from keras import regularizers"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "Mn-OGv91sZSZ",
        "colab_type": "code",
        "outputId": "b39d539b-07f1-4fe1-afba-53b35fb5d12f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1122
        }
      },
      "cell_type": "code",
      "source": [
        "try:\n",
        "    del model\n",
        "except:\n",
        "    pass\n",
        "\n",
        "# Start construction of the Keras Sequential model.\n",
        "\n",
        "\n",
        "# Add an input layer which is similar to a feed_dict in TensorFlow.\n",
        "# Note that the input-shape must be a tuple containing the image-size.\n",
        "inputs = Input(shape=img_shape_full)\n",
        "net = inputs\n",
        "\n",
        "#model.add(GaussianNoise(stddev=0.2))\n",
        "\n",
        "# The input is a flattened array with 784 elements,\n",
        "# but the convolutional layers expect images with shape (28, 28, 1)\n",
        "# model.add(Reshape(img_shape_full))\n",
        "\n",
        "reg_strength = 0.01\n",
        "\n",
        "# Convolutional layer with ReLU-activation and max-pooling.\n",
        "for num_filters in [32, 64] :\n",
        "    net = Conv2D(kernel_size=3, strides=1, filters=num_filters, padding='same')(net)\n",
        "    net = BatchNormalization(axis=-1)(net)\n",
        "    net = Activation('relu')(net)\n",
        "    layer_in = net\n",
        "    net = Conv2D(kernel_size=3, strides=1, filters=num_filters, padding='same')(net)\n",
        "    net = BatchNormalization(axis=-1)(net)\n",
        "    net = Activation('relu')(net)\n",
        "    net = Conv2D(kernel_size=3, strides=1, filters=num_filters, padding='same')(net)\n",
        "    net = BatchNormalization(axis=-1)(net)\n",
        "    net = Activation('relu')(net)\n",
        "    net = add([layer_in, net])\n",
        "    #model.add(Dropout(0.5))\n",
        "    net = MaxPooling2D(pool_size=2, strides=2)(net)\n",
        "\n",
        "\n",
        "# Flatten the 4-rank output of the convolutional layers\n",
        "# to 2-rank that can be input to a fully-connected / dense layer.\n",
        "net = Flatten()(net)\n",
        "\n",
        "# First fully-connected / dense layer with ReLU-activation.\n",
        "net = Dense(512)(net)\n",
        "net = BatchNormalization(axis=-1)(net)\n",
        "net = Activation('relu')(net)\n",
        "# Last fully-connected / dense layer with softmax-activation\n",
        "# for use in classification.\n",
        "net = Dense(num_classes, activation='softmax')(net)\n",
        "\n",
        "outputs = net\n",
        "\n",
        "model = Model(inputs=inputs, outputs=outputs)\n",
        "model.summary()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 28, 28, 1)    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 28, 28, 32)   320         input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 28, 28, 32)   128         conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "activation (Activation)         (None, 28, 28, 32)   0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 28, 28, 32)   9248        activation[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 28, 28, 32)   128         conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "activation_1 (Activation)       (None, 28, 28, 32)   0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 28, 28, 32)   9248        activation_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 28, 28, 32)   128         conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "activation_2 (Activation)       (None, 28, 28, 32)   0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "add (Add)                       (None, 28, 28, 32)   0           activation[0][0]                 \n",
            "                                                                 activation_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 14, 14, 32)   0           add[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 14, 14, 64)   18496       max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 14, 14, 64)   256         conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "activation_3 (Activation)       (None, 14, 14, 64)   0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 14, 14, 64)   36928       activation_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 14, 14, 64)   256         conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "activation_4 (Activation)       (None, 14, 14, 64)   0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 14, 14, 64)   36928       activation_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 14, 14, 64)   256         conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "activation_5 (Activation)       (None, 14, 14, 64)   0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 14, 14, 64)   0           activation_3[0][0]               \n",
            "                                                                 activation_5[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 7, 7, 64)     0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 3136)         0           max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 512)          1606144     flatten[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 512)          2048        dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "activation_6 (Activation)       (None, 512)          0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 10)           5130        activation_6[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 1,725,642\n",
            "Trainable params: 1,724,042\n",
            "Non-trainable params: 1,600\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "P4nFsRLgsZSd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.python.keras.optimizers import Adam\n",
        "\n",
        "optimizer = Adam()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5etpYwaSsZSg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.compile(optimizer=optimizer,\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TAkg6m1isZSj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.image import ImageDataGenerator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "N-ucb8qIsZSm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n",
        "                         height_shift_range=0.08, zoom_range=0.08)\n",
        "test_gen = ImageDataGenerator()\n",
        "batch_size=256\n",
        "shape = (-1, img_size, img_size, num_channels)\n",
        "train_generator  = gen.flow(mnist.train.images.reshape(shape), mnist.train.labels, batch_size=batch_size)\n",
        "validation_generator = test_gen.flow(mnist.validation.images.reshape(shape), mnist.validation.labels, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "oOosGGsDsZSp",
        "colab_type": "code",
        "outputId": "e3e0fc0c-2136-4b89-f849-9b144c0d2d58",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "#model.fit(x=mnist.train.images, y=mnist.train.labels, epochs=100, batch_size=1024,\n",
        "#          validation_data=(mnist.validation.images, mnist.validation.labels),\n",
        "#          verbose=1)\n",
        "history = model.fit_generator(train_generator, epochs=10, steps_per_epoch=mnist.train.num_examples//batch_size,\n",
        "                              validation_data=validation_generator, validation_steps=mnist.test.num_examples//batch_size,\n",
        "                              verbose=1)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "214/214 [==============================] - 28s 131ms/step - loss: 0.1289 - acc: 0.9595 - val_loss: 3.0924 - val_acc: 0.1556\n",
            "Epoch 2/10\n",
            "214/214 [==============================] - 24s 111ms/step - loss: 0.0420 - acc: 0.9870 - val_loss: 0.7212 - val_acc: 0.7230\n",
            "Epoch 3/10\n",
            "214/214 [==============================] - 23s 109ms/step - loss: 0.0317 - acc: 0.9902 - val_loss: 0.0883 - val_acc: 0.9722\n",
            "Epoch 4/10\n",
            "214/214 [==============================] - 23s 109ms/step - loss: 0.0281 - acc: 0.9912 - val_loss: 0.0497 - val_acc: 0.9849\n",
            "Epoch 5/10\n",
            "214/214 [==============================] - 23s 109ms/step - loss: 0.0230 - acc: 0.9930 - val_loss: 0.0422 - val_acc: 0.9882\n",
            "Epoch 6/10\n",
            "214/214 [==============================] - 23s 110ms/step - loss: 0.0216 - acc: 0.9935 - val_loss: 0.0433 - val_acc: 0.9862\n",
            "Epoch 7/10\n",
            "214/214 [==============================] - 23s 109ms/step - loss: 0.0202 - acc: 0.9934 - val_loss: 0.0653 - val_acc: 0.9807\n",
            "Epoch 8/10\n",
            "214/214 [==============================] - 23s 108ms/step - loss: 0.0178 - acc: 0.9942 - val_loss: 0.0208 - val_acc: 0.9930\n",
            "Epoch 9/10\n",
            "214/214 [==============================] - 23s 107ms/step - loss: 0.0169 - acc: 0.9949 - val_loss: 0.1413 - val_acc: 0.9600\n",
            "Epoch 10/10\n",
            "214/214 [==============================] - 23s 107ms/step - loss: 0.0164 - acc: 0.9945 - val_loss: 0.0220 - val_acc: 0.9924\n",
            "CPU times: user 5min 44s, sys: 33.5 s, total: 6min 18s\n",
            "Wall time: 3min 59s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "4Ds7AcDLsZSt",
        "colab_type": "code",
        "outputId": "f17fd23c-f049-4adb-d481-7f10add3703d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 541
        }
      },
      "cell_type": "code",
      "source": [
        "plot_history(history)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8XHWd//HXmUvu9ya9JG1zKfDl\nLhcVWWSBlptalh8gF0VYtrggivhbf67iCnhhgVVWVMQVEJAfogirgKhVUVBQKy6UChToF0qbJk1L\nm7ZpMrlnMrN/zEwmkyZpmubkzOX9fDzymHM/n/kmOZ9zvt/zPceJRqOIiEju8XkdgIiIeEMJQEQk\nRykBiIjkKCUAEZEcpQQgIpKjAl4HMFXt7aFp365UWVlER0fvTIaT0VQeqVQeSSqLVNlQHjU1pc5E\n83LiCiAQ8HsdQlpReaRSeSSpLFJle3nkRAIQEZE9KQGIiOQoJQARkRylBCAikqOUAEREcpQSgIhI\njlICEBHJUVmfADr6d/PgS48xMDzodSgiImklY3oCT9fru97giXVPUu5U8J4F7/Q6HBFJc9/+9jew\n9nV27drJ4OAA8+fXUlZWzs033zrpeitX/pzi4hJOOumUced/61tf5/zzL6K2ts6NsKfFyZQXwkz3\nURCbulr52gvf5r21x/Ghg8+b6bAyUk1NKe3tIa/DSBsqjySVRdLKlT/n7bdbWbHi416Hsl8mexRE\n1l8B1JUsIOgPsrGrxetQRCRDvfjiC/z4xw/S29vL1Vf/C2vWrOYPf3iKSCTC8cefwIoVV3DvvXdR\nUVFBY+MSHn30ERzHx6ZNGzn55GWsWHEFV199BZ/+9Gf5/e+foqenm5aWTbS1beaaa/4fxx9/Ag8+\neD+/+92T1NbWEQ6HueiiiznmGHdrLVxLAMaYIuB+YB5QANxorf3FqPmnAjcDw8BKa+2NbsQR8AVY\nUrkYu2MD/eF+CgIFbuxGRGbYI0+v5/l122d0m+86eC4XLD1gWuu+9dZ6HnroUfLy8lizZjX/9V/3\n4PP5uOCCs7nwwg+nLPvaa6/yox/9lEgkwvnnn8WKFVekzN++fRv/+Z+389xzq/jZz37KYYcdzqOP\n/jcPPfRTenp6uOiic7noooun/T2nys1G4LOAF6y1JwEXALeNmX87cB5wAnC6MeZQtwI5qLqJKFE2\ndW12axcikuUOOOBA8vLyACgoKODqq6/gk5+8kt27d9PV1ZWyrDEHU1BQQFFR0bjbOvLIowCYO3cu\n3d3dbN7cSlPTEvLzC6iqmsMhhxzm7peJc+0KwFr78KjRRcDI0dcY0wTssta2xsdXAsuA19yI5aA5\nTQBs7NqEqZpe9heR2XXB0gOmfbbuhmAwCMDbb2/l4Yd/yH33/ZCioiIuueSCPZb1+yd/iujo+dFo\nlGgUfL7k+bgzYa39zHK9DcAYswpYCCwfNXk+0D5qfDuwZLLtVFYWTfvRrMG+RgDa+tqoqSmd1jay\njcohlcojSWURU1pawNtvx8qjoqKI/PwgNTWlbNu2iZqaaurr5/Hqq6+ybdvblJbmUVycT0lJQcqy\nAI7jUFNTSl5egMrK4pHlampK6egoJi8vwOGHH8imTRupqCggFArxxhvrqKgocv134XoCsNb+nTHm\nKOBBY8w7rLXj3c2z13y3Py9lqKkpZ05BJXbHBrZv78KZrfSapnSnRyqVR5LKIikU6gegvT3E7t29\nDAwM0d4eorp6IcFgPh/84PkcccRR/MM/nMsXvnADRx75DoLB/pRlIXaG394eYnAwTEdHDz09AwSD\n/bS3h+jo6GFwMEw0ms/SpadzzjnnUl/fyMEHH0ooNDAjv4vJkoibjcDHAtutta3W2r8ZYwJADbGz\n/S3ErgIS6uLTXNNQtpjV219ie98O5hXVuLkrEckC73//WSMJ8Zhj3jlyR47f7+e22+6YdN3Rd+/8\n8pdPAXDHHXcD0NSUrNZqajpgZPqiRYtZseIK/H4/l156EQsW1M7o9xmPm1cAfw/UA//XGDMPKAF2\nAFhrm40xZcaYBmJtA8sBV5u8G8vrWb39JZo7W5QARCTt7Ny5kyuu+EeCwTxOP/1M5s6d5/o+3UwA\ndwL3GmP+CBQCnwAuNcZ0WmsfA64CHoov+7C19g0XY6GpvB6ADV2bOG7BsW7uSkRkn11yyWVccsll\ns7pPN+8C6gM+PMn8Z4Hj3dr/WHUlCwj6Amzs3DRbuxQRSWtZ/zC4hIAvwOLShWzpfpv+cL/X4YiI\neC5nEgBAQ/lidQgTEYnLqQTQVBZrB9jYpWogEZGcSgCN8YbgjZ16MJyI7OnKK/+JdeteT5l25513\n8NBDD+6x7IsvvsB1130WgGuv/fQe83/604e59967JtzX+vVv0tISOxn94hc/z8DA7FdN51QCKM8v\no6qgkuauFjLlMdgiMntOO+0Mnn76tynT/vCHpzn11NMnXe8//mPso8727plnnqa1NXYy+uUv30J+\n/uw/qDLrHwc9VmO8Q1h73w7mqj+AiIyybNnpXHXV5Xz849cAsHbtWmpqamhu3sh1132OYDBIaWkp\nX/nKf6Ss94EPLOOXv3yKF174H26//etUVc1hzpzqkUc733TTl2hv305fXx8rVlzB/PkL+NnPHuWZ\nZ56msrKSG274PA888DDd3SFuueUrDA0N4fP5uPba63Ech5tu+hK1tXWsX/8mBx1kuPba62fk++Ze\nAoh3CNvY2aIEIJLGHl3/C9Zsf2VGt3n03CM494DlE86vrKyitraO115by6GHHs6vfvUrTjvtTEKh\nEF/84r9TW1vHjTfewF//+pdxn/R51113cP31N3LggQfxmc9cQ21tHaFQF+9+93t43/uW09a2meuv\nv5b77nuQ4447npNPXsahhx4+sv4999zJ8uVns2zZ6fz+97/jvvvu5vLLr8Ta1/nyl2+msrKKc855\nP6FQiNLS/X9OUE5VAQE0li8GYh3CRETGOu20M3nqqVg10NNPP83JJy+joqKCr37137n66itYs2Y1\nXV2d4667detWDjzwIACOOuoYAEpLy3j99Ve56qoV3HTTlyZcF8Da1zn66FhH1WOOeSdvvmkBqKtb\nxJw51fh8Pqqra+jp6Z6R75pzVwALS2oJ+gI0qyFYJK2de8DySc/W3XLSSafwwAP3cdppZ9DQ0EBZ\nWRm33HIjt976TRoaGrnttq9OuO7oRzon2hl/+9tf09XVxXe+cw9dXV189KOXTLJ3Z2S9oaEwjhPb\n3tjHS89UG2bOXQEEfAEWlS6krXsr/eEBr8MRkTRTVFTMkiUH8sAD3+ess84CoKenm3nz5hMKhXjx\nxdUMDQ2Nu251dQ0tLc1Eo1HWrFkNwO7du1mwoBafz8czzzw9sq7jOAwPD6esf8ghh/Liiy8A8Le/\nrebggw9x62sCOZgAIFYNFCVKS6jV61BEJA2ddtqZPP/8X1m6dCkA5557PldddTlf+9pNXHzxpTz4\n4P3s3Lljj/WuuOLjXHfd5/jc5/5l5GFuJ5+8lFWr/sinPnUVhYWFzJ07l+9//3u84x1H881v3soL\nL/zPyPof/ejH+PWvV3LNNR9j5cpfcPnlV7r6PZ1MuR2yvT007UDHPuP8b9tf4Xtrf8BZTWdyZsPS\nGYkvk+iZ76lUHkkqi1TZUB41NaUTvgAlJ68AGuINwXownIjkspxMABX55VTmV6hDmIjktJxMABB7\nP0D3UA/tfTu9DkVExBM5mwCSzwVSNZCI5KYcTgDxdoAu9QcQkdyUswlgYUktAb0hTERyWM4mgNgb\nwupo697KwPCg1+GIiMy6nE0AAI1l9fE3hKlDmIjkntxOAGoIFpEcluMJINEQrAQgIrknpxNAokPY\nxk51CBOR3OPq46CNMV8DTozv5xZr7aOj5jUDrUDicXgXW2vb3IxnPI3li3lx+8u09+1kblH1bO9e\nRMQzriUAY8wpwOHW2uONMXOANcCjYxZ7n7V2Zt5sME2N5fW8uP1lmrtalABEJKe4WQX0LHB+fHg3\nUGyM8U+yvCcay9QQLCK5ybUrAGvtMNATH70cWBmfNtqdxpgG4E/A5621E1bEV1YWEQhMP3/U1Iz/\n/szKKkNwTYDWns0TLpONcum7ToXKI0llkSqby8P1V0IaY84mlgBOHzPrBuDXwC7gceA84CcTbaej\no3faMeztmd6LSuto7mxl89s7yffnTXs/mSIbnnE+k1QeSSqLVNlQHpMlMFfvAjLGnAF8gVhdf8qb\nkK21D1hrt1trw8BK4Ag3Y5lMY1k9kWhEHcJEJKe4lgCMMeXArcBya+2usfOMMb8xxiROt08C1roV\ny94kXhCjF8WLSC5xswroQqAaeMQYk5j2NPCKtfYxY8xK4DljTB+xO4QmrP5xW1O8R/AGdQgTkRzi\nZiPw3cDdk8z/FvAtt/a/L5IdwjYRjUZxnAlfoSkikjVyuifwaI3li+ke6mFH3669LywikgWUAOJG\nHgynaiARyRFKAHGNZfEHw6khWERyhBJA3MLSOgKOX1cAIpIzlADigr4Ai0oX6g1hIpIzlABGaSxf\nTCQaoUUdwkQkBygBjJJ8Q5jaAUQk+ykBjNI0cieQEoCIZD8lgFEq8supyC8f6RAmIpLNlADGaCyv\nJzTUzc5+dQgTkeymBDBGU7w/wAa9IEZEspwSwBhqCBaRXKEEMIY6hIlIrlACGCPWIayOtu6tDKpD\nmIhkMSWAcTSWJ94QttnrUEREXKMEMA49GVREcoESwDj0ZFARyQVKAOOoLKhQhzARyXpKABNIdgjr\n8DoUERFXKAFMIFkNpHYAEclOSgATUEOwiGQ7JYAJLEp0CNMVgIhkKSWACSQ6hG1WhzARyVIBNzdu\njPkacGJ8P7dYax8dNe9U4GZgGFhprb3RzVimo7G8no1dLbSE2jigotHrcEREZpRrVwDGmFOAw621\nxwNnAt8cs8jtwHnACcDpxphD3YpluhrUECwiWczNKqBngfPjw7uBYmOMH8AY0wTssta2WmsjwEpg\nmYuxTMvIG8KUAEQkC7lWBWStHQZ64qOXE6vmGY6PzwfaRy2+HVgy2fYqK4sIBPzTjqempnTf16GU\nqsIKmrtbqa4uwXGcae8/3UynPLKZyiNJZZEqm8vD1TYAAGPM2cQSwOmTLLbXI2tHR++0Y6ipKaW9\nPTStdetLFrGm/RXWtbZQXVg17RjSyf6URzZSeSSpLFJlQ3lMlsBcvQvIGHMG8AXgfdbazlGzthC7\nCkioi09LO4n+AM2qBhKRLONmI3A5cCuw3Fqb8oJda20zUGaMaTDGBIDlwJNuxbI/EglgQ5ceDCci\n2cXNKqALgWrgEWNMYtrTwCvW2seAq4CH4tMftta+4WIs07aopBa/OoSJSBZysxH4buDuSeY/Cxzv\n1v5nStAfZFFpHS2hzQwOD5HnD3odkojIjFBP4CloLF9MJBqhJaQ3hIlI9lACmILGMvUHEJHsowQw\nBSMdwtQQLCJZRAlgCiryyynPK9MbwkQkqygBTIHjODSW19M1GGKX3hAmIllCCWCKGsv1YDgRyS5K\nAFPUpA5hIpJllACmaFFJHX7HT3OnEoCIZAclgClKdAhr7W5jcHjI63BERPabEsA+aCxThzARyR5K\nAPtADcEikk2UAPZBozqEiUgWUQLYB5X5FeoQJiJZQwlgH6R2CNvtdTgiIvtFCWAfjbQDdKkdQEQy\n25QSgDHmWGPM8vjwTcaYp4wxJ7obWnrSk0FFJFtM9QrgdsDGD/rvAj4JfNm1qNLY4tK6+BvC1BAs\nIpltqgmg31r7JvAPwN3W2teAiHthpa+gP8jC0lp1CBORjDfVBFBsjDkfOAd40hhTBVS6F1Z6ayqr\nJxKN0Bpq8zoUEZFpm2oC+DxwMfBv1tou4BrgNteiSnNqCBaRbDCll8Jba39vjFltre0yxswDngL+\n7G5o6atBDcEikgWmehfQt4Hz41U/q4Crge+6GVg6qyqooDyvVB3CRCSjTbUK6Ghr7b3ABcD91toL\ngQPcCyu9JTqEdapDmIhksClVAQFO/HM5cF18OH9vKxljDgd+BnzDWnvHmHnNQCswHJ90sbU2Y1pV\nG8vr+Vv7Wpq7NjGnMGfbw0Ukg001AbxhjHkNaLfW/s0Ycymwa7IVjDHFwLeJtRdM5H3W2u4pxpBW\nkh3CWjh23lEeRyMisu+mWgX0UeDDwGnx8VeBS/eyzgDwfmDL9EJLb4tK6/A5PjboTiARyVBTvQIo\nBM4CvmKMiQLPAd+cbAVrbRgIG2MmW+xOY0wD8Cfg89bajGlRzfMHWVRSx+bQFoaGhwj6g16HJCKy\nT6aaAL4HbAbuItYecGp82kf2Y983AL8mVpX0OHAe8JOJFq6sLCIQ8E97ZzU1pdNedyKHzj+ATW+2\n0uXv4OCaJTO+fTe5UR6ZTOWRpLJIlc3lMdUEMM9a+6FR478wxvxhf3ZsrX0gMWyMWQkcwSQJoKOj\nd9r7qqkppb09NO31JzI/bwEAa1peZw5zZ3z7bnGrPDKVyiNJZZEqG8pjsgS2L4+CKEqMxBt4C6Yb\nkDGm3BjzG2NMXnzSScDa6W7PK6MbgkVEMs1UrwDuAtYZY16Ijx8LXD/ZCsaYY4GvAw3AkDHmg8AT\nwEZr7WPxs/7njDF9wBomOftPV2M7hDmOs/eVRETSxFQfBXGfMea3wDFAlNjjoD+5l3VWAydPMv9b\nwLemHGkachyHhvJ6XmpfS8fAbqoK1B9ARDLHVK8AsNa2Euu4BYAx5t2uRJRhGssW81L7WjZ2blIC\nEJGMsj+vhFR9B7EewQAbu9QOICKZZX8SQMbcs++mxaUL8Tk+NQSLSMaZtArIGNPK+Ad6B6h2JaIM\nk+gQ1hpqU4cwEckoe2sDeO+sRJHhGsoXsynUSmt3G03lDV6HIyIyJZMmAGutHnQzBU1li3mGP7Oh\nc5MSgIhkjP1pA5C4RENws9oBRCSDKAHMgKqCSsrySnUnkIhkFCWAGZB4Q9jugU469IYwEckQSgAz\npLFsMQAb9KJ4EckQSgAzJNkhTAlARDKDEsAMUYcwEck0SgAzJM8fZGFJLZtDbQxFwl6HIyKyV0oA\nM6ixvJ5wdJjWUJvXoYiI7JUSwAxqijcEb1RDsIhkACWAGdSQaAhWAhCRDKAEMIPmFFRSmleiDmEi\nkhGUAGaQ4zg0lalDmIhkBiWAGaYXxIhIplACmGGNagcQkQyhBDDDFpfWxTuEKQGISHpTAphhef48\nFpYsiL0hTB3CRCSNKQG4INEhbLM6hIlIGlMCcEFjmdoBRCT97e2dwPvFGHM48DPgG9baO8bMOxW4\nGRgGVlprb3QzltmUaAje0NXCUo9jERGZiGtXAMaYYuDbwFMTLHI7cB5wAnC6MeZQt2KZbSMdwnQF\nICJpzM0qoAHg/cCWsTOMMU3ALmttq7U2AqwElrkYy6xyHIdGdQgTkTTnWhWQtTYMhI0x482eD7SP\nGt8OLJlse5WVRQQC/mnHU1NTOu11p+Pw2gN5ecer7KKdg2oWzeq+p2K2yyPdqTySVBapsrk8XG0D\n2AfO3hbo6Oid9sZrakppbw9Ne/3pmOufD8DfWtexpODAWd333nhRHulM5ZGkskiVDeUxWQLz6i6g\nLcSuAhLqGKeqKJPVl+kNYSKS3jxJANbaZqDMGNNgjAkAy4EnvYjFLckOYZvVIUxE0pJrVUDGmGOB\nrwMNwJAx5oPAE8BGa+1jwFXAQ/HFH7bWvuFWLF5pKKunJdTG5lDbyK2hIiLpws1G4NXAyZPMfxY4\n3q39p4PG8sU827aKjZ2blABEJO2oJ7CLmvRoaBFJY0oALppTUEVpsEQNwSKSlpQAXOQ4Do3l9XQM\n7Gb3QKfX4YiIpFACcFlj+WIANuixECKSZpQAXNZYFksAei6QiKQbJQCXLS5bhM/x0ayGYBFJM0oA\nLsv351FXsoCWUBthdQgTkTSiBDALGsvqCUfCtIay6mkXIpLhlABmQaIheGOX2gFEJH0oAcwCvSJS\nRNKREsAsqC6soiRYrA5hIpJWlABmgeM4NJU30DGwm1d3rvM6HBERQAlg1pzZsJSAL8B9a3/E2z3b\nvA5HREQJYLbUly3iIwefT/9wP999+X66h3q8DklEcpwSwCx61/yjOaN+KTv6dnLPKz9gODLsdUgi\nksOUAGbZ8qbTeUfN4by5ewOPvPE40WjU65BEJEcpAcwyn+Pj0kMupK5kAX/a8lee2bzK65BEJEcp\nAXigIJDPx468jNK8En7y5hO8vjPr3oYpIhlACcAjVQWVXHHEP+J3fNz76oNs69nudUgikmOUADzU\nVF7PxYecT1+4nztfvp+eoV6vQxKRHKIE4LF3zz+G0+tPYXvfDu5d+6DuDBKRWaMEkAbOajqDI6sP\nw3as5ydvPuF1OCKSI5QA0oDP8fGPh15EXckCnm37C8/qziARmQUBNzdujPkG8B4gCnzKWvv8qHnN\nQCuQqPO42Frb5mY86awgkM+VR1zG1164nf9+8wnmFtVwcNWBXoclIlnMtSsAY8xJwIHW2uOBy4Hb\nx1nsfdbak+M/OXvwT5hTGLszyIfDPWsfZFtvu9chiUgWc7MKaBnwOIC19nWg0hhT5uL+ssKSigY+\ndPB59IX7uPPl79OrO4NExCVuVgHNB1aPGm+PT+saNe1OY0wD8Cfg89baCZ+LUFlZRCDgn3YwNTWl\n0153tp1Vcwqd0Q6eWPdbHnzzYa498RP4fdP/7uPJpPKYDSqPJJVFqmwuD1fbAMZwxozfAPwa2EXs\nSuE84CcTrdzRMf0z4ZqaUtrbQ9Ne3wunLVjGhvbNvPT269z5l4e44KCzZ2zbmVgeblJ5JKksUmVD\neUyWwNysAtpC7Iw/oRbYmhix1j5grd1urQ0DK4EjXIwl4/gcH/902IeoLZ7PM5v/zB/bnvM6JBHJ\nMm4mgCeBDwIYY44BtlhrQ/HxcmPMb4wxefFlTwLWuhhLRioIFPCxIy+jJFjMI288zhsd670OSUSy\niGsJwFq7ClhtjFlF7A6gTxhjLjPGnGOt7SR21v+cMebPxNoHJqz+yWVzCqv45yMuxcHhe6/8gO29\nO7wOSUSyhJMpz6Nvbw9NO9BsqMdbteV5frjuv5lXNJfPHPsJioKF095WNpTHTFJ5JKksUmVDedTU\nlI5tfx2hnsAZ4u9q38XSRSeyrXc79736Qz0zSET2mxJABjnngA9w2JyDeX3XGzz21i+9DkdEMpwS\nQAaJ3Rn0YeYXz+P3rX/iz1v+6nVIIpLBlAAyTGGggKuOvIziYBE/to/xZsdbXockIhlKCSADVRfO\n4Z8PvwSA7639ATv6dnockYhkIiWADHVg5RIuMufQM9TLd1++n75wv9chiUiGUQLIYCfUHscpi97L\n2z3b+P6rPyISjXgdkohkECWADHfOkg9waJXh1Z3reGy97gwSkalTAshwfp+fFYd/mPlFc3m69Y+s\n2vL83lcSEUEJICsUBgq58sjLKA4U8WP7KOt3b/Q6JBHJAEoAWWJuUTUfPeIjRInyvVceYEffLq9D\nEpE0pwSQRQ6qPIALDvo/dA/1cNfL99OvO4NEZBJKAFnmxLr3cNLCE9jS8zbff/Uh3RkkIhNSAshC\n5x2wnEOqDmLtztf52Vu/8jocEUlTSgBZyO/zs+Kwi5lXVMPvWp7hL1tf8DokEUlDSgBZqihYyMeO\nvIyiQCE/XvdT3trd7HVIIpJmsv6FMNFoFAIBwgODBAP+mQ4r7dld67njpXsoChTy2Xdew5zCyqx4\nycVMUnkk5XJZhCNhNndvYcPuZjZ0bmJjVwsBv5/6kkU0ltfTVF7PwpJa/L7MOo5M9kKYrE8Az6/b\nzncfX4vjQE1FIXXVxdRWF1M7J/Y5f04R+cHM+oXuq2c3/4WH33iMupIFfPqYj7NoQXXO/pOPJ5cP\nemPlUll0D/WwsXMTGzo3saGzmU1drQxFwiPzy/JKwYnSNdA9Mi3PF6S+bBFN5Q00ldfTUL6YkmCx\nF+FPWU4ngM7uAZ5as4U3WjrYsqOH7r6hlPkOUF1RMJIQEj8L5hRRkBeYidDTwsP2cZ5tW8WR1Yfx\nb0s/zs4dPV6HlDZy6aC3N9laFtFolO297bzVuYmNnc281bmJbb3bR+Y7ONSWzB85sC8pb6CqIHa1\n/FpLczxRxK4MtvZsI0rycDSvaC5N8SuEpvJ65hbV4HPSp3Y9pxMAJP+oo9Eood4htuzoYcvOHtp2\n9LB1Rw9bdvTQ1Tu0x3pzygqorS6mrrqYBdVFI1cOhfmZlxiGI8N856V7sR3reffCo5iXN5/iYFHs\nJ1CUHA4Wk+cPeh3urMrWg950ZEtZDA4P0RLaHKvO6YoduHuGekfm5/vzaCyLH7QrGmgoW0xhoGCP\n7YxXHn3hPpo7W0cSQnNXC/3DAyPziwKFI1VGTeX1LC5dREEg370vuxdKAFP4ow71DsYTQ2/sM/7T\n2TO4x7JVZfl7XDHUzimiqCC9D5y9Q7385+r/SjnzGU/QF6A4WJySHIqCqUmiOFCYXCZYRFGgMOPq\nRhOy5aA3XdFolHAkzGBkiMqqQvo6Ixn3u+wcCMUPyLGDcmuojeFo8r3ZVQWVI2f2jeUN1JXMn9JZ\n+lT+NiLRCFt7to3se0PnppR3dPgcH3UlC2IJoayexvIGqgoqcJwJj8szSglgP/7Bu/uG2LozlgxG\nrhh29tIRGthj2YqSvPjVQmo7Q0lh+iSGoeEheoNdbG7fQc9QDz1DvfQO9dIT7qVnKPHTQ89QHz1D\nvfQPT703cWGgIJ4wiikKFiaTxR5XGbGfPH8ePnz4nOSP3/HhOM6sXkKnewKIRqOEo8MMDQ8yMDzI\n4PAgA5FBBoeHGIyPD0aGRuYNDg8xGBkcNT4YHx/aYxtDw7H1RldpODgUBQspCZZQmldMabCEkrwS\nSoPFlOaNGQ6WUBQsnNXf1+gD7lu7Y1UzO/uTjz7xOT4WldTRVFE/UqVTkV8+rX1N92+jazA0qn1h\nEy2hzYRHtS+U55WNXCE0ljewqLSWgM+dmgUlABf+wXv7w2zZmbxSSAzv6tozMZQX51FbXUx1eQGB\ngA+/zyHg8+H3O/h98R+/b4Lh0cv64uPJ4b2tH4iv5/Ml/wb2pTyGI8P0hGNJonsomSR6w6OTRW/q\nT7g35Y99OhycUYnBwef4R5L+ZPZTAAAJlElEQVSD3/HHpuPg8/nwOX58OPid+LDjpCSVZGLxxZcZ\n9YOPgsIg/fG2odEHwsRw8l8kuucyY/5/RtYZteLoLU40L0okfkAfe/COjc9Uj+6A4yfozyPfn0ee\nP0ieL4+8xLgvSEFBHru6OwkN9dA92E3PUG/K9x2Pz/FRHCxKSRSxzxJK8mKJYmQ4WEJhoGCfzn77\nw/00dyWrXDZ2tqScmBQFCkcOpEvK66kvW0SeP2/aZTTaTB07hiJhNofaRhLChs5mugaT2w34AtSX\nLqSpvGGk+qg0r2S/9wseJgBjzDeA9xD7O/+Utfb5UfNOBW4GhoGV1tobJ9tWuiWAifQNhNm6s5e2\nHd1s3dE7khh2dHr7XB4HRpJIwB/7e0j8E/qc2LAz+pPEeGKaQyKH+EYvy57r4kTBFyHqDBL1DxL1\nDxD1DTHsGyDqGyTiGyDiGyTqDMeWJRr7jA9HiYITiQ07USASnxafR5QokZR50ZHP5DB7OXClIx8+\nAk6QgC9IwAkSdIIEE8O+2PSgkxyPzc8bmR70jZruz4st6wuS50tsMxD7YyD+MTIc+91VVBSxe3cv\nvvjfRpQIfcN99A330jvcQ1+4l57hHnrDPfQN99ITjg33hmPDU7li9Dv++BVhCSXB4vhPSfIzr5jB\n4UGaQ7H69S09W1OSUE1hNY1l9bGf8nrmFlbjc2JVVuPlFWfUd0x+8ZGP+DLjHyPdOnZEo1F29XeM\nJISNnc1s7h77PeeMJIR3zjtq3DaKqfAkARhjTgL+1Vq73BhzCHCftfb4UfNfA84A2oBngCutta9N\ntL1MSQAT6R8M0xEaYDgSZXg4ynAkSng4EhuPREamxeZHUobDI+tEUtZPXS85HB5nG8nhKI7PIRwe\nJhqFSPz3H4nG/ihjP7ET1CiJ4eRnJP5bSAxHo/FDbXTsssl53onumWBGjTsj48nFU+35fxONTnbm\n6oyznQmWj44zL+KHaPrcPTItTgQCgzjBQZz4Z2x8ACcwFPscmTaI4x+edHPRiI9ITxmRUCWR7koi\n3RUQnpmz+0m/RuLT2fPPwhnze9vbxcz488fZhi+MU7QbSnbjK+6A4g6cQOxKekneUXz6vR+eavgp\nJksAbt7Osgx4HMBa+7oxptIYU2at7TLGNAG7rLWtAMaYlfHlJ0wAma4gL8CCOelx99BsJ8REUojE\nPyFKJBI/jx/5TF0mNh5NjicSTmTstNTlU7YRT2iRCONsK0o0EhsuLS2gq6s/efYVTf2nT0lko+eN\nmj76PGqflx+1UCKZMma5PaqNprJ8YuspsY2/z8RwQWGQ3t7YjQ+R2FnAyPyRE4JR32Xk9xlNLpNY\nZ6QcRo9HIRoBBqJEByASDRN2+hly+gk7/QzHP8GhIFxNfrgKH36ieUAVRKv2TNijyzf1u6eWM2PK\nasIyG7WdYNDP0FAySe2x9tjt7zlhz/1NsP+YChiOQhdEu6IMB0IM5e/mXYceteeGZoCbR6T5wOpR\n4+3xaV3xz/ZR87YDSybbWGVlEYH96MlbU1M67XWzkcpDRGbzlHQK184T6+jo3dsiE0qHKqB0ovJI\npfJIUlmkyobymOxkz80Kxy3EzvQTaoGtE8yri08TEZFZ4mYCeBL4IIAx5hhgi7U2BGCtbQbKjDEN\nxpgAsDy+vIiIzBLXqoCstauMMauNMauACPAJY8xlQKe19jHgKuCh+OIPW2vfcCsWERHZk6ttANba\na8dMemnUvGeB4xEREU9k+E3HIiIyXUoAIiI5SglARCRHZczD4EREZGbpCkBEJEcpAYiI5CglABGR\nHKUEICKSo5QARERylBKAiEiOUgIQEclR6fGKKhdN9l7iXGSM+RpwIrHf/S3W2kc9DslTxphCYC1w\no7X2fo/D8ZQx5mLgs0AYuMFa+0uPQ/KMMaYEeACoBPKBL1trf+NtVDMvq68A4u8lPjD+LuLLgds9\nDslTxphTgMPj5XEm8E2PQ0oH1wG7vA7Ca8aYOcAXgfcSezz72d5G5LnLAGutPYXYY+2/5W047sjq\nBMCY9xIDlcaYMm9D8tSzwPnx4d1AsTFm+u/ZzHDGmIOBQ4GcPdMd5VTgd9bakLV2q7X2Cq8D8tgO\nYE58uDI+nnWyPQGMffdw4r3EOclaO2yt7YmPXg6stNYOT7ZOlvs68Gmvg0gTDUCRMeYJY8wfjTHL\nvA7IS9baHwOLjTHriZ04fcbjkFyR7QlgrL2+ezgXGGPOJpYArvY6Fq8YYy4F/mKt3eh1LGnCIXbG\ney6x6o/vG2Ny9v/FGPMRoMVaewCwFLjD45Bcke0JYLL3EuckY8wZwBeA91lrO72Ox0MfAM42xjwH\nfBS43hhzqscxeWkbsMpaG7bWvgWEgBqPY/LSCcBvAKy1LwG12Vhdmu13AT0JfBm4a+x7iXORMaYc\nuBU41Vqb0w2f1toLE8PGmC8Bzdba33kXkeeeBO43xnyVWJ13CVla7z1F64HjgJ8aY+qB7mysLs3q\nBDDee4m9jsljFwLVwCPGmMS0S621Ld6FJOnAWttmjPkJ8Fx80iettREvY/LYXcB9xphniB0nP+Zx\nPK7Q+wBERHJUtrcBiIjIBJQARERylBKAiEiOUgIQEclRSgAiIjkqq28DFdkbY0wDYIG/jJn1S2vt\nrTOw/ZOBf7fWvnd/tyUy05QARKDdWnuy10GIzDYlAJEJGGPCwI3AKcR6xl5mrV1rjDmO2IPkhoi9\nZ+Jqa+1rxpgDge8Rq1rtB/4pvim/Mea7wNHAALHHUAD8iFiv2yDwc2vtTbPzzURi1AYgMjE/sDZ+\ndfBd4Cvx6Q8A/xJ/VvxtwHfi0+8EbrXW/j1wH8lHbx8CfMla+x5iSeMM4DQgaK09Efg7oNsYo/9H\nmVW6AhCBGmPMH8ZM+2z8M/EWqD8D/2qMqQDmjXqz3B+AH8eHj4uPJx4nnGgDWGet3RZfZjNQAfwc\n+Iox5hFgJXBPjj96QTygBCAyQRtA/HlJibNyh1h1z9hnpzijpkUZ/6o6PHYda+12Y8w7gOOJvX3r\nBWPMMdbavml9A5Fp0CWnyOSWxj/fC7wcf4T21ng7AMTepJV4gNoqYq/axBhzoTHm5ok2aow5HfiA\ntfbP1trPAt3AXDe+gMhEdAUgMn4VUOJFMUcbY64i1lh7aXzapcBtxphhYBi4Kj79auBuY8wniNX1\nrwCWTLBPC/x/Y8xn49t40lq7aSa+jMhU6WmgIhMwxkSJNdSOrcIRyQqqAhIRyVG6AhARyVG6AhAR\nyVFKACIiOUoJQEQkRykBiIjkKCUAEZEc9b/pPdWJkmWViQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f5c26b0aa58>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XNV99/HPnUX7aluyLdvIm3xs\nA2FNwAXCFhPCEgqBhD4plEBKIJBQ0pbQp2QjITRNCAnQNlCS8FCaFUjYTAKBQCgGAobQsB3LsiXv\ntmxrmdE6mrnPHzOSRrIlj2WN7izfd17OzNxtfnPEnN+9v3vnXMd1XUREJP/4vA5ARES8oQQgIpKn\nlABERPKUEoCISJ5SAhARyVMBrwNIVWtraMKXK1VXl9DW1j2Z4WQ1tcdIao9haouRcqE9amrKnbHm\n5cURQCDg9zqEjKL2GEntMUxtMVKut0deJAAREdmbEoCISJ5SAhARyVNKACIieSqtVwEZYw4DHgFu\nt9beNWreh4BvAlFglbX26+mMRURERkrbEYAxphS4E3hmjEXuAD4GnACcYYxZnq5YRERkb+ksAfUB\nZwFbR88wxiwE9lhrN1lrY8Aq4PQ0xiIiIqOkrQRkrR0ABowx+5o9C2hNer0TWDTe9qqrSw7qmtya\nmvIJr5uLcq09YjGXmOvGHxPPo7GxX0eTlg9t6cB1XVwAFxLPGBwpfcS8xPx9zhucnvR86NeL4623\nnyHZHWcfv+MZ46c9Y/3iZ5/b2IeWXfEfPcUSwcfcxOdIxOsmv058vr2mjV42lWUSf4uIG6E31gWu\nj2KnDBwn0abx5QfjGlwX4n97YO/3ImnZWNL7wdBnG1pvcFvuyL/Rvtpv8KmTaO3kpnUch9FN7TjO\n8N/FGf4bDW7TGfq/vZcdfFj5gXrqZ1eM/YeboEz5JfB+/+s8mF/j1dSU09oamvD6U81NdFbRqMtA\nLBZ/jMYYiLlEo7ExpifmxRLToi7RWOJx1PSi4iBdXf1DX5RY0hdm9Jc0lvi2xEZ/eZO+bPucl/SF\n3Wt60rZHd9oxd1RnPvQYnz7YNvHpwx3Hgf1MPNFL+1xwYolJzvA/Bv/JQXOiOMF+CPTjBPtxEo8E\n+3ECffFpyfN9saFV3f5CYuEqYuEqoqFq3O4KcPPgupVgL76ydvzlbfjK2nFKQjT+4Vj+4bQLJ7S5\n8Xb2vEoAW4kfBQyawz5KRdku3BOhZUeIjTtCbNwRZld7DwOjOu9oolOPd9jD03KTC4FIohPow+eP\n4fhj+Hwujs/F8ScefbHhRycxzYnhT7wOOrGhzttxYvFO3InhOm7iMQYkHp0YbuK5S3To+f45Sf/z\nJf45SY/OqGmJ544vaa5vxOvB5XzO8Lo+xzc03e/48RPE7wTwE8DvBPC5gRGvR8xneJ7P8TNmGtzH\n5LH+CyspKaCnuz+xpxrfmx3cK3Wc+E5ChF763R763B763G763B763R56Y/HnfYnH3lg3Ebd/vy3t\ndwIU+0oo9ldR7C+l2F9CJNbPdv8Wugt24J+2gyAQcALUFs6mrngudcXzmF00h5JAyfBetQM+Z3iv\nPHkPfWjPOrFs8p568udj8Hli2enTy9i9OzzUaIPtNnxkl9TqyfOTjh6GjiiSXgzOjcai7Ozdwebu\nzWzp3sSW7s10RjqG1vXho7ZoJmeZ9+23HSfCkwRgrW02xlQYY+YDm4FzgE96EctkcF2X3Z29bNwR\nHursW3aEaAv1jVjO5zgEAz4Cfge/z8Hvjz8vDAbj0/w+AknT/T7fqOkOAb8Pvy/xmLTMiOn7WDZ5\ne9XVpXR29uAkvjDO6C974tHnDMedPG/4S+bgulG6ot10DYTpinTRNdBFOBImHIm/DkXChBKvw5Eu\nYu74nW9i33xCHBwCPj9+J5B49OH3FRBw/Ph9/sRjAL/jj8/3+fE7fgoK/PT2RYi5saF/UTeGO+ox\n5saIESMWS0zHJepGcd2BxPxo4igmvqyb3COkmc/xUeALEvQHKfAVUOAPEvQFKRjjdXy5kcsX+Auo\nrChm6+5dhPrDhCJd8cfE83B/mK5I99iJJimWsmAptcXTKQ+WUVZQSnlBGeXBsvhjQRllwTLKC0op\nC5ZR6C/YZ4nKdV1297axvqOZpo5mNnS0sDW8ma29m6DtJQBmlc5kUWU9Cyvns7ByPjXF01Mud6Wi\nproEZyA6adsLR7po7tjI+o4W1nc009K5if5YZGh+WbCUw2csZ2FFPQsq66mvmEuBv2DS3n80J123\nhDTGHAPcBswHIsAW4FFgg7X2V8aYDwLfSiz+kLX2O+Nt72AGg5vMElA0FmP77u6hTn7jjhCbdobp\n6h0YsVxlaQGHzCznkJll1CceZ1QVD3WeXtpfe/RF+wn1h+gc/PL3hwj1dxGKxKeF+8NDj10D+y/N\nFfoLEl/+8qEOoLygjCJ/IYHkDnmoYw7s1Wnv9Tp5+USH73MmVh5IV4kw5sYSpa7BhJGUSEb9i7ox\nItEI/bFI4rGf/miESCxC/xivU11+MpQGSigriHfa8U69jPJgvGMvG9W5FweKJvy32J+egR42dGxM\nJIUWmjs30h8dPsooD5axsGo+CyvrWVQ5n3nlcwj4Jr6fezD/bcTcGDu6W1nf0cz6jhY2dLSwo3v4\n1KeDw+zSmSyorGdh4l9N8YxJTWAw/mBwaUsAk82LBNAXibK5NczGHWE27QjRsiPM5tYwkYGRe7G1\n1cUcMrOc+pll8U6/tozKssKJhptW4UgX2wY2s2XXrnjHHgkndfTxzr4/hU6jNFgS79CDpVSM6tgH\nO/uKxOt07sFMhmw7R3QgXNclEhsYThjRfvpjA0QSCaI/2p+UNCKUlAZx+/wj9tjLgqX4fZk5KFo0\nFmVLeBtNHc1DHW1733AJJeALUF8+l4WV81lUNZ8FlfWUBUtT3v6B/LfRO9BLc+cmNnS0sL6zhQ0d\nG+kZ6BmaX+QvZH7FIYnOfj7zK+dRHChO/cNOkBJACn/EcE9kqHyzcWf8cdvuLpKbx+9zmDOjdGjP\n/pCZ5cyrLaO4MFPOpY+vPxrhG6/cxu7ePXvN8zuDX/rSvfbUy4NlIzr5TO4QJiKXE8CByva2cF2X\nPb3tQ8lgfUczW8LbRpStZpbUDpeNquZTO85e91jtES9P7Rnas1/f0bLX+9QUT2dh5fyhPfzZpTPT\ndmQ0nvESQHb0XJPIdV3aQn2J8s1gzT7E7s6R9frCAj+L51QOd/a15cypKSXgz96rEJ7Z+Dy7e/fw\nF4ccy7KKpUOdfUVBGcWB4kk/9BSZao7jML24munF1bx/1lEA9Az00ty5kfXtiVJMZwurt+1k9bZX\ngXjdPX4OoZ5FVfOZVz6X4KiyUSQaYVN4SyKpxBNLqD88ND/oCwxtY7DDLy8om7oPPkE5fwQQjcVo\n3Bbmf+1OWhL1+nDPyBJHRWnBUK1+Xm38saY6M+r1k6Wtt52bX/42hYFC7jznZrraB/a/Up7I9r3e\nyZQPbRGNRdnatT1eNkokhba+9qH5AV+AQ8rnsqhyPoVFft7evo5Noc0MuMMng6sKK0fU7ueW1R3U\nuYZ0yusjgD++s5P/fPydode1VcUsPaSKeUk1+6oMrddPpkeanqQ/FuGihX9JSbCYLnL7Sy4yFr/P\nz7zyOcwrn8Mpc08A4jtIQ+cR2psTZZ1mIH5V09yyuqHOfmHlfKqLqjz8BJMn5xPAkQ0zuP6vjqLA\ngXm15ZQU5fxH3sv6jhZe3fEG88rncPzsY7wORyTjVBdVcWzRkRw780gAegf6aOncxLTqUipi0ynM\n8AsZJirne8PiwgCnHXtIzh/WjiXmxnhw7aMAXNjwUU9OQolkm6JAIWba4pwviak3yHGvbn+DltAm\njqk9gsVVC7wOR0QyiBJADusd6OORplUEfQH+cvFZXocjIhlGCSCHPdXyezr6Q6w85BSmFVV7HY6I\nZBglgBy1q2cPz2z6A1WFlaysP8XrcEQkAykB5KhfrXuCgdgA5y86K+OHYhARbygB5KC1bU38qfXP\nLKys55jEZW0iIqMpAeSYmBvjwcbhyz41vIOIjEUJIMe8uPWPbAlv4/hZx1JfMc/rcEQkgykB5JDu\nSA+Pr/8thf4CPrroTK/DEZEMpwSQQ55s/h3hSBdn1p9OZeHk30BaRHKLEkCO2NG1k+c2v8iMommc\nOu9Er8MRkSygBJAjHlr3ODE3xvkN5xD0B70OR0SygBJADnh793u8vfs9llQv5ogZh3odjohkCSWA\nLBeNRXmo8TEcHC5sOFeXfYpIypQAstzzW1azo7uVE+ccz5yy2V6HIyJZRAkgi4X6w6za8DTFgWLO\nWXCG1+GISJZRAshij294ip6BXs5esJKyglKvwxGRLKMEkKW2hLfx4pZXmFlSywfnrPA6HBHJQkoA\nWch1XR5c+yguLh9rOBe/z+91SCKShZQAstCbu95mbXsTh05fyqHTjdfhiEiWUgLIMpFohIcbH8fn\n+PjY4nO8DkdEspgSQJb5/ab/YXfvHk6ZewIzS2u9DkdEspgSQBbp6OvkyZZnKAuW8pH5H/I6HBHJ\nckoAWeTRpt/QH+3nnIUfpiRY7HU4IpLllACyREvnJl7e/hpzymZzQt0HvA5HRHKAEkAWcF2XX64d\nvs2jz9GfTUQOnnqSLPDajj+xobOFI2sOZ0n1Iq/DEZEcoQSQ4fqi/fy6aRUBX4DzF5/tdTgikkOU\nADLc0y3P0d7XwenzPsiM4mlehyMiOUQJIIPt6W3jdxufo7KgnDPqT/U6HBHJMUoAGezX61YRiQ1w\n3qKzKAoUeh2OiOQYJYAMta59A2t2vkl9xTzeP+sor8MRkRwUSOfGjTG3A8cDLnCdtfbVpHnXAH8N\nRIHXrLV/l85YsknMjfFgoy77FJH0SlvPYow5GWiw1q4ArgDuSJpXAfwjcJK19kRguTHm+HTFkm1e\n3raGTaEtvH/mUSysrPc6HBHJUenctTwd+DWAtfZdoDrR8QP0J/6VGWMCQAmwJ42xZI2egV4ebXqS\nAl+Q8xZ9xOtwRCSHpbMENAtYk/S6NTGt01rba4z5GrAe6AF+Zq1dO97GqqtLCAQmfuOTmpryCa87\nlR5483eEImE+fti5LJk3L23vky3tMVXUHsPUFiPlcnuk9RzAKM7gk8SRwP8FlgCdwLPGmCOstW+O\ntXJbW/eE37imppzW1tCE158qO7t38YR9hurCKlZMPz5tMWdLe0wVtccwtcVIudAe4yWwdJaAthLf\n4x9UB2xLPF8GrLfW7rLW9gMvAMekMZas8Kt1TxB1o5y/+GwK/EGvwxGRHJfOBPAUcCGAMeZoYKu1\ndjCVNgPLjDGDYxofCzSmMZaM996eRv5319ssqlzA0bXv8zocEckDaSsBWWtXG2PWGGNWAzHgGmPM\nZUCHtfZXxphvA783xgwAq621L6QrlkwXjUV5sPFRHBwuWvJRHMfZ/0oiIgcprecArLU3jpr0ZtK8\nu4G70/n+2eJ/tr7Ctq4d/MXsDzCvfI7X4YhIntAvjDzWFenmifVPUeQv4txFH/Y6HBHJI0oAHnti\nw9N0DXTzkQWnU1GQu5ebiUjmUQLw0Nbwdl7Y8hK1xTM4Ze4JXocjInlGCcAjruvyUONjxNwYFzSc\nQ8A3lT/JEBFRAvDMW7vf5b22RpZNW8Jh05d5HY6I5CElAA8MxAZ4qPExfI6PjzWcq8s+RcQTSgAe\neG7zi7T27OakOSuYXTrT63BEJE8pAUyxzv4QT254htJACWcvWOl1OCKSx5QApthjTb+lN9rL2QvP\noDRY4nU4IpLHlACm0KbQFl7a9iqzS2dyYt1xXocjInlOCWCKuK7LL9c+iovLhQ0fxe+b+L0NREQm\ngxLAFHmj9c80dWzg8BnLWTqtwetwRESUAKbKi1teAeCCxWd7HImISJwSwBQYiA2wvqOZ2aUzqS2p\n8TocERFACWBKNHduoj8WYUn1Iq9DEREZogQwBRrbmgBYUqUEICKZQwlgCqxNJIDFVQs9jkREZJgS\nQJpFohE2dLYwp2w2ZQWlXocjIjJECSDNmjs3EokNqPwjIhlHCSDNBss/DToBLCIZRgkgzda2N+Hg\n0FC1wOtQRERGUAJIo/5ohOaOjcwtm02JBn4TkQyjBJBGGzpaGHCjKv+ISEZSAkijte2J6/+VAEQk\nAykBpNHatnj9f7Hq/yKSgZQA0qQv2k9L5ybmlc+hOFDsdTgiIntRAkiT9R3NRN2oyj8ikrH2mwCM\nMUunIpBcM3j9vxKAiGSqQArLPGSMaQN+CPzcWtud5phyQmNbEz7Hx6LK+V6HIiKyT/s9ArDWHgpc\nBSwAnjPG3GOMeX/aI8tivQO9tIQ2U18+l6JAkdfhiIjsU0rnAKy1b1lrvwx8AVgGPGqM+YMxRvc2\n3IemjhZibkzX/4tIRttvCcgYUw9cBvwV8A5wC/Bb4P3AA8BxaYwvK2n8fxHJBqmcA3iOeP3/NGvt\n1qTpfzTG/DEtUWW5tW1N+B0/C6vmex2KiMiYUikBHQGsHez8jTFXGWPKAKy1n0tncNmoZ6CHjaHN\n1FfMo9Bf4HU4IiJjSiUB/BiYlfS6BPiv9IST/da1b8DFZYnu/iUiGS6VBDDNWnvH4Atr7XeBqvSF\nlN0a29YDGv9fRDJfKgmg0BizbPCFMeYYQLWNMaxtbyLg+Fmo6/9FJMOlchL4euARY0wl4AdagUvS\nGlWW6o50szm0lUVV8ynwB70OR0RkXPtNANbaV4AlxpjpgGut3WOM+YtUNm6MuR04HnCB66y1rybN\nmwf8lPjRxOvW2qsm8gEySeNQ/V/lHxHJfKmMBVRhjPkscA1wrTHmFuDBFNY7GWiw1q4ArgDuGLXI\nbcBt1toPAFFjzCEHHH2GaWzX/X9FJHukcg7g58D7gE8B5cA5wNUprHc68GsAa+27QLUxpgLAGOMD\nTgIeTcy/xlq78YCjzzBr25oI+AIsqMj6XCYieSCVcwBF1tqrjDHPWWv/0RhzK3An8Mh+1psFrEl6\n3ZqY1gnUACHgdmPM0cAL1tp/Gm9j1dUlBAL+FMLdt5qa8gmvm4pQX5gt4W0cWruEulnT0vpekyHd\n7ZFt1B7D1BYj5XJ7pJIACo0xpYDPGDPdWrvbGDORGocz6vkc4PtAM/CEMeZsa+0TY63c1jbxQUhr\nasppbQ1NeP1U/GnnnwFYUDo/7e91sKaiPbKJ2mOY2mKkXGiP8RJYKiWg+4G/Be4F3jXGvA1sT2G9\nrYz8AVkdsC3xfBfQYq1tstZGgWeAQ1PYZsZaq/q/iGSZVBLA3dba71lr7weOAv4aOD+F9Z4CLgRI\nlHm2WmtDANbaAWB90miixwD2QIPPJI1t6wn6gtRXzPM6FBGRlKRSAnoWOBXAWrsF2JLKhq21q40x\na4wxq4EYcI0x5jKgw1r7K+DvgPsSJ4T/DDw2gfgzQqg/zNau7SytbiDoS6VJRUS8l0pv9SdjzM3A\naqB/cKK19tn9rWitvXHUpDeT5q0DTkwxzozW2K7hH0Qk+6SSAI5MPJ6UNM0lfmQg6P6/IpKdUvkl\n8KlTEUg2a2xrosBfQH35XK9DERFJWSp3BHuB+B7/CNbaD6YloizT0Rdie/dOlk1bgt838d8piIhM\ntVRKQDclPS8ATgPC6Qkn+wwO/6Dyj4hkm1RKQM+PmvS0MWZVmuLJOqr/i0i2SqUENPrWVvMAk55w\nsk9jWxNF/kLmlc3xOhQRkQOSSgnomaTnLvGxfL6almiyTHtfBzt7dnHo9KWq/4tI1kmlBLTAGOOz\n1sYAjDFBa20k/aFlPpV/RCSbpXI/gI8xcuTPF4wxF6YvpOzROJgAdAMYEclCqYwF9PfEx/8ZdEZi\nWt5b29ZEcaCIueV1XociInLAUkkAjrW2Y/CFtbaT+Ng+eW1Pbxu7evewuGoBPieVZhQRySypnAR+\nzRjzc+A54gnjTEbe6CUvNbbFx/9R+UdEslUqCeDzwCeB44hfBfQA8Mt0BpUNBk8AN1Qv9jgSEZGJ\nSSUBlAD91trPARhjrkpMy+tfA69tb6I0UMKcsln7X1hEJAOlekew5F6uBPiv9ISTHXb17GFPbxuL\nqxeq/i8iWSuV3muatfaOwRfW2u8CVekLKfMNXv7ZUDX6R9IiItkjlQRQaIxZNvjCGHMs8UHh8tZa\nDQAnIjkglXMA1wOPGGMqiSeMXcAlaY0qg7muy9q2JsqCpcwunel1OCIiE7bfIwBr7SvW2iXAscR/\nALYVeDTdgWWq1p7dtPd10FCl+r+IZLdURgM9HvgU8AniCeNK4KE0x5WxBsf/1/1/RSTbjZkAjDE3\nAJcBpcSvBDoW+KW19mdTE1pm0gBwIpIrxjsCuAV4G7jGWvt7AGPMXreGzCeu69LY1kR5QRmzSmq9\nDkdE5KCMlwDmAX8D/MAY4wfuI8+v/tnZ3UpHf4hjao/AcRyvwxEROShjnsW01m631n7LWmuAy4HF\nQL0x5jFjzFlTFmEGWav6v4jkkJQuY7HW/sFaexlQBzwOfDmdQWWq4QHg9AMwEcl+qfwOYIi1NgTc\nnfiXV1zXZW17E5UF5dSW1HgdjojIQdOF7Cna3r2TUH+YhupFqv+LSE5QAkiRLv8UkVyjBJCi4QHg\nlABEJDcoAaQg5sZobF9PVWElNcXTvQ5HRGRSKAGkYFvXDsKRLpao/i8iOUQJIAVD9X+Vf0QkhygB\npKCxPXH9v04Ai0gOUQLYj5gbY13beqYVVTO9eJrX4YiITBolgP3YEt5O10C3yj8iknOUAPajsW0d\noPKPiOQeJYD9GB4ATuP/iEhuUQIYR8yNsa59AzOKpjGtqNrrcEREJtUBDQZ3oIwxtwPHAy5wnbX2\n1X0scyuwwlp7SjpjmYjNoa30DPRyVM3hXociIjLp0nYEYIw5GWiw1q4ArgDu2Mcyy4EPpiuGg6Xx\n/0Ukl6WzBHQ68GsAa+27QLUxpmLUMrcB/5zGGA6KBoATkVyWzhLQLGBN0uvWxLROAGPMZcDzQHMq\nG6uuLiEQ8E84mJqa8gNaPhqLsr6jmdnltTTMnTvh981UB9oeuU7tMUxtMVIut0dazwGMMjSIjjFm\nGvAp4EPAnFRWbmvrnvAb19SU09oaOqB1mjs30jPQy9HlRxzwupluIu2Ry9Qew9QWI+VCe4yXwNJZ\nAtpKfI9/UB2wLfH8NKAGeAH4FXB04oRxxlD5R0RyXToTwFPAhQDGmKOBrYlbSmKtfdBau9xaezxw\nPvC6tfb6NMZywNZq/H8RyXFpSwDW2tXAGmPMauJXAF1jjLnMGHN+ut5zskRjUZo6mplVUktlYe7W\n/0Qkv6X1HIC19sZRk97cxzLNwCnpjONAtYQ20x/tV/lHRHKafgm8D0PlHyUAEclhSgD7MHz/X43/\nIyK5SwlglEhsgKaOZupKZ1FeUOZ1OCIiaaMEMEpL5yYisYjKPyKS85QARmkcuv+vyj8iktuUAEZZ\n29aEg8Nijf8vIjlOCSBJJBphfWcLdWWzKAuWeh2OiEhaKQEk2dC5kYHYgK7/F5G8oASQZLj+rwQg\nIrlPCSDJ2vZE/b9qgdehiIiknRJAQn80QnPHRuaW11ESLPE6HBGRtFMCSFjf0cyAG1X5R0TyhhJA\nQmP7ekDj/4tI/lACSBi8/n9R1XyvQxERmRJKAEBftJ+Wzk0cUj6X4kCx1+GIiEwJJQBgfXszUTeq\n8o+I5BUlAOKXf4LG/xeR/KIEQPwHYD7Hx6LK+V6HIiIyZfI+AfQO9NIS2kx9+VyKAoVehyMiMmXy\nPgE0dTQTc2Mq/4hI3sn7BDB4/1+dABaRfJP3CaCxbT1+x89C1f9FJM/kdQLoGehhY2gz9RXzKPQX\neB2OiMiUyusEsK59Ay6uyj8ikpfyOgGs1fj/IpLH8joBNLY1EXD8LKis9zoUEZEpl7cJoDvSzebw\nNhZU1lPgD3odjojIlMvbBNCYqP83VC30OhQREU/kbwLQ9f8ikucCXgfglbXtTQR9Aear/i8iSe68\n83asfZc9e3bT39/HrFl1VFRU8s1vfnvc9VateozS0jJOPvnUfc7//vdv46KLLqaubk46wp6QvEwA\n4UgXW8LbWFK9mKAvL5tARMbwuc9dD8Q79O3bN3H55Z9Nab2zzjp33PnXXff3Bx3bZMvL3m9dW+L2\nj7r8UyRj/eLZdbz63s5J3eb7l9by8dMWH/B6r7/+Gj/72QN0d3dz7bXX88Yba3juuWeIxWKsWHEC\nl19+JT/84d1UVVWxYMEiHn74FziOj5aWDZxyyulcfvmVXHvtlXzhCzfw+98/Q1dXmI0bW9iyZTOf\n//zfs2LFCTzwwH387ndPUVc3h4GBAS6++JMcffSxk/r5R8vLBDA8/r9OAItIapqa1vHTnz5MQUEB\nb7yxhn//93vx+Xx8/OPn8YlP/J8Ry77zztv85CcPEYvFuOiic7n88itHzN+5cwff+c4dvPzyah55\n5CEOPfQwHn74l/z0pw/R1dXFxRdfwMUXfzLtnyk/E0BbEwW+IPMr5nkdioiM4eOnLZ7Q3nq6LF7c\nQEFBfMiYoqIirr32Svx+P+3t7XR2do5Y1pilFBUVjbmt973vSABqa2sJh8Ns3ryJhQsXUVhYRGFh\nEcuWHZq+D5Ik7xJAqD/Mtq4dLK1uIKD6v4ikKBiM/15o+/Zt/Pzn/82PfvTflJSUcMklH99rWb/f\nP+62kue7rovrgs83fFGm40xS0PuRdz1gY3ui/q/LP0VkAtrb26murqakpARr32P79u1EIpGD2ubs\n2bNZv76JgYEBQqEQ77337iRFO768SwCD4//oBjAiMhENDUsoLi7h6qsv5/DDj+S88y7gttu+xfve\nd8SEtzlt2nRWrjyTv/3bS6mvX8Dy5Yfu9yhiMjiu66b9TSZDa2towoHW1JTT2hoC4OaXv0NbXzvf\nOelr+H3pb+BMlNweovZIprYYaSrbY9Wqx1i58kz8fj+XXnox3/3undTWzjzo7dbUlI9ZUErrEYAx\n5nbgeMAFrrPWvpo071TgViAKWODT1tpYOuPp6OtkR/dOlk8zedv5i0hm2r17N1de+TcEgwWcccaZ\nk9L570/aEoAx5mSgwVq7whizDPgRsCJpkXuAU621m40xvwTOBFalKx5Q/V9EMtcll1zGJZdcNqXv\nmc6xgE4Hfg1grX0XqDbGVCTKtiTTAAAJsUlEQVTNP8ZauznxvBWYnsZYAN3/V0QkWTpLQLOANUmv\nWxPTOgGstZ0AxpjZwBnAl8bbWHV1CYHAxMs2NTXlrP/jBooDRRy1QCWgmppyr0PIKGqPYWqLkXK5\nPabyKqC9TkQYY2qBx4DPWmt3j7dyW1v3hN+4pqacxs2b2RbeyWHTl7Jn98S3lQt0om8ktccwtcVI\nudAe4yWwdCaArcT3+AfVAdsGXyTKQU8C/2ytfSqNcQC6/FNEZLR0ngN4CrgQwBhzNLDVWpucSm8D\nbrfW/iaNMQzR+P8isj+f+cyn9voR1g9+cBc//ekDey37+uuvcdNNNwBw441f2Gv+Qw/9nB/+8O4x\n32vdukY2bmwB4Ctf+Sf6+noPJvQJSdsRgLV2tTFmjTFmNRADrjHGXAZ0AL8FLgUajDGfTqzyE2vt\nPemKZ21bE8WBYuaW1aXrLUQky61c+WGeffZpli5dNjTtueee5c47fzDuev/yL9894Pd6/vlnWbp0\nOYccUs/XvnbrAa8/GdJ6DsBae+OoSW8mPS9M53sn29W1h129ezh8xnJ8Tt7eBE0kqzy87nHe2Pnn\nSd3mUbWHc8Hic8acf/rpZ3D11Vfw2c9+HoC33nqLmpoamps3cNNNXyQYDFJeXs7NN//LiPXOPvt0\nnnjiGV577Y/cccdtTJs2nenTZwwN7XzLLV+ltXUnPT09XH75lcyaNZtHHnmY559/lurqar785X/i\n/vt/Tjgc4tZbbyYSieDz+bjxxi/hOA633PJV6urmsG5dI0uWGG68cdxrZlKWF73hWzstoPKPiIyv\nunoadXVzeOedtwB48sknWbnyTEKhEF/5yje46657KCkp5ZVXXtrn+nfffRdf+tLX+d73/p2OjnYA\nQqFOPvCB47nrrnu4+eZb+eEP72bRosUcd9wKPvOZa1m+/LCh9e+99wecc8553HXXPZx//oX86Efx\nooi17/KZz1zDvffez0svvUgoNDknpvNiLKB3djYCugGMSDa5YPE54+6tp8vKlWfyzDNPs3z5YTz7\n7LPcdde9rFu3lm996xtEo1G2bt3CMce8n5KSkr3W3bZtGw0NSwA48sij6evro7y8gnfffZtHH30Y\nx/HR2dkx5ntb+y5XXXUtAEcffSz33XcvAHPmzGP69BkAzJhRQ1dXmPLyg788NS+OAN7eaSkNlFBX\nNmv/C4tIXjv55FNZvfoF3nvvHebPn09FRQW33vp1rr/+Bu666x5OPPGDY66bPKTz4DhrTz/9Gzo7\nO/m3f7uXb37zO/t5d2dovUhkACdRsh49MNxkjeGW8wlgV88eWrv3sLh6oer/IrJfJSWlLFrUwP33\n/5hzz43f57erK8zMmbMIhUK8/vqaMYd/njGjho0bm3FdlzfeiP8Otr29ndmz6/D5fDz//LND6zqO\nQzQaHbH+smXLef311wD405/WjDgZnQ453yM2tW8AVP4RkdStXHkmr776CqeddhoAF1xwEVdffQX/\n+q+38MlPXsoDD9zH7t279lrvyis/y003fZEvfvH6ocHcTjnlNFavfoHrrrua4uJiamtr+fGP/5Mj\njjiK733v27z22h+H1v/0p6/iN79Zxec/fxWrVj3OFVd8Jq2fM+eHg94a3s5vNj/NRQvPp7ygbLLD\nykq58OvGyaT2GKa2GCkX2mO84aBz/gigrmwWXzzps+r8RURGyfkEICIi+6YEICKSp5QARETylBKA\niEieUgIQEclTSgAiInlKCUBEJE8pAYiI5Kms+SWwiIhMLh0BiIjkKSUAEZE8pQQgIpKnlABERPKU\nEoCISJ5SAhARyVNKACIieSrgdQDpZoy5HTgecIHrrLWvehySp4wx/wqcRPxvf6u19mGPQ/KUMaYY\neAv4urX2Po/D8ZQx5pPADcAA8GVr7RMeh+QZY0wZcD9QDRQCX7PW/tbbqCZfTh8BGGNOBhqstSuA\nK4A7PA7JU8aYU4HDEu1xJvA9j0PKBDcBe7wOwmvGmOnAV4ATgXOA87yNyHOXAdZaeypwIfB9b8NJ\nj5xOAMDpwK8BrLXvAtXGmApvQ/LUH4CLEs/bgVJjjN/DeDxljFkKLAfydk83yYeA31lrQ9babdba\nK70OyGO7gOmJ59WJ1zkn1xPALKA16XVrYlpestZGrbVdiZdXAKustVEvY/LYbcAXvA4iQ8wHSowx\njxpjXjDGnO51QF6y1v4MOMQYs474jtM/eBxSWuR6AhjN8TqATGCMOY94ArjW61i8Yoy5FHjJWrvB\n61gyhEN8j/cC4uWPHxtj8vb7Yoz5a2CjtXYxcBpwl8chpUWuJ4CtjNzjrwO2eRRLRjDGfBj4Z+Aj\n1toOr+Px0NnAecaYl4FPA18yxnzI45i8tANYba0dsNY2ASGgxuOYvHQC8FsAa+2bQF0ulktz/Sqg\np4CvAXcbY44GtlprQx7H5BljTCXwbeBD1tq8PvFprf3E4HNjzFeBZmvt77yLyHNPAfcZY75FvOZd\nRo7WvVO0DjgOeMgYUw+Ec7FcmtMJwFq72hizxhizGogB13gdk8c+AcwAfmGMGZx2qbV2o3chSSaw\n1m4xxjwIvJyY9DlrbczLmDx2N/AjY8zzxPvJqzyOJy10PwARkTyV6+cARERkDEoAIiJ5SglARCRP\nKQGIiOQpJQARkTyV05eBiuyPMWY+YIGXRs16wlr77UnY/inAN6y1Jx7stkQmmxKACLRaa0/xOgiR\nqaYEIDIGY8wA8HXgVOK/jL3MWvuWMeY44gPJRYjfZ+Jaa+07xpgG4D+Jl1Z7gU8lNuU3xvwHcBTQ\nR3wYCoCfEP/VbRB4zFp7y9R8MpE4nQMQGZsfeCtxdPAfwM2J6fcD1yfGiv8u8G+J6T8Avm2t/SDw\nI4aH3l4GfNVaezzxpPFhYCUQtNaeBPwFEDbG6PsoU0pHACJQY4x5btS0GxKPg3eBehH4R2NMFTAz\n6c5yzwE/Szw/LvF6cDjhwXMA71lrdySW2QxUAY8BNxtjfgGsAu7N86EXxANKACJjnANIjJc0uFfu\nEC/3jB47xUma5rLvo+qB0etYa3caY44AVhC/+9ZrxpijrbU9E/oEIhOgQ06R8Z2WeDwR+N/EENrb\nEucBIH4nrcEB1FYTv9UmxphPGGO+OdZGjTFnAGdba1+01t4AhIHadHwAkbHoCEBk3yWgwRvFHGWM\nuZr4ydpLE9MuBb5rjIkCUeDqxPRrgXuMMdcQr/VfDiwa4z0t8P+MMTcktvGUtbZlMj6MSKo0GqjI\nGIwxLvETtaNLOCI5QSUgEZE8pSMAEZE8pSMAEZE8pQQgIpKnlABERPKUEoCISJ5SAhARyVP/H7Je\nrJeE1C6KAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f5c2022a978>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "KB3JEUiwsZSw",
        "colab_type": "code",
        "outputId": "35429f3d-ee2e-4102-826d-68f81a16e5d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "result = model.evaluate(x=mnist.test.images.reshape(shape),\n",
        "                        y=mnist.test.labels)\n",
        "for name, value in zip(model.metrics_names, result) :\n",
        "    print(name, value)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10000/10000 [==============================] - 2s 153us/step\n",
            "loss 0.02147128265624051\n",
            "acc 0.9932\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qlGI26ANsZS0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_pred = model.predict(mnist.test.images.reshape(shape))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "r4sew0lGsZS5",
        "colab_type": "code",
        "outputId": "d53dd89b-f5eb-43a1-96db-979bce597d5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        }
      },
      "cell_type": "code",
      "source": [
        "plot_example_errors(y_pred)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUwAAAD1CAYAAAA73MxWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYFNW5x/EvGgkIiuKCawCv+rpc\nXIPiNcqQ6HVDBUVjFGJcHlRU3HANKhpEIYqYgHEniqK4EiEoFzCoiCACokF4jfq4i+KKCyKo94+u\n08sw01Mz3dU9Pfw+z8NDTXUtb8+ZPv3WqVPnNPvpp58QEZG6rVXuAEREKoUqTBGRmFRhiojEpApT\nRCQmVZgiIjH9rMD918Rb7M3KHUCJqYybPpVxTMowRURiUoUpIhKTKkwRkZgKbcMUEWmQww8/PL38\nr3/9C4BnnnkGgD322KMsMdVFGaaISEyqMEVEYmpW4OAb6o7Q9KmMm76SlvGMGTMAOPjgg9Prvv32\nWwCOP/54AO69996kw1C3IhGRJOmmjzRKzZplEoCjjjoKgHA1tPPOOwPwpz/9qfSBScGuv/56AJYv\nX77aawsXLix1OPWiDFNEJCa1Ydaf2rdKYK21Mt/lIdsMf6vh54ceegjIZKBFpDJOwPPPPw/AgQce\nCORmmKFsd911VwDmz5+fdDhqwxQRSVKjbsNctmwZAJdffjkADz74IADrr79+epulS5cC0KNHDwBu\nvvlmAFq0aFGyOKX4brnlltXWDRw4EIBPPvkEgGuvvRZIJMOUIvr4448BOPPMM4Ga2y6DM844oyQx\nNZQyTBGRmBplhjllyhQAhg8fDkDr1q0BmDt3LgBbbLFFetsXX3wRgM6dOwPwi1/8AoBBgwaVJFZJ\nRt++fVdbN2/ePABuv/32UocjDfDDDz8AcNZZZwHw0ksv1brtyJEjgZrLvTFRhikiElOjzDDDg/ih\n13+fPn1q3dbMADj22GMBePfddxOOTsot3FHdb7/9yhyJ5HPDDTcA8MgjjwC5fWsBTjzxxPRyv379\nShdYAZRhiojEpApTRCSmiu+4vmrVKiAztl6nTp0AGDZsWFKnVKfmMmnXrh2Q6VY0ffp0IJFLc5Vx\nAd566y0AttlmG2D1S/EDDjgAgAkTJqTXNW/evJghxKGO6yIiSWqUN33q44477gDg/fffBzKd26Xp\n6dmzJwC33XYboJs+jcmbb76ZXs4eti1b6A4YHjgoQ1ZZMGWYIiIxVXyG+cEHHwDQpk0bANZbb71y\nhiNFEh55hUxG8thjjwGw0047lSUmWV1orwyPrQK88cYbNW570kknAcWfryfEcPXVVwPwxBNPAPDh\nhx8W9TygDFNEJLaKzzClaXj77bcB2GSTTYDcKQpGjBgBwLrrrgvA008/XeLopLrXX38dyAzVFsov\nW+iBs+222wJw8sknF+XcS5YsATKZZMgsa4qh2JRhiojEVPEZZhhEdp999ilzJFKIvfbaC8g8Tnfd\nddelXwv9+C677DIAdthhhxJHJ8HKlSsBOP/88wF45513gNX7WgKss846QGYQnQ4dOtT7fK+99hoA\nN954Y3rdrbfeWuM5a4qh2JRhiojEVJEZ5ssvv5xeDnfINJxbZXr00UeBzCCzQ4YMyfkZYMcddwQy\nGaaUTxhi75///Ged24ay7N69e53bhkzyhRdeADJt2LNmzQLgq6++qn+wCVCGKSISkypMEZGYKvKS\nPMxrDJl0v0uXLuUKR+ph0aJFQGaMxKFDhwKZBvtevXoBufNTjx8/HoDBgwcDuZ2kpbTizAUf5o2/\n4IIL8m4XbthCZlT2MLBKQ/zXf/0XAP3792/wMeqiDFNEJKaKzDDdPb0cMsz27duXKxypQ3aH4j/+\n8Y9A5jHHrl27Apmbd2GU/W+++Sa9T3gUMsweGrqn9O7dO7mgpUaTJk0CVu/CE7JKgKlTp+a8Fm7g\n3XPPPTn/L168OL1NGKaxIV2DjjnmGACuueYaIJNpJkEZpohITBWZYUpl+f3vf59enjFjBgCbbrop\nkOnUHGb73HjjjQH49ttv0/uEbkVheLeQSYRHJTUveenUNuB49vpwjyEM8Bxmdm3IcWsSBpIODzdk\nzw2UNGWYIiIxVdQUFf/5z38AOOKII9Lrxo4dC8Duu+9eqjA0fUFMYYi2kE1Cps0yZB+FCMd98skn\n0+uKNHSYyrgWYYi20A6Z96BR3VJbu2R2ZhjaOcOAGtUddthh6eWQWRY4zJ+mqBARSVJFtWGG9q8w\nQAOUNLOUegp3wrMzjNAOWQxjxowB4NVXX02vK/bgtJIrtDmHO+FhAO+aVFVVAZnh+Pbcc08g0wYd\nrjYgM+1IbRlmdt/bcg4grQxTRCQmVZgiIjFVxE2fH3/8Ecg8NnfooYemXzv11FNLEUI23RCIKYxE\ndPrpp6fXhW5D4THHhnQJCsc9+uijgdxL/vC3UiCVcR3effddAL744gsg91HW8GBBuAQPHdRbt24N\nQMeOHVc73pdffgnAL3/5SyB3FkrIvQwfMGAAUHB3It30ERFJUkVkmJ9//jkAbdu2BWD27Nnp18JI\n3SWk7KOewpw8kJlHPjwueemllwLxxroMWWkYsCM8Ppl9QyDM71IglXGZPP/88wD86le/AqBz586r\nbTNt2jQAWrVqVciplGGKiCSpIjLMZ599FsgMARVGYQZo2bJlKULIpuyjAGF4t0MOOQTIPDYX2jZr\n0qdPHyDTFhaGAAvdyxJ4NFJl3PQpwxQRSVJFZJgHHXQQAJtvvjkA/fr1S7+mNszEJVLG4aohDKxR\nPcMMmShkyji0d/bt27fGfYpIZdz0KcMUEUlSRWSYYZCFMJiD7pKXlLKPpk9lHJMyTBGRmCpi8I1d\nd90VgC222AKAXXbZpZzhiMgaShmmiEhMqjBFRGKqiJs+jYxuCDR9KuOmTzd9RESSpApTRCQmVZgi\nIjEV2oYpIrLGUIYpIhKTKkwRkZhUYYqIxKQKU0QkprzPkpvZMGAvoAWwO/B89NKd7j4miYDM7I/A\ngVmrdgDOc/f7a9m+CvgHMD9a1QKYB5zj7isbGMNgYJW7D8qzzQnAecC30fnOd/eiTFlYSmUq4w2A\n0cCGQEtgnLsPz7N9FeUp417ARcD3wJdAb3f/vCHnK6dylHF03kuA44EfgTvcfWSebasoQxlnbXsk\nMN7d83Zoz1thuvtF0cE6ADPcvSpusA3l7tcA10Tn3RiYAoyvY7dXQmxm1gx4ADgNqLWACmFmmwI3\nAju7+1IzGwX0Ah5M4nxJKkcZA2cCC919oJk1B940s4fd/Z08+5S6jNsCo4Cd3P1TMxsO9AeuSuJ8\nSSpHGUcV4HGkKuq1gfvN7HZ3X5Fnt5KWcVasGwEXAx/WtW2DRysys0FAR6A9cAFwAzDY3admFcxW\nZrYhcAuwCdAGuMHdx5rZbsAp7n52ntMMBq5z9+Vx43L3n8xsBqnMFDNbBtwJrO3u/c3sbOBYUu99\nMdDP3Zeb2TVAd+Bd4BtgUbT/CGCMu8/NOk0H4B13Xxr9/E/gt1RghZlPgmX8CbBHtNwK+AH4Om5c\nJSrjz4Ht3H1Z9PPHwJZxY6wUCZbx0cDf3f276Ocj6hNXico4GAUMBO6qK65Ch3frCHSN3lxt2wwG\nnnT30WbWClhgZlPc/SWg1srSzLYG9gX61bZNLfu1AA4n9csFaA1McvcpZrYX0BPYP4r5RuBUM5sM\nnAAYqQ/vC0S/aHc/t4bTvAa0N7NtgTeAg4DN6xNnBUmijG8HJpvZm0Bb4GJ3/yxuQKUoY3f/CVgW\nnW9D4BRSl5dNURJlvC3wiZlNIFXJ3u7ud9awXY1K9DnGzH4LfObuT+V572mFVpizoj+sfLoBnc3s\nxOjnlaQKaGntuwCpivL2mO2CncxsetbPE9x9XLTcDHguWq4iVZD/in45raJ4OgFzw+WCmT2T72Tu\n/oWZnQT8nVRmNAv4Lt8+FSyJMj4feAv4X2AjYGb04XszzzlKWsaBmW0BTAKudfc5cfapQEl9jtsD\nR5KqMF80s+fcfXGe7UtaxmbWDhgQvbdYCq0wv89azv6FN89aXkEqXX6xnsfuSSpziyPd9lGLEOcK\n4HF3Pyv7xahxP7tiXruuE7r7RGBitH9vUn8UTVESZfxrYFT0If3EzOYCewP5KsySl7GZbQZMBf7o\n7o/VtX0FS6KMPwCejhKej8zsBVIVWr4Ks9RlfBipynZqVPFubmazgAPcvcYmomKOuL4M2Dpa/nXW\n+hmk2hpeNLOWpNpI+rv7qtoOFN3s2cDd3y5ifJD6hupvZq3d/Wsz60fqrtwiYI/oBsRPQFfg0Tzx\nrQ08S+rb8zPgVFKXLE1dscp4MfA/wD/N7OfAbsCwIsVYlDKOjAUuir4c1xTFKuPxwInAPdH2uwFX\nFCnGopSxu99FVrulmb3l7l3ynbiY/TBHAgPNbAqpWjsYBGwXNeA+A8x391VmtpuZ/bWWY20NLMle\nYWabmdlDhQQYfTuOAqZH8VQBC9x9IakCng08BLyUdd4RZrZnteP8QKoBfBowB3jK3acWEluFKFYZ\nDwZ2iC6ZppPqcjK/MZWxmXUmVakPMLPp0b/a/l6bkmKV8URgsZnNAZ4G/uLuCxtTGTdERQ2+YWa3\nuXvfcschyVEZN32VXMYV86RPlGZPKHcckhyVcdNX6WVcURmmiEg5VUyGKSJSbqowRURiKrRb0Zp4\nPa8ZBZs+lXHTp1kjRUSSpApTRCQmVZgiIjGpwhQRiUkVpohITMUcfCNxJ598MgCjR49Or+vRowcA\njz3WlAeTEZHGQBmmiEhMFZVhNmvWLOf/6ssi0nhMnZoawGvlytQcZhMmpB4h/9vf/lbnvh06dABg\ngw02AODsszODuocrzXJQhikiElNFZZiy5njnncwEkr169QJgzpzcGSIGDBgAwJ///OfSBSZ1euCB\nBwA44YQTAKg+wE+4Ksxev/766wNwySWXALDvvvsCsPfeewPw85//PMGI41OGKSISkzJMaRRmzpwJ\nwJAhQwD48MPMFNHz588HMplJaNc6/vimOoljZZs7NzWT7dZbp2a6yL5aqE3ILC+99NLkAisCZZgi\nIjEpw5SyWLo0NTvruHGpWVQHDhwIwLJly+rc94svvgDg/vvvB2D33XdPIkRpoNCmfOWVVwIwefJk\nAI455pic7bLbJTt37lyi6AqjDFNEJCZlmJKYkC1++umn6XXhiax77rkHgJdffrn0gUlJtG7dGsi0\naVb3619nZvE94IADShJToZRhiojEpApTRCQmXZJL0S1fvhyA3r17AzBx4sTY+x5++OFA7g2Bhx9+\nuIjRSam8/vrrAIwaNQpYvQN7JXYLU4YpIhJTRWSYb7/9NgDvvvtumSOROL777jugfpllly5dALj7\n7rsBmDZtWvo1ZZiV6cknnwTgq6++AjIPHhx88MHA6t2MKoEyTBGRmCoiwwzDRIX/L7744vRr4eF8\nabxCWybAvffem/PaEUccAWQGagiPPd53330lik6ScsMNN9S4vn///kDjGVCjPpRhiojEVBEZZnUn\nnXRSenn77bcvYySSTxh8IbRBA3zwwQc526y33noAtGrVKmf9E088kXB0Ui7hYYXsxyE33HBDANZa\nq3HncI07OhGRRqRRZ5gPPvggAKeeeioAd911F6CssrFbd911gcxAsitWrEi/ttlmm+XdN0xw98MP\nPyQUnZRKGJavU6dOALz//vtAZii37HsRv/vd7wA49NBDc44Rpqr41a9+lWiscSnDFBGJqVn13vf1\nVNDOddlkk02AzOANd955J5DbhlkGa9qsa4mWcXVhwqwwZQHkZqiQyVLDwLQ/+1nRL5RUxkU0Y8YM\nAE4//XQAFi5cWO9jDB06NL180UUXFSOsBpWxMkwRkZhUYYqIxNSob/rImuOVV14BYNCgQQCsWrWq\n1m1D15MELsUlAeGGzYsvvgjAt99+C+Q+nBAem3z88ceBzKj6YZ/LL788ve0vfvELAI477rgkw66R\nMkwRkZga5Vf0e++9B9SeZWSvD0NIVXfGGWcAsN9++6XX/e1vf8vZJtxY2HzzzYHMTYS2bdumtwk3\nHMI8Jd27d4/5LqQ+whw/YUT2qqqq9GvTp08vQ0RSbC1atMj5/+yzz15tm7POOguACRMmAHDkkUcC\n8P3336e3uemmmwBlmCIijVqjzDDDt8yXX36Zsz50T8ieWfC8886r8RibbropANttt116Xc+ePXO2\n2XfffQE48cQTAbj22muBzPBTAB9//DEA8+bNA5RhJqV58+YAnHvuuQDsuOOO6deqZ5iXXXZZyeKS\n0vjmm2+ATFtl6EJYk1122aUkMdVEGaaISEyNquN6GHChX79+QO6gDbUJc1KHrDQInd4TyAjVqTlB\np5xyCpB5RLImYXDhbt26JRWGyrgEwqOSkHn8OcxhXl32YMNhiMB11lmnkNOr47qISJIaVRtm6ItX\nW2YZ7mo/8sgj6XWhrWuLLbZIODpJ0muvvQZk+t/VpEePHgD88pe/rHWb0OYc+vqFwRuk8ZgzZw4A\nxx57bHrdW2+9VeO2YZtx48YlHlccyjBFRGJShSkiElPZL8nHjx+fXl60aFHOa2FsvNDl5MwzzwTg\nN7/5TYmik1Lp2rUrAB999FGt24TLtmHDhgGZbmHZ3cxCs84999wDZMZU1dxPyQjz9myzzTbpdeFh\nkY033hjIPIhyyy23AHD99dcDuaNQhUcjQ6f2Pn36ADBy5MjEYm8IZZgiIjGVvVtR+GbJXv7DH/4A\nwKhRowBo2bJloacpJnU5SUBo1A8jb8fRpk0bIHcU93CTIMxhHx6jC3MHxaQyjmmfffYBYPbs2el1\n7dq1AzI3aT/77DMgM65t+qRZdc/OO+8MwHXXXQeU5AERdSsSEUlS2dswQ7sGwMyZM4FMG0cjyywl\nQeFBg/oIsw6G7kaQ6VYUBkupZ2Yp9RTmlQ/z9wAsWbIEqL09OsxHHobyg8xVZchOGytlmCIiMZW9\nDbMCqX0rAQsWLABg4MCBQO5jb2FYvjAMXzBgwAAgc9ccMnfHDzroIKDBGYvKuJ4efvjh9HKYc6d6\nZ/QwVNtVV10FlHcQDdSGKSKSLGWY9afsowR69+6dXr777ruBTD/c0LcyDMIRBmApIpVx06cMU0Qk\nScow60/ZR9OnMm76lGGKiCRJFaaISEyqMEVEYlKFKSISkypMEZGYVGGKiMRUaLciEZE1hjJMEZGY\nVGGKiMSkClNEJCZVmCIiManCFBGJKe8UFWY2DNgLaAHsDjwfvXSnu49JIiAzWx8YDWwCtAaGu/u9\nebb/AzAUCHP0tgSedPcrC4jhXmCqu/89xrbnAOe5e4eGnq+cVMY1l7GZ7QnckLVqA+Ajdz+ooecs\nlzKV8SDgOGBJtOprd691ZjMzqwL+AYS5LloA84Bz3H1lA2MYDKxy90F5tjkb6A0sB74DTnH392vb\nPm+F6e4XRQftAMxw96p6R11/lwOvuvvlZrYp8JqZPeTuK/LsM8XdewOY2TrA02Y2x90nJhmomW0P\nHJbkOZKmMq6Zu88FqsLPZjYGuD+JcyWtTGUMcF2cpCPLKyE2M2sGPACcBiQyObmZbQYcA+zv7ivM\n7GrgQuDc2vZp8CRo0TdIR6A9cAGpb+PB7j41q2C2MrMNgVtIZRNtgBvcfayZ7UaqNj+72qGvAH6M\nlj8D1iaVheT7MKW5+0ozex7Ywcz+DUwAXgH+7e5DzGwIsC+pLOVp4CJSQz3dCXQC3gZaZb3PB4AL\nqn/rmNla0fs6E3giTmyVZk0v46zX9wE2cvdJceKrJAmWcUHc/SczmwHsEMW5jFT5re3u/aPM8FhS\nddhioJ+7Lzeza4DuwLvAN0RXJWY2AhgTfRGGcywB9o9eXwvYEliYL65CZ43sCHSN3lxt2wwmdfk0\n2sxaAQvMbIq7vwSs9kt29+VZP54NPOXun1bfrjZm1gY4EOgbrdoR6OXubmbHAFu6e9do28dI/XJX\nkCqYzqQ+ZG+Q+nbD3Y+r5VQXRu9rUZ733hSsyWUcXBG9x6aq6GUcOcHMjiP15TTS3cfFDcjMWgCH\nk6okIfWFOsndp5jZXkBPUpnhT2Z2I3CqmU0GTgAM+AF4gajCdPdas8aoWe18YA7wl3xxFVphznL3\nuh4V6gZ0NrMTo59Xkiqgpfl2MrP+pNoWDogRx4FmNj1a/hG43t1nRd+Qn7m7Z8WyT9a2baJY1gFm\nRu/lWzPLzEpfc2w7A4cAv4kRW6VbI8s4K0YD2rn7c3G2r1BJlPEkUl+Ez5hZe2CWmc1399fynKNT\nVrkBTMiqZJsBoQyqgG2Bf0UVfKsonk7A3NC0Y2bP1PGeAHD3m8xsJDAEuJHavwAKrjC/z1rO/oU3\nz1peQSpdfjHuQc3sYlIfom7uvizGLun2rTpiXAHc5u7XVzvfADKXiJC6RMznaKAt8FxUYJtH37YH\nxoi10qypZRwcBYyPuW2lKnoZu/sLWctvm9ksYBcgX4WZbsOsI84VwOPuflb2i2bWi3qUsZltBXR0\n92fd/QczGwuMzbdPMbsVLQO2jpZ/nbV+Bqm2BsyspZndbGa1VtRm1o3UJVT3mB+k+pgBHBXOb2ZX\nmNl2wKtAFzNrZmbrAXvnO4i7X+3uu7h7F3fvAnzYRCvL6taYMs6yL6lLuzVFscp4pJkdGS23AfYA\nXi5SjM8Bh5hZ6+j4/aJ25kXAHmbWPLox2LWO47QB7rFUrw1IlXXeNsxiVpgjgYFmNoWsBnVgELBd\n1ID7DDDf3VeZ2W5m9tcajjOAVMPyZDObHv3bM8/29fEoqV/2zOimQTvgTWAy8A4wG7iLTLcLzOwB\nM9uywPM2FWtiGW9NpmvMmqBYZXwzMMDMngamAoPc/TUz28zMHiokwCjLHQVMj+KpAha4+0JSVwOz\ngYeAl8I+ZjbCUl3Fso+zELgWmBpdvvck9bdZq4oarcjMbnX308odhyRHZdz0mdlt7t637i0bn4p5\n0sfMNqaO9gWpbCrjps/MmpPqBlaRKirDFBEpp4rJMEVEyq3QbkVrYnraoAngK5jKuOlTGcekDFNE\nJCZVmCIiManCFBGJSRWmiEhMqjBFRGJShSkiEpMqTBGRmFRhiojEpApTRCQmVZgiIjEV+mhkvX38\n8ccA3HtvalbVxx9/PP3a008/DUCzZqmnlvbaay8ArrnmGgB+85s1YUaINduMGTNWWzdt2jQArrvu\nOgAOPDA1VnPPnj0B6NatW3rbDh06JByh1NdXX30FwMiRuZM//t///V96edasWQCcd955AFxwwQUA\nbLTRRqUIMTZlmCIiMRU6vFu9d/79738PwH333bf6waJYQoYZtGnTBoDevVNTuvzlL3kndkuaBmao\np5UrV6aXv/zySwBatGgBwPDhwwG4//7UlN+LFy8GVv8bqDGw6O/lkksuSa+79tprCw0XVMZFEeal\nC1eKX3/9de5Js+qe6uXdqlVqsPchQ4YAcNZZOdP3FIMG3xARSVLJ2jDHjUvNlhkyiZqEtqr//u//\nBuCZZ1KzZPbp0weAUaNGAdCrV6/0Pvvvv3/xg5WiOv3009PLo0ePBqB9+/YAvP3227GPE8o6/F1I\n4/TJJ58AcMYZZwCrZ5ZxfPPNNwBcdNFFQG57Z/Z9j1JThikiElPJMsxbb70VgB9//DFn/cknn5xe\nzr7bCXD00Ufn/HzSSScBcOSRR6bXPfdcam73nXbaqXjBSlGEO5533XVXel1oqwqZ5fbbbw/A0KFD\nAdhuu+1ytgO47bbbAJg3b17O8XfbbTcADj744KLHLvUTer8AnHDCCUCm10shVqxYAcDSpUsLPlYx\nKMMUEYlJFaaISEwl77heXbhsyydcmv/sZ6lwQ4dlgHPPPReAf/zjHwC0bNmy2CFKA7Vr167W18IN\nnNC9bMstt8x5fdCgQenl8JDDp59+CoCZAfDkk0/WeR4pjcceeyy9/NRTT9W4zTrrrANkHkTJvmH7\nyCOPAPDnP/85qRCLQhmmiEhMJcswQyfV6h3lx48fn16u68ZNuNmTfdMnZJaTJk0CVr9RJOUTOpRf\neuml6XWh/EPXsSuuuAKA//znPwA8++yzqx1nxx13BODMM88EcrNPaRxCd7F8wg2+8NhjttmzZxc9\npiQowxQRialkGWZVVRWQecj++++/B+Dyyy9PbzNlyhQAevToAWQG2wjZSBDatAD2228/ALbddtsE\nopZiyO5oftRRRwFw880317jtxRdfDOReKYQMs3Xr1kmFKAkK5ReuBmtS06PS2TbddNOixtRQyjBF\nRGIqWYZ55ZVXAvDdd98BMGzYsNW2CR1dQ0YSHsAPd8Uvu+wyAD777LP0PosWLQJg8uTJQGZ4r9Dh\nNZ/mzZsDsMEGG9TjnUh9hasAgD333BPIfdQt2+effw5kHq8DZZaVLlz9dezYMWf9hAkT0ssvvfRS\n3mOcc845xQ+sAZRhiojEVPLh3V599VUARowYAcADDzyQfi08pF/X0F41DQu18847A5nhw957771a\ntw26du0K1N5vrBYa+qsAy5cvBzIDKDz66KNA5u74hx9+CGT63ALssssuQOZu+2GHHQYk2udWZRzT\nggULgNzHU7MfkwTo168fAH/9619z1mf3dvjTn/5U4/FDn9vsK5KtttqqoeFm0/BuIiJJKnmGWd0b\nb7yRXg5ZZ1gX2iVXO2merLE+2yrDjCWRwWWr++ijjwCYOXMmkDsoS7hqCMLwfmEw6u7duxc7HJVx\nTOEJn+whF6sLg3GEQVSuvvpqIDPlCNT+OT7ggAOA2uuCAijDFBFJkipMEZGYyn5JHkcYT3HgwIFA\n5sYA1J7Kb7jhhgBcddVV6XVrrZX6fujcuXPO//Wky7USmzp1KpAZwfv111/PeT0M5gCZrmcFUhnX\n0957751enjNnTryTxmhaC49MNvCzmo8uyUVEklT24d3iCDcAQqf37BnkavtmCp3eE5htTkosNPyH\nG0JjxowBMl1RsrunhCHEwgAP4apCkpX9OYwz42e+/Rsz/TWJiMRUEW2YQRiwITtrDPNbh47OYba5\n4LTTTltt/wJVxldh8ZS9DbM2M2bMAGqeOTTMAbPRRhs15NAq43rKHkA4XxejnJPGaMMMg/WoDVNE\npMJUfIYZho0bMmQIkBmoIzx81IoNAAAHHElEQVSetf7666e3LdIMk8o+qlm1ahWQaWMu1WAZK1eu\nBHKH/wsDEYeHIPr379+QQ6uM6yn7yi5khdWv6F555RUg82BKnAzzt7/9LQBjx44tNMTqlGGKiCSp\nIu6SB9UHEobMkHCh71doz3r44YeB3MfqGsvcxk1NGPw1TGAV+kIef/zxiZ433BGv6U54nOH9pHhC\nrxTIDPwd/g/CUIw1fY5rEya+ayyUYYqIxKQKU0Qkpoq6JA+X29kjpH/xxRdA7SMyZzcsZ4/ULsUT\nZvEcOnQokBmdJox1euGFF6a3zR59vVDhBs+SJUvS60J5N5Y5YCSj+tzzccybNw+A+fPnp9ftvvvu\nRYupvpRhiojEVFEZZjBu3Lj0cvZIz3VZuHAhkOl6JMURMv4XXngBgGOPPRbIjJKdPRhD6O5zyCGH\nALndvuIK3ZfCPOXZN/bCgwyHH354vY8rjU+4KgxzPZWbMkwRkZgqMsPs1q1bejl0XZg2bVqd+/37\n3/9OLCbJdFifNGkSkHl0sU+fPultfve73wGw2WabAXDHHXcAcOihh9Z63DCcX5i3PmSpoV0ru9Nz\neBS2bdu2hbwVkRopwxQRiamiHo2sSZgdcvjw4QDcdNNNOa/36NEjvRzu1nbp0qWQU+qxuXoKs4FC\nJuvs27cvkLnDHe6sh7vb4TE6yAzrtmzZspzj7rrrrkDu3DDh6iPMOd9AKuMEfPXVV0CmjLMfLqjt\n0UjNGikiUqEqPsMMwgAQ4bG8MEVFmMu6iJR9FEG4+7l48eKc9SFbnDhx4mr79O7dG4CjjjoKgH32\n2QeAdu3aFTs8lXGCwuPM2fciqmeY4fHJ8Pk97rjjih2GMkwRkSQ1mQyzhJR9NH0q46ZPGaaISJJU\nYYqIxKQKU0QkJlWYIiIxqcIUEYlJFaaISEyFdisSEVljKMMUEYlJFaaISEyqMEVEYlKFKSISU94R\n181sGLAX0ALYHXg+eulOdx+TREBmtgEwGtgQaAmMc/fhebavAv4BhGnlWgDzgHPcfWUDYxgMrHL3\nQXm2OQE4D/g2Ot/57v5jQ85XTmUq462BO4HmQCtgtLvfnGf7KspTxt8Bs7JWjXT3hxtyvnLS57h4\nn+O8Faa7XxQdtAMww92r6h11/Z0JLHT3gWbWHHjTzB5293fy7PNKiM3MmgEPAKcBI5MI0Mw2BW4E\ndnb3pWY2CugFPJjE+ZJUpjI+Hbjb3e8zs/WB98zsAXfPNw9yScs4sqREv49E6XNcs4Z8jhs8p4+Z\nDQI6Au2BC4AbgMHuPjWrYLYysw2BW4BNgDbADe4+1sx2A05x97OrHfoTYI9ouRXwA/A1Mbn7T2Y2\nA9ghinMZqWxmbXfvb2ZnA8eSeu+LgX7uvtzMrgG6A+8C3wCLov1HAGPcfW7WaToA77j70ujnfwK/\npQIrzHySKmN3/2PWj+2AZaS+4WMpURmvEfQ5rt/nuNBJ0DoCXaM3V9s2g4En3X20mbUCFpjZFHd/\nCaj+Swa4HZhsZm8CbYGL68g8cphZC+BwUr9cgNbAJHefYmZ7AT2B/aOYbwRONbPJwAmAkSrYF4h+\n0e5+bg2neQ1ob2bbAm8ABwGbx42xwiRRxphZG+AJUh/UE939u7gBlaiMAdY3s/uBrYHXgQuzPlxN\niT7HMT/HhVaYs9y9rp7v3YDOZnZi9PNKUgVU2x/e+cBbwP8CGwEzo4J5M885OpnZ9KyfJ7h7mLy8\nGfBctFwFbAv8K/rDaBXF0wmY6+4rAMzsmXxvyN2/MLOTgL+T+tacBcT+wFeYJMoYd/8S+B8z2waY\namZVdVyulbSMI5cA97v7V2Y2FBgO9Kljn0qkz3HMz3GhFeb3WcvZv/DsGahWkEqXX4x5zF8Do6IC\n/MTM5gJ7A/l+0em2jzriXAE87u5nZb9oZr2A7IbetesK0t0nAhOj/XuTulRpiopexmZ2KDDT3b9w\n9zfNbAGpMo7VvlVHnMUs49uyfhwL3FfXPhVKn2PifY6L2a1oGalLF0j9soIZpNoaMLOWZnazmeWr\nqBcD/xNt/3Ngt2hdMTwHHGJmraPj9zOzfUil7XuYWXMzWwfomu8gZra2mc00s03MbG3gVOCxIsXY\nmBWrjE8E/hBtvy6wK/BqkWIsVhnvZGYTo20BDiBzB7cp0+c4j2JWmCOBgWY2hVSKHAwCtosacJ8B\n5rv7KjPbzcz+WsNxBgM7ROn0dOAOd59vZpuZ2UOFBBh9O44CpkfxVAEL3H0hMB6YDTwEvBT2MbMR\nZrZnteP8QKoBfBowB3jK3acWEluFKFYZnwccaGZPk/ogjnD3hY2sjF8F5gKzo7/FKuDCQmKrEPoc\n51FRg2+Y2W3u3rfccUhyVMZNXyWXccU86WOpvlwTyh2HJEdl3PRVehlXVIYpIlJOFZNhioiUmypM\nEZGYVGGKiMSkClNEJCZVmCIiMf0/sRMwELw3k8YAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f5c294c7b00>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "zyINywCVubTT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model_json = model.to_json()\n",
        "with open(\"model.json\", \"w\") as json_file:\n",
        "    json_file.write(model_json)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JktOvZycudVA",
        "colab_type": "code",
        "outputId": "4e3d3084-48c6-444d-acea-64962ed75c60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MNIST_data  model.json\tsample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QlL620OswuJo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Tg6ZAalcwvde",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "with open(\"model.json\", \"r\") as json_file:\n",
        "  m = json.load(json_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Nul7MNhlxFAv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 18547
        },
        "outputId": "4b03639d-aa55-4697-a79b-3792a5ebdb40"
      },
      "cell_type": "code",
      "source": [
        "print(json.dumps(m, sort_keys=True, indent=2, separators=(',', ': ')))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{\n",
            "  \"backend\": \"tensorflow\",\n",
            "  \"class_name\": \"Model\",\n",
            "  \"config\": {\n",
            "    \"input_layers\": [\n",
            "      [\n",
            "        \"input_1\",\n",
            "        0,\n",
            "        0\n",
            "      ]\n",
            "    ],\n",
            "    \"layers\": [\n",
            "      {\n",
            "        \"class_name\": \"InputLayer\",\n",
            "        \"config\": {\n",
            "          \"batch_input_shape\": [\n",
            "            null,\n",
            "            28,\n",
            "            28,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"input_1\",\n",
            "          \"sparse\": false\n",
            "        },\n",
            "        \"inbound_nodes\": [],\n",
            "        \"name\": \"input_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 32,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"input_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 32,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d_1\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_1\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_1\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 32,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d_2\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d_2\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_2\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d_2\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_2\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_2\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_2\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_2\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Add\",\n",
            "        \"config\": {\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"add\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ],\n",
            "            [\n",
            "              \"activation_2\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"add\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"MaxPooling2D\",\n",
            "        \"config\": {\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"max_pooling2d\",\n",
            "          \"padding\": \"valid\",\n",
            "          \"pool_size\": [\n",
            "            2,\n",
            "            2\n",
            "          ],\n",
            "          \"strides\": [\n",
            "            2,\n",
            "            2\n",
            "          ],\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"add\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"max_pooling2d\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 64,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d_3\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"max_pooling2d\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d_3\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_3\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d_3\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_3\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_3\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_3\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_3\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 64,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d_4\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation_3\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d_4\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_4\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d_4\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_4\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_4\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_4\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_4\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Conv2D\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dilation_rate\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"dtype\": \"float32\",\n",
            "          \"filters\": 64,\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"kernel_size\": [\n",
            "            3,\n",
            "            3\n",
            "          ],\n",
            "          \"name\": \"conv2d_5\",\n",
            "          \"padding\": \"same\",\n",
            "          \"strides\": [\n",
            "            1,\n",
            "            1\n",
            "          ],\n",
            "          \"trainable\": true,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation_4\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"conv2d_5\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            3\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_5\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"conv2d_5\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_5\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_5\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_5\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_5\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Add\",\n",
            "        \"config\": {\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"add_1\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation_3\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ],\n",
            "            [\n",
            "              \"activation_5\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"add_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"MaxPooling2D\",\n",
            "        \"config\": {\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"max_pooling2d_1\",\n",
            "          \"padding\": \"valid\",\n",
            "          \"pool_size\": [\n",
            "            2,\n",
            "            2\n",
            "          ],\n",
            "          \"strides\": [\n",
            "            2,\n",
            "            2\n",
            "          ],\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"add_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"max_pooling2d_1\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Flatten\",\n",
            "        \"config\": {\n",
            "          \"data_format\": \"channels_last\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"flatten\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"max_pooling2d_1\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"flatten\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Dense\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"linear\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"name\": \"dense\",\n",
            "          \"trainable\": true,\n",
            "          \"units\": 512,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"flatten\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"dense\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"BatchNormalization\",\n",
            "        \"config\": {\n",
            "          \"axis\": [\n",
            "            1\n",
            "          ],\n",
            "          \"beta_constraint\": null,\n",
            "          \"beta_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"beta_regularizer\": null,\n",
            "          \"center\": true,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"epsilon\": 0.001,\n",
            "          \"gamma_constraint\": null,\n",
            "          \"gamma_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"gamma_regularizer\": null,\n",
            "          \"momentum\": 0.99,\n",
            "          \"moving_mean_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"moving_variance_initializer\": {\n",
            "            \"class_name\": \"Ones\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"name\": \"batch_normalization_6\",\n",
            "          \"scale\": true,\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"dense\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"batch_normalization_6\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Activation\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"relu\",\n",
            "          \"dtype\": \"float32\",\n",
            "          \"name\": \"activation_6\",\n",
            "          \"trainable\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"batch_normalization_6\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"activation_6\"\n",
            "      },\n",
            "      {\n",
            "        \"class_name\": \"Dense\",\n",
            "        \"config\": {\n",
            "          \"activation\": \"softmax\",\n",
            "          \"activity_regularizer\": null,\n",
            "          \"bias_constraint\": null,\n",
            "          \"bias_initializer\": {\n",
            "            \"class_name\": \"Zeros\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\"\n",
            "            }\n",
            "          },\n",
            "          \"bias_regularizer\": null,\n",
            "          \"dtype\": \"float32\",\n",
            "          \"kernel_constraint\": null,\n",
            "          \"kernel_initializer\": {\n",
            "            \"class_name\": \"GlorotUniform\",\n",
            "            \"config\": {\n",
            "              \"dtype\": \"float32\",\n",
            "              \"seed\": null\n",
            "            }\n",
            "          },\n",
            "          \"kernel_regularizer\": null,\n",
            "          \"name\": \"dense_1\",\n",
            "          \"trainable\": true,\n",
            "          \"units\": 10,\n",
            "          \"use_bias\": true\n",
            "        },\n",
            "        \"inbound_nodes\": [\n",
            "          [\n",
            "            [\n",
            "              \"activation_6\",\n",
            "              0,\n",
            "              0,\n",
            "              {}\n",
            "            ]\n",
            "          ]\n",
            "        ],\n",
            "        \"name\": \"dense_1\"\n",
            "      }\n",
            "    ],\n",
            "    \"name\": \"model\",\n",
            "    \"output_layers\": [\n",
            "      [\n",
            "        \"dense_1\",\n",
            "        0,\n",
            "        0\n",
            "      ]\n",
            "    ]\n",
            "  },\n",
            "  \"keras_version\": \"2.1.6-tf\"\n",
            "}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "OY5pVaSRb4Er",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "outputId": "f3fc308b-4013-49bf-c041-2fe461ff6239"
      },
      "cell_type": "code",
      "source": [
        "for l in m[\"config\"][\"layers\"] :\n",
        "  print(l)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'name': 'input_1', 'class_name': 'InputLayer', 'config': {'batch_input_shape': [None, 28, 28, 1], 'dtype': 'float32', 'sparse': False, 'name': 'input_1'}, 'inbound_nodes': []}\n",
            "{'name': 'conv2d', 'class_name': 'Conv2D', 'config': {'name': 'conv2d', 'trainable': True, 'dtype': 'float32', 'filters': 32, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['input_1', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d', 0, 0, {}]]]}\n",
            "{'name': 'activation', 'class_name': 'Activation', 'config': {'name': 'activation', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization', 0, 0, {}]]]}\n",
            "{'name': 'conv2d_1', 'class_name': 'Conv2D', 'config': {'name': 'conv2d_1', 'trainable': True, 'dtype': 'float32', 'filters': 32, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['activation', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_1', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_1', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d_1', 0, 0, {}]]]}\n",
            "{'name': 'activation_1', 'class_name': 'Activation', 'config': {'name': 'activation_1', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_1', 0, 0, {}]]]}\n",
            "{'name': 'conv2d_2', 'class_name': 'Conv2D', 'config': {'name': 'conv2d_2', 'trainable': True, 'dtype': 'float32', 'filters': 32, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['activation_1', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_2', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_2', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d_2', 0, 0, {}]]]}\n",
            "{'name': 'activation_2', 'class_name': 'Activation', 'config': {'name': 'activation_2', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_2', 0, 0, {}]]]}\n",
            "{'name': 'add', 'class_name': 'Add', 'config': {'name': 'add', 'trainable': True, 'dtype': 'float32'}, 'inbound_nodes': [[['activation', 0, 0, {}], ['activation_2', 0, 0, {}]]]}\n",
            "{'name': 'max_pooling2d', 'class_name': 'MaxPooling2D', 'config': {'name': 'max_pooling2d', 'trainable': True, 'dtype': 'float32', 'pool_size': [2, 2], 'padding': 'valid', 'strides': [2, 2], 'data_format': 'channels_last'}, 'inbound_nodes': [[['add', 0, 0, {}]]]}\n",
            "{'name': 'conv2d_3', 'class_name': 'Conv2D', 'config': {'name': 'conv2d_3', 'trainable': True, 'dtype': 'float32', 'filters': 64, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['max_pooling2d', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_3', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_3', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d_3', 0, 0, {}]]]}\n",
            "{'name': 'activation_3', 'class_name': 'Activation', 'config': {'name': 'activation_3', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_3', 0, 0, {}]]]}\n",
            "{'name': 'conv2d_4', 'class_name': 'Conv2D', 'config': {'name': 'conv2d_4', 'trainable': True, 'dtype': 'float32', 'filters': 64, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['activation_3', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_4', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_4', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d_4', 0, 0, {}]]]}\n",
            "{'name': 'activation_4', 'class_name': 'Activation', 'config': {'name': 'activation_4', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_4', 0, 0, {}]]]}\n",
            "{'name': 'conv2d_5', 'class_name': 'Conv2D', 'config': {'name': 'conv2d_5', 'trainable': True, 'dtype': 'float32', 'filters': 64, 'kernel_size': [3, 3], 'strides': [1, 1], 'padding': 'same', 'data_format': 'channels_last', 'dilation_rate': [1, 1], 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['activation_4', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_5', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_5', 'trainable': True, 'dtype': 'float32', 'axis': [3], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['conv2d_5', 0, 0, {}]]]}\n",
            "{'name': 'activation_5', 'class_name': 'Activation', 'config': {'name': 'activation_5', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_5', 0, 0, {}]]]}\n",
            "{'name': 'add_1', 'class_name': 'Add', 'config': {'name': 'add_1', 'trainable': True, 'dtype': 'float32'}, 'inbound_nodes': [[['activation_3', 0, 0, {}], ['activation_5', 0, 0, {}]]]}\n",
            "{'name': 'max_pooling2d_1', 'class_name': 'MaxPooling2D', 'config': {'name': 'max_pooling2d_1', 'trainable': True, 'dtype': 'float32', 'pool_size': [2, 2], 'padding': 'valid', 'strides': [2, 2], 'data_format': 'channels_last'}, 'inbound_nodes': [[['add_1', 0, 0, {}]]]}\n",
            "{'name': 'flatten', 'class_name': 'Flatten', 'config': {'name': 'flatten', 'trainable': True, 'dtype': 'float32', 'data_format': 'channels_last'}, 'inbound_nodes': [[['max_pooling2d_1', 0, 0, {}]]]}\n",
            "{'name': 'dense', 'class_name': 'Dense', 'config': {'name': 'dense', 'trainable': True, 'dtype': 'float32', 'units': 512, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['flatten', 0, 0, {}]]]}\n",
            "{'name': 'batch_normalization_6', 'class_name': 'BatchNormalization', 'config': {'name': 'batch_normalization_6', 'trainable': True, 'dtype': 'float32', 'axis': [1], 'momentum': 0.99, 'epsilon': 0.001, 'center': True, 'scale': True, 'beta_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'gamma_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'moving_mean_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'moving_variance_initializer': {'class_name': 'Ones', 'config': {'dtype': 'float32'}}, 'beta_regularizer': None, 'gamma_regularizer': None, 'beta_constraint': None, 'gamma_constraint': None}, 'inbound_nodes': [[['dense', 0, 0, {}]]]}\n",
            "{'name': 'activation_6', 'class_name': 'Activation', 'config': {'name': 'activation_6', 'trainable': True, 'dtype': 'float32', 'activation': 'relu'}, 'inbound_nodes': [[['batch_normalization_6', 0, 0, {}]]]}\n",
            "{'name': 'dense_1', 'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 10, 'activation': 'softmax', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None, 'dtype': 'float32'}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {'dtype': 'float32'}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'inbound_nodes': [[['activation_6', 0, 0, {}]]]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "IQcFXpCEoABM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import islpy as isl\n",
        "import copy\n",
        "\n",
        "def PrintCode(Schedule) :\n",
        "  context = isl.Set(\"{ : }\")\n",
        "  build = isl.AstBuild.from_context(context)\n",
        "  if isinstance(Schedule, isl.Schedule):\n",
        "    ast = build.node_from_schedule(Schedule)\n",
        "  else:\n",
        "    ast = build.node_from_schedule_map(Schedule)\n",
        "\n",
        "  opts = isl.AstPrintOptions.alloc(isl.DEFAULT_CONTEXT)\n",
        "  #opts, cb_print_user_handle = opts.set_print_user(cb_print_user)\n",
        "  #opts, cb_print_for_handle = opts.set_print_for(cb_print_for)\n",
        "\n",
        "  printer = isl.Printer.to_str(isl.DEFAULT_CONTEXT)\n",
        "  printer = printer.set_output_format(isl.format.C)\n",
        "  #printer.print_str(\"// Start\\n\")\n",
        "  printer = ast.print_(printer, opts)\n",
        "  print(printer.get_str())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yISLf5lOk_sV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2315
        },
        "outputId": "e7a9a50f-e483-44c1-c09d-2fee1df7cbe1"
      },
      "cell_type": "code",
      "source": [
        "batchsize = 128\n",
        "#build a statement for each layers\n",
        "\n",
        "OutputShapes = {}\n",
        "\n",
        "TotalSchedule = None\n",
        "\n",
        "def appendSchedule(Schedule):\n",
        "  global TotalSchedule\n",
        "  if not TotalSchedule:\n",
        "    TotalSchedule = Schedule\n",
        "  else:\n",
        "    TotalSchedule = TotalSchedule.sequence(Schedule)\n",
        "    \n",
        "def createScheduleForElementwiseLayer(Name, StmtName, Shape) :\n",
        "  space = isl.Space.set_alloc(isl.DEFAULT_CONTEXT, 0, len(Shape)).set_tuple_name(isl.dim_type.set, \"{StmtName}_{Name}\".format(StmtName=StmtName, Name=Name))\n",
        "  LS = isl.LocalSpace.from_space(space)\n",
        "  Stmt = isl.Set.universe(space)\n",
        "  for idx, val in enumerate(Shape):\n",
        "    #LB\n",
        "    c = isl.Constraint.alloc_inequality(LS)\n",
        "    c = c.set_coefficient_val(isl.dim_type.set, idx, 1)\n",
        "    c = c.set_constant_val(0)\n",
        "    Stmt = Stmt.add_constraint(c)\n",
        "    \n",
        "    #UB\n",
        "    c = isl.Constraint.alloc_inequality(LS)\n",
        "    c = c.set_coefficient_val(isl.dim_type.set, idx, -1)\n",
        "    c = c.set_constant_val(val - 1)\n",
        "    Stmt = Stmt.add_constraint(c)\n",
        "    \n",
        "  Schedule = isl.Schedule.from_domain(Stmt)\n",
        "  #Add loops\n",
        "  for idx in reversed(range(len(Shape))):\n",
        "    a = isl.Aff.zero_on_domain(LS)\n",
        "    a = a.set_coefficient_val(isl.dim_type.in_, idx, 1)\n",
        "    Schedule = Schedule.insert_partial_schedule(isl.MultiUnionPwAff.from_multi_aff(isl.MultiAff.from_aff(a)))\n",
        "\n",
        "  return Schedule\n",
        "\n",
        "def getInputName(l) :\n",
        "  return l[\"inbound_nodes\"][0][0][0]\n",
        "\n",
        "def getOutputShape(name) :\n",
        "  return OutputShapes.get(name, [batchsize, 0]).copy()\n",
        "\n",
        "def buildStmtForDense(l) :\n",
        "  name = l[\"name\"]\n",
        "  units = l[\"config\"][\"units\"]\n",
        "  #Generate statement with 3d loops\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShape = InputShape.copy()\n",
        "  OutputShape[-1] = units\n",
        "  OutputShapes[name] = OutputShape\n",
        "\n",
        "\n",
        "  #i,j,k -> N,M,L\n",
        "  N = OutputShape[0]\n",
        "  M = OutputShape[1]\n",
        "  L = InputShape[-1]\n",
        "\n",
        "  WeightShape = [L, M]\n",
        "\n",
        "  MulStmt = isl.Set(\"{{Mult_{Name}[i, j, k] : 0 <= i < {N} and 0 <= j < {M} and 0 <= k < {L}}}\".format(N=N,M=M,L=L,Name=name))\n",
        "  RedStmt = isl.Set(\"{{Red_{Name}[i, j] : 0 <= i < {N} and 0 <= j < {M}}}\".format(N=N,M=M,Name=name))\n",
        "\n",
        "  MulSchedule = isl.Schedule.from_domain(MulStmt)\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[i, j, k] -> [(k)] }}]\".format(Name=name)))\n",
        "  RedSchedule = isl.Schedule.from_domain(RedStmt)\n",
        "  Schedule = MulSchedule.sequence(RedSchedule)\n",
        "  Schedule = Schedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[i, j] -> [(j)]; Mult_{Name}[i, j, k] -> [(j)]}}]\".format(Name=name)))\n",
        "  Schedule = Schedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[i, j] -> [(i)]; Mult_{Name}[i, j, k] -> [(i)]}}]\".format(Name=name)))\n",
        "  appendSchedule(Schedule)\n",
        "  \n",
        "def buildStmtForInput(l) :\n",
        "  name = l[\"name\"]\n",
        "  shape = l[\"config\"][\"batch_input_shape\"]\n",
        "  shape[0] = batchsize\n",
        "  OutputShapes[name] = shape\n",
        "\n",
        "def buildStmtForConv2D(l) :\n",
        "  name = l[\"name\"]\n",
        "  filters = l[\"config\"][\"filters\"]\n",
        "  kernel_size = l[\"config\"][\"kernel_size\"]\n",
        "  strides = l[\"config\"][\"strides\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShape = InputShape.copy()\n",
        "  OutputShape[-1] = filters\n",
        "  OutputShapes[name] = OutputShape\n",
        "  #print(InputShape, \"->\", OutputShape)\n",
        "  \n",
        "  CO = filters\n",
        "  CI = InputShape[-1]\n",
        "  W = InputShape[-3]\n",
        "  H = InputShape[-2]\n",
        "  KW = kernel_size[0]\n",
        "  KH = kernel_size[1]\n",
        "  #https://towardsdatascience.com/intuitively-understanding-convolutions-for-deep-learning-1f6f42faee1\n",
        "  #foreach input in batch - b\n",
        "  #  foreach channel in output - co\n",
        "  #    foreach channel in input - ci\n",
        "  #      for W in channel - i\n",
        "  #        for H in channel - j\n",
        "  #          for KW in kernel -ki\n",
        "  #            for KH in kernel -kj\n",
        "  #              ElementwiseMult\n",
        "  #    for W in channel - i\n",
        "  #      for H in channel - j\n",
        "  #        Reduce on input channel   \n",
        "  MulStmt = isl.Set(\"{{Mult_{Name}[b, co, ci, i, j, ki, kj] : 0 <= b < {BatchSize} and 0 <= co < {CO} and 0 <= ci < {CI} and 0 <= i < {W} and 0 <= j < {H} and 0 <= ki < {KW} and 0 <= kj < {KH}}}\".format(BatchSize=batchsize,CO=CO,CI=CI,W=W,H=H,KW=KW,KH=KH,Name=name))\n",
        "  RedStmt = isl.Set(\"{{Red_{Name}[b, co, i, j] : 0 <= b < {BatchSize} and 0 <= co < {CO} and 0 <= i < {W} and 0 <= j < {H} }}\".format(BatchSize=batchsize,CO=CO,W=W,H=H,Name=name))\n",
        "\n",
        "  MulSchedule = isl.Schedule.from_domain(MulStmt)\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(kj)] }}]\".format(Name=name)))\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(ki)] }}]\".format(Name=name)))\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(j)] }}]\".format(Name=name)))\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(i)] }}]\".format(Name=name)))\n",
        "  MulSchedule = MulSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(ci)] }}]\".format(Name=name)))\n",
        "\n",
        "  RedSchedule = isl.Schedule.from_domain(RedStmt)\n",
        "  RedSchedule = RedSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[b, co, i, j] -> [(j)]}}]\".format(Name=name)))\n",
        "  RedSchedule = RedSchedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[b, co, i, j] -> [(i)]}}]\".format(Name=name)))\n",
        "\n",
        "  Schedule = MulSchedule.sequence(RedSchedule)\n",
        "  Schedule = Schedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[b, co, i, j] -> [(co )]; Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(co)]}}]\".format(Name=name)))\n",
        "  Schedule = Schedule.insert_partial_schedule(isl.MultiUnionPwAff(\"[{{Red_{Name}[b, co, i, j] -> [(b)]; Mult_{Name}[b, co, ci, i, j, ki, kj] -> [(b)]}}]\".format(Name=name)))\n",
        "  appendSchedule(Schedule)\n",
        "\n",
        "  \n",
        "def buildStmtForBatchNormalization(l) :\n",
        "  name = l[\"name\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShapes[name] = InputShape\n",
        "  #appendSchedule(createScheduleForElementwiseLayer(name, \"BN\", InputShape))\n",
        "\n",
        "def buildStmtForActivation(l) :\n",
        "  name = l[\"name\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShapes[name] = InputShape\n",
        "  appendSchedule(createScheduleForElementwiseLayer(name, \"Act\", InputShape))\n",
        "    \n",
        "def buildStmtForAdd(l) :\n",
        "  name = l[\"name\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShapes[name] = InputShape\n",
        "  #appendSchedule(createScheduleForElementwiseLayer(name, \"Add\", InputShape))\n",
        "\n",
        "def buildStmtForFlatten(l) :\n",
        "  name = l[\"name\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  while (len(InputShape) > 2) :\n",
        "    last = InputShape.pop(-1)\n",
        "    InputShape[-1] *= last\n",
        "  OutputShapes[name] = InputShape\n",
        "  appendSchedule(createScheduleForElementwiseLayer(name, \"Flatten\", InputShape))\n",
        "\n",
        "  \n",
        "def buildStmtForMaxPooling2D(l) :\n",
        "  name = l[\"name\"]\n",
        "  pool_size = l[\"config\"][\"pool_size\"]\n",
        "  strides = l[\"config\"][\"strides\"]\n",
        "  InputName = getInputName(l)\n",
        "  InputShape = getOutputShape(InputName)\n",
        "  OutputShape = InputShape.copy()\n",
        "  OutputShape[-3 + 0] //= strides[0]\n",
        "  OutputShape[-3 + 1] //= strides[1]\n",
        "  OutputShapes[name] = OutputShape\n",
        "  appendSchedule(createScheduleForElementwiseLayer(name, \"MaxPool\", OutputShape))\n",
        "\n",
        "  \n",
        "for l in m.copy()[\"config\"][\"layers\"] :\n",
        "  class_name = l[\"class_name\"]\n",
        "  if class_name == \"Conv2D\" :\n",
        "    buildStmtForConv2D(l)\n",
        "  elif class_name == \"Activation\" :\n",
        "    buildStmtForActivation(l)\n",
        "  elif class_name == \"Dense\" :\n",
        "    buildStmtForDense(l)\n",
        "  elif class_name == \"MaxPooling2D\" :\n",
        "    buildStmtForMaxPooling2D(l)\n",
        "  elif class_name == \"Flatten\" :\n",
        "    buildStmtForFlatten(l)\n",
        "  elif class_name == \"BatchNormalization\" :\n",
        "    buildStmtForBatchNormalization(l)\n",
        "  elif class_name == \"Add\" :\n",
        "    buildStmtForAdd(l)\n",
        "  elif class_name == \"InputLayer\" :\n",
        "    buildStmtForInput(l)\n",
        "  else :\n",
        "    print(\"unhandled layer: %s\" % class_name)\n",
        "    \n",
        "print(\"Shapes\")\n",
        "print(OutputShapes)\n",
        "\n",
        "PrintCode(TotalSchedule)"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shapes\n",
            "{'input_1': [128, 28, 28, 1], 'conv2d': [128, 28, 28, 32], 'batch_normalization': [128, 28, 28, 32], 'activation': [128, 28, 28, 32], 'conv2d_1': [128, 28, 28, 32], 'batch_normalization_1': [128, 28, 28, 32], 'activation_1': [128, 28, 28, 32], 'conv2d_2': [128, 28, 28, 32], 'batch_normalization_2': [128, 28, 28, 32], 'activation_2': [128, 28, 28, 32], 'add': [128, 28, 28, 32], 'max_pooling2d': [128, 14, 14, 32], 'conv2d_3': [128, 14, 14, 64], 'batch_normalization_3': [128, 14, 14, 64], 'activation_3': [128, 14, 14, 64], 'conv2d_4': [128, 14, 14, 64], 'batch_normalization_4': [128, 14, 14, 64], 'activation_4': [128, 14, 14, 64], 'conv2d_5': [128, 14, 14, 64], 'batch_normalization_5': [128, 14, 14, 64], 'activation_5': [128, 14, 14, 64], 'add_1': [128, 14, 14, 64], 'max_pooling2d_1': [128, 7, 7, 64], 'flatten': [128, 3136], 'dense': [128, 512], 'batch_normalization_6': [128, 512], 'activation_6': [128, 512], 'dense_1': [128, 10]}\n",
            "{\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 31; c1 += 1) {\n",
            "      for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "        for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "          for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "            for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "              Mult_conv2d(c0, c1, 0, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "          Red_conv2d(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 27; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 31; c3 += 1)\n",
            "          Act_activation(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 31; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 31; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "          for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "            for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "              for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "                Mult_conv2d_1(c0, c1, c2, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "          Red_conv2d_1(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 27; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 31; c3 += 1)\n",
            "          Act_activation_1(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 31; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 31; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "          for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "            for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "              for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "                Mult_conv2d_2(c0, c1, c2, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 27; c3 += 1)\n",
            "          Red_conv2d_2(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 27; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 27; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 31; c3 += 1)\n",
            "          Act_activation_2(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 13; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 31; c3 += 1)\n",
            "          MaxPool_max_pooling2d(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 63; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 31; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "            for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "              for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "                Mult_conv2d_3(c0, c1, c2, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          Red_conv2d_3(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 13; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 63; c3 += 1)\n",
            "          Act_activation_3(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 63; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 63; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "            for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "              for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "                Mult_conv2d_4(c0, c1, c2, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          Red_conv2d_4(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 13; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 63; c3 += 1)\n",
            "          Act_activation_4(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 63; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 63; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "            for (int c5 = 0; c5 <= 2; c5 += 1)\n",
            "              for (int c6 = 0; c6 <= 2; c6 += 1)\n",
            "                Mult_conv2d_5(c0, c1, c2, c3, c4, c5, c6);\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "          Red_conv2d_5(c0, c1, c2, c3);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 13; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 63; c3 += 1)\n",
            "          Act_activation_5(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 6; c1 += 1)\n",
            "      for (int c2 = 0; c2 <= 6; c2 += 1)\n",
            "        for (int c3 = 0; c3 <= 63; c3 += 1)\n",
            "          MaxPool_max_pooling2d_1(c0, c1, c2, c3);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 3135; c1 += 1)\n",
            "      Flatten_flatten(c0, c1);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 511; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 3135; c2 += 1)\n",
            "        Mult_dense(c0, c1, c2);\n",
            "      Red_dense(c0, c1);\n",
            "    }\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 511; c1 += 1)\n",
            "      Act_activation_6(c0, c1);\n",
            "  for (int c0 = 0; c0 <= 127; c0 += 1)\n",
            "    for (int c1 = 0; c1 <= 9; c1 += 1) {\n",
            "      for (int c2 = 0; c2 <= 511; c2 += 1)\n",
            "        Mult_dense_1(c0, c1, c2);\n",
            "      Red_dense_1(c0, c1);\n",
            "    }\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "fKwOY3C6L2Xh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "f18e3a56-89e3-47a5-a872-a2d0c63c5ff1"
      },
      "cell_type": "code",
      "source": [
        "ScheduleDomain = TotalSchedule.get_domain()\n",
        "ScheduleMap = TotalSchedule.get_map()\n",
        "print(\"ScheduleMap %s:\" % ScheduleMap)"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ScheduleMap { Red_conv2d_3[b, co, i, j] -> [7, b, co, 1, i, j, 0, 0, 0]; Mult_conv2d_1[b, co, ci, i, j, ki, kj] -> [2, b, co, 0, ci, i, j, ki, kj]; MaxPool_max_pooling2d_1[i0, i1, i2, i3] -> [13, i0, i1, i2, i3, 0, 0, 0, 0]; Act_activation_1[i0, i1, i2, i3] -> [3, i0, i1, i2, i3, 0, 0, 0, 0]; Act_activation_3[i0, i1, i2, i3] -> [8, i0, i1, i2, i3, 0, 0, 0, 0]; Mult_conv2d_4[b, co, ci, i, j, ki, kj] -> [9, b, co, 0, ci, i, j, ki, kj]; Mult_conv2d_2[b, co, ci, i, j, ki, kj] -> [4, b, co, 0, ci, i, j, ki, kj]; MaxPool_max_pooling2d[i0, i1, i2, i3] -> [6, i0, i1, i2, i3, 0, 0, 0, 0]; Red_conv2d_4[b, co, i, j] -> [9, b, co, 1, i, j, 0, 0, 0]; Act_activation_6[i0, i1] -> [16, i0, i1, 0, 0, 0, 0, 0, 0]; Act_activation_2[i0, i1, i2, i3] -> [5, i0, i1, i2, i3, 0, 0, 0, 0]; Act_activation_4[i0, i1, i2, i3] -> [10, i0, i1, i2, i3, 0, 0, 0, 0]; Mult_conv2d_3[b, co, ci, i, j, ki, kj] -> [7, b, co, 0, ci, i, j, ki, kj]; Mult_conv2d_5[b, co, ci, i, j, ki, kj] -> [11, b, co, 0, ci, i, j, ki, kj]; Red_dense[i, j] -> [15, i, j, 1, 0, 0, 0, 0, 0]; Red_conv2d[b, co, i, j] -> [0, b, co, 1, i, j, 0, 0, 0]; Red_conv2d_2[b, co, i, j] -> [4, b, co, 1, i, j, 0, 0, 0]; Flatten_flatten[i0, i1] -> [14, i0, i1, 0, 0, 0, 0, 0, 0]; Mult_dense[i, j, k] -> [15, i, j, 0, k, 0, 0, 0, 0]; Act_activation_5[i0, i1, i2, i3] -> [12, i0, i1, i2, i3, 0, 0, 0, 0]; Red_dense_1[i, j] -> [17, i, j, 1, 0, 0, 0, 0, 0]; Mult_conv2d[b, co, ci, i, j, ki, kj] -> [0, b, co, 0, ci, i, j, ki, kj]; Red_conv2d_1[b, co, i, j] -> [2, b, co, 1, i, j, 0, 0, 0]; Mult_dense_1[i, j, k] -> [17, i, j, 0, k, 0, 0, 0, 0]; Act_activation[i0, i1, i2, i3] -> [1, i0, i1, i2, i3, 0, 0, 0, 0]; Red_conv2d_5[b, co, i, j] -> [11, b, co, 1, i, j, 0, 0, 0] }:\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "uIV_kM-jN0Sh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1992
        },
        "outputId": "d4bf228d-bdec-47db-e5ba-a5e3e21c7bb7"
      },
      "cell_type": "code",
      "source": [
        "RSet = ScheduleMap.range()\n",
        "transformation = RSet.identity()\n",
        "print(transformation)\n",
        "transformation = isl.UnionMap(\"\"\"{[i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8] : i0 < 1;\n",
        "                                  [1, i0, i1, i2, i3, 0, 0, 0, 0]->[0, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 1 and i0 < 3;\n",
        "                                  [3, i0, i1, i2, i3, 0, 0, 0, 0]->[2, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 3 and i0 < 5;\n",
        "                                  [5, i0, i1, i2, i3, 0, 0, 0, 0]->[4, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 5 and i0 < 8;\n",
        "                                  [8, i0, i1, i2, i3, 0, 0, 0, 0]->[7, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 8 and i0 < 10;\n",
        "                                  [10, i0, i1, i2, i3, 0, 0, 0, 0]->[9, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 10 and i0 < 12;\n",
        "                                  [12, i0, i1, i2, i3, 0, 0, 0, 0]->[11, i0, i3, 1, i1, i2, 1, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 12 and i0 < 16;\n",
        "                                  [16, i0, i1, 0, 0, 0, 0, 0, 0]->[15, i0, i1, 1, 0, 0, 0, 0, 0];\n",
        "                                  [i0, i1, i2, i3, i4, i5, i6, i7, i8]->[i0, i1, i2, i3, i4, i5, i6, i7, i8]: i0 > 16}\"\"\")\n",
        "print(transformation)\n",
        "transformed = ScheduleMap.apply_range(transformation)\n",
        "transformed = transformed.intersect_domain(ScheduleDomain)\n",
        "print(\"transformed with loop fusion %s:\" % transformed)\n",
        "PrintCode(transformed)"
      ],
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{ [11, i1, i2, 0, i4, i5, i6, i7, i8] -> [11, i1, i2, 0, i4, i5, i6, i7, i8]; [9, i1, i2, 0, i4, i5, i6, i7, i8] -> [9, i1, i2, 0, i4, i5, i6, i7, i8]; [7, i1, i2, 0, i4, i5, i6, i7, i8] -> [7, i1, i2, 0, i4, i5, i6, i7, i8]; [4, i1, i2, 0, i4, i5, i6, i7, i8] -> [4, i1, i2, 0, i4, i5, i6, i7, i8]; [2, i1, i2, 0, i4, i5, i6, i7, i8] -> [2, i1, i2, 0, i4, i5, i6, i7, i8]; [0, i1, i2, 0, i4, i5, i6, i7, i8] -> [0, i1, i2, 0, i4, i5, i6, i7, i8]; [11, i1, i2, 1, i4, i5, 0, 0, 0] -> [11, i1, i2, 1, i4, i5, 0, 0, 0]; [9, i1, i2, 1, i4, i5, 0, 0, 0] -> [9, i1, i2, 1, i4, i5, 0, 0, 0]; [7, i1, i2, 1, i4, i5, 0, 0, 0] -> [7, i1, i2, 1, i4, i5, 0, 0, 0]; [4, i1, i2, 1, i4, i5, 0, 0, 0] -> [4, i1, i2, 1, i4, i5, 0, 0, 0]; [2, i1, i2, 1, i4, i5, 0, 0, 0] -> [2, i1, i2, 1, i4, i5, 0, 0, 0]; [0, i1, i2, 1, i4, i5, 0, 0, 0] -> [0, i1, i2, 1, i4, i5, 0, 0, 0]; [13, i1, i2, i3, i4, 0, 0, 0, 0] -> [13, i1, i2, i3, i4, 0, 0, 0, 0]; [12, i1, i2, i3, i4, 0, 0, 0, 0] -> [12, i1, i2, i3, i4, 0, 0, 0, 0]; [10, i1, i2, i3, i4, 0, 0, 0, 0] -> [10, i1, i2, i3, i4, 0, 0, 0, 0]; [8, i1, i2, i3, i4, 0, 0, 0, 0] -> [8, i1, i2, i3, i4, 0, 0, 0, 0]; [6, i1, i2, i3, i4, 0, 0, 0, 0] -> [6, i1, i2, i3, i4, 0, 0, 0, 0]; [5, i1, i2, i3, i4, 0, 0, 0, 0] -> [5, i1, i2, i3, i4, 0, 0, 0, 0]; [3, i1, i2, i3, i4, 0, 0, 0, 0] -> [3, i1, i2, i3, i4, 0, 0, 0, 0]; [1, i1, i2, i3, i4, 0, 0, 0, 0] -> [1, i1, i2, i3, i4, 0, 0, 0, 0]; [17, i1, i2, 0, i4, 0, 0, 0, 0] -> [17, i1, i2, 0, i4, 0, 0, 0, 0]; [15, i1, i2, 0, i4, 0, 0, 0, 0] -> [15, i1, i2, 0, i4, 0, 0, 0, 0]; [17, i1, i2, 1, 0, 0, 0, 0, 0] -> [17, i1, i2, 1, 0, 0, 0, 0, 0]; [15, i1, i2, 1, 0, 0, 0, 0, 0] -> [15, i1, i2, 1, 0, 0, 0, 0, 0]; [16, i1, i2, 0, 0, 0, 0, 0, 0] -> [16, i1, i2, 0, 0, 0, 0, 0, 0]; [14, i1, i2, 0, 0, 0, 0, 0, 0] -> [14, i1, i2, 0, 0, 0, 0, 0, 0] }\n",
            "{ [i0, i1, i2, i3, i4, i5, i6, i7, i8] -> [i0, i1, i2, i3, i4, i5, i6, i7, i8] : i0 <= 0 or i0 >= 17 or (13 <= i0 <= 15) or (6 <= i0 <= 7); [i0 = 3, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [2, i1, i4, 1, i2, i3, 1, 0, 0]; [i0 = 1, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [0, i1, i4, 1, i2, i3, 1, 0, 0]; [i0 = 16, i1, i2, i3 = 0, i4 = 0, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [15, i1, i2, 1, 0, 0, 0, 0, 0]; [i0 = 11, i1, i2, i3, i4, i5, i6, i7, i8] -> [11, i1, i2, i3, i4, i5, i6, i7, i8]; [i0 = 9, i1, i2, i3, i4, i5, i6, i7, i8] -> [9, i1, i2, i3, i4, i5, i6, i7, i8]; [i0 = 4, i1, i2, i3, i4, i5, i6, i7, i8] -> [4, i1, i2, i3, i4, i5, i6, i7, i8]; [i0 = 2, i1, i2, i3, i4, i5, i6, i7, i8] -> [2, i1, i2, i3, i4, i5, i6, i7, i8]; [i0 = 12, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [11, i1, i4, 1, i2, i3, 1, 0, 0]; [i0 = 10, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [9, i1, i4, 1, i2, i3, 1, 0, 0]; [i0 = 8, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [7, i1, i4, 1, i2, i3, 1, 0, 0]; [i0 = 5, i1, i2, i3, i4, i5 = 0, i6 = 0, i7 = 0, i8 = 0] -> [4, i1, i4, 1, i2, i3, 1, 0, 0] }\n",
            "transformed with loop fusion { Red_conv2d_3[b, co, i, j] -> [7, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= i <= 13 and 0 <= j <= 13; Mult_conv2d_1[b, co, ci, i, j, ki, kj] -> [2, b, co, 0, ci, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= ci <= 31 and 0 <= i <= 27 and 0 <= j <= 27 and 0 <= ki <= 2 and 0 <= kj <= 2; MaxPool_max_pooling2d_1[i0, i1, i2, i3] -> [13, i0, i1, i2, i3, 0, 0, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 6 and 0 <= i2 <= 6 and 0 <= i3 <= 63; Act_activation_1[i0, i1, i2, i3] -> [2, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 27 and 0 <= i2 <= 27 and 0 <= i3 <= 31; Act_activation_3[i0, i1, i2, i3] -> [7, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 13 and 0 <= i2 <= 13 and 0 <= i3 <= 63; Mult_conv2d_4[b, co, ci, i, j, ki, kj] -> [9, b, co, 0, ci, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= ci <= 63 and 0 <= i <= 13 and 0 <= j <= 13 and 0 <= ki <= 2 and 0 <= kj <= 2; Mult_conv2d_2[b, co, ci, i, j, ki, kj] -> [4, b, co, 0, ci, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= ci <= 31 and 0 <= i <= 27 and 0 <= j <= 27 and 0 <= ki <= 2 and 0 <= kj <= 2; MaxPool_max_pooling2d[i0, i1, i2, i3] -> [6, i0, i1, i2, i3, 0, 0, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 13 and 0 <= i2 <= 13 and 0 <= i3 <= 31; Red_conv2d_4[b, co, i, j] -> [9, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= i <= 13 and 0 <= j <= 13; Act_activation_6[i0, i1] -> [15, i0, i1, 1, 0, 0, 0, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 511; Act_activation_2[i0, i1, i2, i3] -> [4, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 27 and 0 <= i2 <= 27 and 0 <= i3 <= 31; Act_activation_4[i0, i1, i2, i3] -> [9, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 13 and 0 <= i2 <= 13 and 0 <= i3 <= 63; Mult_conv2d_3[b, co, ci, i, j, ki, kj] -> [7, b, co, 0, ci, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= ci <= 31 and 0 <= i <= 13 and 0 <= j <= 13 and 0 <= ki <= 2 and 0 <= kj <= 2; Mult_conv2d_5[b, co, ci, i, j, ki, kj] -> [11, b, co, 0, ci, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= ci <= 63 and 0 <= i <= 13 and 0 <= j <= 13 and 0 <= ki <= 2 and 0 <= kj <= 2; Red_dense[i, j] -> [15, i, j, 1, 0, 0, 0, 0, 0] : 0 <= i <= 127 and 0 <= j <= 511; Red_conv2d[b, co, i, j] -> [0, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= i <= 27 and 0 <= j <= 27; Red_conv2d_2[b, co, i, j] -> [4, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= i <= 27 and 0 <= j <= 27; Flatten_flatten[i0, i1] -> [14, i0, i1, 0, 0, 0, 0, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 3135; Mult_dense[i, j, k] -> [15, i, j, 0, k, 0, 0, 0, 0] : 0 <= i <= 127 and 0 <= j <= 511 and 0 <= k <= 3135; Act_activation_5[i0, i1, i2, i3] -> [11, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 13 and 0 <= i2 <= 13 and 0 <= i3 <= 63; Red_dense_1[i, j] -> [17, i, j, 1, 0, 0, 0, 0, 0] : 0 <= i <= 127 and 0 <= j <= 9; Mult_conv2d[b, co, ci = 0, i, j, ki, kj] -> [0, b, co, 0, 0, i, j, ki, kj] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= i <= 27 and 0 <= j <= 27 and 0 <= ki <= 2 and 0 <= kj <= 2; Red_conv2d_1[b, co, i, j] -> [2, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 31 and 0 <= i <= 27 and 0 <= j <= 27; Mult_dense_1[i, j, k] -> [17, i, j, 0, k, 0, 0, 0, 0] : 0 <= i <= 127 and 0 <= j <= 9 and 0 <= k <= 511; Act_activation[i0, i1, i2, i3] -> [0, i0, i3, 1, i1, i2, 1, 0, 0] : 0 <= i0 <= 127 and 0 <= i1 <= 27 and 0 <= i2 <= 27 and 0 <= i3 <= 31; Red_conv2d_5[b, co, i, j] -> [11, b, co, 1, i, j, 0, 0, 0] : 0 <= b <= 127 and 0 <= co <= 63 and 0 <= i <= 13 and 0 <= j <= 13 }:\n",
            "{\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 31; c2 += 1) {\n",
            "      for (int c5 = 0; c5 <= 27; c5 += 1)\n",
            "        for (int c6 = 0; c6 <= 27; c6 += 1)\n",
            "          for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "            for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "              Mult_conv2d(c1, c2, 0, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 27; c5 += 1) {\n",
            "          Red_conv2d(c1, c2, c4, c5);\n",
            "          Act_activation(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 31; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 31; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 27; c5 += 1)\n",
            "          for (int c6 = 0; c6 <= 27; c6 += 1)\n",
            "            for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "              for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "                Mult_conv2d_1(c1, c2, c4, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 27; c5 += 1) {\n",
            "          Red_conv2d_1(c1, c2, c4, c5);\n",
            "          Act_activation_1(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 31; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 31; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 27; c5 += 1)\n",
            "          for (int c6 = 0; c6 <= 27; c6 += 1)\n",
            "            for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "              for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "                Mult_conv2d_2(c1, c2, c4, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 27; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 27; c5 += 1) {\n",
            "          Red_conv2d_2(c1, c2, c4, c5);\n",
            "          Act_activation_2(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 13; c2 += 1)\n",
            "      for (int c3 = 0; c3 <= 13; c3 += 1)\n",
            "        for (int c4 = 0; c4 <= 31; c4 += 1)\n",
            "          MaxPool_max_pooling2d(c1, c2, c3, c4);\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 63; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 31; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1)\n",
            "          for (int c6 = 0; c6 <= 13; c6 += 1)\n",
            "            for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "              for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "                Mult_conv2d_3(c1, c2, c4, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1) {\n",
            "          Red_conv2d_3(c1, c2, c4, c5);\n",
            "          Act_activation_3(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 63; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 63; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1)\n",
            "          for (int c6 = 0; c6 <= 13; c6 += 1)\n",
            "            for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "              for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "                Mult_conv2d_4(c1, c2, c4, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1) {\n",
            "          Red_conv2d_4(c1, c2, c4, c5);\n",
            "          Act_activation_4(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 63; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 63; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1)\n",
            "          for (int c6 = 0; c6 <= 13; c6 += 1)\n",
            "            for (int c7 = 0; c7 <= 2; c7 += 1)\n",
            "              for (int c8 = 0; c8 <= 2; c8 += 1)\n",
            "                Mult_conv2d_5(c1, c2, c4, c5, c6, c7, c8);\n",
            "      for (int c4 = 0; c4 <= 13; c4 += 1)\n",
            "        for (int c5 = 0; c5 <= 13; c5 += 1) {\n",
            "          Red_conv2d_5(c1, c2, c4, c5);\n",
            "          Act_activation_5(c1, c4, c5, c2);\n",
            "        }\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 6; c2 += 1)\n",
            "      for (int c3 = 0; c3 <= 6; c3 += 1)\n",
            "        for (int c4 = 0; c4 <= 63; c4 += 1)\n",
            "          MaxPool_max_pooling2d_1(c1, c2, c3, c4);\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 3135; c2 += 1)\n",
            "      Flatten_flatten(c1, c2);\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 511; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 3135; c4 += 1)\n",
            "        Mult_dense(c1, c2, c4);\n",
            "      Red_dense(c1, c2);\n",
            "      Act_activation_6(c1, c2);\n",
            "    }\n",
            "  for (int c1 = 0; c1 <= 127; c1 += 1)\n",
            "    for (int c2 = 0; c2 <= 9; c2 += 1) {\n",
            "      for (int c4 = 0; c4 <= 511; c4 += 1)\n",
            "        Mult_dense_1(c1, c2, c4);\n",
            "      Red_dense_1(c1, c2);\n",
            "    }\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}