11_feedforward_neural_network__housing__with_regularization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sklearn.datasets\n",
    "import tensorflow as tf\n",
    "\n",
    "# Prints numpy arrays nicer\n",
    "np.set_printoptions(precision=2, suppress=True, linewidth=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston_housing = sklearn.datasets.load_boston()\n",
    "# Uncomment the following line for a description of the dataset.\n",
    "# print(boston_housing['DESCR'])\n",
    "xs = boston_housing.data\n",
    "ys = boston_housing.target\n",
    "\n",
    "standardize_xs = True\n",
    "if standardize_xs:\n",
    "    xs = (xs - np.mean(xs, axis=0)) / np.std(xs, axis=0)\n",
    "ys = np.expand_dims(ys, -1)  # Adding the extra axis enables tf.matmul later\n",
    "\n",
    "data = list(zip(xs, ys))\n",
    "\n",
    "# Perform 60% / 40% training/test split\n",
    "split_index = int(len(data) * 0.6)\n",
    "train_data = data[:split_index]\n",
    "test_data = data[split_index:]\n",
    "print('Num training examples:', len(train_data))\n",
    "print('Num testing examples:', len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "learning_rate = 0.005\n",
    "num_epochs = 500\n",
    "num_features = len(train_data[0][0])\n",
    "num_hidden = 20\n",
    "l2regularization_weight = 0.05\n",
    "batch_size = 10\n",
    "\n",
    "# Model Definition\n",
    "batch_x = tf.placeholder(tf.float32, shape=[None, num_features])\n",
    "batch_y = tf.placeholder(tf.float32, shape=[None, 1])\n",
    "\n",
    "dense1 = tf.layers.Dense(num_hidden, activation=tf.nn.sigmoid)\n",
    "dense2 = tf.layers.Dense(1)\n",
    "\n",
    "h = dense1(batch_x)\n",
    "y_prediction = dense2(h)\n",
    "\n",
    "W1, b1 = dense1.weights\n",
    "W2, b2 = dense2.weights\n",
    "\n",
    "loss = (tf.losses.mean_squared_error(labels=batch_y, predictions=y_prediction)\n",
    "        + l2regularization_weight * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2)))\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # Training\n",
    "    time_before = time.time()\n",
    "    losses = []  # Storing losses so we can plot them later\n",
    "    for epoch in range(num_epochs):\n",
    "        np.random.shuffle(train_data)\n",
    "        cumulative_loss = 0\n",
    "        for i in range(0, len(train_data), batch_size):\n",
    "            _batch_x, _batch_y = zip(*train_data[i:i + batch_size])\n",
    "            _loss, _train_op = sess.run(\n",
    "                (loss, train_op),\n",
    "                feed_dict={batch_x: _batch_x, batch_y: _batch_y})\n",
    "            cumulative_loss += _loss * len(_batch_x)\n",
    "        average_loss = cumulative_loss / len(train_data)\n",
    "        if epoch % 25 == 24:\n",
    "            print('Epoch: {}, Loss: {}'.format(epoch + 1, average_loss))\n",
    "        losses.append(average_loss)\n",
    "    time_after = time.time()\n",
    "    print('Training took {:.2f}s.'.format(time_after - time_before))\n",
    "\n",
    "    # Prediction\n",
    "    train_xs, train_ys = zip(*train_data)\n",
    "    train_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: train_xs})\n",
    "\n",
    "    test_xs, test_ys = zip(*test_data)\n",
    "    test_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: test_xs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=150)\n",
    "plt.title('Loss over Time')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.plot(range(len(losses)), losses, color='#458588')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mean_squared_error = np.mean((train_ys - train_ys_prediction) ** 2)\n",
    "test_mean_squared_error = np.mean((test_ys - test_ys_prediction) ** 2)\n",
    "\n",
    "print('Mean Squared Error on Training data:', train_mean_squared_error)\n",
    "print('Mean Squared Error on Testing data:', test_mean_squared_error)\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Actual vs Predicted Data Points: Training Set')\n",
    "plt.xlabel('Actual House Price')\n",
    "plt.ylabel('Predicted House Price')\n",
    "plt.plot([min(train_ys), max(train_ys)], [min(train_ys), max(train_ys)],\n",
    "         color='#1D2021', linestyle='--')\n",
    "plt.scatter(train_ys, train_ys_prediction, color='#458588')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Actual vs Predicted Data Points: Testing Set')\n",
    "plt.xlabel('Actual House Price')\n",
    "plt.ylabel('Predicted House Price')\n",
    "plt.plot([min(test_ys), max(test_ys)], [min(test_ys), max(test_ys)],\n",
    "         color='#1D2021', linestyle='--')\n",
    "plt.scatter(test_ys, test_ys_prediction, color='#CC241D')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}