Added NN functions

added the nnCostFunction, nnGradFunction, and sigmoid gradient. I used numpy's 'np.random.rand' function to initialize thetas. Still need to complete the nnCheckGradients, debugInitializeWeights, and fmincg

Author: manvillej

Diff for: __pycache__/ex3helper.cpython-36.pyc

@@ -0,0 +1,148 @@
+## Machine Learning Online Class - Exercise 4 Neural Network Learning
+
+#  Instructions
+#  ------------
+# 
+#  This file contains code that helps you get started on the
+#  linear exercise. You will need to complete the following functions 
+#  in this exericse:
+#
+#     sigmoidGradient.m
+#     randInitializeWeights.m
+#     nnCostFunction.m
+#
+#  For this exercise, you will not need to change any code in this file,
+#  or any other files other than those mentioned above.
+#
+
+## Initialization
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.io as io
+import ex3helper as helper3
+import ex4helper as helper
+
+## Setup the parameters you will use for this exercise
+inputLayerSize  = 400;  # 20x20 Input Images of Digits
+hiddenLayerSize = 25;   # 25 hidden units
+numLabels = 10;         # 10 labels, from 1 to 10   
+                        # (note that we have mapped "0" to label 10)
+
+## =========== Part 1: Loading and Visualizing Data =============
+#  We start the exercise by first loading and visualizing the dataset. 
+#  You will be working with a dataset that contains handwritten digits.
+#
+
+# Load Training Data
+print('Loading and Visualizing Data ...')
+mat = io.loadmat('./data/ex4data1.mat')
+X = mat['X']
+y = np.squeeze(mat['y'])
+
+m = X.shape[0]
+
+# Randomly select 100 data points to display
+perm = np.random.permutation(m)
+sel = X[perm[0:100],:]
+
+helper3.displayData(sel)
+
+input('\nPart 1 completed. Program paused. Press enter to continue: ')
+
+
+## ================ Part 2: Loading Parameters ================
+# In this part of the exercise, we load some pre-initialized 
+# neural network parameters.
+
+print('\nLoading Saved Neural Network Parameters ...')
+
+mat = io.loadmat('./data/ex4weights.mat')
+
+theta1 = mat['Theta1']
+theta2 = mat['Theta2']
+
+nnParams = np.array([theta1.flatten(), theta2.flatten()])
+
+
+## ================ Part 3: Compute Cost (Feedforward) ================
+#  To the neural network, you should first start by implementing the
+#  feedforward part of the neural network that returns the cost only. You
+#  should complete the code in nnCostFunction.m to return cost. After
+#  implementing the feedforward to compute the cost, you can verify that
+#  your implementation is correct by verifying that you get the same cost
+#  as us for the fixed debugging parameters.
+#
+#  We suggest implementing the feedforward cost *without* regularization
+#  first so that it will be easier for you to debug. Later, in part 4, you
+#  will get to implement the regularized cost.
+#
+print('Feedforward Using Neural Network ...')
+
+# Weight regularization parameter (we set this to 0 here).
+lambdaVal = 0
+
+J = helper.nnCostFunction(nnParams, X, y, lambdaVal)
+
+print('Cost at parameters (loaded from ex4weights): {:.6f}'.format(J))
+print('this value should be approx: 0.287629')
+
+input('\nPart 2 & 3 completed. Program paused. Press enter to continue: ')
+
+## =============== Part 4: Implement Regularization ===============
+#  Once your cost function implementation is correct, you should now
+#  continue to implement the regularization with the cost.
+#
+
+print('\nChecking Cost Function (w/ Regularization) ... ')
+
+# Weight regularization parameter (we set this to 1 here).
+lambdaVal = 1
+
+J = helper.nnCostFunction(nnParams, X, y, lambdaVal)
+
+print('Cost at parameters (loaded from ex4weights): {:.6f}'.format(J))
+print('this value should be approx: 0.383770')
+
+input('\nPart 4 completed. Program paused. Press enter to continue: ')
+
+
+## ================ Part 5: Sigmoid Gradient  ================
+#  Before you start implementing the neural network, you will first
+#  implement the gradient for the sigmoid function. You should complete the
+#  code in the sigmoidGradient.m file.
+#
+
+print('\nEvaluating sigmoid gradient...')
+
+g = helper.sigmoidGradient(np.array([-1, -0.5, 0, 0.5, 1]))
+print('Sigmoid gradient evaluated at [-1 -0.5 0 0.5 1]:');
+print(g);
+
+input('\nPart 5 completed. Program paused. Press enter to continue: ')
+
+## ================ Part 6: Initializing Pameters ================
+#  In this part of the exercise, you will be starting to implment a two
+#  layer neural network that classifies digits. You will start by
+#  implementing a function to initialize the weights of the neural network
+#  (randInitializeWeights.m)
+
+print('\nInitializing Neural Network Parameters ...')
+
+initialTheta1 = np.random.rand(inputLayerSize + 1, hiddenLayerSize)
+initialTheta2 = np.random.rand(hiddenLayerSize + 1, num_labels)
+
+# Unroll parameters
+initialNNParams = np.array([initialTheta1.flatten(), initialTheta2.flatten()])
+
+## =============== Part 7: Implement Backpropagation ===============
+#  Once your cost matches up with ours, you should proceed to implement the
+#  backpropagation algorithm for the neural network. You should add to the
+#  code you've written in nnCostFunction.m to return the partial
+#  derivatives of the parameters.
+#
+print('\nChecking Backpropagation... ')
+
+#Check gradients by running checkNNGradients
+#helper.checkNNGradients()
+
+#input('\nPart 6 & 7 completed. Program paused. Press enter to continue: ')

Diff for: __pycache__/ex4helper.cpython-36.pyc

@@ -0,0 +1,94 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.optimize as op
+import ex2helper as helper
+import math
+import matplotlib.image as mpimg
+
+def nnCostFunction(nnParams, X, y, lambdaVal):
+	m = X.shape[0]
+	print(m)
+	theta1 = nnParams[0]
+	n1 = X.shape[1] + 1
+	n2 = theta1.shape[0]
+	theta1 = theta1.reshape(int(n2/n1),n1)
+	theta2 = nnParams[1]
+	n1 = theta1.shape[0] + 1
+	n2 = theta2.shape[0]
+	theta2 = theta2.reshape(int(n2/n1),n1)
+	
+
+	#prepare Y matrix for cost function
+	numLabels = np.unique(y).shape[0]+1
+	#create boolean array of value or not out of 1s and 0s
+	Y = (y==1).astype(int)
+	for i in range(2,numLabels):
+		Y = np.append(Y,(y==i).astype(int))
+	#reshape so first dimension corresponds with label
+	Y = Y.reshape(10,5000)
+
+	X = np.insert(X,0,np.ones(X.shape[0]),axis=1) # adding bias unit
+	h1 = helper.sigmoid(np.matmul(X,theta1.transpose()))
+	h1 = np.insert(h1,0,np.ones(h1.shape[0]),axis=1) # adding bias unit
+	h2 = helper.sigmoid(np.matmul(h1,theta2.transpose())).transpose()
+
+	#getting regulation parameters
+	R1 = theta1[:,1:]
+	R2 = theta2[:,1:]
+	# calculating the cost of regulation
+	costRegulation = lambdaVal*(np.sum(np.square(R1.flatten())) + np.sum(np.square(R2.flatten())))/(2*m)
+	
+	#calculating true cost without regulation
+	cost = np.sum(np.multiply(np.log(h2),Y)) + np.sum(np.multiply(np.log(1-h2),1-Y))
+	cost = -cost/m
+
+	#calculate total cost
+	totalCost = cost + costRegulation
+
+	return totalCost
+
+
+def nnGradFunction(nnParams, X, y, lambdaVal):
+	m = X.shape[0]
+	theta1 = nnParams[0]
+	n1 = X.shape[1] + 1
+	n2 = theta1.shape[0]
+	theta1 = theta1.reshape(int(n2/n1),n1)
+	theta2 = nnParams[1]
+	n1 = theta1.shape[0] + 1
+	n2 = theta2.shape[0]
+	theta2 = theta2.reshape(int(n2/n1),n1)
+	
+
+	#prepare Y matrix for cost function
+	numLabels = np.unique(y).shape[0]+1
+	#create boolean array of value or not out of 1s and 0s
+	Y = (y==1).astype(int)
+	for i in range(2,numLabels):
+		Y = np.append(Y,(y==i).astype(int))
+	#reshape so first dimension corresponds with label
+	Y = Y.reshape(10,5000)
+
+	X = np.insert(X,0,np.ones(X.shape[0]),axis=1) # adding bias unit
+	h1 = helper.sigmoid(np.matmul(X,theta1.transpose()))
+	h1 = np.insert(h1,0,np.ones(h1.shape[0]),axis=1) # adding bias unit
+	h2 = helper.sigmoid(np.matmul(h1,theta2.transpose())).transpose()
+
+
+	#calculate gradients
+	theta2Error = h2-Y
+	theta1Error = np.multiply(np.matmul(theta2Error.transpose(),theta2),np.multiply(h1,1-h1))
+	theta1Grad = np.matmul(theta1Error.transpose(),X)
+	theta1Grad = theta1Grad[1:,:]#drop bias unit error from hiddent layer
+	theta2Grad = np.matmul(theta2Error,h1)
+	
+
+	return np.array([theta1Grad.flatten(), theta2Grad.flatten()])
+
+
+def sigmoidGradient(Z):
+	R = helper.sigmoid(Z)
+	return np.multiply(R,1-R)
+
+
+