Merge pull request manvillej#3 from manvillej/Ex3Multiclass&NeuralNets1

Ex3 multiclass&neural nets1

Author: manvillej

Diff for: README.md

@@ -2,7 +2,7 @@
 Python implementation of Andrew Ng's ML course projects
  - Ex1 (Linear Regression) = Complete
  - Ex2 (Logistic Regression & Regulation) = Complete
- - Ex3 = Incomplete
+ - Ex3 (MultiClass LR and Neural Network Prediction) = Complete
  - Ex4 = Incomplete
  - Ex5 = Incomplete
  - Ex6 = Incomplete

Diff for: __pycache__/ex2helper.cpython-36.pyc

@@ -40,8 +40,10 @@
 
 print('\nPlotting data with \'o\' indicating (y = 1) examples and \'x\' indicating (y = 0) examples.')
 
-helper.plotData(x,y,'Exam Score 1', 'Exam Score 2')
-
+helper.plotData(x,y)
+plt.xlabel('Exam Score 1')
+plt.ylabel('Exam Score 2')
+plt.show()
 
 input('\nPart 1 completed. Program paused. Press enter to continue: ')
 ## ============ Part 2: Compute Cost and Gradient ============
@@ -95,7 +97,10 @@
 print(theta)
 print('Expected theta (approx):')
 print('[ -25.161  0.206  0.201]')
-helper.plotDecisionBoundary(theta,x,y,'Exam Score 1', 'Exam Score 2')
+helper.plotDecisionBoundary(theta,x,y)
+plt.xlabel('Exam Score 1')
+plt.ylabel('Exam Score 2')
+plt.show()
 
 input('\nPart 3 completed. Program paused. Press enter to continue: ')
 

Diff for: __pycache__/ex3helper.cpython-36.pyc

@@ -98,6 +98,9 @@
 lambdaVal = 1
 
 results = helper.optimizeReg(theta,x,y,lambdaVal)
+print(x.shape)
+print(theta.shape)
+print(y.shape)
 theta = results.x
 cost = results.fun
 

Diff for: data/ex3data1.mat

@@ -73,8 +73,11 @@ def mapFeatures(X):
 	return mapped_X
 
 def costFunctionReg(theta, x, y, lambdaVal):
-
+	if(y.ndim>1):
+		y = np.squeeze(y)
 	m = x.shape[0]
+	if(y.shape[0]!=m):
+		raise ValueError('Y & X are not compatible: X.shape = {} &  y.shape = {}'.format(X.shape, y.shape))
 
 	z = sigmoid(np.matmul(x,theta))
 
@@ -91,8 +94,12 @@ def costFunctionReg(theta, x, y, lambdaVal):
 
 
 def gradientReg(theta, x, y, lambdaVal):
+	if(y.ndim>1):
+		y = np.squeeze(y)
 	m = x.shape[0]
-
+	if(y.shape[0]!=m):
+		raise ValueError('Y & X are not compatible: X.shape = {} &  y.shape = {}'.format(X.shape, y.shape))
+	
 	z = sigmoid(np.matmul(x,theta))
 
 	grad = np.matmul(x.transpose(),np.subtract(z,y))/m

Diff for: data/ex3weights.mat

@@ -0,0 +1,98 @@
+## Machine Learning Online Class - Exercise 3 | Part 1: One-vs-all
+
+#  Instructions
+#  ------------
+#
+#  This file contains code that helps you get started on the
+#  linear exercise. You will need to complete the following functions
+#  in this exericse:
+#
+#     lrCostFunction (logistic regression cost function) - complete
+#     oneVsAll - complete
+#     predictOneVsAll - complete
+#     predict - complated
+#
+#  For this exercise, you will not need to change any code in this file,
+#  or any other files other than those mentioned above.
+#
+
+## Initialization
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.io as io
+import ex2helper as helper2
+import ex3helper as helper
+
+## Setup the parameters you will use for this part of the exercise
+input_layer_size  = 400;  # 20x20 Input Images of Digits
+num_labels = 10;          # 10 labels, from 1 to 10
+                          # (note that we have mapped "0" to label 10)
+
+## =========== Part 1: Loading and Visualizing Data =============
+#  We start the exercise by first loading and visualizing the dataset.
+#  You will be working with a dataset that contains handwritten digits.
+#
+
+# Load Training Data
+print('Loading and Visualizing Data ...')
+mat = io.loadmat('./data/ex3data1.mat')
+X = mat['X']
+y = np.squeeze(mat['y'])
+
+
+m = X.shape[0]
+
+# Randomly select 100 data points to display
+perm = np.random.permutation(m)
+sel = X[perm[0:100],:]
+
+#display data as image
+helper.displayData(sel)
+ 
+input('\nPart 1 completed. Program paused. Press enter to continue: ')
+
+## ============ Part 2a: Vectorize Logistic Regression ============
+#  In this part of the exercise, you will reuse your logistic regression
+#  code from the last exercise. You task here is to make sure that your
+#  regularized logistic regression implementation is vectorized. After
+#  that, you will implement one-vs-all classification for the handwritten
+#  digit dataset.
+
+
+# Test case for lrCostFunction
+print('\nTesting lrCostFunction() with regularization')
+
+theta_t = np.array([-2,-1,1,2])
+X_t = np.concatenate((np.array([np.ones(5)]),np.divide(np.arange(1,16,1),10).reshape(3,5)),axis=0).transpose()
+Y_t = np.array([1,0,1,0,1])
+lambda_t = 3
+
+J = helper2.costFunctionReg(theta_t,X_t,Y_t,lambda_t)
+grad = helper2.gradientReg(theta_t,X_t,Y_t,lambda_t)
+
+print('Cost: {:.6f}'.format(J))
+print('Expected cost: 2.534819')
+print('Gradients:')
+print(grad)
+print('Expected gradients:')
+print('[0.146561 -0.548558 0.724722 1.398003]')
+
+
+input('\nPart 2a completed. Program paused. Press enter to continue: ')
+
+## ============ Part 2b: One-vs-All Training ============
+print('\nTraining One-vs-All Logistic Regression...')
+
+lambdaVal = .1
+allTheta = helper.OneVsAll(X, y, np.unique(y), lambdaVal)
+
+
+input('\nPart 2b completed. Program paused. Press enter to continue: ')
+## ================ Part 3: Predict for One-Vs-All ================
+
+p = helper.predictOneVsAll(allTheta,X)
+predictions = np.zeros(p.shape)
+predictions[np.where(p==y)] = 1
+
+print('Train Accuracy: {:.1f}%'.format(np.mean(predictions) * 100))
+print('Expected Accuracy: 96.5%')

Diff for: ex2.py

@@ -0,0 +1,99 @@
+## Machine Learning Online Class - Exercise 3 | Part 2: Neural Networks
+
+#  Instructions
+#  ------------
+# 
+#  This file contains code that helps you get started on the
+#  linear exercise. You will need to complete the following functions 
+#  in this exericse:
+#
+#     lrCostFunction (logistic regression cost function) - completed
+#     oneVsAll - completed
+#     predictOneVsAll - completed
+#     predict - completed
+#
+#  For this exercise, you will not need to change any code in this file,
+#  or any other files other than those mention  d above.
+#
+
+## Initialization
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.io as io
+import ex2helper as helper2
+import ex3helper as helper
+
+## Setup the parameters you will use for this exercise
+input_layer_size  = 400;  # 20x20 Input Images of Digits
+hidden_layer_size = 25;   # 25 hidden units
+num_labels = 10;          # 10 labels, from 1 to 10   
+                          # (note that we have mapped "0" to label 10)
+
+## =========== Part 1: Loading and Visualizing Data =============
+#  We start the exercise by first loading and visualizing the dataset. 
+#  You will be working with a dataset that contains handwritten digits.
+#
+
+# Load Training Data
+print('Loading and Visualizing Data ...')
+
+mat = io.loadmat('./data/ex3data1.mat')
+X = mat['X']
+y = np.squeeze(mat['y'])
+
+
+m = y.shape[0]
+
+# Randomly select 100 data points to display
+perm = np.random.permutation(m)
+sel = X[perm[0:100],:]
+
+#display data as image
+helper.displayData(sel)
+
+
+input('\nPart 1 completed. Program paused. Press enter to continue: ')
+
+
+## ================ Part 2: Loading Pameters ================
+# In this part of the exercise, we load some pre-initialized 
+# neural network parameters.
+
+print('\nLoading Saved Neural Network Parameters ...')
+
+# Load the weights into variables Theta1 and Theta2
+mat = io.loadmat('./data/ex3weights.mat')
+theta1 = mat['Theta1']
+theta2 = mat['Theta2']
+
+## ================= Part 3: Implement Predict =================
+#  After training the neural network, we would like to use it to predict
+#  the labels. You will now implement the "predict" function to use the
+#  neural network to predict the labels of the training set. This lets
+#  you compute the training set accuracy.
+
+p = helper.predict(theta1, theta2, X)
+predictions = np.zeros(p.shape)
+predictions[np.where(p==y)] = 1
+
+print('Train Set Accuracy: {:.1f}%'.format(np.mean(predictions) * 100))
+
+input('\nPart 3 completed. Program paused. Press enter to continue: ')
+
+# Randomly select 100 data points to display
+perm = np.random.permutation(m)
+for i in range(0,m):
+	print('\n    Displaying Example Image...\n')
+	example = X[perm[i],:]
+	example = example[np.newaxis,:]
+
+	helper.displayData(example)
+	p = helper.predict(theta1, theta2, example)
+	print('    Neural Network Prediction: {}'.format(p[0]%10))
+	print('    Correct Answer: {}\n'.format(y[perm[i]]%10))
+
+
+
+	answer = input('Paused - press enter to continue, q to exit:')
+	if(answer=='q'):
+		break

Diff for: ex2_reg.py

@@ -0,0 +1,120 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import scipy.optimize as op
+import ex2helper as helper
+import math
+import matplotlib.image as mpimg
+
+def OneVsAll(X, y, numlabels, lambdaVal):
+	m = X.shape[0] #number of examples
+	n = X.shape[1] #number of data points
+ 
+	X = np.insert(X,0,np.ones(X.shape[0]),axis=1) # adding bias unit
+	theta = np.array([])#initialize theta
+
+
+	for i in numlabels:
+		yTemp = np.zeros(y.shape[0])
+		yTemp[np.where(y==i)] = 1
+		thetaTemp = np.zeros(n + 1)
+
+		#run regularized optimization
+		results = helper.optimizeReg(thetaTemp, X, yTemp, lambdaVal)
+		thetaTemp = results.x
+
+		#get prediction accuracy
+		p = helper.predict(thetaTemp, X)
+		predictions = np.zeros(p.shape)
+		predictions[np.where(p==yTemp)] = 1
+		p = helper.sigmoid(np.matmul(X,thetaTemp))
+
+		#calculating cost and accuracy to validate that the function is working correctly
+		print('Train Accuracy: {:.1f}%'.format(np.mean(predictions) * 100))
+		print('cost for {} = {:.3f}, max = {:.3f}'.format(i%10,results.fun,np.max(p)))
+
+		theta = np.append(theta, thetaTemp)#appending discovered theta to theta
+
+	#struggled on this for awhile. Reshape works from left to right, top to bottom. 
+	#so if your data needs to be in columns instead of rows. It messes it all up, but it still works 
+	theta = np.reshape(theta, (numlabels.shape[0],n + 1))
+	return theta.transpose()
+
+def predictOneVsAll(allTheta, X):
+	X = np.insert(X,0,np.ones(X.shape[0]),axis=1) # adding bias unit
+
+	pred = helper.sigmoid(np.matmul(X,allTheta))#calculate predictions for all thetas
+	
+	#return vector of position of maximum for each row +1 to adjust for arrays initializing at 0
+	return(np.argmax(pred,axis=1)+1)
+
+def displayData(X, **keywordParameters):
+	#set example width automatically if not given
+	if('exampleWidth' in keywordParameters):
+		exampleWidth = keywordParameters['exampleWidth']
+	else:
+		exampleWidth = round(math.sqrt(X.shape[1]))
+
+	#calculate size of rows and columns
+	[m, n] = X.shape
+	exampleHeight = n//exampleWidth #eliminating float with // divide
+
+	#calculate number of items to display
+	displayRows = math.floor(math.sqrt(m))
+	displayColumns = math.ceil(m/displayRows)
+
+	#set padding between images
+	padding = 1
+
+	#set up blank display
+	displayHeight = padding + displayRows * (exampleHeight + padding)
+	displayWidth = padding + displayColumns * (exampleWidth + padding)
+
+	displayArray = - np.ones([displayHeight, displayWidth])
+
+	#Copy each example into a path on the display array
+	currentExample = 0
+	for j in range(0,displayRows):
+		for i in range(0, displayColumns):
+			if(currentExample > m):
+				break
+
+			#Copy the Patch
+
+			#1. get the max value of the patch
+			maxValue = np.amax(np.absolute(X[currentExample,:]))
+			
+			#2. get current example in the correct shape
+			example = np.reshape(X[currentExample,:], [exampleHeight, exampleWidth])/maxValue
+			example = example.transpose()
+
+			#3. calculate current position height and width
+			currentPositionHeight = padding + j * (exampleHeight + padding)
+			currentPositionWidth = padding + i * (exampleWidth + padding)
+			
+			#4. assign current example to correct position in the display array
+			displayArray[currentPositionHeight:currentPositionHeight + exampleHeight, currentPositionWidth:currentPositionWidth + exampleWidth] = example
+
+			#5. iterate current example
+			currentExample = currentExample + 1
+
+		if(currentExample>m):
+			break
+
+	#show image
+	imgplot = plt.imshow(displayArray, cmap='gray')
+	plt.axis('off')
+	plt.show()
+
+def predict(theta1, theta2, X):
+	m = X.shape[0]
+	num_labels = theta2.shape[0]
+
+	X = np.insert(X,0,np.ones(X.shape[0]),axis=1) # adding bias unit
+	a1 = np.matmul(X,theta1.transpose())
+	a1 = helper.sigmoid(a1)
+	a1 = np.insert(a1,0,np.ones(a1.shape[0]),axis=1) # adding bias unit
+	a2 = np.matmul(a1,theta2.transpose())
+	a2 = helper.sigmoid(a2)
+	
+	return(np.argmax(a2,axis=1)+1)
+