[Python] Building NN from scratch, why does my NN not memorize a small sample size of training...

No matter which input I give it after training, it still spits the class distribution.. whereas if I just remove the hidden layer and use a single layer nn, it works much better.

I know the proper math uses vectorizes math all the way, but I wanted to try going at it manually first to really get to know what's happening at each point of training. I suspect that there might be an error in my backpropagation, but I've poured over it many many times to no avail. I'm making this post in hopes an outside perspective can catch the error, thanks a lot!

# %%
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

data = pd.read_csv('train.csv')

# %%
#Training data management
data= np.array(data)

#Train test split 80:20
test_datas = data[int(len(data)*0.8):]
train_datas = data[:int(len(data)*0.8)]

#Separating pixel data and label data
train_labels = train_datas[:,0] #label col
train_datas = (train_datas[:,1:] - np.min(train_datas[:,1:]))/(np.max(train_datas[:,1:])-np.min(train_datas[:,1:])) # pixel data, scaled to 0-1

test_labels = test_datas[:,0] #label col
test_datas = (test_datas[:,1:] - np.min(test_datas[:,1:]))/(np.max(test_datas[:,1:])-np.min(test_datas[:,1:])) # pixel data, scaled to 0-1

# %%
def sigmoid(x): #sigmoid func to squish all inputs into range 0 to 1
return 1 / (1 + np.exp(-x))

# %%
#Initialization

size=[16, 10]
train_data = train_datas[:10]
train_label = train_labels
# --------------------------------------

weights = [] #list to store all the weights for every layer
biases = [] #list to store all the biases for every layer

#Randomly initialize weights and biases to append to list
'''
weights.append(np.random.uniform(-0.1,0.1,size=(size[0],len(train_data[0])))) #First layer
biases.append(np.random.uniform(-0.1,0.1,size[0]))
for i in range(len(size)-1):
weights.append(np.random.uniform(-0.1,0.1,size=(size[i+1],size))) #following layers
biases.append(np.random.uniform(-0.1,0.1,size[i+1]))
'''

#Try using Xavier/Glorot initialization
for i in range(len(size)): #Initialize weights for each layer
if i == 0:
weights.append(np.random.randn(size[0], len(train_data[0])) * np.sqrt(1/len(train_data[0])))
else:
weights.append(np.random.randn(size, size[i-1]) * np.sqrt(1/size[i-1]))

for i in range(len(size)): #Initialize biases for each layer
if i == 0:
biases.append(np.zeros(size[0])) #First layer biases
else:
biases.append(np.zeros(size))



# %%
#Temporarily training on 10 data example for trouble shooting
learning_rate = 0.1
for w in range(1):
train_data = train_datas[w*10w+1)*10]
for o in range(10):
#global cost,z,a,one_hot
#global Zs,As
cost = 0

# Create temporary storage for averaging weights and biases
temp_weights = [] #list to store all the weights for every layer
temp_biases = [] #list to store all the biases for every layer

temp_weights.append(np.zeros(shape=(size[0],len(train_data[0])))) #First layer
temp_biases.append(np.zeros(size[0]))
for i in range(len(size)-1):
temp_weights.append(np.zeros(shape=(size[i+1],size))) #following layers
temp_biases.append(np.zeros(size[i+1]))


for i in range(len(train_data)): #Iterate through every train_data
#Forward propagation
Zs = []
As = [train_data] #TAKE NOTE that As and Zs will be different because we put in initial input as first item for QOL during backprop
z = weights[0] @ train_data + biases[0] #First layer
a = sigmoid(z)
Zs.append(z) #Storing data for backward propagation
As.append(a)

for j in range(len(size)-1):
z = weights[j+1] @ a + biases[j+1] #Following layers
a = sigmoid(z)
Zs.append(z) #Storing data for backward propagation
As.append(a)

#Calculating cost

one_hot = np.zeros(10)
one_hot[train_label]=1

cost = cost + np.sum((a - one_hot)**2) #Just to keep track of model fit

#final/output layer Backpropagation
dC_da = 2*(a - one_hot)
#print("Last layer dC_da=",dC_da,"\n")
dadz = (np.exp(-z) / (1 + np.exp(-z))**2)

for x in range (len(weights[-1][0])): #iterating through weights column by column
# updating weights
dzdw = As[-2][x] #This one input, affects a whole column of weights
dC_dw = dC_da * dadz * dzdw


(temp_weights[-1])[:,x] += -dC_dw*learning_rate/len(train_data) #keeping track of updates to the weights


#updating Biases
dzdb = 1
dC_db = dC_da * dadz * dzdb
temp_biases[-1] += -dC_db*(learning_rate)/len(train_data) #keeping track of updates to the biases

#print("Updates to biases=", temp_biases[-1] ) #DEBUGGING

global dCda_0
#Previous layer Backpropagation
dCda_0 = np.array([])
for x in range (len(weights[-1][0])): #iterating through inputs, a, summing weights column by column,
dzda_0 = weights[-1][:,x] #A whole column of weights affect how ONE prev layer input affects the next layer
dC_da_0 = np.sum(dC_da*dadz*dzda_0)/len(weights[-1]) #Keep track of how previous layer output affect next layer for chain rule later
dCda_0 = np.append(dCda_0,dC_da_0)
#print("second from last layer dCda=\n",dCda_0)

#Previous layer weights
for k in range(len(size)-1): #iterating through layers, starting from the second last
z = Zs[-k-2]
dadz = (np.exp(-z) / (1 + np.exp(-z))**2)

#Updating previous layer weights
for l in range (len(weights[-2-k][0])): #iterating through weights column by column (-2-k because we start from second from last)

dzdw = As[-3-k][l] #This one input, affects a whole column of weights
dC_dw = dCda_0 * dadz * dzdw

(temp_weights[-2-k])[:,l] += -dC_dw*(learning_rate)/len(train_data) #keeping track of updates to the weights


#updating Biases
dzdb = 1
dC_db = dCda_0 * dadz * dzdb
temp_biases[-2-k] += -dC_db*(learning_rate)/len(train_data) #keeping track of updates to the biases

#Keep track of how this layer output affect next layer for chain rule later
temp_dCda_0 = np.array([])
for x in range (len(weights[-2-k][0])): #iterating through inputs, a, summing weights column by column
dzda_0 = weights[-2-k][:,x] #A whole column of weights affect how ONE prev layer input affects the next layer
dC_da_0 = np.sum(dCda_0*dadz*dzda_0)/len(weights[-2-k])
temp_dCda_0 = np.append(temp_dCda_0,dC_da_0)

dCda_0 = temp_dCda_0 #MUtable / unmutable object? Is this going to be problem?

#Updating biases and weights

for i in range(len(size)):
weights += temp_weights
biases += temp_biases

# Analysis of changes to weights
print("weights, iteration",o)
print(temp_weights[0][0][132:136])

print("\n", weights[0][0][132:136])

print("\n",temp_weights[1][0])

print("\n", weights[1][0])

# Analysis of changes to biases
print("biases, iteration",o)
print("\n",temp_biases[0])

print("\n", biases[0])

print("\n", temp_biases[1])

print("\n", biases[1])






# %%
cost

# %%
#Forward propagation, testing training fit
m=0
z = weights[0] @ train_datas[m] + biases[0] #First layer
a = sigmoid(z)
print("\nFirst layer, \nz=",z,"\na=",a )

for j in range(len(size)-1):
z = weights[j+1] @ a + biases[j+1] #Following layers
a = sigmoid(z)
print("\n",j+1,"th layer, \nz=",z,"\na=",a )

print("\nevaluation=",a,"max= ",np.argmax(a)," label= ",train_labels[m])

# %%
#Forward propagation, testing training fit
m=4
z = weights[0] @ train_datas[m] + biases[0] #First layer
a = sigmoid(z)
print("\nFirst layer, \nz=",z,"\na=",a )

for j in range(len(size)-1):
z = weights[j+1] @ a + biases[j+1] #Following layers
a = sigmoid(z)
print("\n",j+1,"th layer, \nz=",z,"\na=",a )

print("\nevaluation=",a,"max= ",np.argmax(a)," label= ",train_labels[m])

# %%
#Check accuracy on training set
correct = 0
k = 100
for i in range(k):
z = weights[0] @ train_datas + biases[0] #First layer
a = sigmoid(z)

for j in range(len(size)-1):
z = weights[j+1] @ a + biases[j+1] #Following layers
a = sigmoid(z)

if train_labels == np.argmax(a): #np.argmax(a)
correct += 1

print(correct/k)


I did it in Jupyter sorry if this is confusing.

Continue reading...