[Python] Errors with MLP neural network made from scratch trained on MNIST dataset

I'm experimenting with neural networks and decided to create an MLP from scratch to classify digits based on the MNIST training dataset. The MNIST data set I'm using is in the csv format and downloaded from here

I understand that my code is very inefficient as I don't support any batching or parallelization(this is just a rough first run through to get a better understanding before making it efficient) but I don't understand why it would make it as inaccurate as it currently is. I was hoping if there were any obvious errors I made in my MLP code below.

import numpy as np
import math
import random
import time
import pickle

class Parameter:
def __init__(self, val):
self.val = val
self.grad = 0

def __str__(self):
return str(self.val)

def __repr__(self):
return str(self)

class Neuron:
def __init__(self, inputs):
self.bias = Parameter(random.uniform(-1, 1))
self.weights = [Parameter(random.uniform(-1, 1)) for i in range(inputs)]

def __str__(self):
return f"weight={self.weights}, bias={self.bias}"

class Layer:
def __init__(self, inputs, outputs):
self.inputs = inputs
self.outputs = outputs
self.neurons = [Neuron(inputs) for _ in range(outputs)]

def __str__(self):
str = ""
for i in range(len(self.neurons)):
str+=f"neuron {i+1}-\n{self.neurons}\n"
return str

class MLP:
def __init__(self, inputs, hidden, output, activation, loss):
if(len(hidden)==0):
self.layers = [Layer(inputs, output)]
else:
self.layers = [Layer(inputs, (hidden[0]))]
if(len(hidden)==1):
self.layers += [Layer(hidden[0], output)]
else:
self.layers += [Layer(input, output) for input, output in zip(hidden, hidden[1:])]
self.layers += [Layer(hidden[-1], output)]

self.activation_func = activation
self.loss_func = loss

def forward(self, input, label):
inputs = input.copy()
z = []
a = []

for i in range(len(self.layers)):
unactivated = []
activated = []
for neuron in self.layers.neurons:
w = [item.val for item in neuron.weights]
val = np.dot(w, inputs) + neuron.bias.val
unactivated.append(val)

activated.append(self.activation_func(val))

inputs = activated.copy()
a.append(activated)
z.append(unactivated)

# a[-1] = MLP.normalize(a[-1])

self.z = z
self.a = a
# print(f"z: {self.z}, a: {self.a}")

self.loss = self.loss_func(a[-1], label)
# print(f"\nLoss: {self.loss}")
self.input = input
self.label = label
self.output = a[-1]

def backward(self, learning_rate):

for layer in self.layers:
for neuron in layer.neurons:
for weight in neuron.weights:
weight.grad = 0
neuron.bias.grad = 0

in_da = []
for i in range(len(self.layers[-1].neurons)):
in_da.append(self.loss_func(self.a[-1], self.label, deriv=True))

for i in range(len(self.layers)-1, -1, -1):
next_da = [0 for i in range(len(self.layers[i-1].neurons))]
for n in range(len(self.layers.neurons)):
dz = in_da[n] * self.activation_func(self.z[n], deriv=True)

if(i==0):
dw = np.multiply(dz, self.input)
else:
dw = np.multiply(dz, self.a[i-1])

db = dz

for w in range(len(self.layers.neurons[n].weights)):
self.layers.neurons[n].weights[w].grad += dw[w]
if i>0:
next_da[w] += dz * self.layers.neurons[n].weights[w].val

self.layers.neurons[n].bias.grad += db

in_da = next_da.copy()

# for i in range(len(self.layers)):
# print(f"LAYER {i}: \n")
# for n in range(len(self.layers.neurons)):
# print(f"Neuron {n}:")
# str = "weights: ["
# for w in self.layers.neurons[n].weights:
# str+=f"{w.grad}, "
# print(f"{str[:-2]}], bias: {self.layers.neurons[n].bias.grad}")
# print()

for layer in self.layers:
for neuron in layer.neurons:
for weight in neuron.weights:
weight.val -= learning_rate * weight.grad
neuron.bias.val -= learning_rate * neuron.bias.grad

def run(self, input):
inputs = input.copy()
for i in range(len(self.layers)):
activated = []
for neuron in self.layers.neurons:
w = [item.val for item in neuron.weights]
val = np.dot(w, inputs) + neuron.bias.val

activated.append(self.activation_func(val))

inputs = activated.copy()

return MLP.normalize(activated)

def mean_square(x, label, deriv=False):
return 2 * (x - label) if deriv else sum(np.subtract(x, label) ** 2)

def relu(x, deriv=False):
return float(np.greater(x, 0)) if deriv else max(0, x)

def tanh(x, deriv=False):
return 1 - (math.tanh(x) ** 2) if deriv else math.tanh(x)

def sigmoid(x, deriv=False):
return MLP.sigmoid(x) * (1 - MLP.sigmoid(x)) if deriv else 1 / (1 + math.exp(-x))

def normalize(x):
if all(item == 0 for item in x):
return x
val = sum(x)
return [item/val for item in x]

def __str__(self):
str = ""
for i in range(len(self.layers)):
str+=f"LAYER {i}:\n\n{self.layers}\n"
return str

def save(self, path):
with open(path+".pickle", "wb") as outfile:
pickle.dump(self, outfile)

def load(path):
with open(path+".pickle", "rb") as infile:
return pickle.load(infile)


The code below is the actual training of the MLP

from NeuralNet import MLP
import csv
import time

def main():
net = MLP(784, [200, 80], 10, activation=[MLP.tanh, MLP.sigmoid, MLP.sigmoid], loss=MLP.mean_square)
# net = MLP.load("network")

# train
start = time.time()
try:
with open("MNISTdata/mnist_train.csv") as file:
train_data = csv.reader(file, delimiter=',')
v = 0
for epoch in range(3):
for row in train_data:
num = [0 for i in range(10)]
num[int(row[0])] = 1

pixels = [int(pixel) for pixel in row[1:]]

# forward pass
net.forward(input=pixels, label=num)

# backward pass
net.backward(learning_rate=0.1)

v+=1
print(f"{v} Loss: {net.loss}")

except KeyboardInterrupt:
print('Interrupted')

# test
try:
with open("MNISTdata/mnist_test.csv") as file:
test_data = csv.reader(file, delimiter=',')
correct = 0
total = 0
for row in test_data:
num = int(row[0])

pixels = [int(pixel) for pixel in row[1:]]

distribution = net.run(input=pixels)
predicted = distribution.index(max(distribution))

if(num==predicted):
correct+=1
total+=1

print(correct, total)

except KeyboardInterrupt:
print('Interrupted')

print(f"Accuracy: {correct/total * 100}%")

net.save("network")

end = time.time()
print(end - start)

if __name__ == "__main__":
main()


I tested my code with test samples and did the math to make sure forward and backward propagation are all working with the correct gradient descent values being calculated. The only problem is that when I attempt to train with MNIST, the maximum accuracy I could get is around 40% with 784 inputs(pixels) and 2 hidden layers with 200 and 80 neurons and then 10 neurons for output. It has the following activation functions performed in the order they are listed: tanh, sigmoid, sigmoid. It also uses the mean square loss function.

Continue reading...