[Python] Neural Network Debugging - Simplistic Network producing weird results and having...

I wanted to work on a simplistic neural network not using numpy or aadvanced type coding to really focus on the pure math and background of what makes a neural net work. I created the model using the XOR problem using 2 inputs, 2 hidden nodes, and 1 output neuron with the hidden nodes using the leaky relu for activation and the output using sigmoid. Sometimes, it'll work fine with perfect accuracy and other times it won't with bad accurary and that won't be fixed through training. Currently, I have it running at a learning rate of 0.5 with 500 epochs though I've played around with both of those a bit with no significant changes. Here is the code I have. It is in Python. Thanks!

import math
import time

import numpy as numpy
import random

Input = [[0, 1], [0, 0], [1, 1], [1, 0]]
expectedoutputs = [1, 0, 0, 1]
alpha=0.01



def leakyrelu(x):
if x >= 0:
return x
else:
return alpha * x

def drelu(x):
if x >= 0:
returndx = 1
else:
returndx = alpha
return returndx


def sigmoid(x):
return 1 / (1 + math.exp(-x))





numofhiddennodes = 2
numofinputs = 2


def initweights():
w1 = []
# Xavier initialization for weights in the hidden layer
for i in range(0, numofhiddennodes):
h = []
for j in range(0, numofinputs):
# Xavier Initialization for weights
val = random.uniform(-1, 1) * math.sqrt(2 / (numofinputs + numofhiddennodes))
h.append(val)
w1.append(h)

# Xavier initialization for biases (usually a small value near zero)
b1 = [random.uniform(-1, 1) * math.sqrt(2 / (numofinputs + numofhiddennodes)) for _ in range(numofhiddennodes)]

# Xavier Initialization for weights in the output layer
w2 = [random.uniform(-1, 1) * math.sqrt(2 / (numofhiddennodes + 1)) for _ in range(numofhiddennodes)]
b2 = random.uniform(-1, 1) * math.sqrt(2 / (numofhiddennodes + 1))

return (w1, b1, w2, b2)

def forward(w1, b1, w2, b2, Input):
hidden = []
nvals = []
for i in range(0, numofhiddennodes):

nval = 0

for j in range(0, numofinputs):

nval = nval + Input[j] * w1[j]

nval = nval + b1
nvals.append(nval)
hidden.append(leakyrelu(nval))

oval = 0

for i in range(0, numofhiddennodes):
oval = oval + hidden * w2

oval = oval + b2

output = sigmoid(oval)

return output, hidden, nvals


def backprop(w1, b1, w2, b2, z, hiddens, inputs, epoch, expectedoutputs, learningrate, nvals):
zzero = expectedoutputs[Input.index(inputs)]

L = (z - zzero) * (z - zzero)

dldz = 2 * (z - zzero)

dzdy = z * (1 - z)

print(f"Gradient w.r.t. output: dldz = {dldz}, dzdy = {dzdy}")

for i in range(0, numofhiddennodes): #update w2

dydwi = hiddens

dldwi = dldz * dzdy * dydwi

print(f"Gradient for w2[{i}]: dldwi = {dldwi}")

w2 = w2 - learningrate * dldwi

dldb = dldz * dzdy #Adjusting B2

b2 = b2 - learningrate * dldb #b2

print(f"Gradient for b2: dldb = {dldb}")

for i in range(0, numofhiddennodes): #updating weight 1

for j in range(0, numofinputs):

dydhi = w2

dhidc = drelu(nvals)

dcdwi = inputs[j]

dcdwi = int(dcdwi)

dLdwone = dldz * dzdy * dydhi * dhidc * dcdwi
print(f"Gradient for w1[{i}][{j}]: dLdwone = {dLdwone}")
w1[j] = w1[j] - learningrate * dLdwone

for i in range(0,numofhiddennodes): #updating bias 1

dydhi = w2

dhidc = drelu(nvals)

dcdbone = 1

dLdbone = dldz * dzdy * dydhi * dhidc * dcdbone

print(f"Gradient for b1[{i}]: dLdbone = {dLdbone}")

b1 = b1 - learningrate * dLdbone

return w1,b1,w2,b2, L



def train(epochs,Input,expectedoutputs, learningrate):
w1, b1, w2, b2 = initweights()
epoch = 1


for i in range(0, epochs):
tack = random.randint(0,3)

inputs = Input[tack]
output, hidden, nvals = forward(w1, b1, w2, b2, inputs)
w1,b1,w2,b2, Loss = backprop(w1, b1, w2, b2, output, hidden, inputs, epoch, expectedoutputs, learningrate, nvals)
epoch = epoch + 1
print("Loss: ", Loss)

return w1, b1, w2, b2




epoch = 1




def test():
learningrate = 0.5
epochs = 500
turn = 0


w1, b1, w2, b2 = train(epochs, Input, expectedoutputs, learningrate)

correctvals = []
for i in range(0,100):
turn = turn + 1
print("turn", turn)

randomvalue = random.randint(0,3)
set = Input[randomvalue]
exoutput = expectedoutputs[randomvalue]

recievedval,h,v = forward(w1, b1, w2, b2, set)



if recievedval >= 0.5:
recievedval = 1
else:
recievedval = 0
correctval = 0
if recievedval != exoutput:
correctval = 0
elif recievedval == exoutput:
correctval = 1
correctvals.append(correctval)

print("Accuracy:", sum(correctvals) / len(correctvals))
time.sleep(0.1)
print(correctvals)
print(w1, b1, w2, b2)

test()

Continue reading...