[Python] My neural network tries to find a lazy solution instead of the optimal solution for...

This is a neural network i tried making, im not too certain about my backpropogation, but it seems to atleast converge on answers, although when given 4 numbers and the solution to find the mean of the 4 numbers, it seems to converge to 50 being "safe" instead of learning the underlying pattern, help, either with problems with my code or with the theory behind the problem would be much appreciated, thanks!

import numpy as np

import random

class NeuralNetwork:
def __init__(self,layers):
self.layersConfig = layers
self.learningRate = 0.01
self.createActivations()
self.createWeights()
self.createBiases()

def reLU(self,x):
return x * (x > 0)
#return np.maximum(0, x)

def lreLU(self,x):
return x*0.99 * (x > 0) + abs(x)*0.01

# return np.maximum(0.01*np.abs(x), x)


def reLUd(self,x):
return 1 * (x > 0)


def lreLUd(self,x):
return 1*0.99 * (x > 0) +1*0.01


def sigmoid(self,x):
pass
def createActivations(self):
self.activations = self.layersConfig.copy() #check
for i in range(len(self.activations)):
if i == 0:
self.activations = np.random.rand(self.activations)# temporary input layer
else:
self.activations = np.zeros(self.activations)
#print(self.activations) #maybe they should all be 0??

def createWeights(self):
self.weights = []
for i in range(len(self.layersConfig)-1):
self.weights.append(np.random.randn(self.layersConfig[i+1],self.layersConfig)) #create matrixes of weight connections between layers
#print(self.weights)
# print(self.weights[1][1])
# print(np.flip(np.rot90((self.weights[1]))))
def createBiases(self):
self.biases = self.layersConfig.copy() #check
for i in range(len(self.biases)-1):
self.biases = np.random.randn(self.biases[i+1])
self.biases = self.biases[:-1]
#self.biases = (np.random.rand(len(self.layersConfig))) #create biases for each layer
#print(self.biases)


def findCost(self,desired,actual):
return (actual-desired)**2

def feedFwd(self,k,n): #n denoting the layer that calculations are being done with, k denoting the next layers node being activated
##print(self.biases[n][k],n,k)
##print("looped")
return self.reLU(np.dot(self.weights[n][k],self.activations[n]) + self.biases[n][k])

def forwardPropogation(self,inputs): #working??
self.activations[0] = np.array(inputs)
self.preActivations = self.activations.copy()
for i in range(len(self.layersConfig)-1):
for j in range(len(self.activations[i+1])):
self.activations[i+1][j] = self.feedFwd(j,i)
self.preActivations[i+1][j] = np.dot(self.weights[j],self.activations) + self.biases[j]


def backPropogation2(self,desiredValues):
self.gradVecB = self.biases.copy()
self.gradVecW = self.weights.copy()
self.errorVec = self.activations.copy()
#print(self.errorVec)
desiredValues = np.array(desiredValues)

# dz = self.reLUd(self.activations) #wil work

# dzda = self.reLUd(self.errorVec)
#update that part of the gradVecs and work backwards
#print(self.errorVec[-1],self.findCost(desiredValues,self.errorVec[-1]))
#self.errorVec[-1] = self.findCost(desiredValues,self.errorVec[-1]) #need to find the desired values <---------------------------------
self.errorVec[-1] = 2*(self.errorVec[-1]-desiredValues)
# reverseWeights = np.flip(np.rot90((self.weights[])))
for i in range(len(self.layersConfig)-1): #find error of every neuron
# -2 -i is the current layer, -1-i is the layer to the right of that
for j in range(len(self.errorVec[-2-i])): #iterates through every neuron in each layer (backwards through layers)
#print(i,j) LOOK THROUGH HOW WEIGHTS WORK
dzda = self.reLUd(self.preActivations[-2-i])
#print(dzda[0])
reverseWeights = np.flip(np.rot90((self.weights[-1-i]))) # flips the weight matrix 90 degrees so it can be accessed backwards
reverseWeights = self.weights[-1-i].T # <--- experimential
self.errorVec[-2-i][j] = dzda[j] * np.dot(reverseWeights[j],self.errorVec[-1-i])# self.weights[-1-i] * self.errorVec[-1-i]) #error is calculated by reluD of nueron * sum of product of weight*erorr of next layer

#print(i,j,self.gradVecW[-1-i][j],self.activations[-2-i]*self.errorVec[-1-i])
#self.gradVecW[layer][neuron] = self.errorVec[neuronL+1]*self.activations[layer][neuron]
# self.gradVecW[-1-i][j] = self.activations[-2-i]*self.errorVec[-1-i]

#print(self.errorVec[-2-i][j],dzda[j],np.dot(reverseWeights[j],self.errorVec[-1-i]))
#This should create the error vec?

#print(self.errorVec)
#print(self.gradVecW)
for i in range(len(self.gradVecW)):
for j in range(len(self.gradVecW[-i-1])):
# print(i,j)
# print(self.activations[-2-i]*self.errorVec[-1-i][j])
# self.gradVecW[j] = self.errorVec[j]*self.activations[-2-i]
self.gradVecW[-1-i][j] = self.activations[-2-i]*self.errorVec[-1-i][j]


self.gradVecB = self.errorVec
#self.gradVecW = np.multiply(self.gradVecB,2)
#print(self.gradVecB)

def gradientDescent(self):
#for i,each in enumerate(self.gradVecB):
# self.gradVecB = each*self.learningRate
self.gradVecB = self.gradVecB[1:]
#print(self.gradVecW)
for i in range(len(self.gradVecW)):
#self.gradVecW = each*self.learningRate
self.weights = self.weights - self.gradVecW*self.learningRate

# print(self.biases)
# print(self.gradVecB)
for i in range(len(self.gradVecB)):
self.biases = self.biases - self.gradVecB*self.learningRate
#print(self.weights)
# self.biases =self.biases - self.gradVecB*self.learningRate
# self.weights =self.weights - self.gradVecW*self.learningRate


nn = NeuralNetwork([4,2,2,1])



for i in range(2000):
rnum1 = random.randint(1,100)
rnum2 = random.randint(1,100)
rnum3 = random.randint(1,100)
rnum4 = random.randint(1,100)
nn.forwardPropogation([rnum1/100,rnum2/100,rnum3/100,rnum4/100])
ans = (rnum1+rnum2+rnum3+rnum4)/4
print(ans,nn.activations[-1],nn.findCost(ans,nn.activations[-1]))
nn.backPropogation2([ans])
nn.gradientDescent()

Continue reading...