[Python] Training loss increases instead of decrease with epochs

I am developing from scratch my first feed-forward fully-connected ANN based on batch learning mode on a toy training set. I am using back-propagation for calculating the gradient of the loss function with respect to weights and biases and using the gradient descent method as a learning rule. But when I print the training loss, it gets bigger as the epoch increases:

E(0) on TrS is: [[7.83898769]]
E(1) on TrS is: [[10.00738465]]
E(2) on TrS is: [[10.76653098]]
E(3) on TrS is: [[15.94001008]]
E(4) on TrS is: [[23.80650667]]
E(5) on TrS is: [[28.65805023]]
E(6) on TrS is: [[29.56550719]]
E(7) on TrS is: [[30.5424694]]
E(8) on TrS is: [[34.26980112]]
E(9) on TrS is: [[39.9948856]]


This is my loss_functions.py file:

import numpy as np

def sum_of_squares(c, t, y, derivative=False):
ret = 0
for k in range(c):
ret += np.square(y - t)
ret = 1 / 2 * ret
if derivative:
return y - t
return ret


this is my activation_functions.py file:

import numpy as np


def sigmoid(a, derivative=False):
f_a = 1 / (1 + np.exp(-a))
df_a = np.multiply(f_a, (1 - f_a))
if derivative:
return df_a
return f_a


def identity(a, derivative=False):
f = a
df = np.ones(np.shape(a))
if derivative:
return df
return f


and this is the main.py file:

from activation_functions import *
from loss_functions import *


class NeuralNetwork:

def _init_(self):
self.layers = []

def add_layer(self, layer):
self.layers.append(layer)

def create(self):
for i, layer in enumerate(self.layers):
if i == 0:
layer.type = "input"
else:
if i == len(self.layers) - 1:
layer.type = "output"
else:
layer.type = "hidden"
layer.configure(self.layers[i - 1].neurons)

def train(self, X, targets):
MAX_EPOCHS = 10
loss_function = sum_of_squares
E = 0 # errore sull'intero DS
for epoch in range(MAX_EPOCHS):
for i, x in enumerate(X):
target = targets
prediction = self.forward_prop(x.T)
E_n = loss_function(c, target, prediction)
E += E_n
self.back_prop(target, local_loss=sum_of_squares)
print("E(%d) on TrS is:" % epoch, E) # increasing!!!
self.learning_rule(l_rate=0.05)

def forward_prop(self, z):
for layer in self.layers:
z = layer.forward_prop_step(z)
return z

def back_prop(self, target, local_loss):
for i, layer in enumerate(self.layers[:0:-1]):
next_layer = self.layers[-i]
prev_layer = self.layers[-i - 2]
layer.back_prop_step(next_layer, prev_layer, target, local_loss)

def learning_rule(self, l_rate):
# GD
for layer in self.layers:
if layer.type != "input":
layer.weight -= l_rate * layer.dE_dW
layer.bias -= l_rate * layer.dE_db


class Layer:

def _init_(self, neurons, type=None, activation=None):
self.dE_dW = 0
self.dE_db = 0
self.dEn_db = None # based on the n-th item
self.dEn_dW = None # based on the n-th item
self.dact_a = None
self.out = None
self.weight = None
self.bias = None
self.w_sum = None
self.neurons = neurons
self.type = type
self.activation = activation
self.deltas = None

def configure(self, prev_layer_neurons):
self.weight = np.asmatrix(np.random.normal(0, 0.5, (self.neurons, prev_layer_neurons)))
self.bias = np.asmatrix(np.random.normal(0, 0.5, self.neurons)).T
if self.activation is None:
if self.type == "hidden":
self.activation = sigmoid
elif self.type == "output":
self.activation = identity

def forward_prop_step(self, z):
if self.type == "input":
self.out = z
else:
self.w_sum = np.dot(self.weight, z) + self.bias
self.out = self.activation(self.w_sum)
return self.out

def back_prop_step(self, next_layer, prev_layer, target, local_loss):
if self.type == "input":
pass
elif self.type == "output":
self.dact_a = self.activation(self.w_sum, derivative=True)
self.deltas = np.multiply(self.dact_a, local_loss(c, target, self.out, derivative=True))
else:
self.dact_a = self.activation(self.w_sum, derivative=True)
self.deltas = np.multiply(self.dact_a, np.dot(next_layer.weight.T, next_layer.deltas))

self.dEn_dW = np.dot(self.deltas, prev_layer.out.T)

self.dEn_db = self.deltas

self.dE_dW += self.dEn_dW

self.dE_db += self.dEn_db


if _name_ == '_main_':
net = NeuralNetwork()

for m in (2, 4, 4, 1):
net.add_layer(Layer(m))

net.create()

X = np.asmatrix([
[1, 0],
[1, 1],
[0, 1],
[0, 0]
])

targets = np.asarray([1, 0, 0, 0])

net.train(X, targets)


What I did for trying to fix the problem is:

1. Check for any bug
2. Decrease the learning rate (l_rate)
3. Increase MAX_EPOCHS value
4. Replace - symbol to + in GD formula

Unfortunately none of these worked. There must be a hidden bug somewhere in the code...

How can I resolve the issue?

Continue reading...