[Python] neural network that is supposed to recognise the formula y = k*k + 2 * l + b is faulty

I have tried to write an NN that should recognise the formula y = k*k + 2 * l + b in python

But there is an error. After training the predicted values are always the same although the input values vary.

This code implements a neural network with the following properties:

• Two input neurons (k and l) One hidden layer with sigmoid activation function
• One output neuron
• Trainable with backpropagation
• Test method for calculating the mean square error
• Consistent random input values due to fixed seed
• Visualisation of the training error and test results

The sigmoid function was chosen as the activation function for the hidden layer as it introduces non-linear properties and is well suited for this use case. The output layer does not use an activation function as we want a continuous output. To run the code, copy it into a Python file and make sure you have numpy and matplotlib installed. The code will train and test the network and visualise the results.

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

# Sigmoid activation function and its derivative
def sigmoid(x):
return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
return x * (1 - x)

# Define the neural network class
class SimpleNeuralNet:
def __init__(self, hidden_neurons, learning_rate, bias):
np.random.seed(42) # For reproducible results
self.hidden_neurons = hidden_neurons
self.learning_rate = learning_rate
self.bias = bias

# Initialize weights and biases
self.weights_input_hidden = np.random.rand(2, self.hidden_neurons) - 0.5
self.bias_hidden = np.random.rand(self.hidden_neurons) - 0.5
self.weights_hidden_output = np.random.rand(self.hidden_neurons, 1) - 0.5
self.bias_output = np.random.rand(1) - 0.5

def forward(self, inputs):
# Hidden layer computation
self.hidden_layer_input = np.dot(inputs, self.weights_input_hidden) + self.bias_hidden
self.hidden_layer_output = sigmoid(self.hidden_layer_input)

# Output layer computation
self.output_layer_input = np.dot(self.hidden_layer_output, self.weights_hidden_output) + self.bias_output
self.output_layer_output = self.output_layer_input # No activation for the output layer (linear)

return self.output_layer_output

def backward(self, inputs, expected_output, actual_output):
# Calculate errors
output_error = expected_output - actual_output
output_delta = output_error # Linear activation derivative is 1

hidden_error = np.dot(output_delta, self.weights_hidden_output.T)
hidden_delta = hidden_error * sigmoid_derivative(self.hidden_layer_output)

# Update weights and biases
self.weights_hidden_output += self.learning_rate * np.dot(self.hidden_layer_output.T, output_delta)
self.bias_output += self.learning_rate * np.sum(output_delta, axis=0)

self.weights_input_hidden += self.learning_rate * np.dot(inputs.T, hidden_delta)
self.bias_hidden += self.learning_rate * np.sum(hidden_delta, axis=0)

def train(self, inputs, expected_output, epochs):
for epoch in range(epochs):
# Forward pass
actual_output = self.forward(inputs)

# Backward pass
self.backward(inputs, expected_output, actual_output)

def test(self, inputs):
return self.forward(inputs)

# Generate input data and expected output
def generate_data(b, size=200):
np.random.seed(42)
k = np.random.randint(0, 10, size)
l = np.random.randint(0, 10, size)
inputs = np.array(list(zip(k, l)))
expected_output = k * k + 2 * l + b
return inputs, expected_output.reshape(-1, 1)


# Create a 3D scatter plot
def create_3d_scatter_plot(inputs, outputs):
k = inputs[:, 0]
l = inputs[:, 1]
output = outputs.flatten()
fig = px.scatter_3d(x=k, y=l, z=output, title="3D Scatter Plot", labels={"x": "k", "y": "l", "z": "Output"})
fig.update_layout(scene=dict(xaxis_title="k", yaxis_title="l", zaxis_title="Output"))
fig.show()

# Main script
if __name__ == "__main__":
# Parameters
hidden_neurons = 5
learning_rate = 0.01
bias = 3
epochs = 1000

# Generate training data
inputs, expected_output = generate_data(bias)

# Initialize and train the neural network
nn = SimpleNeuralNet(hidden_neurons, learning_rate, bias)
nn.train(inputs, expected_output, epochs)

# Create a 3D scatter plot
create_3d_scatter_plot(inputs, expected_output)

# Test the neural network
test_inputs, test_expected_output = generate_data(bias, size=10)
predictions = nn.test(test_inputs)

print("Test Inputs:")
print(test_inputs)
print("Expected Outputs:")
print(test_expected_output.flatten())
print("Predicted Outputs:")
print(predictions.flatten())

# Create a 3D scatter plot
# create_3d_scatter_plot(test_inputs, predictions)


When running a test I get the results:

Test Inputs:
[[6 3]
[3 7]
[7 7]
[4 2]
[6 5]
[9 4]
[2 1]
[6 7]
[7 5]
[4 1]]
Expected Outputs:
[45 26 66 23 49 92 9 53 62 21]
Predicted Outputs:
[-153.04305672 -153.04305672 -153.04305672 -153.04305672 -153.04305672
-153.04305672 -153.04305672 -153.04305672 -153.04305672 -153.04305672]

Continue reading...