[Python] How to mitigate high MAE/MSE in gradient descent linear regression? [closed]

I'm learning about machine learning algorithms by implementing them from scratch. Starting with the basics, I'm working on Linear Regression. However, I'm facing issues with the model performance. Given the simplicity of the dataset, I expected the model to predict outputs closely, aiming for a MAE/MSE smaller than 1.

Here are my current results:

[[29.76307389]] # weight
MAE: 11.860197526386747
MSE: [320.16732974]
RMSE: [17.89322022]


My question is: How can I mitigate the high cost functions? Is this a data scaling problem (which I think it's not)? I can't see any flaws in my code logic.


My implementation:

import numpy as np
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split

class Weights:

def __init__(self, n_features: int):
self.lim = 1 / np.sqrt(n_features)
self.n_features = n_features
self.seed = 42
np.random.seed(self.seed)

def random_uniform(self):
return np.random.uniform(-self.lim, self.lim, (self.n_features, 1))

class Regression:
def __init__(self, n_iter: int, l_rate: float):
self.n_iter = n_iter
self.l_rate = l_rate
self.w = None
self.b = 0
self.error = 0

def predict(self, X):
return np.dot(X, self.w) + self.b

def fit(self, X, y, reg_factor: str = None):
n_samples, n_features = X.shape
if self.w is None: # persisting weights
self.w = Weights(n_features).random_rand()
y = np.reshape(y, (-1, 1)) # transform an array to a column vector
regularization = self.regularize()

for _ in range(self.n_iter):
y_pred = self.predict(X)
self.error = y - y_pred
grad_w = (-2 * np.dot(X.T, self.error) / n_samples)
grad_b = (-2 * np.sum(self.error)) / n_samples
self.w -= self.l_rate * grad_w # l_rate * (grad_w + regularization_factor)
self.b -= self.l_rate * grad_b # l_rate * grad_b


class LinearRegression(Regression):
def __init__(self, n_iter: int, l_rate: float):
super().__init__(n_iter, l_rate)

def gradientDescent(self, X, y):
super().fit(X, y)

class EvaluationMetrics:
def __init__(self, n: int, y, y_pred):
self.n = n
self.y = np.reshape(y, (-1, 1)) # actual value
self.y_pred = y_pred # predicted value

def mae(self):
return np.mean(np.abs(self.y - self.y_pred))
def mse(self):
return sum(np.pow((self.y - self.y_pred), 2)) / self.n
def rmse(self):
return np.sqrt(sum(np.pow((self.y - self.y_pred), 2)) / self.n)

X, y = make_regression(n_samples=1000, n_features=1, noise=15, random_state=4)

X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.4, random_state=42
)

lr = LinearRegression(1000, 0.01)
lr.gradientDescent(X_train, y_train)
lr.printWeights()
y_pred = lr.predict(X_train)

MAE = EvaluationMetrics(n=X_train.shape[0], y=y_train, y_pred=y_pred).mae()
print(f"MAE: {MAE}")
MSE = EvaluationMetrics(n=X_test.shape[0], y=y_train, y_pred=y_pred).mse()
print(f"MSE: {MSE}")
RMSE = EvaluationMetrics(n=X_test.shape[0], y=y_train, y_pred=y_pred).rmse()
print(f"RMSE: {RMSE}")

Continue reading...