[Python] Test Accuracy and ConfMatrix dont match up?

I'm doing the "hello world" project involving the MNIST dataset. But by test accuracy doesn't seem to match with the diagonal of my confusion matrix. I'm not sure if I have a bug somewhere that's causing this or if my accuracy function is not correct.

Here is the model:

class MNISTV1(nn.Module):
def __init__(self,
input_shape: int,
hidden_units: int,
output_shape: int):

super().__init__()
self.layer_stack = nn.Sequential(
nn.Flatten(),
nn.Linear(in_features=input_shape,
out_features=hidden_units),
nn.ReLU(),
nn.Linear(in_features=hidden_units,
out_features=output_shape),
nn.ReLU()
)

def forward(self, x):
return self.layer_stack(x)

torch.manual_seed(42)
model_1 = MNISTV1(input_shape=784,
hidden_units=256,
output_shape=len(train_dataset.classes))
model_1.to(device)


Creating getting predicted class labels:

from tqdm.auto import tqdm

y_preds = []
model_1.eval()
with torch.inference_mode():
for X, y in tqdm(test_dataloader, desc="Making Predictions..."):

X,y = X.to(device), y.to(device)

pred_logits = model_1(X)

y_pred = torch.softmax(pred_logits.squeeze(), dim=0).argmax(dim=1)

y_preds.append(y_pred.cpu())

y_preds_tensor = torch.cat(y_preds)
y_preds_tensor


The matrix:

from torchmetrics import ConfusionMatrix
from mlxtend.plotting import plot_confusion_matrix

confmat = ConfusionMatrix(task="Multiclass",
num_classes=len(classes))
confmat_tensor = confmat(preds=y_preds_tensor,
target=test_dataset.targets)

fig, ax = plot_confusion_matrix(conf_mat=confmat_tensor.numpy(),
class_names=classes,
figsize=(10,7))


torch.manual_seed(42)
def eval_model(model: torch.nn.Module,
data_loader: torch.utils.data.DataLoader,
loss_fn: torch.nn.Module,
accuracy_fn):
""" Returns a dictionary containing the results of model predicting on data_loader. """
loss, acc = 0,0
model.eval()
with torch.inference_mode():
for X,y in tqdm(data_loader):
# make predictions
y_pred = model(X)

# accumulate the loss and acc values per batch
loss += loss_fn(y_pred, y)
acc += accuracy_fn(y_true=y,
y_pred=y_pred.argmax(dim=1))

# scale loss and acc to find the average loss/ acc per batch
loss /= len(data_loader)
acc /= len(data_loader)

return {"model_name": model.__class__.__name__, # only works when model was created with a class
"model_loss": loss.item(),
"model_acc": acc}


model_1_results = eval_model(model=model_1,
data_loader=test_dataloader,
loss_fn=loss_fn,
accuracy_fn=accuracy_fn)
model_1_results


Accuracy function:

def accuracy_fn(y_true, y_pred):

correct = torch.eq(y_true, y_pred).sum().item()
acc = (correct / len(y_pred)) * 100
return acc


My model gets 98.09% which seems very high considering its not a Conv2D network. But the sum of the diagonal of the matrix only gives me about 86% accuracy.

I thought it could be the way I loaded in the data but I checked the dataloader as well as the dataset to make sure that there was no shuffling involved.

Here's the full code on github: https://github.com/nicosaboonchi/Deep-Learning/blob/main/Classification_with_MNIST_Data.ipynb

Continue reading...