[Python] Struggling with Coin Classification Using TensorFlow

I'm working on a model that would classify coin categories from a self-made coin dataset. (17 categories and around 6K augmented images) Coin images are cropped, so background pixels are set to be transparent. (for reference I've uploaded training data to Dropbox (coin_catalog/augmented/crops/...) https://www.dropbox.com/scl/fo/ipyg...ey=cq8x97j9k7e2w6aavcgray5x2&st=lujy2zvf&dl=0)

I expected this task to be similar to image classification tasks like flower recognition. However, despite trying several architectures, including ResNet152V2, the models fail to achieve reasonable accuracy. For example, they often misclassify even clearly distinct coin types.

Training Setup:

• Are my training data or the current ResNet-based model unsuitable for this task?
• Does my training script (see below) have any apparent issues that could cause poor performance?
• Can you recommend any resources or examples of similar image classification tasks, particularly with small datasets or transparent-background images?

Below are scripts I used for this task:

• train.py: Training script
• model_ResNet50V2.py: Model definition
• test.py: Inference script for evaluating the trained model

train.py

import os
from pathlib import Path

import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from tensorflow.keras import layers
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping

from core.utilities.helper import get_directories
from kunstkammer.object_recognition.model_ResNet50V2 import build_model

if __name__=="__main__":
catalog_path = Path("coin_catalog/augmented")

""" Hyperparemeters """
image_shape = (128, 128)

testrun_name = "ResNet152V2"
num_epochs = 50
validation_split = 0.2
batch_size = 1
lr = 1e-3

seed = 42
np.random.seed(seed)
tf.random.set_seed(seed)

""" Directory for storing files """
if not os.path.exists(output_dir := Path(os.path.dirname(__file__), "trained")):
os.makedirs(output_dir)

crops_path = Path(catalog_path, "crops")
model_path = Path(output_dir, f"object_recognition_{testrun_name}.keras")
crops_dataset_path = Path(output_dir, f"crops_dataset_{testrun_name}.tfrecord")
enum_path = f"trained/enumerations/train_enum_{testrun_name}.json"

enumerations = [str(coin.stem) for coin in get_directories(Path(catalog_path, "images"))]

try:
crop_dataset = tf.data.Dataset.load(str(crops_dataset_path))
train_dataset, val_dataset = train_test_split(crop_dataset, test_size=validation_split, random_state=seed)
print("=== Dataset loaded ===")

except:
crop_dataset = tf.keras.utils.image_dataset_from_directory(str(crops_path),
seed=seed,
batch_size=batch_size,
image_size=image_shape)

""" Split dataset into train and validation subsets """
dataset_size = tf.data.experimental.cardinality(crop_dataset).numpy()
val_size = int(dataset_size * validation_split)
train_size = dataset_size - val_size
train_dataset = crop_dataset.take(train_size)
val_dataset = crop_dataset.skip(train_size)

AUTOTUNE = tf.data.AUTOTUNE
train_ds = train_dataset.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)
val_dataset = val_dataset.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)

print("=== Save datasets ===")
crop_dataset.save(str(crops_dataset_path))

""" Model """
if not os.path.exists(model_path):
model = build_model(input_shape=(*image_shape, 3), num_classes=len(enumerations))

else:
print("Model already exists. Loading the model...")
model = tf.keras.models.load_model(model_path)

""" Training """
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=lr,
decay_steps=1000, # Adjust this based on dataset size and batch size
decay_rate=0.96, # Factor by which to decay
staircase=True # If True, the learning rate decays in discrete steps
)

model.compile(
optimizer=tf.keras.optimizers.SGD(learning_rate=lr_schedule),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['accuracy']
)

callbacks = [
ModelCheckpoint(model_path, save_best_only=True),
# ReduceLROnPlateau(factor=0.1, patience=5),
EarlyStopping(patience=10)
]

history = model.fit(
train_dataset,
validation_data=val_dataset,
epochs=num_epochs,
callbacks=callbacks
)

model.save(model_path)


model_ResNet50V2.py

import tensorflow as tf
from tensorflow.keras import layers, Model


def build_model(input_shape=(224, 224, 3), num_classes=10, use_pretrained=True):
base_model = tf.keras.applications.ResNet152V2(
weights="imagenet" if use_pretrained else None,
include_top=False,
input_shape=input_shape
)

base_model.trainable = not use_pretrained

inputs = tf.keras.Input(shape=input_shape)
x = base_model(inputs, training=False)
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(num_classes, activation="softmax")(x)

model = Model(inputs, outputs, name="ResNet50V2_Custom")

return model


test.py

import os
from pathlib import Path

import numpy as np
import tensorflow as tf

from core.utilities.helper import get_directories

if __name__ == "__main__":
catalog_path = Path("coin_catalog/augmented")
shape = (128, 128)

testrun_name = "ResNet152V2"

# enum_path = f"trained/enumerations/train_enum_{testrun_name}.json"
enumerations = [str(coin.parts[-1]) for coin in get_directories(Path(catalog_path, "images"))]

model_path = Path(os.path.dirname(__file__), f"trained/object_recognition_{testrun_name}.keras")
model = tf.keras.models.load_model(model_path)

samples = [
"coin_catalog/augmented/crops/(Czech Republic, 50 Korun, 2008)/0_10.png",
"coin_catalog/augmented/crops/(Czech Republic, 1 Koruna, 2018)/0_19.png",
"coin_catalog/augmented/crops/(USA, 2.5 Dollar, 1909)/1_7.png",
"coin_catalog/augmented/crops/(Great britain, 0.5 Souvereign, 1906)/0_13.png",
"coin_catalog/augmented/crops/(Iran, 0.5 Souvereign, 1925)/4_3.png",
"coin_catalog/augmented/crops/(Austria-Hungary, 20 Korona, 1893)/0_8.png",
"coin_catalog/augmented/crops/(Czech Republic, 10 Korun, 2020)/0_3.png",
"coin_catalog/augmented/crops/(France, 2 Franc, 1917)/0_20.png",
"coin_catalog/augmented/crops/(Czech Republic, 5 Korun, 2002)/0_17.png",
"coin_catalog/augmented/crops/(India, 1 Rupee, 1840)/4_26.png",
"coin_catalog/augmented/crops/(Austria-Hungary, 20 Korona, 1893)/0_11.png",
]

results = []
for sample_path in samples:
input_image = tf.keras.utils.load_img(sample_path, target_size=shape)
input_image_array = tf.keras.utils.img_to_array(input_image)
input_image_exp_dim = tf.expand_dims(input_image_array, 0)

predictions = model.predict(input_image_exp_dim)
result = tf.nn.softmax(predictions)

sample_dir = Path(sample_path).parts[-2]
predict_dir = enumerations[np.argmax(result)]

results.append(f"[{'PASS' if sample_dir == predict_dir else 'FAIL'}] {sample_dir} -> {predict_dir}")

[print(result) for result in results]

Continue reading...