[Python] Accuracy and Loss always stuck at 0.2 acc and 1.6 loss after training model at 1e-3...

so i got this problem when hyperparameter tuning my model in Kaggle and google colab where every time I run my model with a learning rate of 1e−3, and then create a new model using a new learning rate of 1e−5, the accuracy and loss seem to be stuck at the performance levels from the 1e−3 learning rate. Additionally, I tried running the cell with 1e−5 learning rate first (after reseting kaggle), and the model achieved an accuracy of > 0.7 and a loss of < 0.6, does this bug from tensorflow or something wrong with my code.

Here My Full Model Code:

#Model
def create_model(model, model_type, total_classes, learning_rate = LEARNING_RATE, dropout_rate = DROPOUT_RATE, max_len = MAX_LEN, optimizer = OPTIMIZER):
# Select the optimizer based on the input
if optimizer == 'rmsprop':
opt = tf.keras.optimizers.RMSProp(learning_rate=learning_rate)
elif optimizer == 'adamw':
opt = tf.keras.optimizers.AdamW(learning_rate=learning_rate)
elif optimizer == 'adammax':
opt = tf.keras.optimizers.Adamax(learning_rate=learning_rate)
elif optimizer == 'adam':
opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)
else:
raise ValueError("Unsupported optimizer. Please choose 'rmsprop' or 'adamw' or 'adammax'.")
loss = tf.keras.losses.CategoricalCrossentropy(from_logits=False)
accuracy = tf.keras.metrics.CategoricalAccuracy()
input_ids = tf.keras.Input(shape=(max_len,),dtype='int32', name='input_ids')
attention_mask = tf.keras.Input(shape=(max_len,),dtype='int32',name='attention_mask')
output = model(input_ids=input_ids, attention_mask=attention_mask)
if model_type == 'roberta' or model_type == 'albert':
output = output[1]
elif model_type == 'xlnet':
sequence_output = output.last_hidden_state
output = sequence_output[:, 0, :]
# Add a hidden layer
output = tf.keras.layers.Dropout(rate=dropout_rate)(output)
output = tf.keras.layers.Dense(total_classes, activation=tf.nn.softmax)(output)
model = tf.keras.models.Model(inputs = [input_ids,attention_mask],outputs = output)
model.compile(opt, loss=loss, metrics=accuracy)
return model

model_1 = create_model(roberta_model, 'roberta', total_classes=5, learning_rate=1e-3, max_len=roberta_max_raw)
print(f'Current Learning Rate: {tf.keras.backend.get_value(model_1.optimizer.learning_rate)}')
# Train the model
history = model_1.fit(
[train_input_ids_roberta_raw, train_attention_masks_roberta_raw],
train_y_ohe_raw,
validation_data=([val_input_ids_roberta_raw, val_attention_masks_roberta_raw], val_y_ohe_raw),
epochs=1,
batch_size=4,
callbacks=[early_stopping,
time_callback,
F1ScoreCallback(validation_data=(val_input_ids_roberta_raw, val_attention_masks_roberta_raw, val_y_ohe_raw))]
)
plot_training_history(history)

model = create_model(roberta_model, 'roberta', total_classes=5, learning_rate=1e-5, max_len=roberta_max_raw)
print(f'Current Learning Rate: {tf.keras.backend.get_value(model.optimizer.learning_rate)}')
# Train the model
history_2 = model.fit(
[train_input_ids_roberta_raw, train_attention_masks_roberta_raw],
train_y_ohe_raw,
validation_data=([val_input_ids_roberta_raw, val_attention_masks_roberta_raw], val_y_ohe_raw),
epochs=10,
batch_size=4,
callbacks=[early_stopping,
time_callback,
F1ScoreCallback(validation_data=(val_input_ids_roberta_raw, val_attention_masks_roberta_raw, val_y_ohe_raw))]
)
plot_training_history(history)


1e-3 then 1e-5 Training



1e-5 Training First

Continue reading...