[Python] RNN Training - Gradients and model convergence

currently, Im working on a sequence prediction task using a RNN with TensorFlow. encountering issues during training that I think are due to exploding gradients.

loss sometimes jumps to extremely high values (e.g., several orders of magnitude higher than the initial loss). The gradients appear to be excessive

import tensorflow as tf

class SimpleRNNModel(tf.keras.Model):
def __init__(self, input_size, hidden_size, output_size):
super(SimpleRNNModel, self).__init__()
self.rnn = tf.keras.layers.SimpleRNN(hidden_size, return_sequences=True)
self.dense = tf.keras.layers.Dense(output_size)

def call(self, x):
rnn_out = self.rnn(x)
return self.dense(rnn_out)

input_size = 10
hidden_size = 20
output_size = 1
learning_rate = 0.001
epochs = 100

model = SimpleRNNModel(input_size, hidden_size, output_size)

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate), loss='mean_squared_error')

for epoch in range(epochs):
with tf.GradientTape() as tape:
predictions = model(data)
loss = tf.keras.losses.mean_squared_error(labels, predictions)

gradients = tape.gradient(loss, model.trainable_variables)

if any(tf.reduce_max(tf.abs(grad)) > 1e5 for grad in gradients):
print("Exploding gradients detected. Adjusting learning rate.")
clipped_gradients = [tf.clip_by_value(grad, -1e5, 1e5) for grad in gradients]
else:
clipped_gradients = gradients

model.optimizer.apply_gradients(zip(clipped_gradients, model.trainable_variables))

print(f"Epoch {epoch}: Loss = {loss.numpy().mean()}")

Continue reading...