[Python] Python/TensorFlow: Persistent RAM increase in a long-running loop despite...

I'm running a Python script on a VPS in a continuous loop. The script fetches stock data with yfinance, then trains a new TensorFlow/Keras model for each stock.

Problem: The process's RAM usage steadily increases over time and never returns to its baseline, leading to an eventual crash. This happens even though I am using a cleanup routine after each iteration.

import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input # Aggiungi Input qui
from sklearn.preprocessing import MinMaxScaler
import gc
import time
import os
import psutil

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
tf.get_logger().setLevel('ERROR')

def get_ram_usage():
return psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)

def download_stock_data(ticker, start_date="2020-01-01", end_date=None):
try:
stock_data = yf.download(ticker, start=start_date, end=end_date, progress=False)
return stock_data if not stock_data.empty else None
except Exception:
return None

def create_and_predict_model(data, n_past=30, n_future=3, epochs=10, batch_size=32):
if data is None or len(data) < n_past + n_future:
return None, None

scaler = MinMaxScaler(feature_range=(0, 1))
scaled_data_values = data['Close'].values.reshape(-1, 1)
scaled_data = scaler.fit_transform(scaled_data_values)
del scaled_data_values

X_train, y_train = [], []
for i in range(n_past, len(scaled_data) - n_future + 1):
X_train.append(scaled_data[i - n_past:i, 0])
y_train.append(scaled_data[i:i + n_future, 0])

if not X_train:
del scaler, scaled_data
return None, None

X_train, y_train = np.array(X_train), np.array(y_train)

# --- MODIFICA QUI ---
model = Sequential([
Input(shape=(n_past,)), # Usa un layer Input esplicito
Dense(100, activation='relu'),
Dense(50, activation='relu'),
Dense(n_future)
])
# --- FINE MODIFICA ---
model.compile(optimizer='adam', loss='mean_squared_error')

tf.random.set_seed(int(time.time()))
np.random.seed(int(time.time()))
history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=0)
print(f"RAM usage after training: {get_ram_usage():.2f} MB")

last_n_past_days_scaled = scaled_data[-n_past:].reshape(1, n_past)
predicted_scaled_prices = model.predict(last_n_past_days_scaled, verbose=0)
predicted_prices = scaler.inverse_transform(predicted_scaled_prices.reshape(-1,1))
print(f"RAM usage after prediction: {get_ram_usage():.2f} MB")

del X_train, y_train, scaled_data, scaler, last_n_past_days_scaled, predicted_scaled_prices, history
return predicted_prices.flatten(), model

def clean_ram(model_to_delete=None):
if model_to_delete is not None:
try: del model_to_delete
except: pass
tf.keras.backend.clear_session()
gc.collect()
print(f"RAM usage after cleaning: {get_ram_usage():.2f} MB")

if __name__ == "__main__":
stock_tickers = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA"]
N_PAST_DAYS, N_FUTURE_DAYS, EPOCHS = 60, 3, 15
SECONDS_BETWEEN_TICKERS, SECONDS_BETWEEN_GLOBAL_CYCLES = 1, 5

print(f"INITIAL RAM USAGE: {get_ram_usage():.2f} MB")
main_cycle_count = 0

try:
while True:
main_cycle_count += 1
print(f"\n--- GLOBAL CYCLE {main_cycle_count} ---")
print(f"RAM at start of cycle {main_cycle_count}: {get_ram_usage():.2f} MB")

for ticker_index, ticker in enumerate(stock_tickers):
print(f"\nProcessing Ticker: {ticker} (Cycle {main_cycle_count})")
start_ticker_ram = get_ram_usage()
print(f"RAM before {ticker}: {start_ticker_ram:.2f} MB")

historical_data = download_stock_data(ticker, start_date="2018-01-01")
print(f"RAM after download for {ticker}: {get_ram_usage():.2f} MB")

predictions = None
model_for_this_ticker = None

if historical_data is not None:
predictions, model_for_this_ticker = create_and_predict_model(
historical_data.copy(), N_PAST_DAYS, N_FUTURE_DAYS, EPOCHS
)
del historical_data
print(f"RAM after deleting historical_data for {ticker}: {get_ram_usage():.2f} MB")

if predictions is not None:
del predictions

clean_ram(model_for_this_ticker)
if model_for_this_ticker is not None:
del model_for_this_ticker

end_ticker_ram = get_ram_usage()
print(f"RAM after full processing of {ticker}: {end_ticker_ram:.2f} MB (Change: {end_ticker_ram - start_ticker_ram:+.2f} MB)")

if ticker_index < len(stock_tickers) - 1:
time.sleep(SECONDS_BETWEEN_TICKERS)

print(f"RAM at end of cycle {main_cycle_count}: {get_ram_usage():.2f} MB")
time.sleep(SECONDS_BETWEEN_GLOBAL_CYCLES)

except KeyboardInterrupt:
print("\n\nExiting program...")
except Exception as e:
print(f"\n\nUNHANDLED ERROR: {e}")
import traceback
traceback.print_exc()
finally:
print(f"\nFINAL RAM USAGE: {get_ram_usage():.2f} MB")
print("Script terminated.")


What I've tried: Calling tf.keras.backend.clear_session() after each model is processed. Explicitly deleting the model, DataFrames, and NumPy arrays using del. Forcing garbage collection with gc.collect().

Question: what else can cause this 6 MB memory increase per ticker?

yfinance version: 0.2.61 pandas version: 2.2.2 numpy version: 2.0.2 tensorflow version: 2.18.0 scikit-learn version: 1.6.1 psutil version: 5.9.5

Continue reading...