[Python] I created a very simple RL model for trading stocks but its output is the same...

I've been trying to build a basic Neural Net that analyzes a stock's price and chooses whether to buy, sell, or hold.

# Importing Libraries
import torch
import torch.nn as nn
import torch.optim as optim

import random
import numpy as np
from datetime import datetime, timedelta
import pickle

# Defining Hyperparameters
input_num = 8
output_num = 3
episodes = 100000

# Layer architecture parameters
hidden_size = 100
recurrent_size = 50

# For the trading environment
portfolio = []
buying_power = 0
goal = 999999999 # So the model goes as high as possible

# Define neural network class
class NN(nn.Module):
def __init__(self, in_f, out_f, h_size, r_size):
super().__init__()
# Variables
self.epsilon = 1
self.epsilon_end = 0.01
self.decay = 0.9999
# MLP
self.fc1 = nn.Linear(in_features=in_f, out_features=h_size)
self.fc2 = nn.Linear(in_features=h_size, out_features=h_size)
self.fc3 = nn.Linear(in_features=h_size, out_features=h_size)
# LSTM
self.rnn = nn.LSTM(input_size=h_size, hidden_size=r_size,
num_layers=5, batch_first=True)
#FFN
self.ffn = nn.Linear(in_features=r_size, out_features=out_f)

# Defining EGreedy
def epsilon_decay(self):
self.epsilon = (self.epsilon_end + (self.epsilon - self.epsilon_end)
* np.exp(-1. * self.decay))


def forward(self, x):
x = torch.relu(self.fc1(x))
x = torch.relu(self.fc2(x))
x = torch.relu(self.fc3(x))
# LSTM
x, (hn, cn) = self.rnn(x) # separate the output and hidden state
x = torch.tensor(x, requires_grad=True)
x = torch.relu(self.ffn(x))
return x

def select_action(self, x):
x = self.forward(x)
if np.random.rand() < self.epsilon:
action = torch.randint(0, 10, (1, 3), dtype=torch.float32)
else:
action = x
self.epsilon_decay()
return action

model = NN(input_num, output_num, hidden_size, recurrent_size)

# Defining Loss
criterion = nn.MSELoss() # Mean Squared Error loss function
optimizer = optim.Adam(model.parameters(), lr=0.01) # Adam optimizer

# Load stock data
with open('stock_data.pkl', 'rb') as data:
stock_data = pickle.load(data)
# List of stocks
ticker_list = [
'AAPL', 'NVDA', 'MSFT', 'AMZN', 'GOOGL', 'META', 'TSLA', 'TSM', 'BRK-A',
'AVGO', 'WMT', 'LLY', 'JPM', 'V', 'XOM', 'ORCL', 'MA', 'UNH', 'COST', 'HD',
'PG', 'NFLX', 'NVO', 'BAC', 'JNJ', 'CRM', 'ABBV', 'SAP', 'ASML', 'CVX', 'KO',
'TMUS', 'MRK', 'ACN', 'WFC', 'NOW', 'CSCO', 'TM', 'BX', 'PEP', 'IBM', 'BABA',
'MCD', 'AMD', 'AXP', 'ADBE', 'LIN', 'DIS', 'AZN', 'MS'
]

# Defining functions for buying and selling
def buy(stock, date):
global buying_power
close = stock_data[stock].loc[date]['Close']
price = float(close)
if price <= buying_power:
portfolio.append(stock)
buying_power -= price

def sell(stock, date):
global buying_power
close = stock_data[stock].loc[date]['Close']
price = float(close)
if stock in portfolio:
portfolio.remove(stock)
buying_power += price

def portfolio_value(date):
valuation = sum(stock_data[stock].loc[date]['Close'] for stock in portfolio)
return valuation

def model_evaluate(stock, date, input):
# Formatting input
input_list = []
for i in input:
price = i*i # Squaring so its easier for model to see difference in price
input_list.append(price)
port_value = portfolio_value(date)
input_list.append(port_value)
input_list.append(buying_power)
# To Tensor
input_tensor = torch.tensor(input_list, dtype=torch.float32)
input_tensor = input_tensor.unsqueeze(0)
# Passing Data through model
output = model.select_action(input_tensor)
# Updating environment based on model choice
choice = torch.argmax(output)
choice_v = output[0, choice].item()
if choice != 0:
if choice_v < 1:
number_of_purchase = 1
number_of_purchase = round(choice_v)
if choice == 1:
for _ in range(number_of_purchase):
buy(stock, date)
else:
for _ in range(number_of_purchase):
sell(stock, date)

# Index Checking Function
def index_exists(df, index):
try:
df.loc[index]
return True
except KeyError:
return False

# Main Training Loop
for episode in range(episodes):
# Setting up Episode
random.shuffle(ticker_list)
r_stock = ticker_list[0]
start_account_value = random.randint(1000, 100000)
buying_power = start_account_value
portfolio = []
# Finding TimeFrame
start_of_dataset = datetime(2020, 1, 2)
random_start = random.randint(0, 500)
frame_start = start_of_dataset + timedelta(random_start)
business_day = 0
day_format = 0
day_count = 0

# Track progress
print(f"Starting episode {episode + 1}/{episodes}")
# Training on TimeFrame
while business_day < 100:
# Finding the current day
c_day = frame_start + timedelta(day_count)
day_format = c_day.strftime('%Y-%m-%d')
# Checking if it was a business day
Is_Business_Day = index_exists(stock_data[r_stock], day_format)
if Is_Business_Day:
market = stock_data[r_stock].loc[day_format]
model_evaluate(r_stock, day_format, market.values)
business_day += 1
day_count += 1

# Updating model
# Finding last day account value
final_day_format = day_format
p_value = portfolio_value(final_day_format)
value = buying_power + p_value
gain = (value - start_account_value)/start_account_value
# Loss
loss = criterion(torch.tensor(value, dtype=torch.float32, requires_grad=True),
torch.tensor(goal, dtype=torch.float32, requires_grad=True))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Debug info after each episode
print(f'Episode {episode + 1} started with: {start_account_value} finished with portfolio gain: {gain}%')


If you're wondering about the stock_data file, it's just a dictionary where the keys are stock tickers and the values are pandas data frames of prices of that stock. You can create the same file with this program.


#Importing Libraries
import yfinance as yf
import os
import numpy as np
import pandas as pd

# Testing libraries
import matplotlib.pyplot as plt

# The Window we will use for every stock
start_window='2020-01-01'
end_window='2023-12-31'

# Defining dictionary
stock_data = {}

# Data collecting function
def collect(stock):
history = yf.download(stock,
start=start_window,
end=end_window,
progress=False,
)
stock_data[stock] = history

# list of stocks based on -- What I like
ticker_list=['AAPL','NVDA','MSFT','AMZN','GOOGL','META',
'TSLA','TSM','BRK-A','AVGO','WMT','LLY','JPM','V',
'XOM','ORCL','MA','UNH','COST','HD','PG','NFLX',
'NVO','BAC','JNJ','CRM','ABBV','SAP','ASML','CVX',
'KO','TMUS','MRK','ACN','WFC','NOW','CSCO','TM',
'BX','PEP','IBM','BABA','MCD','AMD','AXP','ADBE',
'LIN','DIS','AZN','MS']

# adding histories to stock data
for ticker in ticker_list:
collect(ticker)


# Writing to file
import pickle

with open('stock_data.pkl', 'wb') as f:
pickle.dump(stock_data, f)
print('Data stored')


No matter what input is given to the model, it always gives the same output. Only once in a while does the model give a different output due to the EGreedy Exploration.

I've tried adjusting the learning rate.

I also tried squaring the input prices in the hopes that it would create a difference between prices that model could see.

I've tested the buy and sell functions to make sure they work.

Other than that, I can't think of anything that would mess up the output.

Continue reading...