[Python] Optuna best parameters gives different results when tested manually

I am using Optuna to optimize the hyperparameters for my DQN agent. However, when I manually test the best parameters in a separate script, I get different results.

Here is the best trial output from Optuna:

[I 2025-02-24 18:52:11,252] Trial 46 finished with value: 0.5056228539630224 and parameters: {'gamma': 0.8319693755370036, 'lr': 0.001448230144216603, 'batch_size': 64, 'target_update_freq': 9000, 'memory_size': 4000, 'num_layers': 7, 'layer_0_size': 192, 'layer_1_size': 320, 'layer_2_size': 192, 'layer_3_size': 512, 'layer_4_size': 384, 'layer_5_size': 320, 'layer_6_size': 448, 'w1': 0.2631460842858361, 'w2': 0.6017998205410044}. Best is trial 46 with value: 0.5056228539630224.

However, the results are different when I manually test these same parameters in another script.

## **Manual Testing Script**
import pdb
import random
from collections import deque

import numpy as np
import torch
from agent import DQNAgent
from env import EnergyEnvironment

seed = 42
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

# Hyperparameters
state_size = 21
action_size = 10
days = 11
episodes = 2000
house_name = "House1"

# [I 2025-02-24 18:52:11,252] Trial 46 finished with value: 0.5056228539630224 and parameters: {'gamma': 0.8319693755370036, 'lr': 0.001448230144216603, 'batch_size': 64, 'target_update_freq': 9000, 'memory_size': 4000, 'num_layers': 7, 'layer_0_size': 192, 'layer_1_size': 320, 'layer_2_size': 192, 'layer_3_size': 512, 'layer_4_size': 384, 'layer_5_size': 320, 'layer_6_size': 448, 'w1': 0.2631460842858361, 'w2': 0.6017998205410044}. Best is trial 46 with value: 0.5056228539630224.

hidden_layers = [64, 448]

w1 = 0.5609477599817275
w2 = 0.325477222487064
w3 = 1.0 - (w1 + w2)

# KG-based state space (9 days historical + 1 day averaged)
env = EnergyEnvironment(days, house_name, w_1=w1, w_2=w2, w_3=w3)

# Initialisation
agent = DQNAgent(state_size, action_size, hidden_layers, verbose=True)

agent.gamma = 0.1529759429203543
agent.lr = 0.01227582671086234
agent.batch_size = 128
agent.memory = deque(maxlen=3000)
agent.optimizer = torch.optim.Adam(agent.model.parameters(), lr=0.0005964073269022759)

agent.train_dqn(env, episodes=2000, train_days=days, target_update_freq=9000)

# Testing
agent.test_dqn(env, test_day=days)


Optuna Optimization Script:

import pdb
import random
from collections import deque

import numpy as np
import torch
from agent import DQNAgent, DQN
from env import EnergyEnvironment
import optuna

seed = 42
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

# Hyperparameters
state_size = 21
action_size = 10
days = 11
episodes = 2000
house_name = "House1"


def objective(trial):
gamma = trial.suggest_float("gamma", 0.05, 0.99, log=True)
lr = trial.suggest_float("lr", 0.000001, 0.02, log=True)
batch_size = trial.suggest_int("batch_size", 32, 160, step=32)
target_update_freq = trial.suggest_int("target_update_freq", 500, 15000, step=500)
memory_size = trial.suggest_int("memory_size", 2000, 10000, step=1000)
num_layers = trial.suggest_int("num_layers", 2, 7)
hidden_layers = [trial.suggest_int(f"layer_{i}_size", 64, 512, step=64) for i in range(num_layers)]

# Optimize w1 and w2, and compute w3
w1 = trial.suggest_float("w1", 0.05, 0.9)
w2 = trial.suggest_float("w2", 0.05, 1.0 - w1)
w3 = 1.0 - (w1 + w2) # Compute w3 to satisfy w1 + w2 + w3 = 1

env = EnergyEnvironment(days, house_name, w_1=w1, w_2=w2, w_3=w3)
agent = DQNAgent(state_size, action_size, hidden_layers, verbose=True)

agent.gamma = gamma
agent.lr = lr
agent.batch_size = batch_size
agent.memory = deque(maxlen=memory_size)
agent.optimizer = torch.optim.Adam(agent.model.parameters(), lr=lr)

agent.train_dqn(env, episodes=2000, train_days=days, target_update_freq=target_update_freq)

reward = agent.test_dqn(env, test_day=days)
return reward

pruner = optuna.pruners.MedianPruner(n_warmup_steps=10)
study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42), pruner=pruner)
study.optimize(objective, n_trials=100)

print("Best Parameters:", study.best_params, study.best_value)

# Visualization of parameter importance
optuna.visualization.plot_param_importances(study).show()


Final reward after testing manually: 0.4753161173956163

Continue reading...