I have been trying to implement Reinforcement Learning books exercise 2.5 I have written this piece of code according to this pseudo version class k_arm: def __init__(self, iter, method="incrementally"): # self.iter placeholder self.iter = iter self.k = 10 self.eps = .1 # here is Q(a) and N(a) self.qStar = np.zeros(self.k) self.n = np.zeros(self.k) # Method just for experimenting different functions self.method = method def pull(self): # selecting argmax(Q(A)) action with prob. (1 - eps) eps = np.random.uniform(0, 1, 1) if eps < self.eps or self.qStar.argmax() == 0: a = np.random.randint(10) else: a = self.qStar.argmax() # R bandit(A) r = np.random.normal(0, 0.01, 1) # N(A) <- N(A) + 1 self.n[a] += 1 # Q(A) <- Q(A) i / (N(A)) * (R - Q(A)) if self.method == "incrementally": self.qStar[a] += (r - self.qStar[a]) / self.n[a] return self.qStar[a]` iter = 1000 rewards = np.zeros(iter) c = k_arm(iter, method="incrementally") for i in range(iter): k = c.pull() rewards = k And I get this as a result Where I am expecting this kind of results. I have been trying to understand where am I went missing here, but I couldn't. Continue reading...
