text_html_output_1.png image_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

### Let's create a simple 'Environment' class and 'Agent' class.
### For every action, the instant reward is sampled from Gaussian distribution with unit standard deviation, each one of which however has different mean. 
### For the sake of simplicity, we assume stationarity.

class Envierment:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon  
        self.q_values = list()
        for k in range(self.K): 
            self.q_values.append(np.random.randn())
    def give_reward(self, k):
        if k>= K:
            print('Invalid action!')
            raise ValueError()
        else:
            return np.random.randn() + self.q_values[k]
            

class Agent:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon  
        self.action, self.reward = 0, 0
        self.result = np.zeros((horizon, 1))
        self.result_df = pd.DataFrame()
        
    def take_action(self, k=None):
        if k == None:
            self.action = np.random.choice(self.K)
        else:
            self.action = k
        return self.action    
    
    def record_reward(self, trial, n, reward):
        self.reward = reward
        if self.result.shape[1]<n+1:
            self.result = np.c_[self.result, np.zeros(self.horizon)]            
        self.result[n][trial] = self.reward
            
        
    def turn_result_into_df(self):
        self.result = np.cumsum(self.result, axis=0)     
        for n in range(1, self.horizon): 
            self.result[n] /=n            
        self.result_df = pd.DataFrame(self.result, columns = ['trial_'+ str(x) for x in range(self.result.shape[1])])   
            
            
### Instantiate an Envierment object with K=3           
K, horizon = 3, 100            
envioronment = Envierment(K, horizon)
agent = Agent(envioronment.K, envioronment.horizon)

### The actions (0~K-1) are taken randomly, i.e. with no stretegy.
### We run 10 independent trials and compare them.
trials = 10
Result = np.zeros((horizon, trials))
for trial in range(trials):
    for n in range(horizon):
        action = agent.take_action()
        reward = envioronment.give_reward(action)
        agent.record_reward(trial, n, reward)
        
agent.turn_result_into_df()

### Plot the result
fig, ax = plt.subplots(figsize=(20, 10))
for trial in range(trials):
    ax.plot(agent.result_df['trial_'+ str(trial)])    
ax.legend(agent.result_df.columns, loc='upper right')
ax.set(title='Average reward for each trial')

print(f'K: {envioronment.K}')
print(f'action-value: {envioronment.q_values}')  

class Envierment:
    def __init__(self, K, horizon):
        self.K, self.horizon = (K, horizon)
        self.q_values = list()
        for k in range(self.K):
            self.q_values.append(np.random.randn())
    def give_reward(self, k):
        if k >= K:
            print('Invalid action!')
            raise ValueError()
        else:
            return np.random.randn() + self.q_values[k]
class Agent:
    def __init__(self, K, horizon):
        self.K, self.horizon = (K, horizon)
        self.action, self.reward = (0, 0)
        self.result = np.zeros((horizon, 1))
        self.result_df = pd.DataFrame()
        self.Q_values = [0] * self.K
        self.num_actions = [0] * self.K
        self.step_size = 1
    def take_action(self, k=None):
        if k == None:
            self.action = np.random.choice(self.K)
        else:
            self.action = k
        return self.action
    def incremental_update(self, step_size):
        self.num_actions[self.action] += 1
        if step_size == 'sample_average':
            self.step_size = 1 / self.num_actions[self.action]
        else:
            self.step_size = step_size
        self.Q_values[self.action] = self.Q_values[self.action] + self.step_size * (self.reward - self.Q_values[self.action])
    def record_reward(self, trial, n, reward, step_size):
        self.reward = reward
        if self.result.shape[1] < n + 1:
            self.result = np.c_[self.result, np.zeros(self.horizon)]
        self.result[n][trial] = self.reward
        self.incremental_update(step_size)
    def turn_result_into_df(self):
        self.result = np.cumsum(self.result, axis=0)
        for n in range(1, self.horizon):
            self.result[n] /= n
        self.result_df = pd.DataFrame(self.result, columns=['trial_' + str(x) for x in range(self.result.shape[1])])
K, horizon = (1, 20)
true_q_values = np.array([0, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 4, 0, 4, 0, 4, 0, 4, 0]).reshape((horizon, 1))
envioronment = Envierment(K, horizon)
agent = Agent(envioronment.K, envioronment.horizon)
step_sizes = [1 / 2, 1 / 8, 1, 'sample_average']
Result = np.zeros((horizon, len(step_sizes)))
Q_values = np.zeros((horizon, len(step_sizes)))
for trial, step_size in enumerate(step_sizes):
    for n in range(horizon):
        Q_values[n][trial] = np.array(agent.Q_values).squeeze()
        action = agent.take_action()
        reward = envioronment.give_reward(action)
        agent.record_reward(trial, n, reward, step_size)
Q_values = Q_values.T
fig, axes = plt.subplots(1, 4, figsize=(40, 10))
for trial in range(len(step_sizes)):
    axes[trial].plot(Q_values[trial])