text_plain_output_1.png image_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

### Let's create a simple 'Environment' class and 'Agent' class.
### For every action, the instant reward is sampled from Gaussian distribution with unit standard deviation, each one of which however has different mean. 
### For the sake of simplicity, we assume stationarity.

class Envierment:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon  
        self.q_values = list()
        for k in range(self.K): 
            self.q_values.append(np.random.randn())
    def give_reward(self, k):
        if k>= K:
            print('Invalid action!')
            raise ValueError()
        else:
            return np.random.randn() + self.q_values[k]
            

class Agent:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon  
        self.action, self.reward = 0, 0
        self.result = np.zeros((horizon, 1))
        self.result_df = pd.DataFrame()
        
    def take_action(self, k=None):
        if k == None:
            self.action = np.random.choice(self.K)
        else:
            self.action = k
        return self.action    
    
    def record_reward(self, trial, n, reward):
        self.reward = reward
        if self.result.shape[1]<n+1:
            self.result = np.c_[self.result, np.zeros(self.horizon)]            
        self.result[n][trial] = self.reward
            
        
    def turn_result_into_df(self):
        self.result = np.cumsum(self.result, axis=0)     
        for n in range(1, self.horizon): 
            self.result[n] /=n            
        self.result_df = pd.DataFrame(self.result, columns = ['trial_'+ str(x) for x in range(self.result.shape[1])])   
            
            
### Instantiate an Envierment object with K=3           
K, horizon = 3, 100            
envioronment = Envierment(K, horizon)
agent = Agent(envioronment.K, envioronment.horizon)

### The actions (0~K-1) are taken randomly, i.e. with no stretegy.
### We run 10 independent trials and compare them.
trials = 10
Result = np.zeros((horizon, trials))
for trial in range(trials):
    for n in range(horizon):
        action = agent.take_action()
        reward = envioronment.give_reward(action)
        agent.record_reward(trial, n, reward)
        
agent.turn_result_into_df()

### Plot the result
fig, ax = plt.subplots(figsize=(20, 10))
for trial in range(trials):
    ax.plot(agent.result_df['trial_'+ str(trial)])    
ax.legend(agent.result_df.columns, loc='upper right')
ax.set(title='Average reward for each trial')

print(f'K: {envioronment.K}')
print(f'action-value: {envioronment.q_values}')  

### Let's modity 'Environment' class in a way that we can feed the predefined q_values when instantiating.

class Envierment:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon
        self.q_values = list()
        for k in range(self.K): 
            self.q_values.append(np.random.randn())
      

    def give_reward(self, k):
        if k>= K:
            print('Invalid action!')
            raise ValueError()
        else:
            return np.random.randn() + self.q_values[k]
            

### Let's also modity 'Agent' class so that it can update the estimate of action-values as it goes through the trials.            
class Agent:
    def __init__(self, K, horizon):
        self.K, self.horizon = K, horizon  
        self.action, self.reward = 0, 0
        self.result = np.zeros((horizon, 1))
        self.result_df = pd.DataFrame()
        self.Q_values = [0]*self.K
        self.num_actions = [0]*self.K
        self.step_size = 1
        
    def take_action(self, k=None):
        if k == None:
            self.action = np.random.choice(self.K)
        else:
            self.action = k
        return self.action    
    
    def incremental_update(self, step_size):
        self.num_actions[self.action] += 1
        if step_size == 'sample_average': self.step_size = (1/self.num_actions[self.action])
        else : self.step_size = step_size
        self.Q_values[self.action] = self.Q_values[self.action] + self.step_size*(self.reward-self.Q_values[self.action])    
    
    
    def record_reward(self, trial, n, reward, step_size):
        self.reward = reward
        if self.result.shape[1]<n+1:
            self.result = np.c_[self.result, np.zeros(self.horizon)]            
        self.result[n][trial] = self.reward
        self.incremental_update(step_size)    
        
    def turn_result_into_df(self):
        self.result = np.cumsum(self.result, axis=0)     
        for n in range(1, self.horizon): 
            self.result[n] /=n            
        self.result_df = pd.DataFrame(self.result, columns = ['trial_'+ str(x) for x in range(self.result.shape[1])])   

### Instantiate an Envierment object with K=3          
K, horizon = 1, 20
true_q_values = np.array([0, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 4, 0, 4, 0, 4, 0, 4, 0]).reshape((horizon, 1))
envioronment = Envierment(K, horizon)
agent = Agent(envioronment.K, envioronment.horizon)

### Set K=1, i.e., we have only one action 0.
### To compare the effect of the step size, let's see how well the agent estimates the action-values depending on different stet sizes

step_sizes = [1/2, 1/8, 1, 'sample_average']
Result = np.zeros((horizon, len(step_sizes)))
Q_values = np.zeros((horizon, len(step_sizes)))
for trial, step_size in enumerate(step_sizes):    
    for n in range(horizon):
        Q_values[n][trial] = np.array(agent.Q_values).squeeze()

        action = agent.take_action()
        reward = envioronment.give_reward(action)
        agent.record_reward(trial, n, reward, step_size)
Q_values = Q_values.T

fig, axes = plt.subplots(1, 4, figsize=(40, 10))
for trial in range(len(step_sizes)):
    axes[trial].plot( Q_values[trial] )    
    
# ax.legend(df.columns, loc='upper right')
# ax.set(title='Average reward for each trial')

# print(f'K: {envioronment.K}')
# print(f'action-value: {envioronment.q_values}')  

horizon = 20
q_a = [0, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 4, 0, 4, 0, 4, 0, 4, 0]
step_sizes = [1 / 2, 1 / 8, 1]
Q_a = [0]
df = pd.DataFrame(q_a, columns=['q_a'])
for step_size in step_sizes:
    for n in range(1, horizon):
        reward = np.random.randn() + q_a[n]
        Q_a.append(Q_a[-1] + step_size * (reward - Q_a[-1]))
    df[f'Q_a_{step_size}'] = Q_a
    Q_a = [0]
for n in range(1, horizon):
    reward = np.random.randn() + q_a[n]
    Q_a.append(Q_a[-1] + 1 / n * (reward - Q_a[-1]))
df[f'Q_a_1/n'] = Q_a
fig, axes = plt.subplots(2, 2, figsize=(20, 12))
axes[0][0].plot(df['q_a'], 'o-')
axes[0][0].plot(df['Q_a_0.5'], 'o-')
axes[0][0].legend(['target', 'estimate'], loc='upper right')
axes[0][1].plot(df['q_a'], 'o-')
axes[0][1].plot(df['Q_a_0.125'], 'o-')
axes[0][1].legend(['target', 'estimate'], loc='upper right')
axes[1][0].plot(df['q_a'], 'o-')
axes[1][0].plot(df['Q_a_1'], 'o-')
axes[1][0].legend(['target', 'estimate'], loc='upper right')
axes[1][1].plot(df['q_a'], 'o-')
axes[1][1].plot(df['Q_a_1/n'], 'o-')
axes[1][1].legend(['target', 'estimate'], loc='upper right')
plt.show()
plt.close()
df