<file_sep># -*- coding: utf-8 -*-
"""
Created on Sun Mar 11 11:25:11 2018

@author: <NAME>

The codes at https://github.com/analog-rl/Easy21 were referred to in creating this code
"""

#%% 
import os 
import sys

# get the directory and add its path to python search path for modules
dirpath = os.path.dirname(os.path.abspath(__file__))
sys.path.append(dirpath)

import numpy as np
from easy21 import *
import random
import matplotlib.cm as cm

# dealer card value: 1-10, player sum: 1-21

# note the state S consists of the dealer's showing card and the player's sum 
# initialization 

class MC_Agent:
    def __init__(self,environment,N0):
        self.N0 = N0; 
        self.env = environment;
        # Intialize state value function to zero
        self.V= np.zeros((self.env.dealer_value_count,self.env.player_value_count))
        # Initialize state-action pair to zero
        self.Q = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count))
        # N(s,a) is the number of times that action a has been selected from state s.
        self.N = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count)) 
       
        # return at each state-action pair, using a list of list of list 
        self.G = [];
        for k in range(self.env.dealer_value_count*self.env.player_value_count):
            G1 =[];
            for j in range (self.env.action_count):
                G1.append([])
            self.G.append(G1)
        
        self.episodes = 0
        self.count_wins = 0

    # selecting an action according to the epsilon-greedy policy 
    def select_action(self,state):
        dealer_id = state.dealer -1;
        player_id = state.player - 1;
        
        epsilon = self.N0/(self.N0+sum(self.N[dealer_id, player_id,:]))
        
        if random.random()< epsilon:
            if random.random() < 0.5:
                action = Action.hit;
            else:
                action = Action.stick
        else:
            action = Action.to_action(np.argmax(self.Q[dealer_id, player_id,:]))
            
        return action    
                
    def train(self,episodes):    
        for episode in range(episodes):
            episode_pairs = []
            # random start 
            s = self.env.gen_start_state()
            
            # generate an episode with the epsilon-greedy policy
            while not s.is_terminal:
                a = self.select_action(s)       
                # update N(s,a)
                self.N[s.dealer-1,s.player-1,Action.as_int(a)] += 1
                
                # store action-value pairs
                episode_pairs.append((s,a))
                # execute action
                s, r= self.env.step(s,a)
            
            self.count_wins = self.count_wins+1 if r==1 else self.count_wins 
                
            for s,a in episode_pairs:
                dealer_id = s.dealer-1
                player_id = s.player-1
                # update the state-action-return pair
                idx = dealer_id*10+player_id
                self.G[idx][Action.as_int(a)].append(r)
                
                # update Q(s,a) using a varying step size alpha = 1/N(st,at)
                alpha = 1.0/self.N[dealer_id,player_id,Action.as_int(a)] 
                error = np.mean(self.G[idx][Action.as_int(a)]) - self.Q[dealer_id,player_id,Action.as_int(a)]
                self.Q[dealer_id,player_id,Action.as_int(a)] += alpha*error
                
                # update V(s)
                # ideally update of V(s) should happen only ONCE after update of Q(s,a) for all s,a in the episode;
                # but for coding simplifity, it is updated right after every update of Q(s,a). This should not influence the final results. 
                self.V[dealer_id,player_id] = max(self.Q[dealer_id,player_id,:])
        self.episodes = self.episodes + episodes      
    def plot_frame(self, ax):
        def get_state_val(x,y):
            return self.V[x,y]
        X = np.arange(0,self.env.dealer_value_count,1)
        Y = np.arange(0,self.env.player_value_count,1)
        X,Y = np.meshgrid(X,Y)
        Z = get_state_val(X,Y)
        
        surf = ax.plot_surface(X,Y,Z,cmap=cm.bwr,antialiased=False)
        return surf
        

#%% Train and generate the value function 
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
#import matplotlib.cm as cm
#
#
#N0 = 100
#agent = MC_Agent(Environment(),N0)
#for i in range(1):
#    agent.train(50000)
#    
#fig = plt.figure()
#ax = fig.add_subplot(111,projection ='3d')
#agent.plot_frame(ax)
#plt.title('value function after %d episodes' % agent.episodes)
#ax.set_xlabel('Dealer showing')
#ax.set_ylabel('Player sum')
#ax.set_zlabel('V(s)')
#ax.set_xticks(range(1,agent.env.dealer_value_count+1))
#ax.set_yticks(range(1,agent.env.player_value_count+1))
#plt.show()
#plt.savefig('Value function.png')
#    

#%% animate the learning process 
    #import matplotlib.animation as animation
#def update(frame):
#    agent.train(10000)
#    
#    ax.clear()
#    surf = agent.plot_frame(ax)
#    plt.title('winning perc: %f frame:%s ' % (float(agent.count_wins)/agent.episodes,frame))
#    fig.canvas.draw()
#    return surf
#    
#
#import matplotlib.pyplot as plt
#import matplotlib.animation as animation
#from mpl_toolkits.mplot3d import Axes3D
#import matplotlib.cm as cm
#N0 = 100
#agent = MC_Agent(Environment(),N0)
##fig=plt.figure('N0=%d' % N0)
#fig = plt.figure('N100')
#ax = fig.add_subplot(111,projection ='3d')
#ani = animation.FuncAnimation(fig,update,4,repeat=False)
#
#ani.save('MC_process.gif',writer = 'imagemagick',fps=3)
#plt.show()
# show the gif
#from IPython.display import Image
#Image(url="MC_process.gif")<file_sep># -*- coding: utf-8 -*-
"""
Created on Wed Mar  7 10:12:25 2018

@author: <NAME>

Reference:
https://github.com/analog-rl/Easy21
"""
#%% 
import random
import matplotlib.pyplot as plt
import numpy as np

from enum import Enum
class Action(Enum):
    stick = 0
    hit = 1    
    
class Card:
    def __init__(self,force_black = False):
        self.value = random.randint(1,10)        
        if force_black or random.randint(1,3) != 3:
            self.is_black = True;            
        else:
            self.is_black = False;
            self.value = -self.value;


class State:
    def __init__ (self, dealer, player, is_terminal = False):
        
        self.dealer = dealer    # the summation of the dealer
        self.player = player    # the summation of the player 
        self.is_terminal = is_terminal # whether the state is terminal

def step (state, action):
    new_state = state;
    reward = 0;
    if action == Action.hit:
        new_state.player  += Card().value;
        if new_state.player > 21 or new_state.player <1:
            new_state.is_terminal = True;
            reward = -1
            return new_state, reward
    elif action == Action.stick:
        while not new_state.is_terminal:
            new_state.dealer += Card().value;
            if new_state.dealer > 21 or  new_state.dealer < 1:
                new_state.is_terminal = True;
                reward = 1
            elif new_state.dealer> 17:
                new_state.is_terminal = True;
                if new_state.player > new_state.dealer:
                    reward = 1
                elif new_state.player < new_state.dealer:
                    reward = -1
    return new_state, reward

#%% Test: whether draws follow a prespecified distribution
cards = [None]*10000;
for i in np.arange(len(cards)):
    cards[i] = Card()

f = plt.figure(1);

plt.subplot(2,1,1)
plt.title('Test: the value of the cards follow a uniform distribution \n between 1 and 10')
card_value = [abs(card.value) for card in cards]
plt.hist(card_value)
plt.subplot(2,1,2)
plt.title('Test: the probability of a red card (0) and a black card (1) \n are 1/3 and 2/3, respectively')
card_is_black = [card.is_black for card in cards]
plt.hist(card_is_black)
plt.subplots_adjust(hspace = 0.6 ) # adjust the horizontal space between subplots
#plt.show()
plt.savefig("test1.png");
#%% Test: If the player’s sum exceeds 21, or becomes less than 1, then he “goes bust” and loses the game (reward -1)
def test_player_bust():
    state = State(Card(True).value, Card(True).value)
    action = Action.hit
    while not state.is_terminal:
        state, reward = step(state,action)
    return state, reward

import matplotlib.pyplot as plt
plt.figure(2)
values = [];
for i in range(10000):
    state,reward = test_player_bust()
    if state.player > 21 or state.player < 1:
        values.append(1)
    else:
        values.append(-1)
        print("error! The player's score should be >21 or <1 at the terminal state!")

plt.hist(values)
plt.title("Test: player bust >21 or <1")
plt.savefig("test2.png")


#%% Test: player stick. 

def test_player_stick():
    state = State(Card(True).value, Card(True).value)
    action = Action.stick
    while not state.is_terminal:
        state, reward = step(state,action)
    return state, reward

import matplotlib.pyplot as plt

values = []
for i in range(10000):
    state,reward = test_player_stick()
    if state.dealer > 21 or state.dealer < 1:
        if reward == 1:
            values.append(1)
        else:
            values.append(-1)
            print('error! The player should have won!')
    elif state.player > state.dealer:
        if reward == 1:
            values.append(1)
        else:
            values.append(-2)
            print('error! The player should have won!')
    elif state.player == state.dealer:
        if reward == 0:
            values.append(1)
        else:
            values.append(-3)
            print('error! It should be a tie!')
            
plt.figure(3)
plt.hist(values)
plt.title("Test: player stick")
plt.savefig("test3.png")


#%% play 
def play():
    dealer = Card(True).value
    player = Card(True).value
    state = State(dealer,player)
    reward = 0
    while not state.is_terminal:
#        print('dealer score:%d, player score:%d' %  (state.dealer, state.player))
        if state.player > 17:
            action = Action.stick
        else:
            action = Action.hit
        state, reward = step(state,action)
#    print('dealer score:%d, player score:%d' %  (state.dealer, state.player))   
    print("reward = %d" % reward);
    return reward

<file_sep># -*- coding: utf-8 -*-
"""
Implementation of Sarsa(\lambda) with binary features and linear function approximation for Easy21 
following the algorithm presented in Section~12.7 of [Sutton & Barto, 2017]
@author: Boran (Pan) Zhao

"""

#%% 
import os 
import sys

# get the directory and add its path to python search path for modules
dirpath = os.path.dirname(os.path.abspath(__file__))
sys.path.append(dirpath)

import numpy as np
from easy21 import *
from easy21_mc_control import *
import random

# dealer card value: 1-10, player sum: 1-21

# note the state S consists of the dealer's showing card and the player's summation 
# initialization 

class Sarsa_Approx_Agent:
    def __init__(self,environment,N0,lam):
        self.N0 = N0; 
        self.lam = lam
        self.gamma = 1
        self.mse = float('inf')
        self.env = environment;
        # Intialize state value function to zero
        self.V= np.zeros((self.env.dealer_value_count,self.env.player_value_count))
        # Initialize state-action pair to zero
        self.Q = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count))
        # weight vecotor 
        self.w = np.zeros((3,6,2))
         # eligibility trace for the weight vector
        self.E = np.zeros((3,6,2))
        self.features ={'dealer':[[1,4], [4,7],[7,10]],  'player':[[1, 6],[4, 9],[7,12],[10, 15],[13, 18],[16, 21]], 'action':[Action.hit,Action.stick]}     
        
        
        self.episodes = 0
        self.count_wins = 0
    # determine phi(s,a)
    def feature_eval(self,s,a):
        Phi = np.zeros((3,6,2))
        for dealer_interval_id, dealer_interval in enumerate(self.features['dealer']):
            if dealer_interval[0] <= s.dealer <= dealer_interval[1]:
                for player_interval_id, player_interval in enumerate(self.features['player']):
                    if player_interval[0] <= s.player <= player_interval[1]:
                        Phi[dealer_interval_id,player_interval_id,Action.as_int(a)]=1
        return Phi
                        
                        
    # selecting an action according to the epsilon-greedy policy 
    def select_action(self,state):
        dealer_id = state.dealer -1
        player_id = state.player - 1
        # epsilon for exploration
#        epsilon = self.N0/(self.N0+sum(self.N[dealer_id, player_id,:]))
        # use a constant epsilon for exploration
        epsilon = 0.05        
        if random.random()< epsilon:
            if random.random() < 0.5:
                action = Action.hit;
            else:
                action = Action.stick
        else:
            action = Action.to_action(np.argmax(self.Q[dealer_id, player_id,:]))            
        return action    
                
    def train(self,num_episodes):
        for episode in range(num_episodes):
            # random start 
            s = self.env.gen_start_state()          
            # reset the eligibility traces.... Is this really necessary?
            self.E = np.zeros((3,6,2))
            
            # generate an episode with the epsilon-greedy policy and update Q(s,a) and E(s,a) in each step
            a = self.select_action(s)       
            while not s.is_terminal:
                # update N(s,a)
#                self.N[s.dealer-1,s.player-1,Action.as_int(a)] += 1
                 # execute action a and observe s_new, r
                s_new, r= self.env.step(s,a)
                td_error = r                
                # identify the active features and update their corresponding eligibility trace elements
                Phi =self.feature_eval(s,a)
                td_error -= np.sum(self.w*Phi)
                 # accumulating traces
                self.E += Phi
                # replacing traces 
#                self.E = Phi                
#                for dealer_interval_id, dealer_interval in enumerate(self.features['dealer']):
#                    if dealer_interval[0] <= s.dealer <= dealer_interval[1]:
#                        for player_interval_id, player_interval in enumerate(self.features['player']):
#                            if player_interval[0] <= s.player <= player_interval[1]:
#                                td_error -= self.w[dealer_interval_id,dealer_interval_id,Action.as_int(a)]
#                                # accumulating traces
#                                self.E[dealer_interval_id,dealer_interval_id,Action.as_int(a)] += 1
#                                # replacing traces 
#                                self.E[dealer_interval_id,dealer_interval_id,Action.as_int(a)] = 1

                if not s_new.is_terminal:
                    # select a new action a_new using policy dervied from Q
                    a_new = self.select_action(s_new)                    
                    Phi = self.feature_eval(s_new,a_new)                    
                    td_error += np.sum(self.gamma*(self.w*Phi))                    
                                       
                # using a constant step size for exploration
                alpha = 0.05                           
                #update the weight vector and eligibility trace 
                self.w += alpha*td_error*self.E
                self.E *= self.gamma*self.lam 
                # update s and 
                s = s_new
                if not s_new.is_terminal:
                    a = a_new                            
            self.count_wins = self.count_wins+1 if r==1 else self.count_wins                 
        # report the mean-squared error mean((Q(s,a)-Qmc(s,a))^2 and the winning percentage
        self.episodes = self.episodes + num_episodes       
       
        # update the Q function
        self.update_Q()
        
    
    def update_Q(self):
        for dealer_id in range(self.Q.shape[0]):
            for player_id in range(self.Q.shape[1]):
                for action_id in range(self.Q.shape[2]):
                    Phi = self.feature_eval(State(dealer_id+1,player_id+1), Action.to_action(action_id))
                    self.Q[dealer_id,player_id,action_id] = np.sum(Phi*self.w)
        
    def update_V(self):
        for dealer_id in range(self.V.shape[0]):
            for player_id in range(self.V.shape[1]):
                self.V[dealer_id,player_id] = max(self.Q[dealer_id,player_id,:])
                
                
    def plot_frame(self, ax):
        def get_state_val(x,y):
            return self.V[x,y]
        X = np.arange(0,self.env.dealer_value_count,1)
        Y = np.arange(0,self.env.player_value_count,1)
        X,Y = np.meshgrid(X,Y)
        Z = get_state_val(X,Y)
        
        surf = ax.plot_surface(X,Y,Z,cmap=cm.bwr,antialiased=False)
        return surf    


#%% Train and generate the Q function with Monte Carlo control
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
#import matplotlib.cm as cm
#
#N0 = 100
#mc_agent = MC_Agent(Environment(),N0)
#mc_agent.train(int(1e6))
#print('After %s episodes, winning percentage:%f' % (mc_agent.episodes, mc_agent.count_wins/mc_agent.episodes))
#
#fig = plt.figure(1)
#ax = fig.add_subplot(111,projection ='3d')
#mc_agent.plot_frame(ax)
#plt.title('value function after %d episodes' % mc_agent.episodes)
#ax.set_xlabel('Dealer showing')
#ax.set_ylabel('Player sum')
#ax.set_zlabel('V(s)')
#ax.set_xticks(range(1,mc_agent.env.dealer_value_count+1))
#ax.set_yticks(range(1,mc_agent.env.player_value_count+1))
##plt.savefig('Value function.png')
#plt.show()
#Qmc = mc_agent.Q

#%% Train with Sarsa(lambda) with linear function approximation using different lambda while printing the MSE of Q 
import matplotlib.pyplot as plt
Lambda = np.linspace(0,1,11)
fig = plt.figure('MSE under different lambda values')
mse = []
Color = ['b','g','r','c','m','y','k']
LineStyle =['-','--',':','-.']
for i in range(len(Lambda)):
    mse.append([])

# Learn and plot the result
for lam_id,lam in enumerate(Lambda):
#    print('lambda = %s'% lam)
    agent = Sarsa_Approx_Agent(Environment(),N0,lam)
    for i in range(1000):
        agent.train(1)    
        agent.mse = np.mean((agent.Q-Qmc)**2)
        mse[lam_id].append(agent.mse)      
    print('lambda = %s, MSE: %f, winning percentage:%f' % (agent.lam, agent.mse, agent.count_wins/agent.episodes))
    
X = list(range(1,len(mse[0])+1))    
fig = plt.figure('MSE against lambda')
plt.plot(Lambda, [x[-1] for x in mse])
plt.xlabel('lambda')
plt.ylabel('mean-squared error')
plt.savefig('MSE against lambda under linear approximation')
plt.show()

fig = plt.figure('Learning process')
plt.subplot(211)
plt.plot(X,mse[0],color = Color[0], linestyle=LineStyle[0%4])
plt.xlabel('episode')
plt.ylabel('MSE')
plt.title('lambda = 0')

plt.subplot(212)
plt.plot(X,mse[-1],color = Color[0], linestyle=LineStyle[0%4])
plt.xlabel('episode')
plt.ylabel('MSE')
plt.title('lambda = 1')
plt.savefig('Learning process for lambda 0 and 1 under linear approximation')
plt.show()
<file_sep># -*- coding: utf-8 -*-
"""
Implementation of Sarsa(\lambda) for Easy21 

@author: <NAME>

"""
#%% 
import os 
import sys

# get the directory and add its path to python search path for modules
dirpath = os.path.dirname(os.path.abspath(__file__))
sys.path.append(dirpath)

import numpy as np
from easy21 import *
from easy21_mc_control import *
import random

# dealer card value: 1-10, player sum: 1-21

# note the state S consists of the dealer's showing card and the player's sum 
# initialization 

class Sarsa_Agent:
    def __init__(self,environment,N0,lam):
        self.N0 = N0; 
        self.lam = lam
        self.gamma = 1
        self.mse = float('inf')
        self.env = environment;
        # Intialize state value function to zero
        self.V= np.zeros((self.env.dealer_value_count,self.env.player_value_count))
        # Initialize state-action pair to zero
        self.Q = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count))
        # N(s,a) is the number of times that action a has been selected from state s.
        self.N = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count)) 
        # eligibility trace for every state-action pair
        self.E = np.zeros((self.env.dealer_value_count,self.env.player_value_count,self.env.action_count))
        
        self.episodes = 0
        self.count_wins = 0

    # selecting an action according to the epsilon-greedy policy 
    def select_action(self,state):
        dealer_id = state.dealer -1;
        player_id = state.player - 1;
        
        epsilon = self.N0/(self.N0+sum(self.N[dealer_id, player_id,:]))
        
        if random.random()< epsilon:
            if random.random() < 0.5:
                action = Action.hit;
            else:
                action = Action.stick
        else:
            action = Action.to_action(np.argmax(self.Q[dealer_id, player_id,:]))            
        return action    
                
    def train(self,num_episodes):    
        for episode in range(num_episodes):
            # random start 
            s = self.env.gen_start_state()          
            
            # generate an episode with the epsilon-greedy policy and update Q(s,a) and E(s,a) in each step
            a = self.select_action(s)       
            while not s.is_terminal:
                # update N(s,a)
                self.N[s.dealer-1,s.player-1,Action.as_int(a)] += 1
                # execute action a and observe s_new, r
                s_new, r= self.env.step(s,a)
                dealer_id = s.dealer-1
                player_id = s.player-1
                if s_new.is_terminal:
                    Q_new = 0
                else: 
                    # select a new action a_new using policy dervied from Q
                    a_new = self.select_action(s_new)
                    dealer_id_new = s_new.dealer-1
                    player_id_new = s_new.player-1
                    Q_new = self.Q[dealer_id_new,player_id_new,Action.as_int(a_new)]
                # using a varying step size alpha = 1/N(st,at)
                alpha = 1.0/self.N[dealer_id,player_id,Action.as_int(a)] 
                # calculate TD error
                td_error = r + self.gamma*Q_new - self.Q[dealer_id,player_id,Action.as_int(a)]
                # update the eligibility trace 
                self.E[dealer_id,player_id, Action.as_int(a)] += 1
                #update the Q and E for all state-action pairs 
                self.Q += alpha*td_error*self.E
                self.E *= self.gamma*self.lam
#                for q,e in np.nditer([self.Q,self.E],op_flags =['readwrite']):
#                    q[...] = q + alpha*td_error*e
#                    e[...] = self.gamma*self.lam*e                
                # update s and 
                s = s_new
                if not s_new.is_terminal:
                    a = a_new                            
            self.count_wins = self.count_wins+1 if r==1 else self.count_wins                 
        # report the mean-squared error mean((Q(s,a)-Qmc(s,a))^2 and the winning percentage
        self.episodes = self.episodes + num_episodes       
    def update_V(self):
        for dealer_id in range(self.V.shape[0]):
            for player_id in range(self.V.shape[1]):
                self.V[dealer_id,player_id] = max(self.Q[dealer_id,player_id,:])
                
                
    def plot_frame(self, ax):
        def get_state_val(x,y):
            return self.V[x,y]
        X = np.arange(0,self.env.dealer_value_count,1)
        Y = np.arange(0,self.env.player_value_count,1)
        X,Y = np.meshgrid(X,Y)
        Z = get_state_val(X,Y)
        
        surf = ax.plot_surface(X,Y,Z,cmap=cm.bwr,antialiased=False)
        return surf    

#%% Train and generate the Q function with Monte Carlo control
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.cm as cm

N0 = 100
mc_agent = MC_Agent(Environment(),N0)
mc_agent.train(int(1e6))
print('After %s episodes, winning percentage:%f' % (mc_agent.episodes, mc_agent.count_wins/mc_agent.episodes))

fig = plt.figure(1)
ax = fig.add_subplot(111,projection ='3d')
mc_agent.plot_frame(ax)
plt.title('value function after %d episodes' % mc_agent.episodes)
ax.set_xlabel('Dealer showing')
ax.set_ylabel('Player sum')
ax.set_zlabel('V(s)')
ax.set_xticks(range(1,mc_agent.env.dealer_value_count+1))
ax.set_yticks(range(1,mc_agent.env.player_value_count+1))
plt.savefig('Value function from MC.png')
plt.show()
Qmc = mc_agent.Q

#%% Train with Sarsa(lambda) using different lambda while printing the MSE of Q 
Lambda = np.linspace(0,1,10)
fig = plt.figure('MSE under different lambda values')
mse = []
Color = ['b','g','r','c','m','y','k']
LineStyle =['-','--',':','-.']
for i in range(len(Lambda)):
    mse.append([])

# Learn and plot the result
for lam_id,lam in enumerate(Lambda):
#    print('lambda = %s'% lam)
    agent = Sarsa_Agent(Environment(),N0,lam)
    for i in range(1000):
        agent.train(1)    
        agent.mse = np.mean((agent.Q-Qmc)**2)
        mse[lam_id].append(agent.mse)      
    print('lambda = %s, MSE: %f, winning percentage:%f' % (agent.lam, agent.mse, agent.count_wins/agent.episodes))
    
X = list(range(1,len(mse[0])+1))    
fig = plt.figure('MSE against lambda')
plt.plot(Lambda, [x[-1] for x in mse])
plt.xlabel('lambda')
plt.ylabel('mean-squared error')
plt.savefig('MSE against lambda')
plt.show()

fig = plt.figure('Learning process')
plt.subplot(211)
plt.plot(X,mse[0],color = Color[0], linestyle=LineStyle[0%4])
plt.xlabel('episode')
plt.ylabel('MSE')
plt.title('lambda = 0')

plt.subplot(212)
plt.plot(X,mse[-1],color = Color[0], linestyle=LineStyle[0%4])
plt.xlabel('episode')
plt.ylabel('MSE')
plt.title('lambda = 1')

plt.savefig('Learning process for lambda 0 and 1')
plt.show()

##%% animate the learning process (does not work at this moment)
#import matplotlib.animation as animation
#def update(frame):
#    agent.train(10000)
#    
#    ax.clear()
#    surf = agent.plot_frame(ax)
#    plt.title('winning perc: %f frame:%s ' % (float(agent.count_wins)/agent.episodes,frame))
#    fig.canvas.draw()
#    return surf
#    

#import matplotlib.pyplot as plt
#import matplotlib.animation as animation
#from mpl_toolkits.mplot3d import Axes3D
#import matplotlib.cm as cm
#N0 = 100
#agent = MC_Agent(Environment(),N0)
##fig=plt.figure('N0=%d' % N0)
#fig = plt.figure('N100')
#ax = fig.add_subplot(111,projection ='3d')
#ani = animation.FuncAnimation(fig,update,4,repeat=False)
#
#ani.save('MC_process.gif',writer = 'imagemagick',fps=3)
#plt.show()
## show the gif
##from IPython.display import Image
##Image(url="MC_process.gif")<file_sep># Easy21



1 Implementation of Easy21: easy21-implement.py

2 Monte-Carlo Control in Easy21: easy21_mc_control.py

3 TD Learning in Easy21: easy21_sarsa_lambda.py

4 Linear Function Approximation in Easy21: easy21_sarsa_lambda_approx

5 Discussion

What are the pros and cons of bootstrapping in Easy21?<br>
Pros: no need to wait until the end of an episode, accelerate the learning process, decrease the variance <br>
Cons: may introduce bias 


Would you expect bootstrapping to help more in blackjack or Easy21 ? Why? <br>
Help more in Easy21, because it takes a longer time on average to finish an episode in easy21 due to the fact that a value of a card can be negative depending on its color.

What are the pros and cons of function approximation in Easy21?<br>
Pros: memory saving, learning speed acceleration<br>
Cons: can only solve the problem approximately since a function approximator cannot represent all the state-action values accurately 

How would you modify the function approximator suggested in this section to get better results in Easy21?<br>
Have no idea right now. 
<file_sep># -*- coding: utf-8 -*-
"""
Created on Sun Mar 11 15:21:25 2018

@author: <NAME>
"""

# -*- coding: utf-8 -*-
"""
Created on Wed Mar  7 10:12:25 2018

@author: <NAME>

The codes at https://github.com/analog-rl/Easy21 were referred to in creating this code
"""
#%% 
import random
import matplotlib.pyplot as plt
import copy

from enum import Enum
class Action(Enum):
    hit = 0    
    stick = 1
    
    
    @staticmethod
    def to_action(n):
        return Action.hit if n==0 else Action.stick
        
    @staticmethod
    def as_int(a):
        return 0 if a==Action.hit else 1 
    
    
    
class Card:
    def __init__(self,force_black = False):
        self.value = random.randint(1,10)        
        if force_black or random.randint(1,3) != 3:
            self.is_black = True;            
        else:
            self.is_black = False;
            self.value = -self.value;


class State:
    def __init__ (self, dealer, player, is_terminal = False):        
        self.dealer = dealer    # the summation of the dealer
        self.player = player    # the summation of the player 
        self.is_terminal = is_terminal # whether the state is terminal


class Environment:
    def __init__(self):
        self.dealer_value_count = 10; # [1:10], note that black card is enforced at the start
        self.player_value_count = 21; # [1:21]
        self.action_count = 2; # hit and stick
        
    def gen_start_state(self):
        s = State(Card(True).value,Card(True).value)
        return s
        
    def step(self, state, action):
#        new_state = state does not work because modifying new_state will influence state
        new_state = copy.copy(state)
        reward = 0;
        if action == Action.hit:
            new_state.player  += Card().value;
            if new_state.player > 21 or new_state.player <1:
                new_state.is_terminal = True;
                reward = -1
                return new_state, reward
        elif action == Action.stick:
            while not new_state.is_terminal:
                new_state.dealer += Card().value;
                if new_state.dealer > 21 or  new_state.dealer < 1:
                    new_state.is_terminal = True;
                    reward = 1
                elif new_state.dealer> 17:
                    new_state.is_terminal = True;
                    if new_state.player > new_state.dealer:
                        reward = 1
                    elif new_state.player < new_state.dealer:
                        reward = -1
        return new_state, reward