<file_sep>import gym
import numpy as np
import time
import sys

class Q_learning(object):
    def __init__(self):
        self.env = gym.make('InvertedPendulum-v2')
        self.digitalied_num = 10
        self.steps = 200
        self.episodes = 1000000
        self.goal_ave = 190
        self.moving_ave_num = 10
        self.first_prob = 0.5
        self.action_num = 7  #3~7
        self.moving_ave = np.full(self.moving_ave_num,0)
        self.q_table = np.random.uniform(low=-1,high=1,size=(self.digitalied_num\
                                                        **self.env.observation_space.shape[0],self.action_num))
        self.reward_of_episode = 0
        self.render_flag = False
        self.learning_finish = False
        self.alpha = 0.1
        self.gamma = 0.9
        self.bin_pram = []
        pram_low =  [-0.5,-0.3,-0.3,-0.3]
        pram_high = [0.5,0.3,0.3,0.3]
        for i in range(self.env.observation_space.shape[0]):
            self.bin_pram.append(np.linspace(pram_low[i],pram_high[i],self.digitalied_num)[1:-1])
            #self.bin_pram.append(np.linspace(self.env.observation_space.low[i],self.env.observation_space.high[i],self.digitalied_num)[1:-1])

    def digitalie(self,obs):
        state = 0
        for i in range(self.env.observation_space.shape[0]):
            state += np.digitize(obs[i],self.bin_pram[i]) * (self.digitalied_num ** i)
        return state
    
    def decide_action(self,next_state,episode):
        epsilon = self.first_prob * (1/(episode+1))
        if epsilon < 0.1:
            epsilon = 0.1
        if epsilon <= np.random.uniform(0,1):
            next_action = np.argmax(self.q_table[next_state])
        else:
            # next_action = int(round(self.env.action_space.sample()[0]))
            next_action = np.random.choice(self.action_num)
        return next_action

    def update_Q_table(self,next_state,state,action,reward,q_table,done):
        if not done:
            next_max_q = max(q_table[next_state])
        else:
            next_max_q = 0
        q_table[state,action] = (1 - self.alpha) * (q_table[state,action]) + \
                                    self.alpha * (reward + self.gamma * next_max_q)
        return q_table

    def run(self):
        print()
        max_step = 0
        for episode in range(self.episodes):
            obs = self.env.reset()
            state = self.digitalie(obs)
            action = np.argmax(self.q_table[state])
            self.reward_of_episode = 0

            for i in range(self.steps):
                if self.render_flag or self.learning_finish:
                    self.env.render()
                
                observation ,reward, done, info = self.env.step(action-3)
                self.reward_of_episode += reward
                
                next_state = self.digitalie(observation)
                self.q_table = self.update_Q_table(next_state,state,action,reward,self.q_table,done)
                action = self.decide_action(next_state,episode)
                state = next_state

                if done:
                    if max_step < i:
                        max_step = i
                    self.moving_ave = np.hstack((self.moving_ave[1:],self.reward_of_episode))
                    sys.stdout.write("\repisode:%5d, reward:%3d, step:%3d, max_step:%3d, average:%3d"%(episode+1,self.reward_of_episode,i+1,max_step+1,self.moving_ave.mean()))
                    sys.stdout.flush()
                    time.sleep(0.001)
                    
                    if self.learning_finish:
                        self.render_flag = True
                    break

            if self.moving_ave.mean() >= self.goal_ave or max_step > self.goal_ave:
                if self.learning_finish:
                    break
                print("\nLearning is finished!!")
                print("episode: {}\n".format(episode+1))
                self.learning_finish = True


pendulum = Q_learning()
pendulum.run()
<file_sep>import gym
import math
import numpy as np
import sys
import time

env = gym.make('InvertedPendulum-v2')
Render_Flag = True
max_step = 0
for i in range(20000):
    env.reset()
    Kp = 6.0
    Ki = 0.04
    Kd = 0.03
    goal = 0.0
    e = 0.0
    e1 = 0.0
    e2 = 0.0
    action = -0.01
    action_old = 0.0

    for t in range(200):
        if Render_Flag:
            env.render()
        obs, reward, done, info = env.step(-action)
        action_old = action
        e2 = e1
        e1 = e
        e = goal - obs[1] 
        action = action_old + Kp * (e-e1) + Ki * e + Kd * ((e-e1)-(e1-e2))
        if done or t >= 199:
            if max_step < t:
                max_step = t
            print("episode:%5d, step:%3d, max:%3d"%(i+1,t+1,max_step+1))
            # sys.stdout.write("\repisode:%5d, step:%3d, max:%3d"%(i+1,t+1,max_step+1))
            # sys.stdout.flush()
            # time.sleep(0.1)
            break


<file_sep># mujoco_RL
## version
python 3.7  
mujoco 2.0
## overview  
Studying reinforcement learning with mujoco
## demo


[![IMAGE ALT TEXT HERE](http://img.youtube.com/vi/EgifOjdg_P8/0.jpg)](http://www.youtube.com/watch?v=EgifOjdg_P8)  
https://youtu.be/QK2n01hk4PY  
DDQN
<file_sep>import gym
import numpy as np
import time
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam
from keras.utils import plot_model
from collections import deque
from keras import backend as K
import tensorflow  as tf
import sys

def huberloss(y_true, y_pred):
    err = y_true - y_pred
    cond = K.abs(err) < 1.0
    L2 = 0.5 * K.square(err)
    L1 = (K.abs(err) - 0.5)
    loss = tf.where(cond,L2,L1)
    return K.mean(loss)

class QNetwork:
    def __init__(self,learning_rate=0.01, state_size=4,action_size=7,hidden_size=10):
        self.action_size = action_size
        self.model = Sequential()
        self.model.add(Dense(hidden_size,activation='relu',input_dim=state_size))
        self.model.add(Dense(hidden_size,activation='relu'))
        self.model.add(Dense(action_size,activation='linear'))
        self.optimizer = Adam(lr=learning_rate)
        self.model.compile(loss=huberloss,optimizer=self.optimizer)

    def replay(self, memory, batch_size, gamma, targetQN):
        inputs = np.zeros((batch_size, 4))
        targets = np.zeros((batch_size, self.action_size))
        mini_batch = memory.sample(batch_size)

        for i, (state_b, action_b, reward_b, next_state_b) in enumerate(mini_batch):
            inputs[i:i+1] = state_b
            target = reward_b

            if not (next_state_b == np.zeros(state_b.shape)).all(axis=1):
                retmainQs = self.model.predict(next_state_b)[0]
                next_action = np.argmax(retmainQs)
                target = reward_b + gamma * targetQN.model.predict(next_state_b)[0][next_action]

            targets[i] = self.model.predict(state_b)
            targets[i][action_b] = target

        self.model.fit(inputs, targets, epochs=1, verbose=0)


class Memory:
    def __init__(self,max_size=1000):
        self.buffer = deque(maxlen=max_size)

    def add(self,experience):
        self.buffer.append(experience)

    def sample(self, batch_size):
        idx = np.random.choice(np.arange(len(self.buffer)),size=batch_size,replace = False)
        return [self.buffer[ii] for ii in idx]

    def len(self):
        return len(self.buffer)

class Actor:
    def get_action(self, state, episode, mainQN):
        epislon = 0.01 + 0.9 /(1.0+episode*0.1)
        if epislon <= np.random.uniform(0,1):
            reTargetQs = mainQN.model.predict(state)[0]
            action = np.argmax(reTargetQs)

        else:
            action = np.random.choice(7)

        return action

METHOD_STR = "DDQN" #DQN or DDQN
RENDER_FLAG = True
env = gym.make('InvertedPendulum-v2')
num_episodes = 300000
max_number_of_steps = 200
goal_average_reward = 195
num_consecutive_iterations = 10
total_reward_vec = np.zeros(num_consecutive_iterations)
gamma = 0.99
islearnd = False
isrender = False
hidden_size = 16
learning_rate = 0.0001
memory_size = 10000
batch_size = 100

max_step = 0

mainQN = QNetwork(hidden_size=hidden_size, learning_rate=learning_rate)
targetQN = QNetwork(hidden_size=hidden_size, learning_rate=learning_rate)
#plot_model(mainQN.model, to_file='Qnetwork.png', show_shapes=True) 
memory = Memory(max_size = memory_size)
actor = Actor()

for episode in range(num_episodes):
    env.reset()
    state, reward, done, info = env.step(env.action_space.sample())
    state = np.reshape(state, [1,4])
    episode_reward = 0
    targetQN.model.set_weights(mainQN.model.get_weights())

    for t in range(max_number_of_steps):
        if islearnd and RENDER_FLAG:
            env.render()
            time.sleep(0.01)

        action = actor.get_action(state, episode, mainQN)
        next_state, reward, done , info = env.step((action-3))
        next_state = np.reshape(next_state,[1,4])

        if t == 199:
            reward = 1
        elif done:
            next_state = np.zeros(state.shape)
            reward = -1
        else:
            reward = 0

        episode_reward += 1
        memory.add((state,action,reward,next_state))
        state = next_state
        
        if (memory.len() > batch_size)  and not islearnd:
            mainQN.replay(memory,batch_size,gamma,targetQN)

        if METHOD_STR=="DQN":
            targetQN.model.set_weights(mainQN.model.get_weights())
        else:
            pass

        if done or t >= 199:
            if max_step < t:
                max_step = t
            total_reward_vec = np.hstack((total_reward_vec[1:], episode_reward))
            print('{:5d} Episode finished, {:6.2f} steps, ave: {:6.2f}, max: {:4d}'.format(episode,t+1,total_reward_vec.mean(),max_step+1),flush=True)
            break

    if total_reward_vec.mean() >= goal_average_reward:
        if not islearnd:
            print('Episode {:5d} train agent successfuly!'.format(episode+1))
        islearnd = True
        if not isrender:
            isrender = True
