<file_sep># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""NFSP agents trained on Kuhn Poker."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging
import tensorflow.compat.v1 as tf

from open_spiel.python import policy
from open_spiel.python import rl_environment
from open_spiel.python.algorithms import exploitability
from open_spiel.python.algorithms import nfsp
import glob
import os
import numpy as np
from open_spiel.python.algorithms import random_agent


FLAGS = flags.FLAGS

flags.DEFINE_integer("num_train_episodes", int(3e6),
                     "Number of training episodes.")
flags.DEFINE_integer("eval_every", 100,
                     "Episode frequency at which the agents are evaluated.")
flags.DEFINE_list("hidden_layers_sizes", [
    128,
], "Number of hidden units in the avg-net and Q-net.")
flags.DEFINE_integer("replay_buffer_capacity", int(2e5),
                     "Size of the replay buffer.")
flags.DEFINE_integer("reservoir_buffer_capacity", int(2e6),
                     "Size of the reservoir buffer.")
flags.DEFINE_float("anticipatory_param_agent0", 0,
                   "Prob of using the rl best response as episode policy for agent0.")
flags.DEFINE_float("anticipatory_param_agent1", 0,
                   "Prob of using the rl best response as episode policy for agent1.")
flags.DEFINE_string("experiment_name", "kuhn_poker_0.1_7_27", "Experiment name")
flags.DEFINE_string("load_path", "/home/jxu8/Code_update/open_spiel/sessions_nfsp/", "Path to load the session")

flags.DEFINE_string("save_path", "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/", "Path to load the session")


class NFSPPolicies(policy.Policy):
  """Joint policy to be evaluated."""

  def __init__(self, env, nfsp_policies, mode):
    game = env.game
    player_ids = [0, 1]
    super(NFSPPolicies, self).__init__(game, player_ids)
    self._policies = nfsp_policies
    self._mode = mode
    self._obs = {"info_state": [None, None], "legal_actions": [None, None]}

  def action_probabilities(self, state, player_id=None):
    cur_player = state.current_player()
    legal_actions = state.legal_actions(cur_player)

    self._obs["current_player"] = cur_player
    self._obs["info_state"][cur_player] = (
        state.information_state_tensor(cur_player))
    self._obs["legal_actions"][cur_player] = legal_actions

    info_state = rl_environment.TimeStep(
        observations=self._obs, rewards=None, discounts=None, step_type=None)

    with self._policies[cur_player].temp_mode_as(self._mode):
      p = self._policies[cur_player].step(info_state, is_evaluation=True).probs
    prob_dict = {action: p[action] for action in legal_actions}
    return prob_dict

def eval_against_random_agent1(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  cur_agents = [trained_agents[0], random_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes

def eval_against_random_agent0(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  cur_agents = [random_agents[0], trained_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes

def eval_against_trained_agents(env, trained_agents, num_episodes):
  wins = np.zeros(3)
  for _ in range(num_episodes):
    time_step = env.reset()
    while not time_step.last():
      player_id = time_step.observations["current_player"]
      if env.is_turn_based:
        agents_output = trained_agents[player_id].step(time_step, is_evaluation=True)
        action_list = [agents_output.action]
      else:
        agents_output = [agent.step(time_step, is_evaluation=True) for agent in trained_agents]
        action_list = [agent_output.action for agent_output in agents_output]
      time_step = env.step(action_list)
    if time_step.rewards[0] > 0:
      wins[0] += 1
    elif time_step.rewards[1] > 0:
      wins[1] += 1
    else:
      wins[2] += 1
  return wins / num_episodes

def eval_between_random_agents(env, random_agents, num_episodes):
    wins = np.zeros(3)
    rewards = np.zeros(2)
    for _ in range(num_episodes):
        time_step = env.reset()
        while not time_step.last():
            player_id = time_step.observations["current_player"]
            if env.is_turn_based:
                agents_output = random_agents[player_id].step(time_step, is_evaluation=True)
                action_list = [agents_output.action]
            else:
                agents_output = [agent.step(time_step, is_evaluation=True) for agent in random_agents]
                action_list = [agent_output.action for agent_output in agents_output]
            time_step = env.step(action_list)
        rewards[0] += time_step.rewards[0]
        rewards[1] += time_step.rewards[1]
        if time_step.rewards[0] > 0:
            wins[0] += 1
        elif time_step.rewards[1] > 0:
            wins[1] += 1
        else:
            wins[2] += 1
    return wins / num_episodes, rewards / num_episodes

def main(unused_argv):
  game = "kuhn_poker"
  num_players = 2
  load_path = FLAGS.load_path + FLAGS.experiment_name
  env_configs = {"players": num_players}
  env = rl_environment.Environment(game, **env_configs)
  info_state_size = env.observation_spec()["info_state"][0]
  num_actions = env.action_spec()["num_actions"]

  hidden_layers_sizes = [int(l) for l in FLAGS.hidden_layers_sizes]
  kwargs = {
      "replay_buffer_capacity": FLAGS.replay_buffer_capacity,
      "epsilon_decay_duration": FLAGS.num_train_episodes,
      "epsilon_start": 0.06,
      "epsilon_end": 0.001,
  }

  random_agents = [
      random_agent.RandomAgent(player_id=idx, num_actions=num_actions)
      for idx in range(num_players)
  ]

  model_dirs = sorted(glob.glob(load_path + "/episode-*"), key=lambda x: int(os.path.split(x)[1][8:]))
  for counter, dir in enumerate(model_dirs, 1):
      if counter % 1 == 0:
          tf.reset_default_graph()
          with tf.Session() as sess:
            # pylint: disable=g-complex-comprehension
            agent0 = nfsp.NFSP(sess, 0, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent0,
                          **kwargs)
            agent1 = nfsp.NFSP(sess, 1, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent1,
                          **kwargs)

            saver = tf.train.Saver()
            saver.restore(sess, dir + "/trained_model-10000")
            #expl_policies_avg = NFSPPolicies(env, [agent0, agent1], nfsp.MODE.average_policy)
            #expl_list, expl = exploitability.exploitability(env.game, expl_policies_avg)
            # f3 = open(FLAGS.save_path + "exploitability_list.txt", "a")
            # f3.write(str(expl_list[0]) + ' ' + str(expl_list[1]) + '\n')
            # f4 = open(FLAGS.save_path + "exploitability_avg.txt", "a")
            # f4.write(str(expl) + '\n')

            # logging.info("Episode: %s, Exploitability AVG %s", counter*10000, expl)
            # logging.info("_____________________________________________")

            # win_rates_against_random_agent1 = eval_against_random_agent1(env, [agent0, agent1], random_agents, 1000)
            # win_rates_against_random_agent0 = eval_against_random_agent0(env, [agent0, agent1], random_agents, 1000)
            # win_rates_against_trained_agents = eval_against_trained_agents(env, [agent0, agent1], 1000)
            win_rates_between_random_agents, avg_utility_between_random_agents = eval_between_random_agents(env, random_agents, 1000)

            f1 = open(FLAGS.save_path + "win_rates/eta_0/win_rates_between_random_agents.txt", "a")
            f1.write(str(win_rates_between_random_agents[0]) + ' ' + str(win_rates_between_random_agents[1]) + ' ' + str(win_rates_between_random_agents[2]) + '\n')
            f2 = open(FLAGS.save_path + "avg_utility/eta_0/avg_utility_between_random_agents.txt", "a")
            f2.write(str(avg_utility_between_random_agents[0]) + ' ' + str(avg_utility_between_random_agents[1]) + '\n')

            logging.info("Episode: %s", counter*10000)
            logging.info("Trained_agent0 vs Random_agent1: %s", avg_utility_between_random_agents)


            # f1 = open(FLAGS.save_path + "win_rates_against_random_agent1.txt", "a")
            # f1.write(str(win_rates_against_random_agent1[0]) + ' ' + str(win_rates_against_random_agent1[1]) + ' ' + str(win_rates_against_random_agent1[2]) + '\n')
            # f2 = open(FLAGS.save_path + "win_rates_against_random_agent0.txt", "a")
            # f2.write(str(win_rates_against_random_agent0[0]) + ' ' + str(win_rates_against_random_agent0[1]) + ' ' + str(win_rates_against_random_agent0[2]) + '\n')
            # f3 = open(FLAGS.save_path + "win_rates_against_eachother.txt", "a")
            # f3.write(str(win_rates_against_trained_agents[0]) + ' ' + str(win_rates_against_trained_agents[1]) + ' ' + str(win_rates_against_trained_agents[2]) + '\n')

            # logging.info("Episode: %s", counter*10000)
            # logging.info("Trained_agent0 vs Random_agent1: %s", win_rates_against_random_agent1)
            # logging.info("Random_agent0 vs Trained_agent1: %s", win_rates_against_random_agent0)
            # logging.info("Trained_agent0 vs Trained_agent1 %s", win_rates_against_trained_agents)
            # logging.info("_____________________________________________")



if __name__ == "__main__":
  app.run(main)
<file_sep>from absl import app
from absl import flags
import numpy as np
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt

def read_wr(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    for line in lines[:-1]:
        [str1, str2, str3] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
    return list1, list2, list3

def read_exploitability(txt_name):
    txt_file = open(txt_name, 'r')
    lines = txt_file.read().split('\n')
    num_list = []
    for str in lines[:-1]:
        if str == "NaN":
            num_list.append(1)
        else:
            num_list.append(float(str))
    return num_list

def read_loss(txt_name):
    txt_file = open(txt_name)
    lines = txt_file.read().split('\n')
    list1 = []
    list2 = []
    for line in lines[:-1]:
        [str1, str2] = line.split(' ')
        if str1 != 'None':
            list1.append(float(str1))
        else:
            list1.append(str1)
        if str2 != 'None':
            list2.append(float(str2))
        else:
            list2.append(str2)
    for idx, number in enumerate(list1):
        if number == 'None':
            list1[idx] = list1[idx+1]
    for number, idx in enumerate(list2):
        if number == 'None':
            list2[idx] = list2[idx+1]
    return list1, list2

def read_behavior_probs(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    list4 = []
    list5 = []
    list6 = []
    list7 = []
    list8 = []
    for line in lines[:-1]:
        [str1, str2, str3, str4, str5, str6, str7, str8] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
        list4.append(float(str4))
        list5.append(float(str5))
        list6.append(float(str6))
        list7.append(float(str7))
        list8.append(float(str8))
    return list1, list2, list3, list4, list5, list6, list7, list8

def au_mean(list, start):
    return sum(list[start:]) / len(list[start:])


def main(argv):
    kuhn_poker_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/"
    kuhn_poker_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_1_7_28/"
    ttt_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_26/"
    ttt_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_1_7_29/"

    kuhn_poker_psro = "/home/jxu8/Code/open_spiel/evaluation_data/eval_kuhn_poker_psro_7_2/"

    avg_rewards_0_against_eachother = []
    avg_rewards_1_against_eachother = []
    avg_rewards_0_against_random_agent1 = []
    avg_rewards_0_against_random_agent0 = []
    avg_rewards_1_against_random_agent1 = []
    avg_rewards_1_against_random_agent0 = []
    avg_rewards_0_between_random_agents = []

    au_0_against_eachother_mean = []
    au_1_against_eachother_mean = []
    au_0_against_random_agent1_mean = []
    au_0_against_random_agent0_mean = []
    au_1_against_random_agent1_mean = []
    au_1_against_random_agent0_mean = []
                                                                                                               # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    # load avg_utility against eachother with eta 0 in training
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility_5000/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    au_0_against_eachother_mean.append([au_mean(avg_rewards_0_against_eachother[0][0], 200), au_mean(avg_rewards_0_against_eachother[0][1], 200)])

    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))
    # load avg_utility against eachother with eta 1 in training
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility_5000/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    au_1_against_eachother_mean.append([au_mean(avg_rewards_1_against_eachother[0][0], 200), au_mean(avg_rewards_1_against_eachother[0][1], 200)])

    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    avg_rewards_0_against_random_agent1.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility_5000/eta_0/avg_utility_against_random_agent1.txt'))
    avg_rewards_0_against_random_agent0.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility_5000/eta_0/avg_utility_against_random_agent0.txt'))
    avg_rewards_1_against_random_agent1.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility_5000/eta_0/avg_utility_against_random_agent1.txt'))
    avg_rewards_1_against_random_agent0.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility_5000/eta_0/avg_utility_against_random_agent0.txt'))

    au_0_against_random_agent1_mean.append([au_mean(avg_rewards_0_against_random_agent1[0][0], 200), au_mean(avg_rewards_0_against_random_agent1[0][1], 200)])
    au_0_against_random_agent0_mean.append([au_mean(avg_rewards_0_against_random_agent0[0][0], 200), au_mean(avg_rewards_0_against_random_agent0[0][1], 200)])
    au_1_against_random_agent1_mean.append([au_mean(avg_rewards_1_against_random_agent1[0][0], 200), au_mean(avg_rewards_1_against_random_agent1[0][1], 200)])
    au_1_against_random_agent0_mean.append([au_mean(avg_rewards_1_against_random_agent0[0][0], 200), au_mean(avg_rewards_1_against_random_agent0[0][1], 200)])

    avg_rewards_0_between_random_agents.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0/avg_utility_between_random_agents.txt'))


    # plt avg utility in kuhn_poker_nfsp_0.1_7_27
    plt.figure(figsize=(15, 10))
    ax2 = plt.subplot(211)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility with eta 0 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[0][0], "b", label="agent0, {}".format(au_0_against_eachother_mean[0][0]))
    line2, = plt.plot(avg_rewards_0_against_eachother[0][1], "r", label="agent1, {}".format(au_0_against_eachother_mean[0][1]))
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(212)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility with eta 0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[0][0], "b", label="agent0, {}".format(au_1_against_eachother_mean[0][0]))
    line2, = plt.plot(avg_rewards_1_against_eachother[0][1], "r", label="agent1, {}".format(au_1_against_eachother_mean[0][1]))
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()


    # ax2 = plt.subplot(512)
    # ax2.set_title("average utility with eta 1 in evaluation (0.1 in training)")
    # line1, = plt.plot(avg_rewards_0_against_eachother[1][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_0_against_eachother[1][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(513)
    # ax2.set_title("average utility with eta 0.1 in evaluation (0.1 in training)")
    # line1, = plt.plot(avg_rewards_0_against_eachother[2][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_0_against_eachother[2][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(514)
    # ax2.set_title("average utility with eta 0_1 in evaluation (0.1 in training)")
    # line1, = plt.plot(avg_rewards_0_against_eachother[3][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_0_against_eachother[3][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(515)
    # ax2.set_title("average utility with eta 1_0 in evaluation (0.1 in training)")
    # line1, = plt.plot(avg_rewards_0_against_eachother[4][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_0_against_eachother[4][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    # plt.show()


    # plt avg utility in kuhn_poker_nfsp_1_7_28
    # plt.figure(figsize=(15, 20))
    # ax2 = plt.subplot(511)
    # y_ticks = np.arange(-0.20, 0.25, 0.05)
    # ax2.set_title("average utility with eta 0 in evaluation (1 in training)")
    # line1, = plt.plot(avg_rewards_1_against_eachother[0][0], "b", label="agent0, {}".format(au_1_against_eachother_mean[0][0]))
    # line2, = plt.plot(avg_rewards_1_against_eachother[0][1], "r", label="agent1, {}".format(au_1_against_eachother_mean[0][1]))
    # plt.yticks(y_ticks)
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(512)
    # ax2.set_title("average utility with eta 1 in evaluation (1 in training)")
    # line1, = plt.plot(avg_rewards_1_against_eachother[1][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_1_against_eachother[1][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(513)
    # ax2.set_title("average utility with eta 0.1 in evaluation (1 in training)")
    # line1, = plt.plot(avg_rewards_1_against_eachother[2][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_1_against_eachother[2][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(514)
    # ax2.set_title("average utility with eta 0_1 in evaluation (1 in training)")
    # line1, = plt.plot(avg_rewards_1_against_eachother[3][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_1_against_eachother[3][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    #
    # ax2 = plt.subplot(515)
    # ax2.set_title("average utility with eta 1_0 in evaluation (1 in training)")
    # line1, = plt.plot(avg_rewards_1_against_eachother[4][0], "b", label="agent0")
    # line2, = plt.plot(avg_rewards_1_against_eachother[4][1], "r", label="agent1")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('avg utility')
    # plt.xlabel('episode(*1e4)')
    # plt.show()

    plt.figure(figsize=(15, 10))
    ax2 = plt.subplot(211)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility(trained agent0 vs random agent1) with eta 0 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_random_agent1[0][0], "b", label="trained_agent0, {}".format(au_0_against_random_agent1_mean[0][0]))
    line2, = plt.plot(avg_rewards_0_against_random_agent1[0][1], "r", label="random_agent1, {}".format(au_0_against_random_agent1_mean[0][1]))
    #plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(212)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility(random agent0 vs trained agent1) with eta 0 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_random_agent0[0][0], "r", label="random_agent0, {}".format(au_0_against_random_agent0_mean[0][0]))
    line2, = plt.plot(avg_rewards_0_against_random_agent0[0][1], "b", label="trained_agent1, {}".format(au_0_against_random_agent0_mean[0][1]))
    #plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 10))
    ax2 = plt.subplot(211)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility(trained agent0 vs random agent1) with eta 0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_random_agent1[0][0], "b", label="trained_agent0, {}".format(au_1_against_random_agent1_mean[0][0]))
    line2, = plt.plot(avg_rewards_1_against_random_agent1[0][1], "r", label="random_agent1, {}".format(au_1_against_random_agent1_mean[0][1]))
    #plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(212)
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility(random agent0 vs trained agent1) with eta 0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_random_agent0[0][0], "r", label="random agent0, {}".format(au_1_against_random_agent0_mean[0][0]))
    line2, = plt.plot(avg_rewards_1_against_random_agent0[0][1], "b", label="trained_agent1, {}".format(au_1_against_random_agent0_mean[0][1]))
    #plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 5))
    y_ticks = np.arange(-0.20, 0.25, 0.05)
    ax2.set_title("average utility(trained agent0 vs random agent1) with eta 0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_0_between_random_agents[0][0], "b", label="random_agent0, {}".format(au_mean(avg_rewards_0_between_random_agents[0][0], 200)))
    line2, = plt.plot(avg_rewards_0_between_random_agents[0][1], "r", label="random_agent1, {}".format(au_mean(avg_rewards_0_between_random_agents[0][1], 200)))
    #plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()



if __name__ == "__main__":
    app.run(main)<file_sep># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""NFSP agents trained on Kuhn Poker."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging
import tensorflow.compat.v1 as tf

from open_spiel.python import policy
from open_spiel.python import rl_environment
from open_spiel.python.algorithms import exploitability
from open_spiel.python.algorithms import nfsp
import glob
import os
import numpy as np
from open_spiel.python.algorithms import random_agent


FLAGS = flags.FLAGS

flags.DEFINE_integer("num_train_episodes", int(3e6),
                     "Number of training episodes.")
flags.DEFINE_integer("eval_every", 100,
                     "Episode frequency at which the agents are evaluated.")
flags.DEFINE_list("hidden_layers_sizes", [
    128,
], "Number of hidden units in the avg-net and Q-net.")
flags.DEFINE_integer("replay_buffer_capacity", int(2e5),
                     "Size of the replay buffer.")
flags.DEFINE_integer("reservoir_buffer_capacity", int(2e6),
                     "Size of the reservoir buffer.")
flags.DEFINE_float("anticipatory_param_agent0", 0,
                   "Prob of using the rl best response as episode policy for agent0.")
flags.DEFINE_float("anticipatory_param_agent1", 0,
                   "Prob of using the rl best response as episode policy for agent1.")
flags.DEFINE_string("experiment_name", "tic_tac_toe_0.1_7_26", "Experiment name")
flags.DEFINE_string("load_path", "/home/jxu8/Code_update/open_spiel/sessions_nfsp/", "Path to load the session")

flags.DEFINE_string("save_path", "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_26/", "Path to load the session")


class NFSPPolicies(policy.Policy):
  """Joint policy to be evaluated."""

  def __init__(self, env, nfsp_policies, mode):
    game = env.game
    player_ids = [0, 1]
    super(NFSPPolicies, self).__init__(game, player_ids)
    self._policies = nfsp_policies
    self._mode = mode
    self._obs = {"info_state": [None, None], "legal_actions": [None, None]}

  def action_probabilities(self, state, player_id=None):
    cur_player = state.current_player()
    legal_actions = state.legal_actions(cur_player)

    self._obs["current_player"] = cur_player
    self._obs["info_state"][cur_player] = (
        state.information_state_tensor(cur_player))
    self._obs["legal_actions"][cur_player] = legal_actions

    info_state = rl_environment.TimeStep(
        observations=self._obs, rewards=None, discounts=None, step_type=None)

    with self._policies[cur_player].temp_mode_as(self._mode):
      p = self._policies[cur_player].step(info_state, is_evaluation=True).probs
    prob_dict = {action: p[action] for action in legal_actions}
    return prob_dict

def eval_against_random_agent1(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  rewards = np.zeros(2)
  cur_agents = [trained_agents[0], random_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      rewards[0] += time_step.rewards[0]
      rewards[1] += time_step.rewards[1]
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def eval_against_random_agent0(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  rewards = np.zeros(2)
  cur_agents = [random_agents[0], trained_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      rewards[0] += time_step.rewards[0]
      rewards[1] += time_step.rewards[1]
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def eval_against_trained_agents(env, trained_agents, num_episodes):
  wins = np.zeros(3)
  rewards = np.zeros(2)
  for _ in range(num_episodes):
    time_step = env.reset()
    while not time_step.last():
      player_id = time_step.observations["current_player"]
      if env.is_turn_based:
        agents_output = trained_agents[player_id].step(time_step, is_evaluation=True)
        action_list = [agents_output.action]
      else:
        agents_output = [agent.step(time_step, is_evaluation=True) for agent in trained_agents]
        action_list = [agent_output.action for agent_output in agents_output]
      time_step = env.step(action_list)
    rewards[0] += time_step.rewards[0]
    rewards[1] += time_step.rewards[1]
    if time_step.rewards[0] > 0:
      wins[0] += 1
    elif time_step.rewards[1] > 0:
      wins[1] += 1
    else:
      wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def action_probability(env, trained_agents, num_episodes, outcome0, outcome1):
    #state_list = []
    acti_list = []
    prob_list = []
    rewards_list = []
    tmp_act = []
    tmp_probs = []
    agent0_1 = np.zeros(2)
    agent1_1 = np.zeros(4)
    agent0_2 = np.zeros(2)
    agent0_1_probs = np.zeros(2)
    agent1_1_probs = np.zeros(4)
    agent0_2_probs = np.zeros(2)
    for _ in range(num_episodes):
        time_step = env.reset_jx(outcome0, outcome1)
        #state_list.append(outcome_list)
        while not time_step.last():
            player_id = time_step.observations["current_player"]
            if env.is_turn_based:
                agents_output = trained_agents[player_id].step(
                    time_step, is_evaluation=True)
                action_list = [agents_output.action]
            else:
                agents_output = [
                    agent.step(time_step, is_evaluation=True) for agent in trained_agents
                ]
                action_list = [agent_output.action for agent_output in agents_output]
            tmp_act.append(agents_output.action)
            tmp_probs.append(agents_output.probs)
            time_step = env.step(action_list)
        rewards = time_step.rewards
        rewards_list.append(rewards)
        acti_list.append(tmp_act)
        prob_list.append(tmp_probs)
        tmp_act = []
        tmp_probs = []
    for action in acti_list:
        if action[0] == 0:
            agent0_1[0] += 1
            if action[1] == 0:
                agent1_1[0] += 1
            else:
                if action[2] == 0:
                    agent0_2[0] += 1
        else:
            if action[1] == 0:
                agent1_1[2] += 1
    agent0_1[1] = num_episodes - agent0_1[0]
    agent1_1[3] = agent0_1[1] - agent1_1[2]
    agent1_1[1] = agent0_1[0] - agent1_1[0]
    agent0_2[1] = agent1_1[1] - agent0_2[0]
    agent0_1_probs[0] = agent0_1[0] / num_episodes
    agent0_1_probs[1] = agent0_1[1] / num_episodes
    agent1_1_probs[agent1_1 != 0] = agent1_1[agent1_1 != 0] / np.array([agent0_1[0], agent0_1[0], agent0_1[1], agent0_1[1]])[agent1_1 != 0]
    agent0_2_probs[agent0_2 != 0] = agent0_2[agent0_2 != 0] / agent1_1[1]
    return acti_list, prob_list, rewards_list, agent0_1_probs, agent1_1_probs[:2], agent1_1_probs[2:], agent0_2_probs

def policy_porbs(env, trained_agents, outcome0, outcome1):
    time_step = env.reset_jx(outcome0, outcome1)
    agents_output_0 = trained_agents[0].step(
        time_step, is_evaluation=True)
    print(env.get_state)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([0])
    agents_output_1 = trained_agents[1].step(
        time_step, is_evaluation=True)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([1])
    agents_output_2 = trained_agents[1].step(
        time_step, is_evaluation=True)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([0])
    time_step = env.step([1])
    agents_output_3 = trained_agents[0].step(
        time_step, is_evaluation=True)
    return agents_output_0.probs, agents_output_1.probs, agents_output_2.probs, agents_output_3.probs

def main(unused_argv):
  game = "kuhn_poker"
  num_players = 2
  load_path = FLAGS.load_path + FLAGS.experiment_name
  env_configs = {"players": num_players}
  env = rl_environment.Environment(game, **env_configs)
  info_state_size = env.observation_spec()["info_state"][0]
  num_actions = env.action_spec()["num_actions"]

  hidden_layers_sizes = [int(l) for l in FLAGS.hidden_layers_sizes]
  kwargs = {
      "replay_buffer_capacity": FLAGS.replay_buffer_capacity,
      "epsilon_decay_duration": FLAGS.num_train_episodes,
      "epsilon_start": 0.06,
      "epsilon_end": 0.001,
  }

  random_agents = [
      random_agent.RandomAgent(player_id=idx, num_actions=num_actions)
      for idx in range(num_players)
  ]

  model_dirs = sorted(glob.glob(load_path + "/episode-*"), key=os.path.getmtime)
  for counter, dir in enumerate(model_dirs, 1):
      if counter % 1 == 0:
          tf.reset_default_graph()
          with tf.Session() as sess:
            # pylint: disable=g-complex-comprehension
            agent0 = nfsp.NFSP(sess, 0, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent0,
                          **kwargs)
            agent1 = nfsp.NFSP(sess, 1, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent1,
                          **kwargs)

            saver = tf.train.Saver()
            saver.restore(sess, dir + "/trained_model-10000")
            expl_policies_avg = NFSPPolicies(env, [agent0, agent1], nfsp.MODE.average_policy)
            # expl_list, expl = exploitability.exploitability(env.game, expl_policies_avg)
            # f4 = open("/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_24/eval_eta_0/win_rates_against_random_agent1.txt", "a")
            # f4.write(str(expl) + '\n')
            #
            # logging.info("Episode: %s, Exploitability AVG %s", counter*10000, expl)
            # logging.info("_____________________________________________")

            win_rates_against_random_agent1, avg_utility_against_random_agent1 = eval_against_random_agent1(env, [agent0, agent1], random_agents, 1000)
            win_rates_against_random_agent0, avg_utility_against_random_agent0 = eval_against_random_agent0(env, [agent0, agent1], random_agents, 1000)
            win_rates_against_trained_agents, avg_utility = eval_against_trained_agents(env, [agent0, agent1], 1000)
            behavior_probabilities_0_1 = action_probability(env, [agent0, agent1], 100, 0, 1)
            behavior_probabilities_0_2 = action_probability(env, [agent0, agent1], 100, 0, 2)
            behavior_probabilities_1_0 = action_probability(env, [agent0, agent1], 100, 1, 0)
            behavior_probabilities_1_2 = action_probability(env, [agent0, agent1], 100, 1, 2)
            behavior_probabilities_2_0 = action_probability(env, [agent0, agent1], 100, 2, 0)
            behavior_probabilities_2_1 = action_probability(env, [agent0, agent1], 100, 2, 1)
            policy_probabilities_0_1 = policy_porbs(env, [agent0, agent1], 0, 1)
            '''
            f1 = open(FLAGS.save_path + "avg_utility_against_random_agent1.txt", "a")
            f1.write(str(avg_utility_against_random_agent1[0]) + ' ' + str(avg_utility_against_random_agent1[1]) + '\n')
            f2 = open(FLAGS.save_path + "avg_utility_against_random_agent0.txt", "a")
            f2.write(str(avg_utility_against_random_agent0[0]) + ' ' + str(avg_utility_against_random_agent0[1]) + '\n')
            f3 = open(FLAGS.save_path + "avg_utility_against_eachother.txt", "a")
            f3.write(str(avg_utility[0]) + ' ' + str(avg_utility[1]) + '\n')
            '''
            logging.info("Episode: %s", counter*10000)
            logging.info("Trained_agent0 vs Random_agent1: %s", avg_utility_against_random_agent1)
            logging.info("Random_agent0 vs Trained_agent1: %s", avg_utility_against_random_agent0)
            logging.info("Trained_agent0 vs Trained_agent1 %s", avg_utility)
            logging.info("_____________________________________________")



if __name__ == "__main__":
  app.run(main)
<file_sep>from absl import app
from absl import flags
import numpy as np
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt

def read_wr(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    for line in lines[:-1]:
        [str1, str2, str3] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
    return list1, list2, list3

def read_exploitability(txt_name):
    txt_file = open(txt_name, 'r')
    lines = txt_file.read().split('\n')
    num_list = []
    for str in lines[:-1]:
        if str == "NaN":
            num_list.append(1)
        else:
            num_list.append(float(str))
    return num_list

def read_loss(txt_name):
    txt_file = open(txt_name)
    lines = txt_file.read().split('\n')
    list1 = []
    list2 = []
    for line in lines[:-1]:
        [str1, str2] = line.split(' ')
        if str1 != 'None':
            list1.append(float(str1))
        else:
            list1.append(str1)
        if str2 != 'None':
            list2.append(float(str2))
        else:
            list2.append(str2)
    for idx, number in enumerate(list1):
        if number == 'None':
            list1[idx] = list1[idx+1]
    for number, idx in enumerate(list2):
        if number == 'None':
            list2[idx] = list2[idx+1]
    return list1, list2
def read_behavior_probs(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    list4 = []
    list5 = []
    list6 = []
    list7 = []
    list8 = []
    for line in lines[:-1]:
        [str1, str2, str3, str4, str5, str6, str7, str8] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
        list4.append(float(str4))
        list5.append(float(str5))
        list6.append(float(str6))
        list7.append(float(str7))
        list8.append(float(str8))
    return list1, list2, list3, list4, list5, list6, list7, list8

def main(argv):
    kuhn_poker_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/"
    kuhn_poker_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_1_7_28/"
    ttt_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_26/"
    ttt_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_1_7_29/"

    kuhn_poker_psro = "/home/jxu8/Code/open_spiel/evaluation_data/eval_kuhn_poker_psro_7_2/"

    bp_jk_cb = []
    bp_jq_cb = []
    bp_kj_cb = []
    bp_kq_cb = []
    bp_qj_cb = []
    bp_qk_cb = []

    bp_jk_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/JK.txt'))
    bp_jq_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/JQ.txt'))
    bp_kj_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/KJ.txt'))
    bp_kq_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/KQ.txt'))
    bp_qj_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/QJ.txt'))
    bp_qk_cb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/competition_based/QK.txt'))

    bp_jk_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/JK.txt'))
    bp_jq_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/JQ.txt'))
    bp_kj_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/KJ.txt'))
    bp_kq_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/KQ.txt'))
    bp_qj_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/QJ.txt'))
    bp_qk_cb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/competition_based/QK.txt'))

    #plt alpha in kuhn_poker_nfsp_0.1(eta 0)
    tmp_list = [bp_jk_cb[0], bp_jq_cb[0], bp_kj_cb[0], bp_kq_cb[0], bp_qj_cb[0], bp_qk_cb[0]]
    alpha_1 = [1 - tmp_list[0][0][i] for i in range(len(tmp_list[0][0]))]
    alpha_2 = [1 - tmp_list[1][0][i] for i in range(len(tmp_list[1][0]))]
    alpha_3 = [(1/3) * (1 - tmp_list[2][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_4 = [(1/3) * (1 - tmp_list[3][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_5 = [tmp_list[4][7][i] - 1/3 for i in range(len(tmp_list[4][7]))]
    alpha_6 = [tmp_list[5][7][i] - 1/3 for i in range(len(tmp_list[5][7]))]

    ax2 = plt.figure(figsize=(10, 5))
    #ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    line1, = plt.plot(alpha_1, "b-", label="JK & JQ")
    #line2, = plt.plot(alpha_2, "b*", label="2")
    line3, = plt.plot(alpha_3, "g-", label="KJ & KQ")
    #line4, = plt.plot(alpha_4, "g*", label="4")
    line5, = plt.plot(alpha_5, "y-", label="QJ & QK")
    #line6, = plt.plot(alpha_6, "y*", label="6")
    #plt.legend(handles=[line1, line2, line3, line4, line5, line6], loc='upper right')
    plt.legend(handles=[line1, line3, line5], loc='upper right')
    plt.ylabel('alpha')
    plt.xlabel('episode(*1e4)')
    plt.show()

    #plt alpha in kuhn_poker_nfsp_1(eta 0)
    tmp_list = [bp_jk_cb[1], bp_jq_cb[1], bp_kj_cb[1], bp_kq_cb[1], bp_qj_cb[1], bp_qk_cb[1]]
    alpha_1 = [1 - tmp_list[0][0][i] for i in range(len(tmp_list[0][0]))]
    alpha_2 = [1 - tmp_list[1][0][i] for i in range(len(tmp_list[1][0]))]
    alpha_3 = [(1/3) * (1 - tmp_list[2][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_4 = [(1/3) * (1 - tmp_list[3][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_5 = [tmp_list[4][7][i] - 1/3 for i in range(len(tmp_list[4][7]))]
    alpha_6 = [tmp_list[5][7][i] - 1/3 for i in range(len(tmp_list[5][7]))]

    ax2 = plt.figure(figsize=(10, 5))
    #ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    line1, = plt.plot(alpha_1, "b-", label="JK & JQ")
    #line2, = plt.plot(alpha_2, "b*", label="2")
    line3, = plt.plot(alpha_3, "g-", label="KJ & KQ")
    #line4, = plt.plot(alpha_4, "g*", label="4")
    line5, = plt.plot(alpha_5, "y-", label="QJ & QK")
    #line6, = plt.plot(alpha_6, "y*", label="6")
    #plt.legend(handles=[line1, line2, line3, line4, line5, line6], loc='upper right')
    plt.legend(handles=[line1, line3, line5], loc='upper right')
    plt.ylabel('alpha')
    plt.xlabel('episode(*1e4)')
    plt.show()


    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_jk_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_jk_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_jk_cb[0][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_jq_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_jq_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_jq_cb[0][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_kj_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_kj_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_kj_cb[0][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_kq_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_kq_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_kq_cb[0][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_qj_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_qj_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_qj_cb[0][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_cb[0][0], "b", label="1")
    line2, = plt.plot(bp_qk_cb[0][2], "r", label="2")
    line3, = plt.plot(bp_qk_cb[0][4], "g", label="3")
    line4, = plt.plot(bp_qk_cb[0][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plt bp for kuhn_poker_nfsp_0.1, eta1 in evaluation
    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_jk_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_jk_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_jk_cb[1][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_jq_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_jq_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_jq_cb[1][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_kj_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_kj_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_kj_cb[1][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_kq_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_kq_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_kq_cb[1][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_qj_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_qj_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_qj_cb[1][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_cb[1][0], "b", label="1")
    line2, = plt.plot(bp_qk_cb[1][2], "r", label="2")
    line3, = plt.plot(bp_qk_cb[1][4], "g", label="3")
    line4, = plt.plot(bp_qk_cb[1][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

if __name__ == "__main__":
    app.run(main)<file_sep># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""NFSP agents trained on Kuhn Poker."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging
import tensorflow.compat.v1 as tf

from open_spiel.python import policy
from open_spiel.python import rl_environment
from open_spiel.python.algorithms import exploitability
from open_spiel.python.algorithms import nfsp
import numpy as np
from open_spiel.python.algorithms import random_agent


FLAGS = flags.FLAGS

flags.DEFINE_integer("num_train_episodes", int(3e6),
                     "Number of training episodes.")
flags.DEFINE_integer("eval_every", 100,
                     "Episode frequency at which the agents are evaluated.")
flags.DEFINE_list("hidden_layers_sizes", [
    128,
], "Number of hidden units in the avg-net and Q-net.")
flags.DEFINE_integer("replay_buffer_capacity", int(2e5),
                     "Size of the replay buffer.")
flags.DEFINE_integer("reservoir_buffer_capacity", int(2e6), #
                     "Size of the reservoir buffer.")
flags.DEFINE_float("anticipatory_param", 0.1,
                   "Prob of using the rl best response as episode policy.")
flags.DEFINE_integer("save_every", 100,
                     "Episode frequency at which the agents are evaluated.")



class NFSPPolicies(policy.Policy):
  """Joint policy to be evaluated."""

  def __init__(self, env, nfsp_policies, mode):
    game = env.game
    player_ids = [0, 1]
    super(NFSPPolicies, self).__init__(game, player_ids)
    self._policies = nfsp_policies
    self._mode = mode
    self._obs = {"info_state": [None, None], "legal_actions": [None, None]}

  def action_probabilities(self, state, player_id=None):
    cur_player = state.current_player()
    legal_actions = state.legal_actions(cur_player)

    self._obs["current_player"] = cur_player
    self._obs["info_state"][cur_player] = (
        state.observation_tensor(cur_player))
    self._obs["legal_actions"][cur_player] = legal_actions

    info_state = rl_environment.TimeStep(
        observations=self._obs, rewards=None, discounts=None, step_type=None)

    with self._policies[cur_player].temp_mode_as(self._mode):
      p = self._policies[cur_player].step(info_state, is_evaluation=True).probs
    prob_dict = {action: p[action] for action in legal_actions}
    return prob_dict

def eval_against_trained_agents(env, trained_agents, num_episodes):
  wins = np.zeros(3)
  for _ in range(num_episodes):
    time_step = env.reset()
    while not time_step.last():
      player_id = time_step.observations["current_player"]
      if env.is_turn_based:
        agents_output = trained_agents[player_id].step(time_step, is_evaluation=True)
        action_list = [agents_output.action]
      else:
        agents_output = [agent.step(time_step, is_evaluation=True) for agent in trained_agents]
        action_list = [agent_output.action for agent_output in agents_output]
      time_step = env.step(action_list)
    if time_step.rewards[0] > 0:
      wins[0] += 1
    elif time_step.rewards[1] > 0:
      wins[1] += 1
    else:
      wins[2] += 1
  return wins / num_episodes


def eval_against_random_agent1(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  cur_agents = [trained_agents[0], random_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes

def eval_against_random_agent0(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  cur_agents = [random_agents[0], trained_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes

def main(unused_argv):
  game = "tic_tac_toe"
  num_players = 2
  path = '/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_0.1'
  env_configs = {"players": num_players}
  env = rl_environment.Environment(game)
  info_state_size = env.observation_spec()["info_state"][0]
  num_actions = env.action_spec()["num_actions"]

  hidden_layers_sizes = [int(l) for l in FLAGS.hidden_layers_sizes]
  kwargs = {
      "replay_buffer_capacity": FLAGS.replay_buffer_capacity,
      "epsilon_decay_duration": FLAGS.num_train_episodes,
      "epsilon_start": 0.06,
      "epsilon_end": 0.001,
  }

  random_agents = [
      random_agent.RandomAgent(player_id=idx, num_actions=num_actions)
      for idx in range(num_players)
  ]

  with tf.Session() as sess:
    # pylint: disable=g-complex-comprehension
    agents = [
        nfsp.NFSP(sess, idx, info_state_size, num_actions, hidden_layers_sizes,
                  FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param,
                  **kwargs) for idx in range(num_players)
    ]
    expl_policies_avg = NFSPPolicies(env, agents, nfsp.MODE.average_policy)
    saver = tf.train.Saver(max_to_keep = 2000)
    sess.run(tf.global_variables_initializer())

    for ep in range(FLAGS.num_train_episodes):
      if (ep + 1) % FLAGS.eval_every == 0:
        losses = [agent.loss for agent in agents]
        win_rates_against_random_agent1 = eval_against_random_agent1(env, agents, random_agents, 1000)
        win_rates_against_random_agent0 = eval_against_random_agent0(env, agents, random_agents, 1000)
        win_rates_against_eachother = eval_against_trained_agents(env, agents, 1000)
        logging.info("Losses: %s", losses)
        # expl = exploitability.exploitability(env.game, expl_policies_avg)
        # logging.info("[%s] Exploitability AVG %s", ep + 1, expl)
        # logging.info("_____________________________________________")
        logging.info("Win rates: Trained_agent0 vs Random_agent1  %s", win_rates_against_random_agent1)
        logging.info("Win rates: Random_agent0 vs Trained_agent1  %s", win_rates_against_random_agent0)
        logging.info("Win rates: Trained_agent0 vs Trained_agent1 %s", win_rates_against_eachother)
        logging.info("__________________________________________________________________")

      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = agents[player_id].step(time_step)
        action_list = [agent_output.action]
        time_step = env.step(action_list)

      # Episode is over, step all agents with final info state.
      for agent in agents:
        agent.step(time_step)

      if (ep + 1) % FLAGS.save_every == 0:
          saver.save(sess, path + "/episode-" + str(ep+1) + "/trained_model", global_step = 100)



if __name__ == "__main__":
  app.run(main)
<file_sep>from absl import app
from absl import flags
import numpy as np
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt

def read_wr(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    for line in lines[:-1]:
        [str1, str2, str3] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
    return list1, list2, list3

def read_exploitability(txt_name):
    txt_file = open(txt_name, 'r')
    lines = txt_file.read().split('\n')
    num_list = []
    for str in lines[:-1]:
        if str == "NaN":
            num_list.append(1)
        else:
            num_list.append(float(str))
    return num_list

def read_loss(txt_name):
    txt_file = open(txt_name)
    lines = txt_file.read().split('\n')
    list1 = []
    list2 = []
    for line in lines[:-1]:
        [str1, str2] = line.split(' ')
        if str1 != 'None':
            list1.append(float(str1))
        else:
            list1.append(str1)
        if str2 != 'None':
            list2.append(float(str2))
        else:
            list2.append(str2)
    for idx, number in enumerate(list1):
        if number == 'None':
            list1[idx] = list1[idx+1]
    for number, idx in enumerate(list2):
        if number == 'None':
            list2[idx] = list2[idx+1]
    return list1, list2
def read_behavior_probs(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    list4 = []
    list5 = []
    list6 = []
    list7 = []
    list8 = []
    for line in lines[:-1]:
        [str1, str2, str3, str4, str5, str6, str7, str8] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
        list4.append(float(str4))
        list5.append(float(str5))
        list6.append(float(str6))
        list7.append(float(str7))
        list8.append(float(str8))
    return list1, list2, list3, list4, list5, list6, list7, list8

def main(argv):
    kuhn_poker_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/"
    kuhn_poker_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_1_7_28/"
    ttt_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_26/"
    ttt_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_1_7_29/"

    kuhn_poker_psro = "/home/jxu8/Code/open_spiel/evaluation_data/eval_kuhn_poker_psro_7_2/"

    bp_jk_pb = []
    bp_jq_pb = []
    bp_kj_pb = []
    bp_kq_pb = []
    bp_qj_pb = []
    bp_qk_pb = []

    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/QK.txt'))

    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/QK.txt'))


    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/QK.txt'))

    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/QK.txt'))

    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/QK.txt'))

    bp_jk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/JK.txt'))
    bp_jq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/JQ.txt'))
    bp_kj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/KJ.txt'))
    bp_kq_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/KQ.txt'))
    bp_qj_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/QJ.txt'))
    bp_qk_pb.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/QK.txt'))

    #plt alpha in kuhn_poker_nfsp_0.1(eta 0)
    tmp_list = [bp_jk_pb[4], bp_jq_pb[4], bp_kj_pb[4], bp_kq_pb[4], bp_qj_pb[4], bp_qk_pb[4]]
    alpha_1 = [1 - tmp_list[0][0][i] for i in range(len(tmp_list[0][0]))]
    alpha_2 = [1 - tmp_list[1][0][i] for i in range(len(tmp_list[1][0]))]
    alpha_3 = [(1/3) * (1 - tmp_list[2][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_4 = [(1/3) * (1 - tmp_list[3][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_5 = [tmp_list[4][7][i] - 1/3 for i in range(len(tmp_list[4][7]))]
    alpha_6 = [tmp_list[5][7][i] - 1/3 for i in range(len(tmp_list[5][7]))]

    ax2 = plt.figure(figsize=(10, 5))
    #ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    line1, = plt.plot(alpha_1, "b-", label="JK & JQ")
    #line2, = plt.plot(alpha_2, "b*", label="2")
    line3, = plt.plot(alpha_3, "g-", label="KJ & KQ")
    #line4, = plt.plot(alpha_4, "g*", label="4")
    line5, = plt.plot(alpha_5, "y-", label="QJ & QK")
    #line6, = plt.plot(alpha_6, "y*", label="6")
    #plt.legend(handles=[line1, line2, line3, line4, line5, line6], loc='upper right')
    plt.legend(handles=[line1, line3, line5], loc='upper right')
    plt.ylabel('alpha')
    plt.xlabel('episode(*1e4)')
    plt.show()

    #plt alpha in kuhn_poker_nfsp_1(eta 0)
    tmp_list = [bp_jk_pb[5], bp_jq_pb[5], bp_kj_pb[5], bp_kq_pb[5], bp_qj_pb[5], bp_qk_pb[5]]
    alpha_1 = [1 - tmp_list[0][0][i] for i in range(len(tmp_list[0][0]))]
    alpha_2 = [1 - tmp_list[1][0][i] for i in range(len(tmp_list[1][0]))]
    alpha_3 = [(1/3) * (1 - tmp_list[2][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_4 = [(1/3) * (1 - tmp_list[3][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_5 = [tmp_list[4][7][i] - 1/3 for i in range(len(tmp_list[4][7]))]
    alpha_6 = [tmp_list[5][7][i] - 1/3 for i in range(len(tmp_list[5][7]))]

    ax2 = plt.figure(figsize=(10, 5))
    #ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    line1, = plt.plot(alpha_1, "b-", label="JK & JQ")
    #line2, = plt.plot(alpha_2, "b*", label="2")
    line3, = plt.plot(alpha_3, "g-", label="KJ & KQ")
    #line4, = plt.plot(alpha_4, "g*", label="4")
    line5, = plt.plot(alpha_5, "y-", label="QJ & QK")
    #line6, = plt.plot(alpha_6, "y*", label="6")
    #plt.legend(handles=[line1, line2, line3, line4, line5, line6], loc='upper right')
    plt.legend(handles=[line1, line3, line5], loc='upper right')
    plt.ylabel('alpha')
    plt.xlabel('episode(*1e4)')
    plt.show()


    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[0][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[0][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[0][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()



    # plt bp for kuhn_poker_nfsp_0.1, eta1 in evaluation
    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[1][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[1][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[1][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[4][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[4][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[4][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[4][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[4][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[4][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[4][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[4][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[4][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()


    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[2][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[2][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[2][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[3][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[3][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[3][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_jk_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_jk_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_jk_pb[5][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_jq_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_jq_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_jq_pb[5][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_kj_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_kj_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_kj_pb[5][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_kq_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_kq_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_kq_pb[5][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_qj_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_qj_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_qj_pb[5][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk_pb[5][0], "b", label="1")
    line2, = plt.plot(bp_qk_pb[5][2], "r", label="2")
    line3, = plt.plot(bp_qk_pb[5][4], "g", label="3")
    line4, = plt.plot(bp_qk_pb[5][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()


if __name__ == "__main__":
    app.run(main)<file_sep>from absl import app
from absl import flags
import numpy as np
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt

def read_wr(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    for line in lines[:-1]:
        [str1, str2, str3] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
    return list1, list2, list3

def read_exploitability(txt_name):
    txt_file = open(txt_name, 'r')
    lines = txt_file.read().split('\n')
    num_list = []
    for str in lines[:-1]:
        if str == "NaN":
            num_list.append(1)
        else:
            num_list.append(float(str))
    return num_list

def merge_expl(txt_name_1, txt_name_2):
    num_list_1 = read_exploitability(txt_name_1)
    num_list_2 = read_exploitability(txt_name_2)
    return num_list_1 + num_list_2

def read_loss(txt_name):
    txt_file = open(txt_name)
    lines = txt_file.read().split('\n')
    list1 = []
    list2 = []
    for line in lines[:-1]:
        [str1, str2] = line.split(' ')
        if str1 != 'None':
            list1.append(float(str1))
        else:
            list1.append(str1)
        if str2 != 'None':
            list2.append(float(str2))
        else:
            list2.append(str2)
    for idx, number in enumerate(list1):
        if number == 'None':
            list1[idx] = list1[idx+1]
    for number, idx in enumerate(list2):
        if number == 'None':
            list2[idx] = list2[idx+1]
    return list1, list2
def read_behavior_probs(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    list4 = []
    list5 = []
    list6 = []
    list7 = []
    list8 = []
    for line in lines[:-1]:
        [str1, str2, str3, str4, str5, str6, str7, str8] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
        list4.append(float(str4))
        list5.append(float(str5))
        list6.append(float(str6))
        list7.append(float(str7))
        list8.append(float(str8))
    return list1, list2, list3, list4, list5, list6, list7, list8


def main(argv):
    kuhn_poker_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/"
    kuhn_poker_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_1_7_28/"
    ttt_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_8_6/"
    ttt_nfsp_0_2 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.2_8_6/"
    ttt_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_1_7_29/"

    kuhn_poker_psro = "/home/jxu8/Code/open_spiel/evaluation_data/eval_kuhn_poker_psro_7_2/"

    expl_0 = []
    expl_1 = []
    expl_0_2 = []
    expl_0_5 = []
    expl_0_0_1 = []
    loss_agent0 = []
    loss_agent1 = []

    win_rates_against_random1_eta0 = [] #load win rates against random agent1, trained with eta 0.1
    win_rates_against_random0_eta0 = [] #load win rates against random agetn0, trained with eta 0.1
    win_rates_against_random1_eta1 = []
    win_rates_against_random0_eta1 = []
    win_rates_against_eachother_eta0 = [] #load win rates against each other, trained with eta 0.1
    win_rates_against_eachother_eta1 = []

    expl_0.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/' + 'exploitabilities.txt'))
    expl_0.append(merge_expl(ttt_nfsp_0 + 'exploitabilities.txt', "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_8_12/exploitabilities_from_1506e4.txt"))
    expl_1.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/' + 'exploitability.txt'))
    expl_1.append(read_exploitability(ttt_nfsp_1 + 'exploitabilities.txt'))
    expl_0_2.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.2_8_6/exploitability.txt'))
    expl_0_2.append(merge_expl(ttt_nfsp_0_2 + 'exploitabilities.txt', "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.2_8_12/exploitabilities_from_1506e4.txt"))
    expl_0_5.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.5_8_6/exploitability.txt'))
    expl_0_0_1.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.01_8_6/exploitability.txt'))

    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/loss_agent1.txt'))
    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/loss_agent1.txt'))

    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_0.1_7_26/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_0.1_7_26/loss_agent1.txt'))
    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_1_7_29/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_1_7_29/loss_agent1.txt'))
    # load win_rates with eta0.1 in training
    win_rates_against_random1_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta0.append(read_wr(ttt_nfsp_0_2 + 'win_rates/eta_0/win_rates_against_random_agent1.txt'))
    win_rates_against_random0_eta0.append(read_wr(ttt_nfsp_0_2 + 'win_rates/eta_0/win_rates_against_random_agent0.txt'))
    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)
    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0_2 + 'win_rates/eta_0/win_rates_against_eachother.txt'))

    #load win rates with eta1 in training
    win_rates_against_random1_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    # episode = range(10, 3010, 10)
    # plot exploitability, avg utility in kuhn_poker_nfsp_0.1_7_27
    plt.figure(figsize=(10, 3))
    #plt.subplot(311)
    plt.xlim(-10, 300)
    y_ticks = np.arange(0, 0.50, 0.05)
    line1, = plt.plot(expl_0[0], "b", label="kuhn_poker_nfsp_0.1")
    line2, = plt.plot(expl_0_0_1[0], 'm', label='kuhn_poker_nfsp_0.01')
    line3, = plt.plot(expl_0_2[0], 'y', label="kuhn_poker_nfsp_0.2")
    line4, = plt.plot(expl_0_5[0], 'g', label='kuhn_poker_nfsp_0.5')
    line5, = plt.plot(expl_1[0], "r", label="kuhn_poker_nfsp_1")

    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4, line5], loc='upper right')
    plt.ylabel('exploitability')
    plt.xlabel('episode(*1e4)')
    plt.show()

    #plt win rates in kuhn_poker_nfsp_0.1_7_27
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[1][0], 'b', label='kp_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[1][1], 'r', label='kp_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[1][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[1][0], 'r', label='kp_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[1][1], 'b', label='kp_nfsp_0.1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[1][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[1][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[1][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[1][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot random agent0 against trained agent1 in ttt_nfsp_0.1_7_24
    # plot win rates of trained agents against each other in ttt_nfsp_0.1_7_24

    # plt win rates in kuhn_poker_nfsp_1_7_28
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[1][0], 'b', label='kp_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[1][1], 'r', label='kp_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[1][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[1][0], 'r', label='kp_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[1][1], 'b', label='kp_nfsp_1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[1][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[1][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[1][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[1][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()
    
    # plt.subplot(312)
    # line1, = plt.plot(loss_agent0[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent0[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('loss_agent0')
    # plt.xlabel('episode(*1e4)')
    # 
    # plt.subplot(313)
    # line1, = plt.plot(loss_agent1[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent1[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('loss_agent1')
    # plt.xlabel('episode(*1e4)')
    # plt.show()

    # plot exploitability in tic_tac_toe_nfsp_0.1_7_26, tic_tac_toe_nfsp_1_7_29
    plt.figure(figsize=(10,3))
    #plt.subplot(311)
    plt.ylim(0, 1.05)
    y_ticks = np.arange(0, 1.1, 0.1)
    x_range_0 = [6*(x+1) for x in range(len(expl_0[1]))]
    line1, = plt.plot(x_range_0, expl_0[1], "b", label="ttt_nfsp_0.1")
    x_range_1 = [6*(x+1) for x in range(len(expl_0_2[1]))]
    line2, = plt.plot(x_range_1, expl_0_2[1], "g", label="ttt_nfsp_0.2")
    x_range_2 = [4*(x+1) for x in range(len(expl_1[1]))]
    line3, = plt.plot(x_range_2, expl_1[1], "r", label="ttt_nfsp_1")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3], loc='upper right')
    plt.ylabel('exploitability')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plt.subplot(312)
    # line1, = plt.plot(loss_agent0[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent0[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='lower right')
    # plt.ylabel('loss_agent0')
    # plt.xlabel('episode(*1e4)')
    # 
    # plt.subplot(313)
    # line1, = plt.plot(loss_agent1[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent1[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='lower right')
    # plt.ylabel('loss_agent1')
    # plt.xlabel('episode(*1e4)')
    # plt.show()
    
    # plot trained agent0 against random agent1 in tic_tac_toe_nfsp_0.1_7_26
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[0][0], 'b', label='ttt_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[0][1], 'r', label='ttt_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[0][0], 'r', label='ttt_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[0][1], 'b', label='ttt_nfsp_0.1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[0][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[0][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[2][0], 'b', label='ttt_nfsp_0.2_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[2][1], 'r', label='ttt_nfsp_0.2_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[2][2], 'g', label='ttt_nfsp_0.2_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[2][0], 'r', label='ttt_nfsp_0.2_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[2][1], 'b', label='ttt_nfsp_0.2_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[2][2], 'g', label='ttt_nfsp_0.2_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[2][0], 'b', label='ttt_nfsp_0.2_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[2][1], 'r', label='ttt_nfsp_0.2_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[2][2], 'g', label='ttt_nfsp_0.2_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot random agent0 against trained agent1 in ttt_nfsp_0.1_7_26

    # plt win rates of ttt_nfsp_1_7_29, trained agent0 against random agent1
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[0][0], 'b', label='ttt_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[0][1], 'r', label='ttt_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[0][0], 'r', label='ttt_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[0][1], 'b', label='ttt_nfsp_1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[0][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[0][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

if __name__ == "__main__":
    app.run(main)<file_sep>from absl import app
from absl import flags
import numpy as np
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt

def read_wr(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    for line in lines[:-1]:
        [str1, str2, str3] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
    return list1, list2, list3

def read_exploitability(txt_name):
    txt_file = open(txt_name, 'r')
    lines = txt_file.read().split('\n')
    num_list = []
    for str in lines[:-1]:
        if str == "NaN":
            num_list.append(1)
        else:
            num_list.append(float(str))
    return num_list

def read_loss(txt_name):
    txt_file = open(txt_name)
    lines = txt_file.read().split('\n')
    list1 = []
    list2 = []
    for line in lines[:-1]:
        [str1, str2] = line.split(' ')
        if str1 != 'None':
            list1.append(float(str1))
        else:
            list1.append(str1)
        if str2 != 'None':
            list2.append(float(str2))
        else:
            list2.append(str2)
    for idx, number in enumerate(list1):
        if number == 'None':
            list1[idx] = list1[idx+1]
    for number, idx in enumerate(list2):
        if number == 'None':
            list2[idx] = list2[idx+1]
    return list1, list2
def read_behavior_probs(txt_name):
    text_file = open(txt_name, "r")
    lines = text_file.read().split("\n")
    list1 = []
    list2 = []
    list3 = []
    list4 = []
    list5 = []
    list6 = []
    list7 = []
    list8 = []
    for line in lines[:-1]:
        [str1, str2, str3, str4, str5, str6, str7, str8] = line.split(" ")
        list1.append(float(str1))
        list2.append(float(str2))
        list3.append(float(str3))
        list4.append(float(str4))
        list5.append(float(str5))
        list6.append(float(str6))
        list7.append(float(str7))
        list8.append(float(str8))
    return list1, list2, list3, list4, list5, list6, list7, list8




def main(argv):
    kuhn_poker_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/"
    kuhn_poker_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_1_7_28/"
    ttt_nfsp_0 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_26/"
    ttt_nfsp_1 = "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_1_7_29/"

    kuhn_poker_psro = "/home/jxu8/Code/open_spiel/evaluation_data/eval_kuhn_poker_psro_7_2/"


    # exploitabilities_kuhn_poker_psro = read_eval(kuhn_poker_psro + 'exploitabilities.txt')
    # array = np.array(exploitabilities_kuhn_poker_psro)
    # print(array[:,0])

    expl_0 = []
    expl_1 = []
    loss_agent0 = []
    loss_agent1 = []
    win_rates_against_random1_eta0 = [] #load win rates against random agent1, trained with eta 0.1
    win_rates_against_random0_eta0 = [] #load win rates against random agetn0, trained with eta 0.1
    win_rates_against_random1_eta1 = [] #load win rates against random agent1, trained with eta 1
    win_rates_against_random0_eta1 = [] #load win rates against random agent0, trained with eta 1
    win_rates_against_eachother_eta0 = [] #load win rates against each other, trained with eta 0.1
    win_rates_against_eachother_eta1 = [] #load win rates against each other, trained with eta 1
    bp_jk = []
    bp_jq = []
    bp_kj = []
    bp_kq = []
    bp_qj = []
    bp_qk = []

    avg_rewards_0_against_eachother = []
    avg_rewards_1_against_eachother = []

    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0.1/policy_based/QK.txt'))

    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_1/policy_based/QK.txt'))


    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0.1/policy_based/QK.txt'))

    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_1/policy_based/QK.txt'))

    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_0 + 'behavior_probs/eta_0/policy_based/QK.txt'))

    bp_jk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/JK.txt'))
    bp_jq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/JQ.txt'))
    bp_kj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/KJ.txt'))
    bp_kq.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/KQ.txt'))
    bp_qj.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/QJ.txt'))
    bp_qk.append(read_behavior_probs(kuhn_poker_nfsp_1 + 'behavior_probs/eta_0/policy_based/QK.txt'))

    tmp_list = [bp_jk[4], bp_jq[4], bp_kj[4], bp_kq[4], bp_qj[4], bp_qk[4]]
    alpha_1 = [1 - tmp_list[0][0][i] for i in range(len(tmp_list[0][0]))]
    alpha_2 = [1 - tmp_list[1][0][i] for i in range(len(tmp_list[1][0]))]
    alpha_3 = [(1/3) * (1 - tmp_list[2][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_4 = [(1/3) * (1 - tmp_list[3][0][i]) for i in range(len(tmp_list[2][0]))]
    alpha_5 = [2/3 - tmp_list[4][6][i] for i in range(len(tmp_list[4][6]))]
    alpha_6 = [2/3 - tmp_list[5][6][i] for i in range(len(tmp_list[5][6]))]

    ax2 = plt.figure(figsize=(10, 10))
    #ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    line1, = plt.plot(alpha_1, "b-", label="1")
    line2, = plt.plot(alpha_2, "b*", label="2")
    line3, = plt.plot(alpha_3, "g-", label="3")
    line4, = plt.plot(alpha_4, "g*", label="4")
    line5, = plt.plot(alpha_5, "y-", label="5")
    line6, = plt.plot(alpha_5, "y*", label="6")
    plt.legend(handles=[line1, line2, line3, line4, line5, line6], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    expl_0.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/' + 'exploitabilities.txt'))
    expl_0.append(read_exploitability(ttt_nfsp_0 + 'exploitabilities.txt'))
    expl_1.append(read_exploitability('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/' + 'exploitability.txt'))
    expl_1.append(read_exploitability(ttt_nfsp_1 + 'exploitabilities.txt'))

    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_0.1_7_27/loss_agent1.txt'))
    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/kuhn_poker_1_7_28/loss_agent1.txt'))

    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_0.1_7_26/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_0.1_7_26/loss_agent1.txt'))
    loss_agent0.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_1_7_29/loss_agent0.txt'))
    loss_agent1.append(read_loss('/home/jxu8/Code_update/open_spiel/sessions_nfsp/tic_tac_toe_1_7_29/loss_agent1.txt'))
    # load win_rates with eta0.1 in training
    win_rates_against_random1_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0.1 in evaluation process
    win_rates_against_random0_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0.1 in evaluation process

    win_rates_against_random1_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_1/win_rates_against_random_agent1.txt'))
    win_rates_against_random0_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_1/win_rates_against_random_agent0.txt'))

    win_rates_against_random1_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0.1 in evaluation process
    win_rates_against_random0_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0.1 in evaluation process

    win_rates_against_random1_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_1/win_rates_against_random_agent1.txt'))
    win_rates_against_random0_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_1/win_rates_against_random_agent0.txt'))


    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 1 for trained agent1
                                                                                                                # (both trained agents use best response policy only)
    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0.1 for both agents
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_0_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0 for trained agent0, and eta 1 for trained agent1
                                                                                                                  # (trained agetn0 use average policy only and trained agent 1 use best response policy only)
    win_rates_against_eachother_eta0.append(read_wr(ttt_nfsp_0 + 'win_rates/eta_1_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 0 for trained agent1
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 1 for trained agent1
                                                                                                                # (both trained agents use best response policy only)
    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0.1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0.1 for both agents
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_0_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0 for trained agent0, and eta 1 for trained agent1
                                                                                                                  # (trained agetn0 use average policy only and trained agent 1 use best response policy only)
    win_rates_against_eachother_eta0.append(read_wr(kuhn_poker_nfsp_0 + 'win_rates/eta_1_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 0 for trained agent1
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    #load win rates with eta1 in training
    win_rates_against_random1_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0.1 in evaluation process
    win_rates_against_random0_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0.1 in evaluation process

    win_rates_against_random1_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_1/win_rates_against_random_agent1.txt'))
    win_rates_against_random0_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_1/win_rates_against_random_agent0.txt'))

    win_rates_against_random1_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0 in evaluation process (average policy only)
    win_rates_against_random0_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0 in evaluation process (average policy only)

    win_rates_against_random1_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_random_agent1.txt')) # load win rates of trained agent0 against random agent1, with eta 0.1 in evaluation process
    win_rates_against_random0_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_random_agent0.txt')) # load win rates of trained agent1 against random agent0, with eta 0.1 in evaluation process

    win_rates_against_random1_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_1/win_rates_against_random_agent1.txt'))
    win_rates_against_random0_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_1/win_rates_against_random_agent0.txt'))


    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 1 for trained agent1
                                                                                                                # (both trained agents use best response policy only)
    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0.1 for both agents
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_0_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0 for trained agent0, and eta 1 for trained agent1
                                                                                                                  # (trained agetn0 use average policy only and trained agent 1 use best response policy only)
    win_rates_against_eachother_eta1.append(read_wr(ttt_nfsp_1 + 'win_rates/eta_1_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 0 for trained agent1
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent1, with eta 0 for both in the evaluation process (both use average policy only)

    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 1 for trained agent1
                                                                                                                # (both trained agents use best response policy only)
    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0.1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0.1 for both agents
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_0_1/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 0 for trained agent0, and eta 1 for trained agent1
                                                                                                                  # (trained agetn0 use average policy only and trained agent 1 use best response policy only)
    win_rates_against_eachother_eta1.append(read_wr(kuhn_poker_nfsp_1 + 'win_rates/eta_1_0/win_rates_against_eachother.txt')) # load win rates of trained agent0 against trained agent0, with eta 1 for trained agent0, and eta 0 for trained agent1
                                                                                                                # (trained agetn0 use best response policy only and trained agent 1 use average policy only)
    # load avg_utility against eachother with eta 0 in training
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(kuhn_poker_nfsp_0 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_0_against_eachother.append(read_loss(ttt_nfsp_0 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))
    # load avg_utility against eachother with eta 1 in training
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(kuhn_poker_nfsp_1 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0.1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_0_1/avg_utility_against_eachother.txt'))
    avg_rewards_1_against_eachother.append(read_loss(ttt_nfsp_1 + 'avg_utility/eta_1_0/avg_utility_against_eachother.txt'))

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[0][0], "b", label="1")
    line2, = plt.plot(bp_jk[0][2], "r", label="2")
    line3, = plt.plot(bp_jk[0][4], "g", label="3")
    line4, = plt.plot(bp_jk[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[0][0], "b", label="1")
    line2, = plt.plot(bp_jq[0][2], "r", label="2")
    line3, = plt.plot(bp_jq[0][4], "g", label="3")
    line4, = plt.plot(bp_jq[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[0][0], "b", label="1")
    line2, = plt.plot(bp_kj[0][2], "r", label="2")
    line3, = plt.plot(bp_kj[0][4], "g", label="3")
    line4, = plt.plot(bp_kj[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[0][0], "b", label="1")
    line2, = plt.plot(bp_kq[0][2], "r", label="2")
    line3, = plt.plot(bp_kq[0][4], "g", label="3")
    line4, = plt.plot(bp_kq[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[0][0], "b", label="1")
    line2, = plt.plot(bp_qj[0][2], "r", label="2")
    line3, = plt.plot(bp_qj[0][4], "g", label="3")
    line4, = plt.plot(bp_qj[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[0][0], "b", label="1")
    line2, = plt.plot(bp_qk[0][2], "r", label="2")
    line3, = plt.plot(bp_qk[0][4], "g", label="3")
    line4, = plt.plot(bp_qk[0][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()



    # plt bp for kuhn_poker_nfsp_0.1, eta1 in evaluation
    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[1][0], "b", label="1")
    line2, = plt.plot(bp_jk[1][2], "r", label="2")
    line3, = plt.plot(bp_jk[1][4], "g", label="3")
    line4, = plt.plot(bp_jk[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[1][0], "b", label="1")
    line2, = plt.plot(bp_jq[1][2], "r", label="2")
    line3, = plt.plot(bp_jq[1][4], "g", label="3")
    line4, = plt.plot(bp_jq[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[1][0], "b", label="1")
    line2, = plt.plot(bp_kj[1][2], "r", label="2")
    line3, = plt.plot(bp_kj[1][4], "g", label="3")
    line4, = plt.plot(bp_kj[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[1][0], "b", label="1")
    line2, = plt.plot(bp_kq[1][2], "r", label="2")
    line3, = plt.plot(bp_kq[1][4], "g", label="3")
    line4, = plt.plot(bp_kq[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[1][0], "b", label="1")
    line2, = plt.plot(bp_qj[1][2], "r", label="2")
    line3, = plt.plot(bp_qj[1][4], "g", label="3")
    line4, = plt.plot(bp_qj[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[1][0], "b", label="1")
    line2, = plt.plot(bp_qk[1][2], "r", label="2")
    line3, = plt.plot(bp_qk[1][4], "g", label="3")
    line4, = plt.plot(bp_qk[1][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_0.1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[4][0], "b", label="1")
    line2, = plt.plot(bp_jk[4][2], "r", label="2")
    line3, = plt.plot(bp_jk[4][4], "g", label="3")
    line4, = plt.plot(bp_jk[4][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[4][0], "b", label="1")
    line2, = plt.plot(bp_jq[4][2], "r", label="2")
    line3, = plt.plot(bp_jq[4][4], "g", label="3")
    line4, = plt.plot(bp_jq[4][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[4][0], "b", label="1")
    line2, = plt.plot(bp_kj[4][2], "r", label="2")
    line3, = plt.plot(bp_kj[4][4], "g", label="3")
    line4, = plt.plot(bp_kj[4][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[4][0], "b", label="1")
    line2, = plt.plot(bp_kq[4][2], "r", label="2")
    line3, = plt.plot(bp_kq[4][4], "g", label="3")
    line4, = plt.plot(bp_kq[4][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[4][0], "b", label="1")
    line2, = plt.plot(bp_qj[4][2], "r", label="2")
    line3, = plt.plot(bp_qj[4][4], "g", label="3")
    line4, = plt.plot(bp_qj[4][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[4][0], "b", label="1")
    line2, = plt.plot(bp_qk[4][2], "r", label="2")
    line3, = plt.plot(bp_qk[4][4], "g", label="3")
    line4, = plt.plot(bp_qk[4][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()


    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta0.1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[2][0], "b", label="1")
    line2, = plt.plot(bp_jk[2][2], "r", label="2")
    line3, = plt.plot(bp_jk[2][4], "g", label="3")
    line4, = plt.plot(bp_jk[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[2][0], "b", label="1")
    line2, = plt.plot(bp_jq[2][2], "r", label="2")
    line3, = plt.plot(bp_jq[2][4], "g", label="3")
    line4, = plt.plot(bp_jq[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[2][0], "b", label="1")
    line2, = plt.plot(bp_kj[2][2], "r", label="2")
    line3, = plt.plot(bp_kj[2][4], "g", label="3")
    line4, = plt.plot(bp_kj[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[2][0], "b", label="1")
    line2, = plt.plot(bp_kq[2][2], "r", label="2")
    line3, = plt.plot(bp_kq[2][4], "g", label="3")
    line4, = plt.plot(bp_kq[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[2][0], "b", label="1")
    line2, = plt.plot(bp_qj[2][2], "r", label="2")
    line3, = plt.plot(bp_qj[2][4], "g", label="3")
    line4, = plt.plot(bp_qj[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[2][0], "b", label="1")
    line2, = plt.plot(bp_qk[2][2], "r", label="2")
    line3, = plt.plot(bp_qk[2][4], "g", label="3")
    line4, = plt.plot(bp_qk[2][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta1 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[3][0], "b", label="1")
    line2, = plt.plot(bp_jk[3][2], "r", label="2")
    line3, = plt.plot(bp_jk[3][4], "g", label="3")
    line4, = plt.plot(bp_jk[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[3][0], "b", label="1")
    line2, = plt.plot(bp_jq[3][2], "r", label="2")
    line3, = plt.plot(bp_jq[3][4], "g", label="3")
    line4, = plt.plot(bp_jq[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[3][0], "b", label="1")
    line2, = plt.plot(bp_kj[3][2], "r", label="2")
    line3, = plt.plot(bp_kj[3][4], "g", label="3")
    line4, = plt.plot(bp_kj[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[3][0], "b", label="1")
    line2, = plt.plot(bp_kq[3][2], "r", label="2")
    line3, = plt.plot(bp_kq[3][4], "g", label="3")
    line4, = plt.plot(bp_kq[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[3][0], "b", label="1")
    line2, = plt.plot(bp_qj[3][2], "r", label="2")
    line3, = plt.plot(bp_qj[3][4], "g", label="3")
    line4, = plt.plot(bp_qj[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[3][0], "b", label="1")
    line2, = plt.plot(bp_qk[3][2], "r", label="2")
    line3, = plt.plot(bp_qk[3][4], "g", label="3")
    line4, = plt.plot(bp_qk[3][6], "y", label="4")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("JK (kuhn_poker_nfsp_1, eta0 in evaluation)")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jk[5][0], "b", label="1")
    line2, = plt.plot(bp_jk[5][2], "r", label="2")
    line3, = plt.plot(bp_jk[5][4], "g", label="3")
    line4, = plt.plot(bp_jk[5][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("JQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_jq[5][0], "b", label="1")
    line2, = plt.plot(bp_jq[5][2], "r", label="2")
    line3, = plt.plot(bp_jq[5][4], "g", label="3")
    line4, = plt.plot(bp_jq[5][6], "y", label="4")
    plt.axhline(y=2/3,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("KJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kj[5][0], "b", label="1")
    line2, = plt.plot(bp_kj[5][2], "r", label="2")
    line3, = plt.plot(bp_kj[5][4], "g", label="3")
    line4, = plt.plot(bp_kj[5][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(10, 10))
    ax2 = plt.subplot(311)
    ax2.set_title("KQ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_kq[5][0], "b", label="1")
    line2, = plt.plot(bp_kq[5][2], "r", label="2")
    line3, = plt.plot(bp_kq[5][4], "g", label="3")
    line4, = plt.plot(bp_kq[5][6], "y", label="4")
    plt.axhline(y=0,ls=":",c="blue")
    plt.axhline(y=1,ls=":",c="blue")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title("QJ")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qj[5][0], "b", label="1")
    line2, = plt.plot(bp_qj[5][2], "r", label="2")
    line3, = plt.plot(bp_qj[5][4], "g", label="3")
    line4, = plt.plot(bp_qj[5][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title("QK")
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 1.1, 0.1)
    line1, = plt.plot(bp_qk[5][0], "b", label="1")
    line2, = plt.plot(bp_qk[5][2], "r", label="2")
    line3, = plt.plot(bp_qk[5][4], "g", label="3")
    line4, = plt.plot(bp_qk[5][6], "y", label="4")
    plt.axhline(y=1/3,ls=":",c="yellow")
    plt.axhline(y=2/3,ls=":",c="yellow")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2, line3, line4], loc='upper right')
    plt.ylabel('behavior_probs')
    plt.xlabel('episode(*1e4)')
    plt.show()

    '''
    # episode = range(10, 3010, 10)
    # plot exploitability, avg utility in kuhn_poker_nfsp_0.1_7_27
    plt.figure(figsize=(10, 3))
    #plt.subplot(311)
    #plt.ylim(0, 0.35)
    y_ticks = np.arange(0, 0.35, 0.05)
    line1, = plt.plot(expl_0[0], "b", label="kuhn_poker_nfsp_0.1")
    line2, = plt.plot(expl_1[0], "r", label="kuhn_poker_nfsp_1")
    plt.yticks(y_ticks)
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('exploitability')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 20))
    ax2 = plt.subplot(511)
    ax2.set_title("average utility with eta 0 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[0][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_0_against_eachother[0][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title("average utility with eta 1 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[1][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_0_against_eachother[1][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title("average utility with eta 0.1 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[2][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_0_against_eachother[2][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(514)
    ax2.set_title("average utility with eta 0_1 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[3][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_0_against_eachother[3][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title("average utility with eta 1_0 in evaluation (0.1 in training)")
    line1, = plt.plot(avg_rewards_0_against_eachother[4][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_0_against_eachother[4][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()

    #plt win rates in kuhn_poker_nfsp_0.1_7_27
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[3][0], 'b', label='kp_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[3][1], 'r', label='kp_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[3][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('trained agent0 against random agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[4][0], 'b', label='kp_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[4][1], 'r', label='kp_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[4][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('trained agent0 against random agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[5][0], 'b', label='kp_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[5][1], 'r', label='kp_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[5][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot random agent0 against trained agent1 in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[3][0], 'r', label='kp_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[3][1], 'b', label='kp_nfsp_0.1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[3][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[4][0], 'r', label='kp_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[4][1], 'b', label='kp_nfsp_0.1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[4][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('random agent0 against trained agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[5][0], 'r', label='kp_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[5][1], 'b', label='kp_nfsp_0.1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[5][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot win rates of trained agents against each other in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 20))
    # fig.suptitle("win rates against each other")
    ax2 = plt.subplot(511)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[5][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[5][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[5][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title('win rates against each other(eta 1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[6][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[6][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[6][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title('win rates against each other(eta 0.1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[7][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[7][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[7][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')


    ax2 = plt.subplot(514)
    ax2.set_title('win rates against each other(eta 0 for agent0 and eta 1 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[8][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[8][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[8][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title('win rates against each other(eta 1 for agent0 and eta 0 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[9][0], 'b', label='kp_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[9][1], 'r', label='kp_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[9][2], 'g', label='kp_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plt avg utility, win rates in kuhn_poker_nfsp_1_7_28
    plt.figure(figsize=(15, 20))
    ax2 = plt.subplot(511)
    ax2.set_title("average utility with eta 0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[0][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_1_against_eachother[0][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title("average utility with eta 1 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[1][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_1_against_eachother[1][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title("average utility with eta 0.1 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[2][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_1_against_eachother[2][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(514)
    ax2.set_title("average utility with eta 0_1 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[3][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_1_against_eachother[3][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title("average utility with eta 1_0 in evaluation (1 in training)")
    line1, = plt.plot(avg_rewards_1_against_eachother[4][0], "b", label="agent0")
    line2, = plt.plot(avg_rewards_1_against_eachother[4][1], "r", label="agent1")
    plt.legend(handles=[line1, line2], loc='upper right')
    plt.ylabel('avg utility')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[3][0], 'b', label='kp_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[3][1], 'r', label='kp_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[3][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('trained agent0 against random agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[4][0], 'b', label='kp_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[4][1], 'r', label='kp_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[4][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('trained agent0 against random agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[5][0], 'b', label='kp_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[5][1], 'r', label='kp_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[5][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[3][0], 'r', label='kp_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[3][1], 'b', label='kp_nfsp_1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[3][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[4][0], 'r', label='kp_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[4][1], 'b', label='kp_nfsp_1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[4][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('random agent0 against trained agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[5][0], 'r', label='kp_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[5][1], 'b', label='kp_nfsp_1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[5][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    plt.figure(figsize=(15, 20))
    # fig.suptitle("win rates against each other")
    ax2 = plt.subplot(511)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[5][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[5][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[5][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title('win rates against each other(eta 1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[6][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[6][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[6][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title('win rates against each other(eta 0.1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[7][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[7][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[7][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')


    ax2 = plt.subplot(514)
    ax2.set_title('win rates against each other(eta 0 for agent0 and eta 1 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[8][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[8][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[8][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title('win rates against each other(eta 1 for agent0 and eta 0 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[9][0], 'b', label='kp_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[9][1], 'r', label='kp_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[9][2], 'g', label='kp_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    
    # plt.subplot(312)
    # line1, = plt.plot(loss_agent0[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent0[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('loss_agent0')
    # plt.xlabel('episode(*1e4)')
    # 
    # plt.subplot(313)
    # line1, = plt.plot(loss_agent1[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent1[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='upper right')
    # plt.ylabel('loss_agent1')
    # plt.xlabel('episode(*1e4)')
    # plt.show()
    
    # plot trained agent0 against random agent1 in kp_nfsp_0.1_6_30

    # plot exploitability, avg_utility in tic_tac_toe_nfsp_0.1_7_24
    plt.figure(figsize=(10,3))
    #plt.subplot(311)
    plt.ylim(0, 1.05)
    x_range_0 = [4*(x+1) for x in range(len(expl_0[1]))]
    line1, = plt.plot(x_range_0, expl_0[1], "b", label="ttt_nfsp_0.1")
    x_range_1 = [4*(x+1) for x in range(len(expl_1[1]))]
    line2, = plt.plot(x_range_1, expl_1[1], "r", label="ttt_nfsp_1")
    plt.legend(handles=[line1, line2], loc='lower right')
    plt.ylabel('exploitability')
    plt.xlabel('episode(*1e4)')
    plt.show()

    
    # plt.subplot(312)
    # line1, = plt.plot(loss_agent0[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent0[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='lower right')
    # plt.ylabel('loss_agent0')
    # plt.xlabel('episode(*1e4)')
    # 
    # plt.subplot(313)
    # line1, = plt.plot(loss_agent1[0][0], "b", label="supervised learning loss")
    # line2, = plt.plot(loss_agent1[0][1], "r", label="reinforcement learning loss")
    # plt.legend(handles=[line1, line2], loc='lower right')
    # plt.ylabel('loss_agent1')
    # plt.xlabel('episode(*1e4)')
    # plt.show()
    
    # plot trained agent0 against random agent1 in tic_tac_toe_nfsp_0.1_7_24
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[0][0], 'b', label='ttt_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[0][1], 'r', label='ttt_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('trained agent0 against random agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[1][0], 'b', label='ttt_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[1][1], 'r', label='ttt_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[1][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('trained agent0 against random agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta0[2][0], 'b', label='ttt_nfsp_0.1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta0[2][1], 'r', label='ttt_nfsp_0.1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta0[2][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot random agent0 against trained agent1 in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[0][0], 'r', label='ttt_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[0][1], 'b', label='ttt_nfsp_0.1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[1][0], 'r', label='ttt_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[1][1], 'b', label='ttt_nfsp_0.1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[1][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('random agent0 against trained agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta0[2][0], 'r', label='ttt_nfsp_0.1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta0[2][1], 'b', label='ttt_nfsp_0.1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta0[2][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot win rates of trained agents against each other in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 20))
    # fig.suptitle("win rates against each other")
    ax2 = plt.subplot(511)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[0][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[0][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[0][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title('win rates against each other(eta 1 for both in evaluation)')
    plt.ylim(-0.05, 1.05)
    line1, = plt.plot(win_rates_against_eachother_eta0[1][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[1][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[1][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title('win rates against each other(eta 0.1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[2][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[2][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[2][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')


    ax2 = plt.subplot(514)
    ax2.set_title('win rates against each other(eta 0 for agent0 and eta 1 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[3][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[3][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[3][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title('win rates against each other(eta 1 for agent0 and eta 0 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta0[4][0], 'b', label='ttt_nfsp_0.1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta0[4][1], 'r', label='ttt_nfsp_0.1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta0[4][2], 'g', label='ttt_nfsp_0.1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plt win rates of ttt_nfsp_1_7_29 with eta 1 in training
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title("trained agent0 against random agent1(eta 0 in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[0][0], 'b', label='ttt_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[0][1], 'r', label='ttt_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('trained agent0 against random agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[1][0], 'b', label='ttt_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[1][1], 'r', label='ttt_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[1][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('trained agent0 against random agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random1_eta1[2][0], 'b', label='ttt_nfsp_1_trained_agent0')
    line2, = plt.plot(win_rates_against_random1_eta1[2][1], 'r', label='ttt_nfsp_1_random_agent1')
    line3, = plt.plot(win_rates_against_random1_eta1[2][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot random agent0 against trained agent1 in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 12))
    ax2 = plt.subplot(311)
    ax2.set_title('random agent0 against trained agent1(eta 0 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[0][0], 'r', label='ttt_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[0][1], 'b', label='ttt_nfsp_1_traomed_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(312)
    ax2.set_title('random agent0 against trained agent1(eta 0.1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[1][0], 'r', label='ttt_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[1][1], 'b', label='ttt_nfsp_1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[1][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(313)
    ax2.set_title('random agent0 against trained agent1(eta 1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_random0_eta1[2][0], 'r', label='ttt_nfsp_1_random_agent0')
    line2, = plt.plot(win_rates_against_random0_eta1[2][1], 'b', label='ttt_nfsp_1_trained_agent1')
    line3, = plt.plot(win_rates_against_random0_eta1[2][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()

    # plot win rates of trained agents against each other in ttt_nfsp_0.1_7_24
    plt.figure(figsize=(15, 20))
    # fig.suptitle("win rates against each other")
    ax2 = plt.subplot(511)
    ax2.set_title("win rates against each other(eta 0 for both in evaluation)")
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[0][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[0][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[0][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(512)
    ax2.set_title('win rates against each other(eta 1 for both in evaluation)')
    plt.ylim(-0.05, 1.05)
    line1, = plt.plot(win_rates_against_eachother_eta1[1][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[1][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[1][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(513)
    ax2.set_title('win rates against each other(eta 0.1 for both in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[2][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[2][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[2][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')


    ax2 = plt.subplot(514)
    ax2.set_title('win rates against each other(eta 0 for agent0 and eta 1 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[3][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[3][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[3][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')

    ax2 = plt.subplot(515)
    ax2.set_title('win rates against each other(eta 1 for agent0 and eta 0 for agent1 in evaluation)')
    plt.ylim(0, 1)
    line1, = plt.plot(win_rates_against_eachother_eta1[4][0], 'b', label='ttt_nfsp_1_agent0')
    line2, = plt.plot(win_rates_against_eachother_eta1[4][1], 'r', label='ttt_nfsp_1_agent1')
    line3, = plt.plot(win_rates_against_eachother_eta1[4][2], 'g', label='ttt_nfsp_1_draw')
    plt.legend(handles=[line1, line2, line3], loc='lower right', prop={'size': 9})
    plt.ylabel('win_rate')
    plt.xlabel('episode(*1e4)')
    plt.show()
    '''



if __name__ == "__main__":
    app.run(main)<file_sep># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""NFSP agents trained on Kuhn Poker."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app
from absl import flags
from absl import logging
import tensorflow.compat.v1 as tf

from open_spiel.python import policy
from open_spiel.python import rl_environment
from open_spiel.python.algorithms import exploitability
from open_spiel.python.algorithms import nfsp
import glob
import os
import numpy as np
from open_spiel.python.algorithms import random_agent


FLAGS = flags.FLAGS

flags.DEFINE_integer("num_train_episodes", int(3e6),
                     "Number of training episodes.")
flags.DEFINE_integer("eval_every", 100,
                     "Episode frequency at which the agents are evaluated.")
flags.DEFINE_list("hidden_layers_sizes", [
    128,
], "Number of hidden units in the avg-net and Q-net.")
flags.DEFINE_integer("replay_buffer_capacity", int(2e5),
                     "Size of the replay buffer.")
flags.DEFINE_integer("reservoir_buffer_capacity", int(2e6),
                     "Size of the reservoir buffer.")
flags.DEFINE_float("anticipatory_param_agent0", 0,
                   "Prob of using the rl best response as episode policy for agent0.")
flags.DEFINE_float("anticipatory_param_agent1", 0,
                   "Prob of using the rl best response as episode policy for agent1.")
flags.DEFINE_string("experiment_name", "kuhn_poker_0.1_7_27", "Experiment name")
flags.DEFINE_string("load_path", "/home/jxu8/Code_update/open_spiel/sessions_nfsp/", "Path to load the session")

flags.DEFINE_string("save_path", "/home/jxu8/Code_update/open_spiel/evaluation_data/eval_kp_nfsp_0.1_7_27/", "Path to load the session")


class NFSPPolicies(policy.Policy):
  """Joint policy to be evaluated."""

  def __init__(self, env, nfsp_policies, mode):
    game = env.game
    player_ids = [0, 1]
    super(NFSPPolicies, self).__init__(game, player_ids)
    self._policies = nfsp_policies
    self._mode = mode
    self._obs = {"info_state": [None, None], "legal_actions": [None, None]}

  def action_probabilities(self, state, player_id=None):
    cur_player = state.current_player()
    legal_actions = state.legal_actions(cur_player)

    self._obs["current_player"] = cur_player
    self._obs["info_state"][cur_player] = (
        state.information_state_tensor(cur_player))
    self._obs["legal_actions"][cur_player] = legal_actions

    info_state = rl_environment.TimeStep(
        observations=self._obs, rewards=None, discounts=None, step_type=None)

    with self._policies[cur_player].temp_mode_as(self._mode):
      p = self._policies[cur_player].step(info_state, is_evaluation=True).probs
    prob_dict = {action: p[action] for action in legal_actions}
    return prob_dict

def eval_against_random_agent1(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  rewards = np.zeros(2)
  cur_agents = [trained_agents[0], random_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      rewards[0] += time_step.rewards[0]
      rewards[1] += time_step.rewards[1]
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def eval_against_random_agent0(env, trained_agents, random_agents, num_episodes):
  """Evaluates `trained_agents` against `random_agents` for `num_episodes`."""
  wins = np.zeros(3)
  rewards = np.zeros(2)
  cur_agents = [random_agents[0], trained_agents[1]]
  for _ in range(num_episodes):
      time_step = env.reset()
      while not time_step.last():
        player_id = time_step.observations["current_player"]
        agent_output = cur_agents[player_id].step(time_step, is_evaluation=True)
        time_step = env.step([agent_output.action])
      rewards[0] += time_step.rewards[0]
      rewards[1] += time_step.rewards[1]
      if time_step.rewards[0] > 0:
        wins[0] += 1
      elif time_step.rewards[1] > 0:
        wins[1] += 1
      else:
        wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def eval_against_trained_agents(env, trained_agents, num_episodes):
  wins = np.zeros(3)
  rewards = np.zeros(2)
  for _ in range(num_episodes):
    time_step = env.reset()
    while not time_step.last():
      player_id = time_step.observations["current_player"]
      if env.is_turn_based:
        agents_output = trained_agents[player_id].step(time_step, is_evaluation=True)
        action_list = [agents_output.action]
      else:
        agents_output = [agent.step(time_step, is_evaluation=True) for agent in trained_agents]
        action_list = [agent_output.action for agent_output in agents_output]
      time_step = env.step(action_list)
    rewards[0] += time_step.rewards[0]
    rewards[1] += time_step.rewards[1]
    if time_step.rewards[0] > 0:
      wins[0] += 1
    elif time_step.rewards[1] > 0:
      wins[1] += 1
    else:
      wins[2] += 1
  return wins / num_episodes, rewards / num_episodes

def eval_between_random_agents(env, random_agents, num_episodes):
    wins = np.zeros(3)
    rewards = np.zeros(2)
    for _ in range(num_episodes):
        time_step = env.reset()
        while not time_step.last():
            player_id = time_step.observations["current_player"]
            if env.is_turn_based:
                agents_output = random_agents[player_id].step(time_step, is_evaluation=True)
                action_list = [agents_output.action]
            else:
                agents_output = [agent.step(time_step, is_evaluation=True) for agent in random_agents]
                action_list = [agent_output.action for agent_output in agents_output]
            time_step = env.step(action_list)
        rewards[0] += time_step.rewards[0]
        rewards[1] += time_step.rewards[1]
        if time_step.rewards[0] > 0:
            wins[0] += 1
        elif time_step.rewards[1] > 0:
            wins[1] += 1
        else:
            wins[2] += 1
    return wins / num_episodes, rewards / num_episodes


def action_probability(env, trained_agents, num_episodes, outcome0, outcome1):
    #state_list = []
    acti_list = []
    prob_list = []
    rewards_list = []
    tmp_act = []
    tmp_probs = []
    agent0_1 = np.zeros(2)
    agent1_1 = np.zeros(4)
    agent0_2 = np.zeros(2)
    agent0_1_probs = np.zeros(2)
    agent1_1_probs = np.zeros(4)
    agent0_2_probs = np.zeros(2)
    for _ in range(num_episodes):
        time_step = env.reset_jx(outcome0, outcome1)
        #state_list.append(outcome_list)
        while not time_step.last():
            player_id = time_step.observations["current_player"]
            if env.is_turn_based:
                agents_output = trained_agents[player_id].step(
                    time_step, is_evaluation=True)
                action_list = [agents_output.action]
            else:
                agents_output = [
                    agent.step(time_step, is_evaluation=True) for agent in trained_agents
                ]
                action_list = [agent_output.action for agent_output in agents_output]
            tmp_act.append(agents_output.action)
            tmp_probs.append(agents_output.probs)
            time_step = env.step(action_list)
        rewards = time_step.rewards
        rewards_list.append(rewards)
        acti_list.append(tmp_act)
        prob_list.append(tmp_probs)
        tmp_act = []
        tmp_probs = []
    for action in acti_list:
        if action[0] == 0:
            agent0_1[0] += 1
            if action[1] == 0:
                agent1_1[0] += 1
            else:
                if action[2] == 0:
                    agent0_2[0] += 1
        else:
            if action[1] == 0:
                agent1_1[2] += 1
    agent0_1[1] = num_episodes - agent0_1[0]
    agent1_1[3] = agent0_1[1] - agent1_1[2]
    agent1_1[1] = agent0_1[0] - agent1_1[0]
    agent0_2[1] = agent1_1[1] - agent0_2[0]
    agent0_1_probs[0] = agent0_1[0] / num_episodes
    agent0_1_probs[1] = agent0_1[1] / num_episodes
    agent1_1_probs[agent1_1 != 0] = agent1_1[agent1_1 != 0] / np.array([agent0_1[0], agent0_1[0], agent0_1[1], agent0_1[1]])[agent1_1 != 0]
    agent0_2_probs[agent0_2 != 0] = agent0_2[agent0_2 != 0] / agent1_1[1]
    return acti_list, prob_list, rewards_list, agent0_1_probs, agent1_1_probs[:2], agent1_1_probs[2:], agent0_2_probs

def policy_porbs(env, trained_agents, outcome0, outcome1):
    time_step = env.reset_jx(outcome0, outcome1)
    # print(env.get_state)
    agents_output_0 = trained_agents[0].step(
        time_step, is_evaluation=True)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([0])
    agents_output_1 = trained_agents[1].step(
        time_step, is_evaluation=True)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([1])
    agents_output_2 = trained_agents[1].step(
        time_step, is_evaluation=True)

    time_step = env.reset_jx(outcome0, outcome1)
    time_step = env.step([0])
    time_step = env.step([1])
    agents_output_3 = trained_agents[0].step(
        time_step, is_evaluation=True)
    return agents_output_0.probs, agents_output_1.probs, agents_output_2.probs, agents_output_3.probs


def main(unused_argv):
  game = "kuhn_poker"
  num_players = 2
  load_path = FLAGS.load_path + FLAGS.experiment_name
  env_configs = {"players": num_players}
  env = rl_environment.Environment(game, **env_configs)
  info_state_size = env.observation_spec()["info_state"][0]
  num_actions = env.action_spec()["num_actions"]

  hidden_layers_sizes = [int(l) for l in FLAGS.hidden_layers_sizes]
  kwargs = {
      "replay_buffer_capacity": FLAGS.replay_buffer_capacity,
      "epsilon_decay_duration": FLAGS.num_train_episodes,
      "epsilon_start": 0.06,
      "epsilon_end": 0.001,
  }

  random_agents = [
      random_agent.RandomAgent(player_id=idx, num_actions=num_actions)
      for idx in range(num_players)
  ]

  if FLAGS.anticipatory_param_agent0 == 0 and FLAGS.anticipatory_param_agent1 == 0:
      eta_name = 'eta_0'
  elif FLAGS.anticipatory_param_agent0 == 0.1 and FLAGS.anticipatory_param_agent1 == 0.1:
      eta_name = 'eta_0.1'
  elif FLAGS.anticipatory_param_agent0 == 1 and FLAGS.anticipatory_param_agent1 == 1:
      eta_name = 'eta_1'
  elif FLAGS.anticipatory_param_agent0 == 0 and FLAGS.anticipatory_param_agent1 == 1:
      eta_name = 'eta_0_1'
  elif FLAGS.anticipatory_param_agent0 == 1 and FLAGS.anticipatory_param_agent1 == 0:
      eta_name = 'eta_1_0'

  #model_dirs = sorted(glob.glob(load_path + "/episode-*"), key=os.path.getmtime)
  model_dirs = sorted(glob.glob(load_path + "/episode-*"), key=lambda x: int(os.path.split(x)[1][8:]))
  #model_dirs = glob.glob(load_path + '/episodes-*')
  print(model_dirs[0:4])
  for counter, dir in enumerate(model_dirs, 1):
      if counter % 1 == 0:
          tf.reset_default_graph()
          with tf.Session() as sess:
            # pylint: disable=g-complex-comprehension
            agent0 = nfsp.NFSP(sess, 0, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent0,
                          **kwargs)
            agent1 = nfsp.NFSP(sess, 1, info_state_size, num_actions, hidden_layers_sizes,
                          FLAGS.reservoir_buffer_capacity, FLAGS.anticipatory_param_agent1,
                          **kwargs)

            saver = tf.train.Saver()
            saver.restore(sess, dir + "/trained_model-10000")
            expl_policies_avg = NFSPPolicies(env, [agent0, agent1], nfsp.MODE.average_policy)
            # expl_list, expl = exploitability.exploitability(env.game, expl_policies_avg)
            # f4 = open("/home/jxu8/Code_update/open_spiel/evaluation_data/eval_ttt_nfsp_0.1_7_24/eval_eta_0/win_rates_against_random_agent1.txt", "a")
            # f4.write(str(expl) + '\n')
            #
            # logging.info("Episode: %s, Exploitability AVG %s", counter*10000, expl)
            # logging.info("_____________________________________________")

            win_rates_against_random_agent1, avg_utility_against_random_agent1 = eval_against_random_agent1(env, [agent0, agent1], random_agents, 1000)
            win_rates_against_random_agent0, avg_utility_against_random_agent0 = eval_against_random_agent0(env, [agent0, agent1], random_agents, 1000)
            win_rates_against_trained_agents, avg_utility = eval_against_trained_agents(env, [agent0, agent1], 1000)
            behavior_probabilities_0_1 = action_probability(env, [agent0, agent1], 100, 0, 1)
            behavior_probabilities_0_2 = action_probability(env, [agent0, agent1], 100, 0, 2)
            behavior_probabilities_1_0 = action_probability(env, [agent0, agent1], 100, 1, 0)
            behavior_probabilities_1_2 = action_probability(env, [agent0, agent1], 100, 1, 2)
            behavior_probabilities_2_0 = action_probability(env, [agent0, agent1], 100, 2, 0)
            behavior_probabilities_2_1 = action_probability(env, [agent0, agent1], 100, 2, 1)
            policy_probabilities_0_1 = policy_porbs(env, [agent0, agent1], 0, 1)
            policy_probabilities_0_2 = policy_porbs(env, [agent0, agent1], 0, 2)
            policy_probabilities_1_0 = policy_porbs(env, [agent0, agent1], 1, 0)
            policy_probabilities_1_2 = policy_porbs(env, [agent0, agent1], 1, 2)
            policy_probabilities_2_0 = policy_porbs(env, [agent0, agent1], 2, 0)
            policy_probabilities_2_1 = policy_porbs(env, [agent0, agent1], 2, 1)


            f1 = open(FLAGS.save_path + "avg_utility/" + eta_name + "/avg_utility_against_random_agent1.txt", "a")
            f1.write(str(avg_utility_against_random_agent1[0]) + ' ' + str(avg_utility_against_random_agent1[1]) + '\n')
            f2 = open(FLAGS.save_path + "avg_utility/" + eta_name + "/avg_utility_against_random_agent0.txt", "a")
            f2.write(str(avg_utility_against_random_agent0[0]) + ' ' + str(avg_utility_against_random_agent0[1]) + '\n')
            f3 = open(FLAGS.save_path + "avg_utility/" + eta_name + "/avg_utility_against_eachother.txt", "a")
            f3.write(str(avg_utility[0]) + ' ' + str(avg_utility[1]) + '\n')

            f4 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "JQ.txt", "a")
            f4.write(str(behavior_probabilities_0_1[3][0]) + ' ' + str(behavior_probabilities_0_1[3][1]) + ' ' +
                    str(behavior_probabilities_0_1[4][0]) + ' ' + str(behavior_probabilities_0_1[4][1]) + ' ' +
                    str(behavior_probabilities_0_1[5][0]) + ' ' + str(behavior_probabilities_0_1[5][1]) + ' ' +
                    str(behavior_probabilities_0_1[6][0]) + ' ' + str(behavior_probabilities_0_1[6][1]) + '\n')
            f5 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "JK.txt", "a")
            f5.write(str(behavior_probabilities_0_2[3][0]) + ' ' + str(behavior_probabilities_0_2[3][1]) + ' ' +
                    str(behavior_probabilities_0_2[4][0]) + ' ' + str(behavior_probabilities_0_2[4][1]) + ' ' +
                    str(behavior_probabilities_0_2[5][0]) + ' ' + str(behavior_probabilities_0_2[5][1]) + ' ' +
                    str(behavior_probabilities_0_2[6][0]) + ' ' + str(behavior_probabilities_0_2[6][1]) + '\n')
            f6 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "QJ.txt", "a")
            f6.write(str(behavior_probabilities_1_0[3][0]) + ' ' + str(behavior_probabilities_1_0[3][1]) + ' ' +
                    str(behavior_probabilities_1_0[4][0]) + ' ' + str(behavior_probabilities_1_0[4][1]) + ' ' +
                    str(behavior_probabilities_1_0[5][0]) + ' ' + str(behavior_probabilities_1_0[5][1]) + ' ' +
                    str(behavior_probabilities_1_0[6][0]) + ' ' + str(behavior_probabilities_1_0[6][1]) + '\n')
            f7 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "QK.txt", "a")
            f7.write(str(behavior_probabilities_1_2[3][0]) + ' ' + str(behavior_probabilities_1_2[3][1]) + ' ' +
                    str(behavior_probabilities_1_2[4][0]) + ' ' + str(behavior_probabilities_1_2[4][1]) + ' ' +
                    str(behavior_probabilities_1_2[5][0]) + ' ' + str(behavior_probabilities_1_2[5][1]) + ' ' +
                    str(behavior_probabilities_1_2[6][0]) + ' ' + str(behavior_probabilities_1_2[6][1]) + '\n')
            f8 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "KJ.txt", "a")
            f8.write(str(behavior_probabilities_2_0[3][0]) + ' ' + str(behavior_probabilities_2_0[3][1]) + ' ' +
                    str(behavior_probabilities_2_0[4][0]) + ' ' + str(behavior_probabilities_2_0[4][1]) + ' ' +
                    str(behavior_probabilities_2_0[5][0]) + ' ' + str(behavior_probabilities_2_0[5][1]) + ' ' +
                    str(behavior_probabilities_2_0[6][0]) + ' ' + str(behavior_probabilities_2_0[6][1]) + '\n')
            f9 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/competition_based/" + "KQ.txt", "a")
            f9.write(str(behavior_probabilities_2_1[3][0]) + ' ' + str(behavior_probabilities_2_1[3][1]) + ' ' +
                    str(behavior_probabilities_2_1[4][0]) + ' ' + str(behavior_probabilities_2_1[4][1]) + ' ' +
                    str(behavior_probabilities_2_1[5][0]) + ' ' + str(behavior_probabilities_2_1[5][1]) + ' ' +
                    str(behavior_probabilities_2_1[6][0]) + ' ' + str(behavior_probabilities_2_1[6][1]) + '\n')

            f10 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "JQ.txt", "a")
            f10.write(str(policy_probabilities_0_1[0][0]) + ' ' + str(policy_probabilities_0_1[0][1]) + ' ' +
                    str(policy_probabilities_0_1[1][0]) + ' ' + str(policy_probabilities_0_1[1][1]) + ' ' +
                    str(policy_probabilities_0_1[2][0]) + ' ' + str(policy_probabilities_0_1[2][1]) + ' ' +
                    str(policy_probabilities_0_1[3][0]) + ' ' + str(policy_probabilities_0_1[3][1]) + '\n')
            f11 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "JK.txt", "a")
            f11.write(str(policy_probabilities_0_2[0][0]) + ' ' + str(policy_probabilities_0_2[0][1]) + ' ' +
                    str(policy_probabilities_0_2[1][0]) + ' ' + str(policy_probabilities_0_2[1][1]) + ' ' +
                    str(policy_probabilities_0_2[2][0]) + ' ' + str(policy_probabilities_0_2[2][1]) + ' ' +
                    str(policy_probabilities_0_2[3][0]) + ' ' + str(policy_probabilities_0_2[3][1]) + '\n')
            f12 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "QJ.txt", "a")
            f12.write(str(policy_probabilities_1_0[0][0]) + ' ' + str(policy_probabilities_1_0[0][1]) + ' ' +
                    str(policy_probabilities_1_0[1][0]) + ' ' + str(policy_probabilities_1_0[1][1]) + ' ' +
                    str(policy_probabilities_1_0[2][0]) + ' ' + str(policy_probabilities_1_0[2][1]) + ' ' +
                    str(policy_probabilities_1_0[3][0]) + ' ' + str(policy_probabilities_1_0[3][1]) + '\n')
            f13 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "QK.txt", "a")
            f13.write(str(policy_probabilities_1_2[0][0]) + ' ' + str(policy_probabilities_1_2[0][1]) + ' ' +
                    str(policy_probabilities_1_2[1][0]) + ' ' + str(policy_probabilities_1_2[1][1]) + ' ' +
                    str(policy_probabilities_1_2[2][0]) + ' ' + str(policy_probabilities_1_2[2][1]) + ' ' +
                    str(policy_probabilities_1_2[3][0]) + ' ' + str(policy_probabilities_1_2[3][1]) + '\n')
            f14 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "KJ.txt", "a")
            f14.write(str(policy_probabilities_2_0[0][0]) + ' ' + str(policy_probabilities_2_0[0][1]) + ' ' +
                    str(policy_probabilities_2_0[1][0]) + ' ' + str(policy_probabilities_2_0[1][1]) + ' ' +
                    str(policy_probabilities_2_0[2][0]) + ' ' + str(policy_probabilities_2_0[2][1]) + ' ' +
                    str(policy_probabilities_2_0[3][0]) + ' ' + str(policy_probabilities_2_0[3][1]) + '\n')
            f15 = open(FLAGS.save_path + "behavior_probs/" + eta_name + "/policy_based/" + "KQ.txt", "a")
            f15.write(str(policy_probabilities_2_1[0][0]) + ' ' + str(policy_probabilities_2_1[0][1]) + ' ' +
                    str(policy_probabilities_2_1[1][0]) + ' ' + str(policy_probabilities_2_1[1][1]) + ' ' +
                    str(policy_probabilities_2_1[2][0]) + ' ' + str(policy_probabilities_2_1[2][1]) + ' ' +
                    str(policy_probabilities_2_1[3][0]) + ' ' + str(policy_probabilities_2_1[3][1]) + '\n')

            logging.info("Episode: %s", counter*10000)
            logging.info("Trained_agent0 vs Random_agent1: %s", avg_utility_against_random_agent1)
            logging.info("Random_agent0 vs Trained_agent1: %s", avg_utility_against_random_agent0)
            logging.info("Trained_agent0 vs Trained_agent1 %s", avg_utility)
            logging.info("_____________________________________________")



if __name__ == "__main__":
  app.run(main)
