import sys

from elegantrl.demo import *


def demo_continuous_action_on_policy():
    gpu_id = int(sys.argv[1]) if len(sys.argv) > 1 else 0  # >=0 means GPU ID, -1 means CPU
    drl_id = 1  # int(sys.argv[2])
    env_id = 4  # int(sys.argv[3])

    env_name = ['Pendulum-v0',
                'Pendulum-v1',
                'LunarLanderContinuous-v2',
                'BipedalWalker-v3',
                'Hopper-v2',
                'Humanoid-v3', ][env_id]
    agent = [AgentPPO, AgentPPO_H, AgentSAC_H][drl_id]

    print('agent', agent.__name__)
    print('gpu_id', gpu_id)
    print('env_name', env_name)

    if env_name in {'Pendulum-v0', 'Pendulum-v1'}:
        env = PendulumEnv(env_name, target_return=-500)
        "TotalStep: 1e5, TargetReward: -200, UsedTime: 600s"
        args = Arguments(agent, env=env)
        args.reward_scale = 2 ** -1  # RewardRange: -1800 < -200 < -50 < 0
        args.gamma = 0.97
        args.target_step = args.max_step * 8
        args.eval_times = 2 ** 3
    elif env_name == 'LunarLanderContinuous-v2':
        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  8.12e+03 -133.73 | -133.73   51.5     74    12 |   -0.59   9.57   0.02  -0.50
        1  1.86e+05   80.11 |  -23.68  161.1    214   111 |    0.05   1.99   0.04  -0.63
        1  9.88e+05  245.42 |  240.23   62.3    267   132 |    0.51   9.22   0.15  -1.13
        1  1.77e+06  271.45 |  246.60   79.0    208    39 |    0.72   2.41   0.11  -1.03
        1  3.55e+06  286.92 |  281.84   25.7    185    72 |    0.88   1.19   0.01  -1.00
        1  3.87e+06  286.98 |  286.98   17.9    160    14 |    0.77   1.05  -0.00  -1.02
        | UsedTime:    3900 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  8.17e+03 -236.66 | -236.66  109.5    115    32 |   -0.81  15.13   0.15  -0.50
        1  9.54e+04 -128.74 | -128.74   94.7    335   243 |    0.02   7.12  -0.01  -0.57
        1  1.71e+05  110.53 |  110.53  166.0    345   121 |    0.06   0.92   0.05  -0.64
        1  5.07e+05  218.86 |  210.64  111.2    297   112 |    0.47   7.01   0.02  -1.08
        1  6.43e+05  268.47 |  268.47   26.2    239   104 |    0.42   6.52  -0.05  -1.13
        1  1.06e+06  284.71 |  275.28   19.1    180    22 |    0.77   1.64  -0.06  -1.16
        1  1.33e+06  290.29 |  271.59   25.3    216   116 |    0.61   2.17   0.03  -1.17
        1  3.89e+06  293.44 |  282.32   19.0    160    13 |    0.91   2.07   0.09  -1.02
        | UsedTime:    3742 |
        """
        env_func = gym.make
        env_args = {'env_num': 1,
                    'env_name': 'LunarLanderContinuous-v2',
                    'max_step': 1000,
                    'state_dim': 8,
                    'action_dim': 2,
                    'if_discrete': False,
                    'target_return': 200,

                    'id': 'LunarLanderContinuous-v2'}
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.target_step = args.max_step * 2
        args.reward_scale = 2 ** -1
        args.gamma = 0.99

        args.net_dim = 2 ** 7
        args.layer_num = 3
        args.batch_size = int(args.net_dim * 2)
        args.repeat_times = 2 ** 4

        args.eval_times = 2 ** 5

        args.lambda_h_term = 2 ** -5
    elif env_name == 'BipedalWalker-v3':
        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        0  2.86e+04  -99.74 |  -99.74    1.4    112     5 |   -0.09   0.92   0.05  -0.53
        0  1.71e+05  -23.46 |  -23.46    4.6   1600     0 |   -0.01   0.03   0.06  -0.87
        0  3.12e+05   91.61 |   91.61   72.6   1600     0 |    0.04   0.07   0.07  -1.19
        0  4.59e+05  207.14 |  207.14  105.8   1405   363 |    0.06   0.48  -0.08  -1.28
        0  7.43e+05  317.28 |   31.39   58.2    487   185 |    0.08   0.73   0.07  -1.21
        0  2.24e+06  326.64 |  325.26    1.0    693    10 |    0.20   0.95   0.04  -1.14
        0  3.90e+06  329.56 |  327.68    0.4    625     3 |    0.22   1.07   0.01  -1.12
        | UsedTime:    3835 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  2.74e+04  -33.96 |  -33.96    1.3   1600     0 |   -0.05   0.29   0.01  -0.53
        1  1.69e+05  102.58 |  102.58    2.3   1600     0 |    0.01   0.03   0.04  -0.88
        1  4.50e+05  206.61 |  206.61    4.0   1600     0 |    0.05   0.21  -0.06  -1.33
        1  8.83e+05  256.48 |  245.42  126.5   1320   351 |    0.08   0.23   0.13  -1.29
        1  1.03e+06  315.47 |  315.47    1.2   1204    22 |    0.11   0.28   0.08  -1.30
        1  2.79e+06  320.63 |  319.06    1.2    899    19 |    0.16   0.40   0.11  -1.21
        1  3.98e+06  320.63 |  319.25    0.2    740     7 |    0.18   0.96  -0.01  -1.19
        | UsedTime:    3704 |
        """
        env_func = gym.make
        env_args = {'env_num': 1,
                    'env_name': 'BipedalWalker-v3',
                    'max_step': 1600,
                    'state_dim': 24,
                    'action_dim': 4,
                    'if_discrete': False,
                    'target_return': 300, }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.gamma = 0.98
        args.eval_times = 2 ** 4
        args.reward_scale = 2 ** -1

        args.target_step = args.max_step * 4
        args.worker_num = 2
        args.net_dim = 2 ** 7
        args.layer_num = 3
        args.batch_size = int(args.net_dim * 2)
        args.repeat_times = 2 ** 4
        args.ratio_clip = 0.25
        args.lambda_gae_adv = 0.96
        args.lambda_entropy = 0.02
        args.if_use_gae = True

        args.lambda_h_term = 2 ** -5
    elif env_name == 'Hopper-v2':
        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        5  1.61e+04  131.99 |  131.99    3.6     81     2 |    0.03   0.09   0.03  -0.54
        5  2.20e+05  391.44 |  391.44    0.3    158     0 |    0.08   0.01  -0.06  -0.75
        5  4.25e+05  860.96 |  860.96   11.9    280     5 |    0.09   0.11   0.12  -0.84
        5  6.27e+05 3001.43 | 3001.43    7.9   1000     0 |    0.10   0.78  -0.01  -0.85
        5  1.64e+06 3203.09 | 3103.14    0.0   1000     0 |    0.10   1.82  -0.06  -0.76
        5  2.86e+06 3256.43 | 3152.72    0.0   1000     0 |    0.10   0.75   0.01  -0.67
        5  3.88e+06 3256.43 | 1549.69    0.0    512     0 |    0.10   0.86   0.00  -0.71
        | UsedTime:    2565 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  1.60e+04  328.68 |  328.68    6.2    262     6 |    0.02   0.01  -0.02  -0.54
        2  2.16e+05 2460.57 | 2460.57   14.3   1000     0 |    0.09   0.86   0.20  -0.74
        2  6.22e+05 2789.97 | 2788.28   30.9   1000     0 |    0.10   0.40  -0.11  -1.04
        2  1.23e+06 3263.16 | 3216.96    0.0   1000     0 |    0.10   1.06   0.12  -1.05
        2  2.46e+06 3378.50 | 3364.02    0.0   1000     0 |    0.11   0.87   0.02  -0.92
        2  3.90e+06 3397.88 | 3302.80    0.0   1000     0 |    0.11   0.46   0.01  -0.93
        | UsedTime:    2557 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        4  2.41e+04  222.39 |  222.39    1.5    120     1 |    0.94   8.45   0.05  -0.55
        4  5.34e+05  344.58 |  344.58    0.4    142     0 |    2.41   1.91   0.02  -0.94
        4  8.74e+05  540.69 |  540.69   20.1    180     4 |    2.96   5.82   0.00  -1.10
        4  1.39e+06  989.51 |  989.51    2.2    308     2 |    3.20  16.75   0.07  -1.08
        4  1.73e+06 3161.60 | 3149.35    0.0   1000     0 |    3.26  43.84  -0.02  -1.08
        4  2.06e+06 3367.27 | 3105.77    0.0   1000     0 |    3.32  44.14   0.00  -1.13
        4  3.92e+06 3604.42 | 3565.39    0.0   1000     0 |    3.44  30.54   0.04  -1.04
        4  5.76e+06 3717.06 | 3607.94    0.0   1000     0 |    3.40  51.92   0.07  -0.95
        4  6.26e+06 3840.95 | 3409.25    0.0   1000     0 |    3.32  66.48  -0.02  -0.94
        | UsedTime:    6251 |
        """
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'Hopper-v2',
            'max_step': 1000,
            'state_dim': 11,
            'action_dim': 3,
            'if_discrete': False,
            'target_return': 3800.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)
        args.eval_times = 2 ** 1
        args.reward_scale = 2 ** -4

        args.target_step = args.max_step * 4  # 6
        args.worker_num = 2

        args.net_dim = 2 ** 7
        args.layer_num = 3
        args.batch_size = int(args.net_dim * 2)
        args.repeat_times = 2 ** 4
        args.ratio_clip = 0.25
        args.gamma = 0.993
        args.lambda_entropy = 0.02
        args.lambda_h_term = 2 ** -5

        args.if_allow_break = False
        args.break_step = int(8e6)
    elif env_name == 'Humanoid-v3':
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.target_step = args.max_step * 16
        args.if_cri_target = False
        args.repeat_times = 2 ** 6
        args.lambda_entropy = 2 ** -6

        args.worker_num = 2
        args.batch_size = args.net_dim * 8
        args.gamma = 0.995
        args.eval_times = 2 ** 4
        args.max_step = int(8e7)
    else:
        raise ValueError('env_name:', env_name)

    args.learner_gpus = gpu_id
    args.random_seed += gpu_id

    if_check = 0
    if if_check:
        train_and_evaluate(args)
    else:
        train_and_evaluate_mp(args)


def save_txt_data_in_python(cwd_path):
    os.makedirs(cwd_path, exist_ok=True)

    txt_path = f"{cwd_path}/Hopper-v2_PPO_11-step_epi_r_s_ary.txt"
    txt_data = """4.049900000000000000e+04 2.343297576904296875e+02 1.305000000000000000e+02
7.304200000000000000e+04 2.852064208984375000e+02 1.600000000000000000e+02
1.062080000000000000e+05 2.591396484375000000e+02 1.850000000000000000e+02
1.390400000000000000e+05 2.934655761718750000e+02 3.060000000000000000e+02
1.723500000000000000e+05 2.349243652343750000e+02 8.875000000000000000e+02
2.068830000000000000e+05 2.600360839843750000e+02 1.000000000000000000e+03
2.413420000000000000e+05 2.491331054687500000e+02 1.000000000000000000e+03
2.773520000000000000e+05 2.729860839843750000e+02 1.000000000000000000e+03
3.118300000000000000e+05 2.614294921875000000e+03 1.000000000000000000e+03
3.456300000000000000e+05 2.654938476562500000e+03 1.000000000000000000e+03
3.794470000000000000e+05 2.707160156250000000e+03 9.315000000000000000e+02
4.138960000000000000e+05 1.674616943359375000e+03 5.095000000000000000e+02
4.485750000000000000e+05 2.801560546875000000e+03 1.000000000000000000e+03
4.839890000000000000e+05 2.850665039062500000e+03 1.000000000000000000e+03
5.186710000000000000e+05 2.762765136718750000e+03 1.000000000000000000e+03
5.529980000000000000e+05 2.843449951171875000e+03 1.000000000000000000e+03
5.873710000000000000e+05 2.795828125000000000e+03 1.000000000000000000e+03
6.223780000000000000e+05 2.766553466796875000e+03 1.000000000000000000e+03
6.565900000000000000e+05 2.981401367187500000e+03 1.000000000000000000e+03
6.913900000000000000e+05 2.860203613281250000e+03 1.000000000000000000e+03
7.261090000000000000e+05 1.805011718750000000e+03 5.455000000000000000e+02
7.611750000000000000e+05 2.920603027343750000e+03 1.000000000000000000e+03
7.958310000000000000e+05 2.883319335937500000e+03 1.000000000000000000e+03
8.304140000000000000e+05 2.870545654296875000e+03 1.000000000000000000e+03
8.644110000000000000e+05 3.131284179687500000e+03 1.000000000000000000e+03
8.976670000000000000e+05 3.206085693359375000e+03 1.000000000000000000e+03
9.314880000000000000e+05 2.991028076171875000e+03 1.000000000000000000e+03
9.657730000000000000e+05 2.890010009765625000e+03 1.000000000000000000e+03
1.000629000000000000e+06 3.044442382812500000e+03 1.000000000000000000e+03
1.034650000000000000e+06 2.968379394531250000e+03 8.605000000000000000e+02
1.067850000000000000e+06 3.000930175781250000e+03 8.675000000000000000e+02
1.102683000000000000e+06 3.261527343750000000e+03 1.000000000000000000e+03
1.137264000000000000e+06 3.216011230468750000e+03 1.000000000000000000e+03
1.172448000000000000e+06 3.012333984375000000e+03 1.000000000000000000e+03
1.208181000000000000e+06 3.093447998046875000e+03 1.000000000000000000e+03
1.243587000000000000e+06 3.170104003906250000e+03 1.000000000000000000e+03
1.278630000000000000e+06 3.163046875000000000e+03 1.000000000000000000e+03
1.315842000000000000e+06 3.096404296875000000e+03 1.000000000000000000e+03
1.350143000000000000e+06 3.090926757812500000e+03 1.000000000000000000e+03
1.384570000000000000e+06 3.203481445312500000e+03 1.000000000000000000e+03
1.419623000000000000e+06 3.207410156250000000e+03 1.000000000000000000e+03
1.455522000000000000e+06 3.214802246093750000e+03 1.000000000000000000e+03
1.490694000000000000e+06 3.204271484375000000e+03 1.000000000000000000e+03
1.525003000000000000e+06 3.202755859375000000e+03 1.000000000000000000e+03
1.559745000000000000e+06 3.309057617187500000e+03 1.000000000000000000e+03
1.596452000000000000e+06 3.125462890625000000e+03 1.000000000000000000e+03
1.632525000000000000e+06 3.020158203125000000e+03 1.000000000000000000e+03
1.667831000000000000e+06 3.027861083984375000e+03 1.000000000000000000e+03
1.703798000000000000e+06 3.259697998046875000e+03 1.000000000000000000e+03
1.740323000000000000e+06 3.233373291015625000e+03 1.000000000000000000e+03
1.776653000000000000e+06 3.193263671875000000e+03 1.000000000000000000e+03
1.812747000000000000e+06 3.220758300781250000e+03 1.000000000000000000e+03
1.848930000000000000e+06 3.294957519531250000e+03 1.000000000000000000e+03
1.882801000000000000e+06 3.317345214843750000e+03 1.000000000000000000e+03
1.918358000000000000e+06 3.309782714843750000e+03 1.000000000000000000e+03
1.952457000000000000e+06 3.236909667968750000e+03 1.000000000000000000e+03
1.987754000000000000e+06 3.240661621093750000e+03 1.000000000000000000e+03
2.022822000000000000e+06 3.346376464843750000e+03 1.000000000000000000e+03
2.057356000000000000e+06 3.282309570312500000e+03 1.000000000000000000e+03
2.093345000000000000e+06 3.323498291015625000e+03 1.000000000000000000e+03
2.129110000000000000e+06 3.343395019531250000e+03 1.000000000000000000e+03
2.165020000000000000e+06 3.329417724609375000e+03 1.000000000000000000e+03
2.199398000000000000e+06 3.338149902343750000e+03 1.000000000000000000e+03
2.234600000000000000e+06 8.908327636718750000e+02 3.040000000000000000e+02
2.270499000000000000e+06 3.406991210937500000e+03 1.000000000000000000e+03
2.305328000000000000e+06 3.410513427734375000e+03 1.000000000000000000e+03
2.339906000000000000e+06 3.389066894531250000e+03 1.000000000000000000e+03
2.374336000000000000e+06 3.315009277343750000e+03 1.000000000000000000e+03
2.411586000000000000e+06 3.312118652343750000e+03 1.000000000000000000e+03
2.447841000000000000e+06 3.367446289062500000e+03 1.000000000000000000e+03
2.482763000000000000e+06 3.339838378906250000e+03 1.000000000000000000e+03
2.517461000000000000e+06 3.391653808593750000e+03 1.000000000000000000e+03
2.552029000000000000e+06 3.375048828125000000e+03 1.000000000000000000e+03
2.586992000000000000e+06 3.329795898437500000e+03 1.000000000000000000e+03
2.620328000000000000e+06 3.314772949218750000e+03 1.000000000000000000e+03
2.654643000000000000e+06 3.318156005859375000e+03 1.000000000000000000e+03
2.690867000000000000e+06 3.339491455078125000e+03 1.000000000000000000e+03
2.724514000000000000e+06 3.318162841796875000e+03 1.000000000000000000e+03
2.758983000000000000e+06 3.343368164062500000e+03 1.000000000000000000e+03
2.794711000000000000e+06 3.348853027343750000e+03 1.000000000000000000e+03
2.829387000000000000e+06 3.375925781250000000e+03 1.000000000000000000e+03
2.865196000000000000e+06 3.373903808593750000e+03 1.000000000000000000e+03
2.900485000000000000e+06 3.322830078125000000e+03 1.000000000000000000e+03
2.935792000000000000e+06 3.301686279296875000e+03 1.000000000000000000e+03
2.970673000000000000e+06 3.347768554687500000e+03 1.000000000000000000e+03
3.005777000000000000e+06 3.374974121093750000e+03 1.000000000000000000e+03
3.040929000000000000e+06 3.345122070312500000e+03 1.000000000000000000e+03
3.076667000000000000e+06 3.361543212890625000e+03 1.000000000000000000e+03
3.111995000000000000e+06 3.355489746093750000e+03 1.000000000000000000e+03
3.148459000000000000e+06 3.379013183593750000e+03 1.000000000000000000e+03
3.184294000000000000e+06 3.372624511718750000e+03 1.000000000000000000e+03
3.218807000000000000e+06 3.310897949218750000e+03 1.000000000000000000e+03
3.253090000000000000e+06 3.150296386718750000e+03 1.000000000000000000e+03
3.288491000000000000e+06 3.277668945312500000e+03 1.000000000000000000e+03
3.324082000000000000e+06 3.285313232421875000e+03 1.000000000000000000e+03
3.361703000000000000e+06 3.286468261718750000e+03 1.000000000000000000e+03
3.396646000000000000e+06 3.317630371093750000e+03 1.000000000000000000e+03
3.433272000000000000e+06 3.369835449218750000e+03 1.000000000000000000e+03
3.467858000000000000e+06 3.345429931640625000e+03 1.000000000000000000e+03
3.503880000000000000e+06 3.349427734375000000e+03 1.000000000000000000e+03
3.539323000000000000e+06 3.354180664062500000e+03 1.000000000000000000e+03
3.573933000000000000e+06 3.352566162109375000e+03 1.000000000000000000e+03
3.608345000000000000e+06 3.364336914062500000e+03 1.000000000000000000e+03
3.642641000000000000e+06 3.389349609375000000e+03 1.000000000000000000e+03
3.678900000000000000e+06 3.377368652343750000e+03 1.000000000000000000e+03
3.713476000000000000e+06 3.392540039062500000e+03 1.000000000000000000e+03
3.747388000000000000e+06 3.389166503906250000e+03 1.000000000000000000e+03
3.781640000000000000e+06 3.381977050781250000e+03 1.000000000000000000e+03
3.818631000000000000e+06 3.340476318359375000e+03 1.000000000000000000e+03
3.852690000000000000e+06 3.233296386718750000e+03 1.000000000000000000e+03
3.887771000000000000e+06 3.295815429687500000e+03 1.000000000000000000e+03
3.923722000000000000e+06 3.347664550781250000e+03 1.000000000000000000e+03
3.958361000000000000e+06 3.277573486328125000e+03 1.000000000000000000e+03
3.992748000000000000e+06 3.266129638671875000e+03 1.000000000000000000e+03
"""
    with open(txt_path, 'w') as f:
        f.writelines(txt_data)

    txt_path = f"{cwd_path}/Hopper-v2_PPO_12-step_epi_r_s_ary.txt"
    txt_data = """4.034300000000000000e+04 2.178105010986328125e+02 9.600000000000000000e+01
7.269800000000000000e+04 2.261656646728515625e+02 1.015000000000000000e+02
1.051110000000000000e+05 3.550444335937500000e+02 1.320000000000000000e+02
1.377520000000000000e+05 3.939020690917968750e+02 1.390000000000000000e+02
1.703880000000000000e+05 4.149125366210937500e+02 1.425000000000000000e+02
2.029270000000000000e+05 4.312397766113281250e+02 1.495000000000000000e+02
2.355160000000000000e+05 4.288845825195312500e+02 1.490000000000000000e+02
2.679090000000000000e+05 4.519456176757812500e+02 1.540000000000000000e+02
3.005570000000000000e+05 2.245014953613281250e+02 9.800000000000000000e+01
3.329620000000000000e+05 4.644690246582031250e+02 1.610000000000000000e+02
3.656170000000000000e+05 1.272360107421875000e+03 4.435000000000000000e+02
3.987730000000000000e+05 1.273014892578125000e+03 4.445000000000000000e+02
4.318940000000000000e+05 1.337710449218750000e+03 4.375000000000000000e+02
4.656550000000000000e+05 1.192656250000000000e+03 3.825000000000000000e+02
4.990770000000000000e+05 1.419962280273437500e+03 4.480000000000000000e+02
5.329740000000000000e+05 2.075947753906250000e+03 6.490000000000000000e+02
5.671280000000000000e+05 1.480293945312500000e+03 4.590000000000000000e+02
6.012430000000000000e+05 1.688222167968750000e+03 5.160000000000000000e+02
6.360050000000000000e+05 2.921617675781250000e+03 1.000000000000000000e+03
6.737880000000000000e+05 2.548560546875000000e+03 7.790000000000000000e+02
7.097350000000000000e+05 3.172675781250000000e+03 1.000000000000000000e+03
7.455150000000000000e+05 3.088864746093750000e+03 1.000000000000000000e+03
7.804770000000000000e+05 3.117426025390625000e+03 9.740000000000000000e+02
8.148750000000000000e+05 1.631784423828125000e+03 4.975000000000000000e+02
8.488610000000000000e+05 1.457217529296875000e+03 4.400000000000000000e+02
8.836600000000000000e+05 3.263509765625000000e+03 1.000000000000000000e+03
9.179160000000000000e+05 3.142808837890625000e+03 1.000000000000000000e+03
9.527050000000000000e+05 3.126567626953125000e+03 1.000000000000000000e+03
9.869020000000000000e+05 3.124933105468750000e+03 1.000000000000000000e+03
1.022791000000000000e+06 3.000129638671875000e+03 1.000000000000000000e+03
1.058127000000000000e+06 3.127746093750000000e+03 1.000000000000000000e+03
1.093428000000000000e+06 3.137821533203125000e+03 1.000000000000000000e+03
1.128301000000000000e+06 3.175414550781250000e+03 1.000000000000000000e+03
1.162200000000000000e+06 3.243176269531250000e+03 1.000000000000000000e+03
1.197099000000000000e+06 3.166359375000000000e+03 1.000000000000000000e+03
1.232841000000000000e+06 3.130691162109375000e+03 1.000000000000000000e+03
1.267744000000000000e+06 3.034230957031250000e+03 1.000000000000000000e+03
1.303360000000000000e+06 3.028932128906250000e+03 1.000000000000000000e+03
1.337639000000000000e+06 3.069020263671875000e+03 1.000000000000000000e+03
1.371672000000000000e+06 3.146883056640625000e+03 1.000000000000000000e+03
1.407156000000000000e+06 3.128060546875000000e+03 1.000000000000000000e+03
1.442504000000000000e+06 3.086258789062500000e+03 1.000000000000000000e+03
1.478779000000000000e+06 3.001636474609375000e+03 1.000000000000000000e+03
1.514164000000000000e+06 3.009221191406250000e+03 1.000000000000000000e+03
1.548919000000000000e+06 3.074035644531250000e+03 1.000000000000000000e+03
1.583975000000000000e+06 3.026476074218750000e+03 1.000000000000000000e+03
1.620203000000000000e+06 3.004805175781250000e+03 1.000000000000000000e+03
1.653711000000000000e+06 2.960291503906250000e+03 1.000000000000000000e+03
1.687564000000000000e+06 3.085207519531250000e+03 1.000000000000000000e+03
1.723309000000000000e+06 3.122834228515625000e+03 1.000000000000000000e+03
1.758058000000000000e+06 3.169125488281250000e+03 1.000000000000000000e+03
1.794122000000000000e+06 3.134137695312500000e+03 1.000000000000000000e+03
1.829048000000000000e+06 3.047645996093750000e+03 1.000000000000000000e+03
1.862487000000000000e+06 3.067165771484375000e+03 1.000000000000000000e+03
1.897178000000000000e+06 3.121096191406250000e+03 1.000000000000000000e+03
1.932096000000000000e+06 2.073406005859375000e+03 6.145000000000000000e+02
1.967315000000000000e+06 2.307780273437500000e+03 7.020000000000000000e+02
2.003490000000000000e+06 1.845058593750000000e+03 5.475000000000000000e+02
2.039739000000000000e+06 1.740868774414062500e+03 5.250000000000000000e+02
2.076504000000000000e+06 2.393298828125000000e+03 7.110000000000000000e+02
2.112265000000000000e+06 3.126540527343750000e+03 9.355000000000000000e+02
2.146547000000000000e+06 3.305196533203125000e+03 1.000000000000000000e+03
2.180829000000000000e+06 3.273039062500000000e+03 1.000000000000000000e+03
2.215654000000000000e+06 3.184907714843750000e+03 1.000000000000000000e+03
2.250004000000000000e+06 3.234335205078125000e+03 1.000000000000000000e+03
2.283861000000000000e+06 3.187615722656250000e+03 1.000000000000000000e+03
2.320530000000000000e+06 3.081823730468750000e+03 1.000000000000000000e+03
2.354926000000000000e+06 2.760846679687500000e+03 8.550000000000000000e+02
2.391374000000000000e+06 3.190476806640625000e+03 1.000000000000000000e+03
2.425785000000000000e+06 3.155068847656250000e+03 1.000000000000000000e+03
2.461396000000000000e+06 3.153059326171875000e+03 1.000000000000000000e+03
2.496985000000000000e+06 3.103491210937500000e+03 1.000000000000000000e+03
2.531599000000000000e+06 3.080425292968750000e+03 1.000000000000000000e+03
2.565529000000000000e+06 3.099724853515625000e+03 1.000000000000000000e+03
2.599884000000000000e+06 3.137273681640625000e+03 1.000000000000000000e+03
2.635263000000000000e+06 3.114508056640625000e+03 1.000000000000000000e+03
2.670445000000000000e+06 3.185020019531250000e+03 1.000000000000000000e+03
2.707394000000000000e+06 3.276217529296875000e+03 9.775000000000000000e+02
2.741960000000000000e+06 3.247000488281250000e+03 1.000000000000000000e+03
2.779080000000000000e+06 3.173932617187500000e+03 1.000000000000000000e+03
2.814215000000000000e+06 3.251589355468750000e+03 1.000000000000000000e+03
2.848610000000000000e+06 1.652526123046875000e+03 5.065000000000000000e+02
2.882155000000000000e+06 1.524916259765625000e+03 4.635000000000000000e+02
2.915883000000000000e+06 3.280160156250000000e+03 1.000000000000000000e+03
2.951219000000000000e+06 3.207485351562500000e+03 1.000000000000000000e+03
2.986366000000000000e+06 3.155824218750000000e+03 1.000000000000000000e+03
3.020292000000000000e+06 3.089792724609375000e+03 1.000000000000000000e+03
3.055520000000000000e+06 3.121609863281250000e+03 1.000000000000000000e+03
3.091008000000000000e+06 3.122458740234375000e+03 1.000000000000000000e+03
3.125894000000000000e+06 1.715688720703125000e+03 5.440000000000000000e+02
3.159967000000000000e+06 2.777770019531250000e+03 8.675000000000000000e+02
3.193956000000000000e+06 2.418841796875000000e+03 7.310000000000000000e+02
3.230057000000000000e+06 3.313566894531250000e+03 1.000000000000000000e+03
3.264603000000000000e+06 3.165643798828125000e+03 9.515000000000000000e+02
3.299125000000000000e+06 2.373316894531250000e+03 7.165000000000000000e+02
3.335127000000000000e+06 2.504906250000000000e+03 7.665000000000000000e+02
3.371780000000000000e+06 2.444432128906250000e+03 7.450000000000000000e+02
3.406291000000000000e+06 1.973151611328125000e+03 5.890000000000000000e+02
3.440874000000000000e+06 3.311587890625000000e+03 1.000000000000000000e+03
3.477605000000000000e+06 3.232211425781250000e+03 1.000000000000000000e+03
3.511972000000000000e+06 3.177491455078125000e+03 1.000000000000000000e+03
3.547243000000000000e+06 3.125782958984375000e+03 1.000000000000000000e+03
3.580968000000000000e+06 3.143778808593750000e+03 1.000000000000000000e+03
3.615682000000000000e+06 3.046243164062500000e+03 1.000000000000000000e+03
3.650647000000000000e+06 3.123518066406250000e+03 1.000000000000000000e+03
3.684709000000000000e+06 3.178277832031250000e+03 1.000000000000000000e+03
3.720366000000000000e+06 3.079544677734375000e+03 1.000000000000000000e+03
3.754421000000000000e+06 3.188493164062500000e+03 1.000000000000000000e+03
3.788235000000000000e+06 3.101297851562500000e+03 1.000000000000000000e+03
3.822854000000000000e+06 3.239887939453125000e+03 1.000000000000000000e+03
3.856989000000000000e+06 2.595781250000000000e+03 7.895000000000000000e+02
3.892340000000000000e+06 3.044919921875000000e+03 9.355000000000000000e+02
3.927460000000000000e+06 3.268782714843750000e+03 1.000000000000000000e+03
3.962711000000000000e+06 2.800395996093750000e+03 8.240000000000000000e+02
3.996988000000000000e+06 2.317447265625000000e+03 6.785000000000000000e+02
"""

    with open(txt_path, 'w') as f:
        f.writelines(txt_data)

    txt_path = f"{cwd_path}/Hopper-v2_PPO_13-step_epi_r_s_ary.txt"
    txt_data = """4.038600000000000000e+04 2.096177978515625000e+02 1.210000000000000000e+02
7.330100000000000000e+04 3.454100646972656250e+02 1.580000000000000000e+02
1.059890000000000000e+05 3.492969360351562500e+02 1.595000000000000000e+02
1.387370000000000000e+05 3.601017456054687500e+02 1.605000000000000000e+02
1.715520000000000000e+05 3.728549194335937500e+02 1.570000000000000000e+02
2.040730000000000000e+05 3.881669921875000000e+02 1.570000000000000000e+02
2.367740000000000000e+05 3.996542968750000000e+02 1.555000000000000000e+02
2.695070000000000000e+05 4.300065917968750000e+02 1.605000000000000000e+02
3.019850000000000000e+05 5.090229187011718750e+02 1.790000000000000000e+02
3.346920000000000000e+05 6.336318359375000000e+02 2.130000000000000000e+02
3.675850000000000000e+05 7.116786499023437500e+02 2.325000000000000000e+02
4.002830000000000000e+05 1.451473022460937500e+03 4.780000000000000000e+02
4.328550000000000000e+05 1.196189208984375000e+03 3.895000000000000000e+02
4.660900000000000000e+05 2.626837646484375000e+03 8.575000000000000000e+02
4.991500000000000000e+05 1.724975830078125000e+03 5.465000000000000000e+02
5.328460000000000000e+05 3.032342773437500000e+03 1.000000000000000000e+03
5.669410000000000000e+05 3.003476074218750000e+03 1.000000000000000000e+03
6.002150000000000000e+05 3.049793945312500000e+03 1.000000000000000000e+03
6.350470000000000000e+05 3.121761230468750000e+03 1.000000000000000000e+03
6.686510000000000000e+05 3.020493652343750000e+03 1.000000000000000000e+03
7.031160000000000000e+05 3.021985839843750000e+03 1.000000000000000000e+03
7.384750000000000000e+05 2.973490722656250000e+03 1.000000000000000000e+03
7.743140000000000000e+05 3.165480712890625000e+03 1.000000000000000000e+03
8.111250000000000000e+05 3.197477294921875000e+03 1.000000000000000000e+03
8.456670000000000000e+05 3.130427734375000000e+03 1.000000000000000000e+03
8.810600000000000000e+05 3.050742187500000000e+03 1.000000000000000000e+03
9.170100000000000000e+05 3.170666015625000000e+03 1.000000000000000000e+03
9.514860000000000000e+05 3.157179931640625000e+03 1.000000000000000000e+03
9.857920000000000000e+05 3.198711669921875000e+03 1.000000000000000000e+03
1.022731000000000000e+06 3.152378906250000000e+03 1.000000000000000000e+03
1.057025000000000000e+06 1.284374755859375000e+03 3.895000000000000000e+02
1.093212000000000000e+06 3.049642089843750000e+03 1.000000000000000000e+03
1.129925000000000000e+06 9.194798583984375000e+02 2.740000000000000000e+02
1.165271000000000000e+06 3.264453125000000000e+03 1.000000000000000000e+03
1.199547000000000000e+06 3.217057128906250000e+03 1.000000000000000000e+03
1.235490000000000000e+06 3.101316894531250000e+03 1.000000000000000000e+03
1.270412000000000000e+06 2.999381347656250000e+03 1.000000000000000000e+03
1.305862000000000000e+06 2.926777832031250000e+03 1.000000000000000000e+03
1.341146000000000000e+06 2.625652343750000000e+03 1.000000000000000000e+03
1.376825000000000000e+06 2.416450439453125000e+03 7.065000000000000000e+02
1.412669000000000000e+06 3.165876464843750000e+03 1.000000000000000000e+03
1.448318000000000000e+06 3.330312500000000000e+03 1.000000000000000000e+03
1.484338000000000000e+06 2.183000976562500000e+03 6.360000000000000000e+02
1.519751000000000000e+06 3.233037109375000000e+03 1.000000000000000000e+03
1.555464000000000000e+06 3.146699218750000000e+03 1.000000000000000000e+03
1.590504000000000000e+06 3.184873535156250000e+03 1.000000000000000000e+03
1.625613000000000000e+06 3.123329589843750000e+03 1.000000000000000000e+03
1.659122000000000000e+06 3.057064941406250000e+03 1.000000000000000000e+03
1.695150000000000000e+06 2.971835693359375000e+03 9.535000000000000000e+02
1.729667000000000000e+06 8.780496826171875000e+02 3.080000000000000000e+02
1.763547000000000000e+06 1.404875488281250000e+03 4.660000000000000000e+02
1.799355000000000000e+06 1.599153320312500000e+03 4.785000000000000000e+02
1.834939000000000000e+06 1.521260253906250000e+03 4.555000000000000000e+02
1.869402000000000000e+06 3.244150390625000000e+03 1.000000000000000000e+03
1.903636000000000000e+06 3.197366699218750000e+03 1.000000000000000000e+03
1.939465000000000000e+06 2.418268310546875000e+03 7.340000000000000000e+02
1.975135000000000000e+06 2.566557617187500000e+03 7.880000000000000000e+02
2.010278000000000000e+06 3.249409423828125000e+03 1.000000000000000000e+03
2.045251000000000000e+06 3.211192382812500000e+03 1.000000000000000000e+03
2.081594000000000000e+06 3.228671875000000000e+03 1.000000000000000000e+03
2.115974000000000000e+06 3.238759765625000000e+03 1.000000000000000000e+03
2.150083000000000000e+06 6.280198364257812500e+02 2.215000000000000000e+02
2.185257000000000000e+06 2.725798828125000000e+03 8.225000000000000000e+02
2.219861000000000000e+06 1.974407958984375000e+03 6.250000000000000000e+02
2.255847000000000000e+06 1.935783325195312500e+03 6.120000000000000000e+02
2.290266000000000000e+06 3.052675537109375000e+03 1.000000000000000000e+03
2.325528000000000000e+06 6.136882934570312500e+02 2.175000000000000000e+02
2.361066000000000000e+06 9.726678466796875000e+02 3.290000000000000000e+02
2.396269000000000000e+06 2.197631103515625000e+03 6.655000000000000000e+02
2.429965000000000000e+06 1.229930786132812500e+03 4.110000000000000000e+02
2.467120000000000000e+06 3.176313476562500000e+03 1.000000000000000000e+03
2.501296000000000000e+06 3.160648681640625000e+03 1.000000000000000000e+03
2.537039000000000000e+06 3.157404296875000000e+03 1.000000000000000000e+03
2.572222000000000000e+06 3.166119628906250000e+03 1.000000000000000000e+03
2.607773000000000000e+06 3.129809570312500000e+03 1.000000000000000000e+03
2.644483000000000000e+06 3.220138427734375000e+03 1.000000000000000000e+03
2.678567000000000000e+06 3.214668945312500000e+03 1.000000000000000000e+03
2.713373000000000000e+06 3.155075927734375000e+03 1.000000000000000000e+03
2.746919000000000000e+06 3.121135253906250000e+03 1.000000000000000000e+03
2.783766000000000000e+06 3.217287841796875000e+03 1.000000000000000000e+03
2.818700000000000000e+06 3.040480468750000000e+03 1.000000000000000000e+03
2.852920000000000000e+06 3.129872558593750000e+03 1.000000000000000000e+03
2.889285000000000000e+06 3.114858642578125000e+03 1.000000000000000000e+03
2.923766000000000000e+06 3.149004882812500000e+03 1.000000000000000000e+03
2.958806000000000000e+06 3.084910888671875000e+03 1.000000000000000000e+03
2.994974000000000000e+06 3.162165527343750000e+03 1.000000000000000000e+03
3.030624000000000000e+06 3.100338623046875000e+03 1.000000000000000000e+03
3.065421000000000000e+06 3.140790039062500000e+03 1.000000000000000000e+03
3.100444000000000000e+06 3.162745117187500000e+03 1.000000000000000000e+03
3.137352000000000000e+06 3.159432128906250000e+03 1.000000000000000000e+03
3.171814000000000000e+06 2.478965087890625000e+03 7.905000000000000000e+02
3.206195000000000000e+06 3.085448242187500000e+03 1.000000000000000000e+03
3.241860000000000000e+06 3.121942871093750000e+03 1.000000000000000000e+03
3.277475000000000000e+06 3.001067382812500000e+03 1.000000000000000000e+03
3.314567000000000000e+06 3.057093261718750000e+03 1.000000000000000000e+03
3.350053000000000000e+06 3.044020996093750000e+03 1.000000000000000000e+03
3.386022000000000000e+06 3.206770996093750000e+03 1.000000000000000000e+03
3.421481000000000000e+06 3.222163574218750000e+03 1.000000000000000000e+03
3.458254000000000000e+06 3.272282714843750000e+03 1.000000000000000000e+03
3.492957000000000000e+06 3.169154785156250000e+03 1.000000000000000000e+03
3.528549000000000000e+06 3.185240234375000000e+03 1.000000000000000000e+03
3.564777000000000000e+06 1.941793334960937500e+03 6.320000000000000000e+02
3.598804000000000000e+06 3.132753662109375000e+03 1.000000000000000000e+03
3.633438000000000000e+06 3.187485839843750000e+03 1.000000000000000000e+03
3.668377000000000000e+06 3.133473144531250000e+03 1.000000000000000000e+03
3.703681000000000000e+06 3.238643066406250000e+03 1.000000000000000000e+03
3.738852000000000000e+06 3.249842529296875000e+03 1.000000000000000000e+03
3.773558000000000000e+06 3.195123046875000000e+03 1.000000000000000000e+03
3.809921000000000000e+06 2.850768554687500000e+03 8.890000000000000000e+02
3.847106000000000000e+06 3.151384765625000000e+03 1.000000000000000000e+03
3.882035000000000000e+06 2.048037597656250000e+03 6.710000000000000000e+02
3.917560000000000000e+06 2.068314697265625000e+03 6.540000000000000000e+02
3.951965000000000000e+06 2.866084228515625000e+03 9.040000000000000000e+02
3.986791000000000000e+06 3.172588378906250000e+03 1.000000000000000000e+03
"""
    with open(txt_path, 'w') as f:
        f.writelines(txt_data)

    txt_path = f"{cwd_path}/Hopper-v2_PPO_14-step_epi_r_s_ary.txt"
    txt_data = """4.034000000000000000e+04 2.211443481445312500e+02 1.000000000000000000e+02
7.279500000000000000e+04 2.408919677734375000e+02 1.075000000000000000e+02
1.053960000000000000e+05 3.183980712890625000e+02 1.285000000000000000e+02
1.378630000000000000e+05 3.431957092285156250e+02 1.350000000000000000e+02
1.703370000000000000e+05 3.602852783203125000e+02 1.405000000000000000e+02
2.027910000000000000e+05 3.873166809082031250e+02 1.420000000000000000e+02
2.355660000000000000e+05 4.275813598632812500e+02 1.500000000000000000e+02
2.683730000000000000e+05 4.831182250976562500e+02 1.615000000000000000e+02
3.008600000000000000e+05 5.469335937500000000e+02 1.785000000000000000e+02
3.336880000000000000e+05 5.881808471679687500e+02 1.890000000000000000e+02
3.661410000000000000e+05 6.619787597656250000e+02 2.120000000000000000e+02
3.991880000000000000e+05 8.490818481445312500e+02 2.585000000000000000e+02
4.320630000000000000e+05 1.031094482421875000e+03 3.220000000000000000e+02
4.647680000000000000e+05 1.125526123046875000e+03 3.630000000000000000e+02
4.984240000000000000e+05 2.797930664062500000e+03 1.000000000000000000e+03
5.323370000000000000e+05 2.912880859375000000e+03 1.000000000000000000e+03
5.663470000000000000e+05 2.895129394531250000e+03 1.000000000000000000e+03
5.999310000000000000e+05 2.849255615234375000e+03 1.000000000000000000e+03
6.336850000000000000e+05 2.991338134765625000e+03 1.000000000000000000e+03
6.669080000000000000e+05 2.825023437500000000e+03 1.000000000000000000e+03
6.997510000000000000e+05 9.913345947265625000e+02 3.240000000000000000e+02
7.332820000000000000e+05 2.802958496093750000e+03 9.910000000000000000e+02
7.675780000000000000e+05 2.854897216796875000e+03 1.000000000000000000e+03
8.015400000000000000e+05 2.950930664062500000e+03 1.000000000000000000e+03
8.363160000000000000e+05 2.933717773437500000e+03 1.000000000000000000e+03
8.705260000000000000e+05 2.792624511718750000e+03 1.000000000000000000e+03
9.047580000000000000e+05 2.844742187500000000e+03 1.000000000000000000e+03
9.394990000000000000e+05 3.111558105468750000e+03 1.000000000000000000e+03
9.746240000000000000e+05 3.042947509765625000e+03 1.000000000000000000e+03
1.010625000000000000e+06 3.008312500000000000e+03 1.000000000000000000e+03
1.046211000000000000e+06 2.979475097656250000e+03 1.000000000000000000e+03
1.081053000000000000e+06 3.033517578125000000e+03 1.000000000000000000e+03
1.115302000000000000e+06 3.072668945312500000e+03 1.000000000000000000e+03
1.150583000000000000e+06 3.012201171875000000e+03 9.325000000000000000e+02
1.185437000000000000e+06 3.054317871093750000e+03 1.000000000000000000e+03
1.221382000000000000e+06 3.168616699218750000e+03 1.000000000000000000e+03
1.255134000000000000e+06 3.168064941406250000e+03 1.000000000000000000e+03
1.290540000000000000e+06 3.035512207031250000e+03 1.000000000000000000e+03
1.325192000000000000e+06 3.103938476562500000e+03 1.000000000000000000e+03
1.359811000000000000e+06 3.036659423828125000e+03 1.000000000000000000e+03
1.395002000000000000e+06 2.857463378906250000e+03 1.000000000000000000e+03
1.430328000000000000e+06 3.065263916015625000e+03 1.000000000000000000e+03
1.464736000000000000e+06 2.987133789062500000e+03 1.000000000000000000e+03
1.499668000000000000e+06 3.004864257812500000e+03 1.000000000000000000e+03
1.534728000000000000e+06 3.056497070312500000e+03 1.000000000000000000e+03
1.569529000000000000e+06 2.938567382812500000e+03 8.660000000000000000e+02
1.604380000000000000e+06 2.335118164062500000e+03 6.860000000000000000e+02
1.639286000000000000e+06 3.068312988281250000e+03 1.000000000000000000e+03
1.673950000000000000e+06 3.174589599609375000e+03 1.000000000000000000e+03
1.707314000000000000e+06 3.118711914062500000e+03 1.000000000000000000e+03
1.743082000000000000e+06 3.090447753906250000e+03 1.000000000000000000e+03
1.778745000000000000e+06 3.039876464843750000e+03 1.000000000000000000e+03
1.813512000000000000e+06 3.125157470703125000e+03 1.000000000000000000e+03
1.848108000000000000e+06 3.124626953125000000e+03 1.000000000000000000e+03
1.883285000000000000e+06 3.052273193359375000e+03 1.000000000000000000e+03
1.919102000000000000e+06 3.139608886718750000e+03 1.000000000000000000e+03
1.953801000000000000e+06 3.105018066406250000e+03 1.000000000000000000e+03
1.988803000000000000e+06 2.177119140625000000e+03 7.445000000000000000e+02
2.022982000000000000e+06 2.965227783203125000e+03 1.000000000000000000e+03
2.059200000000000000e+06 3.025616699218750000e+03 1.000000000000000000e+03
2.093670000000000000e+06 3.093546386718750000e+03 1.000000000000000000e+03
2.129038000000000000e+06 3.129179199218750000e+03 1.000000000000000000e+03
2.164140000000000000e+06 3.117252929687500000e+03 1.000000000000000000e+03
2.199017000000000000e+06 3.228673095703125000e+03 1.000000000000000000e+03
2.233698000000000000e+06 3.221529052734375000e+03 1.000000000000000000e+03
2.268771000000000000e+06 3.141163574218750000e+03 1.000000000000000000e+03
2.302500000000000000e+06 3.123974609375000000e+03 1.000000000000000000e+03
2.337167000000000000e+06 3.086194335937500000e+03 1.000000000000000000e+03
2.372399000000000000e+06 3.062706054687500000e+03 1.000000000000000000e+03
2.409524000000000000e+06 3.089461425781250000e+03 1.000000000000000000e+03
2.444108000000000000e+06 2.471277343750000000e+03 7.200000000000000000e+02
2.479404000000000000e+06 3.384738037109375000e+03 9.865000000000000000e+02
2.514404000000000000e+06 3.278693359375000000e+03 1.000000000000000000e+03
2.550419000000000000e+06 3.282059326171875000e+03 1.000000000000000000e+03
2.585057000000000000e+06 3.298626953125000000e+03 1.000000000000000000e+03
2.620024000000000000e+06 2.878383789062500000e+03 8.510000000000000000e+02
2.654867000000000000e+06 2.646894042968750000e+03 7.780000000000000000e+02
2.689306000000000000e+06 2.000447021484375000e+03 5.790000000000000000e+02
2.723580000000000000e+06 2.532476806640625000e+03 7.310000000000000000e+02
2.757964000000000000e+06 3.247326171875000000e+03 1.000000000000000000e+03
2.792070000000000000e+06 3.298198730468750000e+03 1.000000000000000000e+03
2.824986000000000000e+06 3.808172302246093750e+02 1.455000000000000000e+02
2.858046000000000000e+06 2.504661376953125000e+03 7.365000000000000000e+02
2.892083000000000000e+06 2.634037109375000000e+03 7.685000000000000000e+02
2.925931000000000000e+06 3.271656982421875000e+03 1.000000000000000000e+03
2.961413000000000000e+06 3.611966857910156250e+02 1.470000000000000000e+02
2.995856000000000000e+06 3.203354980468750000e+03 1.000000000000000000e+03
3.031119000000000000e+06 3.291353515625000000e+03 1.000000000000000000e+03
3.065825000000000000e+06 3.255465576171875000e+03 1.000000000000000000e+03
3.100168000000000000e+06 3.268739746093750000e+03 1.000000000000000000e+03
3.135449000000000000e+06 3.232864746093750000e+03 1.000000000000000000e+03
3.170194000000000000e+06 3.211734619140625000e+03 1.000000000000000000e+03
3.203552000000000000e+06 3.206069580078125000e+03 1.000000000000000000e+03
3.237814000000000000e+06 3.222251953125000000e+03 1.000000000000000000e+03
3.272346000000000000e+06 3.244475585937500000e+03 1.000000000000000000e+03
3.306821000000000000e+06 3.257669189453125000e+03 1.000000000000000000e+03
3.342612000000000000e+06 3.273170410156250000e+03 1.000000000000000000e+03
3.377660000000000000e+06 3.163279541015625000e+03 1.000000000000000000e+03
3.412441000000000000e+06 3.152978027343750000e+03 1.000000000000000000e+03
3.447871000000000000e+06 3.208136718750000000e+03 1.000000000000000000e+03
3.484439000000000000e+06 2.568400390625000000e+03 7.970000000000000000e+02
3.519827000000000000e+06 3.211446777343750000e+03 1.000000000000000000e+03
3.554257000000000000e+06 3.098612304687500000e+03 1.000000000000000000e+03
3.590585000000000000e+06 3.246329345703125000e+03 1.000000000000000000e+03
3.625588000000000000e+06 2.485878417968750000e+03 7.400000000000000000e+02
3.661566000000000000e+06 3.275436035156250000e+03 1.000000000000000000e+03
3.696290000000000000e+06 3.251919433593750000e+03 1.000000000000000000e+03
3.731917000000000000e+06 3.174267822265625000e+03 1.000000000000000000e+03
3.767323000000000000e+06 3.160889160156250000e+03 1.000000000000000000e+03
3.802805000000000000e+06 3.220461914062500000e+03 1.000000000000000000e+03
3.837795000000000000e+06 3.234273925781250000e+03 1.000000000000000000e+03
3.872850000000000000e+06 3.144645507812500000e+03 1.000000000000000000e+03
3.909805000000000000e+06 2.323069091796875000e+03 7.175000000000000000e+02
3.945609000000000000e+06 1.904869995117187500e+03 6.160000000000000000e+02
3.982376000000000000e+06 3.203207763671875000e+03 1.000000000000000000e+03
"""
    with open(txt_path, 'w') as f:
        f.writelines(txt_data)


def draw_learning_curve():
    from elegantrl.agent import AgentPPO
    from elegantrl.evaluator import demo_evaluate_actors

    gpu_id = 0
    cwd_path = 'hopper_temp'  # todo
    net_dim = 2 ** 7
    agent = AgentPPO
    env_args = {'env_num': 1,
                'env_name': 'Hopper-v2',
                'max_step': 1000,
                'state_dim': 11,
                'action_dim': 3,
                'if_discrete': False,
                'eval_times': 2 ** 1,
                'target_return': 3800, }

    env_name = env_args['env_name']
    eval_times = env_args['eval_times']

    print('agent', agent.__name__)
    print('gpu_id', gpu_id)
    print('net_dim', net_dim)
    print('env_name', env_name)

    '''txt step_epi_r_s_ary'''
    save_txt_data_in_python(cwd_path)

    '''save step_epi_r_s_ary'''
    dir_names = [name for name in os.listdir(cwd_path)
                 if name.find(env_name) >= 0 and os.path.isdir(f"{cwd_path}/{name}")]
    for dir_name in dir_names:
        dir_path = f"{cwd_path}/{dir_name}"
        txt_path = f"{dir_path}-step_epi_r_s_ary.txt"
        if os.path.exists(txt_path):
            continue
        step_epi_r_s_ary = demo_evaluate_actors(dir_path, gpu_id, agent, env_args, eval_times, net_dim)
        np.savetxt(txt_path, step_epi_r_s_ary)

    '''load step_epi_r_s_ary'''
    import matplotlib.pyplot as plt
    if_plot_single = False  # todo
    if_show_episode_step = True  # todo
    plot_smoothing_number = 16  # todo

    step_epi_r_s_ary = list()

    ary_names = [name for name in os.listdir(cwd_path)
                 if name.find(env_name) >= 0 and name[-4:] == '.txt']
    for i, ary_name in enumerate(ary_names):
        ary_path = f"{cwd_path}/{ary_name}"
        ary = np.loadtxt(ary_path)
        step_epi_r_s_ary.append(ary)
        print('label', i, ary_path)

    if if_plot_single:
        for i, ary in enumerate(step_epi_r_s_ary):
            plt.plot(ary[:, 0], ary[:, 1], label=str(i)) if if_plot_single else None
        plt.legend()
        plt.grid()
        plt.show()
    step_epi_r_s_ary = np.vstack(step_epi_r_s_ary)
    step_epi_r_s_ary = step_epi_r_s_ary[step_epi_r_s_ary[:, 0].argsort()]
    print('step_epi_r_s_ary.shape', step_epi_r_s_ary.shape)

    '''plot'''
    plot_x_y_up_dw_step = list()
    for i in range(0, len(step_epi_r_s_ary), plot_smoothing_number):
        y_ary = step_epi_r_s_ary[i:i + plot_smoothing_number, 1]
        if y_ary.shape[0] <= 1:
            continue

        y_avg = y_ary.mean()
        y_up = y_ary[y_ary > y_avg].mean()
        y_dw = y_ary[y_ary <= y_avg].mean()

        y_step = step_epi_r_s_ary[i:i + plot_smoothing_number, 2].mean()
        x_avg = step_epi_r_s_ary[i:i + plot_smoothing_number, 0].mean()
        plot_x_y_up_dw_step.append((x_avg, y_avg, y_up, y_dw, y_step))

    color0 = 'royalblue'
    color1 = 'lightcoral'
    # color2 = 'darkcyan'
    # colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',
    #           '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']

    title = f"{env_name}_{agent.__name__}_ElegantRL"

    fig, ax = plt.subplots(1)

    plot_x = [item[0] for item in plot_x_y_up_dw_step]
    plot_y = [item[1] for item in plot_x_y_up_dw_step]
    plot_y_up = [item[2] for item in plot_x_y_up_dw_step]
    plot_y_dw = [item[3] for item in plot_x_y_up_dw_step]
    ax.plot(plot_x, plot_y, label='Episode Return', color=color0)
    ax.fill_between(plot_x, plot_y_up, plot_y_dw, facecolor=color0, alpha=0.3)
    ax.set_ylabel('Episode Return', color=color0)
    ax.tick_params(axis='y', labelcolor=color0)
    ax.grid(True)

    if if_show_episode_step:
        ax_twin = ax.twinx()
        plot_y_step = [item[4] for item in plot_x_y_up_dw_step]
        ax_twin.fill_between(plot_x, 0, plot_y_step, facecolor=color1, alpha=0.3)
        ax_twin.set_ylabel('Episode Step', color=color1)
        ax_twin.tick_params(axis='y', labelcolor=color1)
        ax_twin.set_ylim(0, np.max(plot_y_step) * 2)

    print('title', title)
    plt.title(title)
    plt.show()


if __name__ == '__main__':
    draw_learning_curve()
    # demo_continuous_action_on_policy()
