import os
import shutil
import subprocess
import yaml
from yacs.config import CfgNode
import json
from pathlib import Path
import soundfile as sf
from typing import List
import numpy as np
import paddle
import re

from paddlespeech.t2s.exps.syn_utils import am_to_static
from paddlespeech.t2s.exps.syn_utils import get_am_inference
from paddlespeech.t2s.exps.syn_utils import get_am_output
from paddlespeech.t2s.exps.syn_utils import get_frontend
from paddlespeech.t2s.exps.syn_utils import run_frontend
from paddlespeech.t2s.exps.syn_utils import get_predictor
from paddlespeech.t2s.exps.syn_utils import get_voc_output
from paddlespeech.t2s.models.fastspeech2 import FastSpeech2
from paddlespeech.t2s.modules.nets_utils import make_pad_mask
from paddlespeech.t2s.modules.normalizer import ZScore

# 配置路径信息
# 这里的路径是提前写好在 Aistudio 上的路径信息，如果需要迁移到其它的环境，请自行配置
cwd_path = "./"
pretrained_model_dir = "/home/aistudio/PaddleSpeech/examples/other/tts_finetune/tts3/models/fastspeech2_mix_ckpt_1.2.0"
config_path = "/home/aistudio/PaddleSpeech/examples/other/tts_finetune/tts3/conf/finetune.yaml"
inference_dir = "./inference"
exp_base = "./work/exp_"

os.makedirs(inference_dir, exist_ok=True)

pwg_inference_dir = os.path.join(cwd_path, "inference/pwgan_aishell3_static_1.1.0")
hifigan_inference_dir = os.path.join(cwd_path, "inference/hifigan_aishell3_static_1.1.0")
wavernn_inference_dir = os.path.join(cwd_path, "inference/wavernn_csmsc_static_0.2.0")

# 命令行执行函数，可以进入指定路径下执行
def run_cmd(cmd, cwd_path, message, check_result_path=None):
    p = subprocess.Popen(cmd, shell=True, cwd=cwd_path)
    res = p.wait()
    print(cmd)
    print("运行结果：", res)

    # 检查是否有对应文件生成
    if check_result_path:
        file_num = [filename for filename in os.listdir(check_result_path)]
        if file_num == 0:
            res = 1

    if res == 0:
        # 运行成功
        return True
    else:
        # 运行失败
        print(message)
        print(f"你可以新建终端，打开命令行，进入：{cwd_path}")
        print(f"执行：{cmd}")
        print("查看命令行中详细的报错信息")
        return False

def status_change(status_path, status):
    with open(status_path, "w", encoding="utf8") as f:
        status = {
            'on_status': status
        }
        json.dump(status, f, indent=4)

# 找到最新生成的模型
def find_max_ckpt(model_path):
    max_ckpt = 0
    for filename in os.listdir(model_path):
        if filename.endswith('.pdz'):
            files = filename[:-4]
            a1, a2, it = files.split("_")
            if int(it) > max_ckpt:
                max_ckpt = int(it)
    return max_ckpt

# Step1 : 生成标准数据集
def step1_generate_standard_dataset(label_path, exp_path, data_dir):
    error_code = 0

    print("Step1 开始执行: 生成标注数据集")
    # 生成 data 数据路径
    exp_data_path = data_dir
    os.makedirs(exp_data_path, exist_ok=True)
    exp_data_label_file = os.path.join(exp_data_path, "labels.txt")

    if not os.path.exists(label_path):
        print(f"标注文件: {label_path} 不存在，请检查数据预标注是否成功！")
        error_code = 1
        return None, error_code

    with open(label_path, "r", encoding='utf8') as f:
        ann_result = json.load(f)

    if len(ann_result) == 0:
        print(f"标注文件: {label_path} 内容为空，请检查数据预标注是否成功！")
        error_code = 2
        return None, error_code

    if len(ann_result) < 5:
        print(f"标注文件: {label_path} 中，音频数据小于5条，请增加上传的音频数量！")
        error_code = 3
        return None, error_code

    # 复制音频文件
    for label in ann_result:
        if os.path.exists(label['filepath']):
            shutil.copy(label['filepath'], exp_data_path)
        else:
            print(f"音频文件：{label['filepath']} 不存在，请重新检查数据是否正确")
            error_code = 4
            return None, error_code

    # 生成标准标注文件
    with open(exp_data_label_file, "w", encoding="utf8") as f:
        for ann in ann_result:
            f.write(f"{ann['filename'].split('.')[0]}|{ann['pinyin']}\n")

    # 检查 data 里面是否为空，为空则显示异常
    wav_file = [filename for filename in os.listdir(exp_data_path) if filename.endswith(".wav")]
    if len(wav_file) == 0:
        # 为空则是异常
        shutil.rmtree(exp_data_path)
        error_code = 5
        return None, error_code
    return exp_data_path, error_code

# Step2: 检查非法数据
def step2_check_oov(data_dir, new_dir, lang="zh"):
    print("Step2 开始执行: 检查数据集是否合法")
    cmd = f"""
            python3 /home/aistudio/util/check_oov.py \
                --input_dir={data_dir} \
                --pretrained_model_dir={pretrained_model_dir} \
                --newdir_name={new_dir} \
                --lang={lang}
            """
    if not run_cmd(cmd, cwd_path, message="Step2 检查非法数据 执行失败，请检查数据集是否配置正确"):
        if os.path.exists(new_dir):
            shutil.rmtree(new_dir)
        return False
    else:
        return True

# Step3: 生成 MFA 对齐结果
def step3_get_mfa(new_dir, mfa_dir, lang="zh"):
    print("Step3 开始执行: 使用 MFA 对数据进行对齐")
    cmd = f"""
        python3 /home/aistudio/util/get_mfa_result.py \
            --input_dir={new_dir} \
            --mfa_dir={mfa_dir} \
            --lang={lang}
        """
    if not run_cmd(cmd, cwd_path, message="Step3 MFA对齐 执行失败，请执行命令行，查看 MFA 详细报错", check_result_path=mfa_dir):
        if os.path.exists(mfa_dir):
            shutil.rmtree(mfa_dir)
        return False
    else:
        return True

# Step4: 生成 Duration
def step4_duration(mfa_dir):
    if not os.path.exists(mfa_dir):
        print("Step3 执行失败，未生成 MFA 对齐文件夹")
        return False
    print("Step4 开始执行: 生成 Duration 文件")
    cmd = f"""
        python3 /home/aistudio/util/generate_duration.py \
            --mfa_dir={mfa_dir}
        """
    if not run_cmd(cmd, cwd_path, message="Step4 生成 Duration 执行失败，请执行命令行"):
        if os.path.exists("./durations.txt"):
            os.remove("./durations.txt")
        return False
    else:
        return True

# Step5: 数据预处理
def step5_extract_feature(data_dir, dump_dir):
    print("Step5 开始执行: 开始数据预处理")
    cmd = f"""
        python3 /home/aistudio/util/extract_feature.py \
            --duration_file="./durations.txt" \
            --input_dir={data_dir} \
            --dump_dir={dump_dir}\
            --pretrained_model_dir={pretrained_model_dir}\
        """
    if not run_cmd(cmd, cwd_path, message="Step5 执行失败，请执行命令行"):
        if os.path.exists(dump_dir):
            shutil.rmtree(dump_dir)
        return False
    else:
        return True

# Step6: 准备微调环境
def step6_prepare_env(output_dir):
    print("Step6 开始执行: 准备微调环境")
    cmd = f"""
        python3 /home/aistudio/util/prepare_env.py \
            --pretrained_model_dir={pretrained_model_dir} \
            --output_dir={output_dir}
    """
    if not run_cmd(cmd, cwd_path, message="Step6 准备训练环境 执行失败，请执行命令行"):
        return False
    else:
        return True

# Step7: 微调训练
def step7_finetune(dump_dir, output_dir, max_finetune_step, batch_size=None, learning_rate=None):
    # # 读取默认的yaml文件
    # with open(config_path) as f:
    #     finetune_config = yaml.safe_load(f)
    # # 多少个 step 保存一次
    # finetune_config['num_snapshots'] = num_snapshots
    # # 1. 自动调整 batch ，通过 dump/train里面文件数量判断
    train_data_dir = os.path.join(dump_dir, "train/norm/data_speech")
    file_num = len([filename for filename in os.listdir(train_data_dir) if filename.endswith(".npy")])
    if not batch_size:
        if file_num <= 32:
            batch_size = file_num
        else:
            batch_size = 32
    else:
        # 指定 batch size 的情况下还是需要看音频文件的数量
        if file_num <= batch_size:
            batch_size = file_num

    # # 2. 支持调整 learning_rate
    if not learning_rate:
        learning_rate = 0.001

    print("Step7 开始执行: 微调试验开始")

    cmd = f"""
        python3 /home/aistudio/util/finetuneTrain.py \
            --dump_dir={dump_dir} \
            --output_dir={output_dir} \
            --max_step={max_finetune_step} \
            --batch_size={batch_size} \
            --learning_rate={learning_rate}
        """
    if not run_cmd(cmd, cwd_path, message="Step7 微调试验失败 执行失败，请执行命令行"):
        return False
    else:
        return True

# 导出成静态图
def step8_export_static_model(exp_name):
    exp_path = os.path.join(exp_base + exp_name)
    model_path = os.path.join(exp_path, "output/checkpoints")
    dump_dir = os.path.join(exp_path, "dump")
    output_dir = os.path.join(exp_path, "output")
    ckpt = find_max_ckpt(model_path)

    am_config = f"{cwd_path}/models/fastspeech2_mix_ckpt_1.2.0/default.yaml"

    with open(am_config) as f:
        am_config = CfgNode(yaml.safe_load(f))
    finetune_model_path = f"{output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz"
    am_inference = get_am_inference(
        am="fastspeech2_mix",
        am_config=am_config,
        am_ckpt=finetune_model_path,
        am_stat=f"{cwd_path}/models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy",
        phones_dict=f"{dump_dir}/phone_id_map.txt",
        tones_dict=None,
        speaker_dict=f"{dump_dir}/speaker_id_map.txt")

    out_inference_dir = os.path.join(inference_dir, exp_name)
    # 静态图导出
    am_inference = am_to_static(
            am_inference=am_inference,
            am="fastspeech2_mix",
            inference_dir=out_inference_dir,
            speaker_dict=f"{dump_dir}/speaker_id_map.txt")

    # 把 phone_dict 也复制过去
    shutil.copy(f"{dump_dir}/phone_id_map.txt", out_inference_dir)

    # 把预训练模型中 npy 文件，yaml 文件，txt文件都放进去
    for filename in os.listdir(pretrained_model_dir):
        fileend = os.path.splitext(filename)[1]
        if fileend in [".npy", ".yaml", ".txt"]:
            shutil.copy(f"{pretrained_model_dir}/{filename}", out_inference_dir)

    # 动态图模型也放进去
    if os.path.exists(finetune_model_path):
        shutil.copy(finetune_model_path, out_inference_dir)

# 使用微调后的模型生成音频
def step9_generateTTS(text_dict, wav_output_dir, exp_name, voc="PWGan"):
    # 配置一下参数信息
    exp_path = os.path.join(exp_base + exp_name)
    model_path = os.path.join(exp_path, "output/checkpoints")
    output_dir = os.path.join(exp_path, "output")
    dump_dir = os.path.join(exp_path, "dump")

    text_file = os.path.join(exp_path, "sentence.txt")
    status_json = os.path.join(exp_path, "generate_status.json")
    # 写一个物理锁
    status_change(status_json, True)

    with open(text_file, "w", encoding="utf8") as f:
        for k,v in sorted(text_dict.items(), key=lambda x:x[0]):
            f.write(f"{k} {v}\n")

    ckpt = find_max_ckpt(model_path)

    if voc == "PWGan":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="pwgan_aishell3" \
                        --voc_config=models/pwg_aishell3_ckpt_0.5/default.yaml \
                        --voc_ckpt=models/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
                        --voc_stat=models/pwg_aishell3_ckpt_0.5/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --inference_dir
                        --ngpu=1
        """
    elif voc == "WaveRnn":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="wavernn_csmsc" \
                        --voc_config=models/wavernn_csmsc_ckpt_0.2.0/default.yaml \
                        --voc_ckpt=models/wavernn_csmsc_ckpt_0.2.0/snapshot_iter_400000.pdz \
                        --voc_stat=models/wavernn_csmsc_ckpt_0.2.0/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --ngpu=1
        """
    elif voc == "HifiGan":
        cmd = f"""
        python3 /home/aistudio/PaddleSpeech/paddlespeech/t2s/exps/fastspeech2/../synthesize_e2e.py \
                        --am=fastspeech2_mix \
                        --am_config=models/fastspeech2_mix_ckpt_1.2.0/default.yaml \
                        --am_ckpt={output_dir}/checkpoints/snapshot_iter_{ckpt}.pdz \
                        --am_stat=models/fastspeech2_mix_ckpt_1.2.0/speech_stats.npy \
                        --voc="hifigan_aishell3" \
                        --voc_config=models/hifigan_aishell3_ckpt_0.2.0/default.yaml \
                        --voc_ckpt=models/hifigan_aishell3_ckpt_0.2.0/snapshot_iter_2500000.pdz \
                        --voc_stat=models/hifigan_aishell3_ckpt_0.2.0/feats_stats.npy \
                        --lang=mix \
                        --text={text_file} \
                        --output_dir={wav_output_dir} \
                        --phones_dict={dump_dir}/phone_id_map.txt \
                        --speaker_dict={dump_dir}/speaker_id_map.txt \
                        --spk_id=0 \
                        --ngpu=1
        """
    else:
        print("声码器不符合要求，请重新选择")
        status_change(status_json, False)
        return False

    if not run_cmd(cmd, cwd_path, message="Step9 生成音频 执行失败，请执行命令行"):
        status_change(status_json, False)
        return False
    else:
        status_change(status_json, False)
        return True

# 使用静态图生成
def step9_generateTTS_inference(text_dict, exp_name, voc="PWGan"):
    exp_path = exp_base + exp_name
    wav_output_dir = os.path.join(exp_path, "wav_out")
    am_inference_dir = os.path.join(inference_dir, exp_name)

    device = "gpu"

    status_json = os.path.join(exp_path, "generate_status.json")
    # 写一个物理锁
    status_change(status_json, True)

    # frontend
    frontend = get_frontend(
        lang="mix",
        phones_dict=os.path.join(am_inference_dir, "phone_id_map.txt"),
        tones_dict=None
    )

    # am_predictor
    am_predictor = get_predictor(
        model_dir=am_inference_dir,
        model_file="fastspeech2_mix" + ".pdmodel",
        params_file="fastspeech2_mix" + ".pdiparams",
        device=device)

    # voc_predictor
    if voc == "PWGan":
        voc_predictor = get_predictor(
            model_dir=pwg_inference_dir,
            model_file="pwgan_aishell3" + ".pdmodel",
            params_file="pwgan_aishell3" + ".pdiparams",
            device=device)
    elif voc == "WaveRnn":
        voc_predictor = get_predictor(
            model_dir=wavernn_inference_dir,
            model_file="wavernn_csmsc" + ".pdmodel",
            params_file="wavernn_csmsc" + ".pdiparams",
            device=device)
    elif voc == "HifiGan":
        voc_predictor = get_predictor(
            model_dir=hifigan_inference_dir,
            model_file="hifigan_aishell3" + ".pdmodel",
            params_file="hifigan_aishell3" + ".pdiparams",
            device=device)

    output_dir = Path(wav_output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    sentences = list(text_dict.items())

    merge_sentences = True
    fs = 24000
    for utt_id, sentence in sentences:
        am_output_data = get_am_output(
            input=sentence,
            am_predictor=am_predictor,
            am="fastspeech2_mix",
            frontend=frontend,
            lang="mix",
            merge_sentences=merge_sentences,
            speaker_dict=os.path.join(am_inference_dir, "phone_id_map.txt"),
            spk_id=0, )
        wav = get_voc_output(
                voc_predictor=voc_predictor, input=am_output_data)
        # 保存文件
        sf.write(output_dir / (utt_id + ".wav"), wav, samplerate=fs)

    status_change(status_json, False)
    return True

def freeze_layer(model, layers: List[str]):
    """freeze layers

    Args:
        layers (List[str]): frozen layers
    """
    for layer in layers:
        for param in eval("model." + layer + ".parameters()"):
            param.trainable = False
# 前端
def get_tts_phone_ids(sentence, exp_name, exp_fun=False):
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    # 检测有没有SSML标记
    if sentence.strip() != "" and re.match(r".*?<speak>.*?</speak>.*", sentence,
                                           re.DOTALL):
        # 有SSML标记，只支持中文修改多音字
        lang = "zh"
        merge_sentences = False
        frontend = get_frontend(lang=lang, phones_dict=phones_dict, tones_dict=None)
    else:
        # 没有 SSML 标记，按照中英双语处理
        lang = "mix"
        frontend = get_frontend(lang=lang, phones_dict=phones_dict, tones_dict=None)
        if exp_fun:
            merge_sentences = True
        else:
            merge_sentences = False
    print(f"merge_sentences: {merge_sentences}")
    frontend_dict = run_frontend(
                    frontend=frontend,
                    text=sentence,
                    merge_sentences=merge_sentences,
                    get_tone_ids=False,
                    lang=lang)
    phone_ids = frontend_dict['phone_ids']
    return phone_ids

def get_idx2ph_dict(exp_name):
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    id2phs_dict = {}

    with open(phones_dict, "r", encoding="utf8") as f:
        for line in f.readlines():
            if line.strip():
                ph, idx = line.strip().split()
                id2phs_dict[idx] = ph
    return id2phs_dict

def fs2_inference(model, text, spk_id, alpha=1.0, duration=None, return_duration=False):
    x = paddle.cast(text, 'int64')
    xs = x.unsqueeze(0)
    ilens = paddle.shape(x)[0]
    # forward encoder
    x_masks = model._source_mask(ilens)
    # (B, Tmax, adim)
    hs, _ = model.encoder(xs, x_masks)
    # spk_id 添加进去了
    spk_emb = model.spk_embedding_table(spk_id)
    hs = model._integrate_with_spk_embed(hs, spk_emb)
    # forward duration predictor and variance predictors
    d_masks = make_pad_mask(ilens)

    # True
    if model.stop_gradient_from_pitch_predictor:
        p_outs = model.pitch_predictor(hs.detach(), d_masks.unsqueeze(-1))
    else:
        p_outs = model.pitch_predictor(hs, d_masks.unsqueeze(-1))

    # False
    if model.stop_gradient_from_energy_predictor:
        e_outs = model.energy_predictor(hs.detach(), d_masks.unsqueeze(-1))
    else:
        e_outs = model.energy_predictor(hs, d_masks.unsqueeze(-1))

    # duration
    # 输出已经是整数(float32)了
    if duration is None:
        d_outs = model.duration_predictor.inference(hs, d_masks)
    else:
        d_outs = duration

    if return_duration:
        return d_outs

    # print(d_outs)

    # 往下添加 pitch, energy 信息
    p_embs = model.pitch_embed(p_outs.transpose((0, 2, 1))).transpose(
                (0, 2, 1))
    e_embs = model.energy_embed(e_outs.transpose((0, 2, 1))).transpose(
        (0, 2, 1))
    hs = hs + e_embs + p_embs

    # (B, Lmax, adim)
    hs = model.length_regulator(hs, d_outs, alpha, is_inference=True)

    # decoder
    h_masks = None
    zs, _ = model.decoder(hs, h_masks)
    # (B, Lmax, odim)
    before_outs = model.feat_out(zs).reshape(
        (paddle.shape(zs)[0], -1, model.odim))
    after_outs = before_outs + model.postnet(
                before_outs.transpose((0, 2, 1))).transpose((0, 2, 1))
    return before_outs, after_outs, d_outs, p_outs, e_outs

def duration_phones_to_list(duration, phone_ids, id2phs_dict):
    phs_lists = []
    phs_lens = phone_ids[0].shape[0]
    for i in range(phs_lens):
        ph = id2phs_dict[str(phone_ids[0].numpy()[i])]
        dur = int(duration[0].numpy()[i])
        phs_lists.append({
            "ph": ph,
            "dur": dur
        })
        # print(f"{i}\t{ph}\t{dur}")
    return phs_lists

def list_to_durations(phs_lists):
    duration = []
    for k in phs_lists:
        dur = k['dur']
        duration.append(int(dur))
    duration = paddle.to_tensor([duration], dtype=paddle.float32)
    return duration

def load_fs2_model(exp_name):
    am_inference_dir = os.path.join(inference_dir, exp_name)

    ckpt = find_max_ckpt(am_inference_dir)
    if ckpt == 0 or ckpt is None:
        print("ckpt 模型不存在，请将模型放入inference路径")
        return None
    model_path = f"{am_inference_dir}/snapshot_iter_{ckpt}.pdz"
    speaker_dict = f"{am_inference_dir}/speaker_id_map.txt"
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"

    # 加载fs2模型
    default_config_file = f"{am_inference_dir}/default.yaml"
    with open(default_config_file) as f:
        config = CfgNode(yaml.safe_load(f))

    with open(phones_dict, "r") as f:
        phn_id = [line.strip().split() for line in f.readlines()]
    vocab_size = len(phn_id)
    print("vocab_size:", vocab_size)
    odim = config.n_mels

    with open(speaker_dict, 'rt') as f:
        spk_id = [line.strip().split() for line in f.readlines()]
    spk_num = len(spk_id)
    model = FastSpeech2(
        idim=vocab_size,
        odim=odim,
        spk_num=spk_num,
        **config["model"])

    archive = paddle.load(model_path)
    model.set_state_dict(archive['main_params'])
    model.eval()
    return model

def load_voc_model(voc):
    device = "gpu"
    # voc_predictor
    if voc == "PWGan":
        voc_predictor = get_predictor(
            model_dir=pwg_inference_dir,
            model_file="pwgan_aishell3" + ".pdmodel",
            params_file="pwgan_aishell3" + ".pdiparams",
            device=device)
    elif voc == "WaveRnn":
        voc_predictor = get_predictor(
            model_dir=wavernn_inference_dir,
            model_file="wavernn_csmsc" + ".pdmodel",
            params_file="wavernn_csmsc" + ".pdiparams",
            device=device)
    elif voc == "HifiGan":
        voc_predictor = get_predictor(
            model_dir=hifigan_inference_dir,
            model_file="hifigan_aishell3" + ".pdmodel",
            params_file="hifigan_aishell3" + ".pdiparams",
            device=device)
    else:
        return None

    return voc_predictor

def generateTTS_inference_adjust_duration(text_dict, exp_name, voc="PWGan", alpha=1.0, duration=None, merge_sentences=False, exp_fun=False):
    exp_path = exp_base + exp_name
    if not os.path.exists(exp_path):
        os.makedirs(exp_path, exist_ok=True)

    wav_output_dir = os.path.join(exp_path, "wav_out")
    if not os.path.exists(wav_output_dir):
        os.makedirs(wav_output_dir, exist_ok=True)
    am_inference_dir = os.path.join(inference_dir, exp_name)
    phones_dict = f"{am_inference_dir}/phone_id_map.txt"
    status_json = os.path.join(exp_path, "generate_status.json")

    # 写一个物理锁
    status_change(status_json, True)

    # 前端
    # frontend = get_frontend(
    #             lang="mix", phones_dict=phones_dict, tones_dict=None)

    # 加载fs2模型
    model = load_fs2_model(exp_name)
    if not model:
        status_change(status_json, False)
        print("微调模型加载失败！")
        return False

    fs = 24000

    # 加载声码器
    # voc_predictor
    voc_predictor = load_voc_model(voc)
    if not voc:
        status_change(status_json, False)
        print("声码器加载失败！")
        return False

    am_stat = os.path.join(am_inference_dir, "speech_stats.npy")
    am_mu, am_std = np.load(am_stat)
    am_mu = paddle.to_tensor(am_mu)
    am_std = paddle.to_tensor(am_std)
    am_normalizer = ZScore(am_mu, am_std)

    for utt_id, sentence in text_dict.items():
        phone_ids = get_tts_phone_ids(sentence, exp_name, exp_fun=exp_fun)
        wavs = []
        for phone_id in phone_ids:
            spk_id = paddle.to_tensor(0)
            before_outs, after_outs, d_outs, p_outs, e_outs = fs2_inference(model, text=phone_id, alpha=alpha, spk_id=spk_id,duration=duration)
            normalized_mel = after_outs[0]
            logmel = am_normalizer.inverse(normalized_mel)
            am_output_data = logmel.numpy()
            # 给到 vocoder 进行推理
            wav = get_voc_output(
                voc_predictor=voc_predictor, input=am_output_data)
            wavs.append(wav)
        wavs = np.concatenate(wavs)
        # 保存文件
        sf.write(f"{wav_output_dir}/{utt_id}.wav", wavs, samplerate=fs)

    status_change(status_json, False)
    return True

# 全流程微调训练
def finetuneTTS(label_path, exp_name, start_step=1, end_step=100, max_finetune_step=100, batch_size=None, learning_rate=None):
    exp_path = os.path.join(exp_base + exp_name)
    os.makedirs(exp_path, exist_ok=True)
    data_dir = os.path.join(exp_path, "data")
    finetune_status_json = os.path.join(exp_path, "finetune_status.json")
    status_change(finetune_status_json, True)

    if start_step <= 1 and end_step >= 1:
        # Step1 : 生成标准数据集
        if not os.path.exists(data_dir):
            exp_data_path, error_code = step1_generate_standard_dataset(label_path, exp_path, data_dir)
            if exp_data_path is None:
                status_change(finetune_status_json, False)
                # 步骤一出错
                return
        else:
            print(f"{data_dir} 已存在，跳过此步骤！")

    # Step2: 检查非法数据
    new_dir = os.path.join(exp_path, "new_dir")
    if start_step <= 2 and end_step >= 2:
        if not os.path.exists(new_dir):
            if not step2_check_oov(data_dir, new_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{new_dir} 已存在，跳过此步骤")

    mfa_dir = os.path.join(exp_path, "mfa")
    if start_step <= 3 and end_step >= 3:
        # Step3: MFA 对齐
        if not os.path.exists(mfa_dir):
            if not step3_get_mfa(new_dir, mfa_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{mfa_dir} 已存在，跳过此步骤")

    if start_step <= 4 and end_step >= 4:
        # Step4: 生成时长信息文件
        if not step4_duration(mfa_dir):
            status_change(finetune_status_json, False)
            return

    dump_dir = os.path.join(exp_path, "dump")
    if start_step <= 5 and end_step >= 5:
        # Step5: 数据预处理
        if not os.path.exists(dump_dir):
            if not step5_extract_feature(data_dir, dump_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{dump_dir} 已存在，跳过此步骤")

    output_dir = os.path.join(exp_path, "output")
    if start_step <= 6 and end_step >= 6:
        # Step6: 生成训练环境
        if not os.path.exists(output_dir):
            if not step6_prepare_env(output_dir):
                status_change(finetune_status_json, False)
                return
        else:
            print(f"{output_dir} 已存在，跳过此步骤")

    if start_step <= 7 and end_step >= 7:
        # Step7: 微调训练
        if not step7_finetune(dump_dir, output_dir, max_finetune_step, batch_size=None, learning_rate=None):
            status_change(finetune_status_json, False)
            return

    if start_step <= 8 and end_step >= 8:
        # 这一步不需要运行中
        status_change(finetune_status_json, False)
        # Step8: 导出静态图模型
        if not step8_export_static_model(exp_name):
            return

    status_change(finetune_status_json, False)
    return True

# 生成音频
def generateTTS(text_dict, exp_name, voc="PWGan"):
    if not step9_generateTTS_inference(text_dict, exp_name, voc):
        print("音频生成失败，请微调模型是否成功!")
        return None
    else:
        return True


#!/usr/bin/env python3
import argparse

class CreateCli():

    def __init__(self):
        self.args = ''
        self.set_args()

    def set_args(self):
        parser = argparse.ArgumentParser("Create Cli")
        parser.add_argument("--fix",
                            dest="fix",
                            default=False,
                            help="Fix or Create Venv",
                            type=bool)
        parser.add_argument("--list",
                            dest="list",
                            default='',
                            help="all / alive / at ",
                            type=str)
        parser.add_argument("--test",
                            dest="test",
                            default='',
                            help="xxx.xxx.xxx.xxx:xxxx",
                            type=str)
        self.args = parser.parse_args()

    def get_args(self):
        return self.args


from sys import argv

USAGE = f'''Usage: python operations.py <number1> <number2>
Example:
    python operations.py 10 3'''
MANY_ARG = f'''InputError: too many arguments
'''
ONLY_NUM = f'''InputError: only numbers
'''

if len(argv) < 3:
	print(USAGE)
	exit()
if len(argv) > 3:
	print(MANY_ARG)
	print(USAGE)
	exit()
if not argv[1].isdigit() or not argv[2].isdigit():
	print(ONLY_NUM)
	print(USAGE)
	exit()

sum = int(argv[1]) + int(argv[2])
diff = int(argv[1]) - int(argv[2])
prod = int(argv[1]) * int(argv[2])
if (argv[2] != "0"):
	quot = int(argv[1]) / int(argv[2])
	rema = int(argv[1]) % int(argv[2])
else:
	quot = "ERROR (div by zero)"
	rema = "ERROR (modulo by zero)"

print(f'''Sum:         {sum}
Difference:  {diff}
Product:     {prod}
Quotient:    {quot}
Remainder:   {rema}''')


import sys
from copy import deepcopy

user_input = map(lambda line: line.strip(), sys.stdin.readlines())

lava_cubes = set()

for line in user_input:
    lava_cubes.add(tuple(map(int,line.split(','))))

def get_cube_faces_set(cube):
    result = set()
    for i in range(len(cube)):
        result.add(cube[:i] + ((cube[i],cube[i] + 1),)+ cube[i+1:])
        result.add(cube[:i] + ((cube[i] - 1,cube[i]),)+ cube[i+1:])
    return result
cube_faces_set_list = list(map(get_cube_faces_set,lava_cubes))

faces_union = set.union(*cube_faces_set_list)
result = deepcopy(faces_union)

for i in range(len(lava_cubes)):
    for j in range(len(lava_cubes)):
        if i != j:
            result.difference_update(cube_faces_set_list[i].intersection(cube_faces_set_list[j]))

print(len(result))

def check_max_min(index_1,index_2,index_to_max):
    result_min = {}
    result_max = {}
    for cube in lava_cubes:
        i, j, current = cube[index_1], cube[index_2], cube[index_to_max]
        if (i,j) in result_min:
            result_min[(i,j)] = min(result_min[(i,j)],current)
            result_max[(i,j)] = max(result_max[(i,j)],current)
        else:
            result_min[(i,j)] = current
            result_max[(i,j)] = current
    return result_min, result_max

min_z, max_z = check_max_min(0,1,2)
min_x, max_x = check_max_min(1,2,0)
min_y, max_y = check_max_min(0,2,1)

abs_max_z = max(max_z.values())
abs_max_y = max(max_y.values())
abs_max_x = max(max_x.values())
abs_min_z = min(min_z.values())
abs_min_y = min(min_y.values())
abs_min_x = min(min_x.values())

air_cubes = set()

for face in faces_union:
    tuple_index = None
    for i in range(len(face)):
        if type(face[i]) == tuple:
            tuple_index = i
    possible_cubes = [face[:tuple_index] + (face[tuple_index][0],) + face[tuple_index + 1:],face[:tuple_index] + (face[tuple_index][1],) + face[tuple_index + 1:]]
    for pos_cube in possible_cubes:
        if not pos_cube in lava_cubes:
            x, y , z = pos_cube
            if abs_min_x < x < abs_max_x and abs_min_y < y < abs_max_y and abs_min_z < z < abs_max_z:
                if (x,y) in min_z and min_z[(x,y)] < z < max_z[(x,y)]:
                    if (x,z) in min_y and min_y[(x,z)] < y < max_y[(x,z)]:
                        if (y,z) in min_x and min_x[(y,z)] < x < max_x[(y,z)]:
                            air_cubes.add(pos_cube)

to_remove_faces = set.union(*list(map(get_cube_faces_set,air_cubes)))

print(len(result.difference(to_remove_faces)))


# 问题描述
# 给定一个数组，在数组中寻找2个数，使它们的和最接近但不超过目标值，并返回它们的和

# 示例
# 输入：target = 15, array = [1,3,5,11,7]
# 输出：14
# 解释：11 + 3 = 14 < 15

# 输入：target = 16, array = [1,3,5,11,7]
# 输出：16
# 解释：11 + 5 = 16 <= 16

# 源码实现
class Solution:
    def closestTargetNumber(self, target, array):
        size = len(array)
        if size < 2:
            return -1

        array.sort()
        diff = 0x7fffffff
        left = 0
        right = size - 1

        while left < right:
            if array[left] + array[right] > target:
                right -= 1
            else:
                diff = min(diff, target - array[left] - array[right])
                left += 1

        if diff == 0x7fffffff:
            return -1
        else:
            return target - diff

# 主函数
if __name__ == '__main__':
    array = [1,3,5,11,7]
    target = 15
    solution = Solution()
    print("Input: array = ", array, ", target = ", target)
    print("Output: ", solution.closestTargetNumber(target,array))

# 运行结果
# Input: array =  [1, 3, 5, 11, 7] , target =  15
# Output:  14


import re
import os
import json
import docx
import matplotlib

matplotlib.use("Agg")

from collections import defaultdict
from docx import Document
from docx.shared import Cm
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_BREAK
from docx.oxml.shared import OxmlElement, qn

from api.map_client import *
from api.charts import get_pie_chart, get_line_chart

DATA_DIR = os.getenv("DATA_DIR", "./public/data")

TEMPLATE = "api/template.docx"

# TODO: Improve custom table style in template.docx
DOC_STYLE = "SALCC_style"

# Example placeholder: {{label:title}}
PLACEHOLDER_REGEX = re.compile(r"{{(?P<scope>\w+):(?P<key>\w+)}}")

TOTAL_ROW_COLOR = "cacdd1"

def create_report(unit_id, path, config):
    """
        Creates report for summary unit in docx format.

    Parameters
    ----------
    unit_id: string id of a summary unit
    path: string location where docx to be saved
    config: dict of json data from src/config

    Returns
    -------
    report: docx file

    """
    doc = Document(TEMPLATE)

    # per convention, all marine units begin with M, e.g., M100
    is_marine = unit_id[0].lower() == 'm'
    context = generate_report_context(unit_id, config, is_marine)

    table_counter = 0

    for p in doc.paragraphs:
        # Assumption: placeholders are always completely contained within a run
        for r in p.runs:
            # a run may have multiple placeholders
            for match in re.finditer(PLACEHOLDER_REGEX, r.text):
                scope, key = match.groups()

                item = _resolve(scope, key, context)

                if isinstance(item, str):
                    # match.group() has the original text to replace
                    if scope == "table":
                        r.add_break(WD_BREAK.LINE)
                    r.text = r.text.replace(match.group(), item)

                    if scope in {"table", "chart"}:
                        r.font.italic = True

                elif scope in {"map", "chart"}:
                    # Pictures are added at the run level
                    r.text = ""
                    size = 11.0
                    if scope == "map":
                        size = 16.5
                    if scope == "chart" and key == "priorities":  # If chart is present, put on its own page
                        r.add_break(WD_BREAK.PAGE)
                    if not isinstance(context[scope][key], str):
                        run = p.add_run()
                        run.add_picture(context[scope][key], width=Cm(size))

                        # add a caption if we have one in context
                        caption_name = "{0}_{1}".format(scope, key)
                        # if key in context["caption"]:
                        if caption_name in context["caption"]:
                            caption = doc.add_paragraph(
                                context["caption"][caption_name], style="TableCaption"
                            )
                            _move_p_after(caption, p)

                elif scope == "table":
                    r.text = ""
                    table_counter += 1
                    create_table(doc, item, p)

        if "{{INDICATORS}}" in p.text:
            # Remove '{{INDICATORS}}'placeholder and add the report content above empty p.text
            p.text = ""

            for ecosystem in context["ecosystem_indicators"]:
                name = ecosystem["ecosystem_name"]
                percent = ecosystem["ecosystem_percentage"]

                if percent is not "":
                    heading = "{0}: {1}% of area".format(name, percent)
                    p.insert_paragraph_before(heading, style="Heading13")
                else:
                    heading = name
                    p.insert_paragraph_before(heading, style="Heading13")

                if name == 'Inland waterbodies':
                    p.insert_paragraph_before("These are lakes, reservoirs, and ponds not included in the Blueprint 2.2 priorities. Ecosystem-specific indicators have not yet been developed for this area", style="NoInfo")

                elif ecosystem["indicators"]:

                    for indicator in ecosystem["indicators"]:
                        table_counter += 1

                        p.insert_paragraph_before(
                            indicator["value"]["indicator_name"], style="Heading11"
                        )
                        p.insert_paragraph_before(
                            indicator["value"]["indicator_description"]
                        )

                        caption_text = "Table {0}: {1}".format(
                            table_counter, indicator["value"]["indicator_caption"]
                        )
                        caption = p.insert_paragraph_before(
                            caption_text, style="TableCaption"
                        )

                        create_table(
                            doc,
                            indicator["table"]["indicator_table"],
                            caption,
                        )

                else:
                    p.insert_paragraph_before("No indicators are present in this area", style="NoInfo")

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{THREATS}}" in p.text:
            # Remove '{{THREATS}}'placeholder and add the report content
            p.text = ""
            if is_marine:
                # Marine units do not have any threats data
                p.insert_paragraph_before(context["no_info"]["threats_marine_blocks"], style="NoInfo")

            else:
                for key in ("slr", "urban"):
                    p.insert_paragraph_before(context["heading"][key], style="Heading14")
                    if isinstance(context["chart"][key], str):
                        # unit does not have information for this threat
                        p.insert_paragraph_before(context["chart"][key], style="NoInfo")

                    else:
                        i = p.insert_paragraph_before()
                        run = i.add_run()
                        run.add_picture(context["chart"][key], width=Cm(10))

                        chart_key = "chart_{}".format(key)
                        caption = doc.add_paragraph(
                            context["caption"][chart_key], style="TableCaption"
                        )
                        if key == "slr":
                            add_hyperlink(caption, "https://coast.noaa.gov/digitalcoast/data/slr.html", "NOAA sea-level rise inundation data.")
                        else:
                            add_hyperlink(caption, "http://www.basic.ncsu.edu/dsl/urb.html", "SLEUTH urban growth model.")

                        _move_p_after(caption, i)

                    # insert blank line
                    p.insert_paragraph_before()

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{PARTNERS}}" in p.text:
            # Remove '{{PARTNERS}}'placeholder and add the report content
            p.text = ""

            if is_marine:
                # Marine units do not have partner info available
                p.insert_paragraph_before(context["no_info"]["partners_marine_blocks"],
                                          style="NoInfo")

            else:
                if context["partners"]:
                    for category in context["partners"]:
                        p.insert_paragraph_before(
                            context["partner_headers"][category], style="Heading14"
                        )

                        for partner in context["partners"][category]:
                            text, url = partner

                            if url:
                                part = p.insert_paragraph_before(style="List Bullet 2")
                                add_hyperlink(part, url, text)
                            else:
                                p.insert_paragraph_before(style="List Bullet 2").text = text

                else:
                    # no partner info available
                    p.insert_paragraph_before(context["no_info"]["partners"], style="NoInfo")

                # Counties
                if "counties" in context:
                    p.insert_paragraph_before("Land Trusts (by county)", style="Heading14")
                    for county in context["counties"]:
                        county_name = p.insert_paragraph_before(style="List Bullet 2")
                        add_hyperlink(county_name, county["url"], county["name"])

            # Delete the placeholder para
            delete_paragraph(p)

        if "{{OWNERSHIP}}" in p.text:
            # Remove '{{OWNERSHIP}}'placeholder and add the report content
            p.text = ""

            if is_marine:
                # Marine units do not have ownership info
                p.insert_paragraph_before(context["no_info"]["ownership_marine_blocks"], style="NoInfo")

            else:
                # Add source info
                para = p.insert_paragraph_before("Values derived from: ")
                add_hyperlink(para,
                              "https://www.conservationgateway.org/ConservationByGeography/NorthAmerica/UnitedStates/edc/reportsdata/terrestrial/secured/Pages/default.aspx",
                              "Secured Lands From TNC Eastern Division - 2015 Edition")
                p.insert_paragraph_before()

                p.insert_paragraph_before("Conserved lands ownership", style="Heading14")
                if not isinstance(context["table"]["ownership"], str):
                    para = p.insert_paragraph_before(context["caption"]["table_ownership"], style="TableCaption")
                    create_table(doc, context["table"]["ownership"], para)
                else:
                    p.insert_paragraph_before(context["no_info"]["no_info"], style="NoInfo")

                p.insert_paragraph_before("Land protection status", style="Heading14")
                if not isinstance(context["table"]["protection"], str):
                    para = p.insert_paragraph_before(context["caption"]["table_protection"], style="TableCaption")
                    create_table(doc, context["table"]["protection"], para)
                else:
                    p.insert_paragraph_before(context["no_info"]["no_info"], style="NoInfo")

            # Delete the placeholder para
            delete_paragraph(p)

    report = doc.save(path)

    return report

def generate_report_context(unit_id, config, is_marine):
    """
        Create json of data points required for report

    Parameters
    ----------
    unit_id: string id of a summary unit
    config: dict of json data from src/config

    Returns
    -------
    context: dict of data in report structure

    """

    with open(os.path.join(DATA_DIR, "{0}.json".format(unit_id))) as json_file:
        data = json.loads(json_file.read())

    total_acres = data["acres"]
    summary_unit_type = "marine lease block" if "M" in unit_id else "subwatershed"

    context = dict()

    context["value"] = {
        "summary_unit_name": data["name"],
        "summary_unit_type": summary_unit_type,
        "acres": "{:,}".format(data["acres"]),
    }
    context["table"] = {}
    context["chart"] = {}
    context["map"] = {}
    context["caption"] = {
        "map_priorities": "Figure 1: Map of Blueprint priority categories within the {0} {1}.".format(
            data["name"], summary_unit_type
        )
    }
    figure_num = 2
    context["heading"] = {
        "slr": "Sea level rise",
        "urban": "Urban growth"
    }
    context["no_info"] = {
        "threats_marine_blocks": "No information on threats is available for marine lease blocks.",
        "ownership_marine_blocks": "No information on ownership is available for marine lease blocks.",
        "partners_marine_blocks": "No information on partners is available for marine lease blocks.",
        "partners": "No information on partners is available for this {0}.".format(summary_unit_type),
        "slr": "No sea level rise data are available for this {0}.".format(summary_unit_type),
        "urban": "No urban growth data are available for this {0}".format(summary_unit_type),
        "priority": "No priority information is available for this {0}".format(summary_unit_type),
        "no_info": "No information available"
    }

    # Priorities map

    priorities_config = config["priorities"]
    priorities_map = get_map(unit_id, data, priorities_config)

    context["map"]["priorities"] = priorities_map

    # Priorities table
    table_num = 1

    if data["blueprint"]:
        priorities = dict(col_names=["Priority Category", "Acres", "Percent of Area"])

        priorities["rows"] = [
            (
                priorities_config[str(i)]["label"],
                "{:,}".format(percent_to_acres(percentage, total_acres)),
                "{0}%".format(percentage),
            )
            for i, percentage in enumerate(data.get("blueprint"))
        ]
        priorities["rows"].reverse()

        context["table"]["priorities"] = priorities

        indices = [i for i, v in enumerate(data["blueprint"]) if v > 0]
        indices.reverse()
        values = [data["blueprint"][i] for i in indices]
        colors = [priorities_config[str(i)]["color"] for i in indices]
        labels = [
            "{0} ({1}%)".format(
                priorities_config[str(i)]["label"], data["blueprint"][i]
            )
            for i in indices
        ]
        chart = get_pie_chart(values, colors=colors, labels=labels)
        context["chart"]["priorities"] = chart
        context["caption"]["chart_priorities"] = "Figure {0}: Proportion of each Blueprint category within the {1} {2}.".format(
            figure_num, data["name"], summary_unit_type)
        figure_num += 1
        context["caption"]["table_priorities"] = "Table {0}: Extent of each Blueprint priority category within the {1} {2}".format(
            table_num, data["name"], summary_unit_type)
        table_num += 1

    else:
        context["table"]["priorities"] = context["no_info"]["priority"]
        context["chart"]["priorities"] = ""
        context["caption"]["table_priorities"] = ""

    # Ecosystem table

    ecosystems_table = dict(col_names=["Ecosystems", "Acres", "Percent of Area"])
    ecosystems = data.get("ecosystems", [])  # make sure we always have a list
    context["caption"]["table_ecosystems"] = "Table {0}: Extent of each ecosystem within the {1} {2}".format(
        table_num, data["name"], summary_unit_type
        )
    table_num += 1

    ecosystems_with_area = [
        (e, ecosystems[e]["percent"]) for e in ecosystems if "percent" in ecosystems[e]
    ]
    # filter the list to those with percents, and make a new list of tuples of ecosystem ID and percent
    ecosystems_with_area = sorted(
        ecosystems_with_area, key=lambda x: x[1], reverse=True
    )  # sort by percent, from largest percent to smallest

    ecosystems_table["rows"] = [
        (
            config["ecosystems"][e]["label"],
            "{:,}".format(percent_to_acres(percent, total_acres)),
            "{0}%".format(percent),
        )
        for e, percent in ecosystems_with_area
    ]

    context["table"]["ecosystems"] = ecosystems_table

    # Indicators

    context["ecosystem_indicators"] = []

    ind_ecosystems = [
        (key, data["ecosystems"][key].get("percent", None))
        for key in data["ecosystems"]
    ]

    ecosystems_with_percent = [e for e in ind_ecosystems if e[1] is not None]
    ecosystems_with_percent = sorted(
        ecosystems_with_percent, key=lambda x: x[1], reverse=True
    )  # sort descending percent

    ecosystems_without_percent = [e for e in ind_ecosystems if e[1] is None]
    ecosystems_without_percent = sorted(
        ecosystems_without_percent, key=lambda x: x[0]
    )  # sort ascending label

    ecosystems = ecosystems_with_percent + ecosystems_without_percent

    for ecosystem in ecosystems:
        key = ecosystem[0]
        ecosystem_data = data["ecosystems"][key]
        ecosystem_config = config["ecosystems"][key]

        ecosystem_indicators = {
            "ecosystem_name": ecosystem_config["label"],
            "indicators": [],
            "ecosystem_percentage": ecosystem_data.get("percent", ""),
        }

        # Indicator tables

        for indicator in ecosystem_data.get("indicators", []):
            indicator_config = ecosystem_config["indicators"][indicator]
            indicator_data = ecosystem_data["indicators"][indicator]
            indicator_context = {
                "value": {
                    "indicator_name": indicator_config["label"],
                    "indicator_description": indicator_config["description"],
                    "indicator_caption": indicator_config["reportCaption"],
                },
                "table": {
                    "indicator_table": {
                        "col_names": [
                            "",
                            "Indicator Values",
                            "Acres",
                            "Percent of Area",
                        ],
                        "rows": [],
                    }
                },
            }

            good_threshold = indicator_config.get("goodThreshold", None)

            indicator_keys = [int(v) for v in indicator_config["valueLabels"].keys()]
            indicator_keys.sort()  # make sure keys are sorted in ascending order
            indicator_values = list(zip(indicator_keys, indicator_data["percent"]))
            indicator_values.reverse()  # Reverse order so highest priorities at top of table

            if good_threshold is not None:
                # Find threshold position in indicator values so total will be added after
                threshold_position = indicator_keys[::-1].index(good_threshold)

                total_good_percent = 0
                total_not_good_percent = 0
                rows = []

                for label_key, percent in indicator_values:
                    if label_key >= good_threshold:
                        total_good_percent += percent
                        rows.append(
                            [
                                "",
                                indicator_config["valueLabels"][str(label_key)],
                                "{:,}".format(percent_to_acres(percent, total_acres)),
                                "{0}%".format(percent),
                            ]
                        )
                    else:
                        total_not_good_percent += percent
                        rows.append(
                            [
                                "",
                                indicator_config["valueLabels"][str(label_key)],
                                "{:,}".format(percent_to_acres(percent, total_acres)),
                                "{0}%".format(percent),
                            ]
                        )

                # Add lows & highs
                rows[0][0] = "High"
                rows[-1][0] = "Low"

                # Add condition rows

                good_total_row = [
                    "",
                    "Total in good condition",
                    "{:,}".format(percent_to_acres(total_good_percent, total_acres)),
                    "{0}%".format(round(total_good_percent, 1)),
                ]
                not_good_total_row = [
                    "",
                    "Total not in good condition",
                    "{:,}".format(
                        percent_to_acres(total_not_good_percent, total_acres)
                    ),
                    "{0}%".format(round(total_not_good_percent, 1)),
                ]

                # Insert total_good row at threshold and total_not_good at table end
                rows.insert(int(threshold_position) + 1, good_total_row)
                rows.append(not_good_total_row)

                indicator_context["table"]["indicator_table"]["rows"] = rows

            else:
                rows = [
                    [
                        "",
                        indicator_config["valueLabels"][str(label_key)],
                        "{:,}".format(percent_to_acres(percent, total_acres)),
                        "{0}%".format(percent),
                    ]
                    for label_key, percent in indicator_values
                ]

                # Add lows & highs
                rows[0][0] = "High"
                rows[-1][0] = "Low"

                indicator_context["table"]["indicator_table"]["rows"] = rows

            ecosystem_indicators["indicators"].append(indicator_context)

        context["ecosystem_indicators"].append(ecosystem_indicators)

    # Partners list - name and url separated by categories

    context["partner_headers"] = {
        "regional": "Regional Conservation Plans",
        "state": "Statewide Conservation Plans",
        "marine": "Marine Conservation Plans",
    }

    context["partners"] = defaultdict(list)
    for plan_key in data["plans"]:
        plan = config["plans"][plan_key]
        label = plan["label"]
        url = plan.get("url", "")
        context["partners"][plan["type"]].append((label, url))

    # Counties list - name and url

    if "counties" in data:
        context["counties"] = [
            {"name": value, "url": "http://findalandtrust.org/counties/{0}".format(key)}
            for key, value in data["counties"].items()
        ]  # key is FIPS and value is county with state

    # Ownership table

    if "owner" in data:
        owners = dict(col_names=["Ownership", "Acres", "Percent of Area"])

        owners["rows"] = [
            [
                config["owners"][key]["label"],
                "{:,}".format(percent_to_acres(percent, total_acres)),
                "{0}%".format(percent),
            ]
            for key, percent in data["owner"].items()
        ]

        own_perc_sum = sum(data["owner"].values())

        if own_perc_sum < 100:
            perc_remainder = round(100 - own_perc_sum, 1)
            owners["rows"].append(
                [
                    "Not conserved",
                    "{:,}".format(percent_to_acres(perc_remainder, total_acres)),
                    "{0}%".format(perc_remainder),
                ]
            )

        context["table"]["ownership"] = owners
    else:
        context["table"]["ownership"] = context["no_info"]["no_info"]

        # Protections table

    protection = dict(col_names=["Land Protection Status", "Acres", "Percent of Area"])

    if "gap" in data:
        protection["rows"] = [
            [
                config["protection"][key]["label"],
                "{:,}".format(percent_to_acres(percent, total_acres)),
                "{0}%".format(percent),
            ]
            for key, percent in data["gap"].items()
        ]

        pro_perc_sum = sum(data["gap"].values())

        if pro_perc_sum < 100:
            perc_remainder = round(100 - pro_perc_sum, 1)
            protection["rows"].append(
                [
                    "Not conserved",
                    "{:,}".format(percent_to_acres(perc_remainder, total_acres)),
                    "{0}%".format(perc_remainder),
                ]
            )

        context["table"]["protection"] = protection
    else:
        context["table"]["protection"] = context["no_info"]["no_info"]

        # Threats

    if is_marine:
        context["chart"]["slr"] = context["no_info"]["threats_marine_blocks"]

    else:
        if "slr" in data and data["slr"]:
            chart = get_line_chart(
                config["slr"],
                data["slr"],
                x_label="Amount of sea level rise (feet)",
                y_label="Percent of area",
                color="#004da8",
                alpha=0.3,
            )
            context["chart"]["slr"] = chart
            context["caption"]["chart_slr"] = "Figure {0}: Extent of inundation by projected sea level rise within the {1} {2}.  Values from the ".format(
                figure_num, data["name"], summary_unit_type)
            figure_num += 1
        else:
            context["chart"]["slr"] = context["no_info"]["slr"]

        if "urban" in data and data["urban"]:
            chart = get_line_chart(
                config["urbanization"],
                data["urban"],
                x_label="Decade",
                y_label="Percent of area",
                color="#D90000",
                alpha=0.3,
            )
            context["chart"]["urban"] = chart
            context["caption"]["chart_urban"] = "Figure {0}: Extent of projected urbanization within the {1} {2}.  Values from the ".format(
                figure_num, data["name"], summary_unit_type
            )
            figure_num += 1
        else:
            context["chart"]["urban"] = context["no_info"]["urban"]

    # Table captions for last two tables, Ownership and Protection status

    for ecosystem in context["ecosystem_indicators"]:
        for indicator in ecosystem.get("indicators", []):
            table_num += 1

    if "rows" in context["table"]["ownership"]:
        context["caption"]["table_ownership"] = "Table {0}: Extent of ownership class within the {1} {2}.".format(
            table_num, data["name"], summary_unit_type
        )
        table_num += 1
    else:
        context["caption"]["table_ownership"] = ""

    if "rows" in context["table"]["protection"]:
        context["caption"]["table_protection"] = "Table {0}: Extent of land protection status within the {1} {2}.".format(
            table_num, data["name"], summary_unit_type
        )
        table_num += 1
    else:
        context["caption"]["table_protection"] = ""

    return context

def percent_to_acres(percentage, total_acres):
    return round(total_acres * (percentage / 100))

def create_table(doc, data, para):
    """
    Builds table at end of doc

    Parameters
    ----------
    doc: docx.Document object
    data: dict
        table data; 'data' contains a list of column names ('col_names') and a list of rows ('rows'),
        where each row is a list of values for each column
    para: paragraph object
        Where the table needs to be moved

    Returns
    -------
    end_buffer
        location that the last item was added to the document

    """
    # TODO: ensure tables don't run over a page break

    styles = doc.styles
    ncols = len(data["col_names"])

    # Create table with one row for column headings
    table = doc.add_table(rows=1, cols=ncols)
    table.style = styles[DOC_STYLE]

    for index, heading in enumerate(data["col_names"]):
        cell = table.cell(0, index)
        cell.text = heading
        # Change justification of first heading
        if ncols is 3 and index is 0:
            cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
        elif ncols is 4 and index is 1:
            cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT

    for datarow in data["rows"]:
        row = table.add_row()
        condition_style = (
            str(datarow[1]) == "Total in good condition"
            or str(datarow[1]) == "Total not in good condition"
        )
        for index, datacell in enumerate(datarow):
            row.cells[index].text = str(datacell)
            if ncols is 4 and index is 1:
                row.cells[index].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.LEFT
            if condition_style is True:
                # A new shade must be generated for each cell
                new_shade = shade_generator(TOTAL_ROW_COLOR)
                row.cells[index]._tc.get_or_add_tcPr().append(new_shade)

    if "Indicator Values" in data["col_names"]:
        table_type = "indicator"
    else:
        table_type = "other"

    set_col_widths(table, table_type)

    _move_table_after(table, para)

    end_buffer = doc.add_paragraph("")
    _move_p_after_t(table, end_buffer)

    return end_buffer

def shade_generator(color):
    return docx.oxml.parse_xml(
        r'<w:shd {0} w:fill="{1}"/>'.format(docx.oxml.ns.nsdecls("w"), color)
    )

def set_col_widths(table, table_type):
    """
    Set widths for standard table columns

    Parameters
    ----------
    table: table object

    """
    if table_type is "indicator":
        widths = (Cm(1), Cm(9), Cm(3.5), Cm(3.5))
    else:
        widths = (Cm(9), Cm(4), Cm(4))

    for row in table.rows:
        for idx, width in enumerate(widths):
            row.cells[idx].width = width

def _move_p_after(p_move, p_destination):
    """
    Move one paragraph after another

    Parameters
    ----------
    heading
        heading to be moved
    paragraph
        location where table will be moved

    """
    p_destination._p.addnext(p_move._p)

def _move_table_after(table, paragraph):
    """
    Move the table after 'paragraph.'
    Normally, tables are created at the end of the document.
    This function allows you to move them to a different location in the document.
    The paragraph passed in is not modified, it is just used as a reference point for inserting the table.

    Parameters
    ----------
    table
        table to be moved
    paragraph
        location where table will be moved

    """
    paragraph._p.addnext(table._tbl)

def _move_p_after_t(table, para):
    """
    Move a paragraph after a table

    Parameters
    ----------
    table
        table to be moved
    para: paragraph object
        location where table will be moved

    """
    table._tbl.addnext(para._p)

def add_hyperlink(paragraph, url, text):
    """
    A function that places a hyperlink within a paragraph object.

    Parameters
    ----------
    paragraph: The paragraph we are adding the hyperlink to.
    url: A string containing the required url
    text: The text displayed for the url

    Returns
    -------
    hyperlink: hyperlink object

    """

    # This gets access to the document.xml.rels file and gets a new relation id value
    part = paragraph.part
    r_id = part.relate_to(
        url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True
    )

    # Create the w:hyperlink tag and add needed values
    hyperlink = OxmlElement("w:hyperlink")
    hyperlink.set(qn("r:id"), r_id)

    # Create a w:r element
    new_run = OxmlElement("w:r")

    # Create a new w:rPr element
    rPr = OxmlElement("w:rPr")

    # Style it
    c = OxmlElement("w:color")
    c.set(qn("w:val"), "4F81BD")
    rPr.append(c)
    u = OxmlElement("w:u")
    u.set(qn("w:val"), "none")
    rPr.append(u)

    # Join all the xml elements together add add the required text to the w:r element
    new_run.append(rPr)
    new_run.text = text
    hyperlink.append(new_run)

    paragraph._p.append(hyperlink)

    return hyperlink

def delete_paragraph(paragraph):
    p = paragraph._element
    p.getparent().remove(p)
    p._p = p._element = None

def get_map(unit_id, data, priorities):
    bounds = data["bounds"]

    # convert priorities to legend, in descending priority (dropping Not a Priority class)
    legend = [
        (int(k), v["color"], v["label"]) for k, v in priorities.items() if k != "0"
    ]
    legend = sorted(legend, key=lambda x: x[0], reverse=True)
    # strip off the value used for sorting
    legend = [entry[1:] for entry in legend]

    m = get_map_image(unit_id, bounds, legend)

    buffer = BytesIO()
    m.save(buffer, "PNG")
    return buffer

def _resolve(scope, key, context):
    """
    Resolve scope and key to a context item
    Provides very minimal validation of presence

    Parameters
    ----------
    scope: str
        singular variant of scope in context
    key: str
        key to lookup context item in context within scope
    context: dict

    Returns
    -------
    context item: str, dict, etc
    """

    if scope not in context:
        raise ValueError("Scope {0} is not found in context!".format(scope))

    if key not in context[scope]:
        raise ValueError("Key {0} is not found in context!".format(key))

    return context[scope][key]

if __name__ == "__main__":
    id = "I1"
    outpath = "/tmp/{0}.docx".format(id)
    create_report(id, outpath)


import matplotlib.pyplot as plt
import serial
import collections

PORT_NAME = 'COM28'
BAUD_RATE = 460800

MAX_ITEMS = 1000

# d = collections.deque([0]*MAX_DEQUE_ITEMS , maxlen=MAX_DEQUE_ITEMS)
arr = [0]*MAX_ITEMS
index = 0;
plt.style.use('ggplot')

plt.ion()

fig, ax  = plt.subplots()
line, = ax.plot(arr)

with serial.Serial(PORT_NAME, BAUD_RATE) as ser:
    # Read loop
    while True:
        # Update deque
        value = float(ser.readline().decode("ascii"))
        arr[index] = value
        index += 1
        if index >= MAX_ITEMS:
            index = 0

        # Update plot
        line.set_ydata(arr);
        fig.canvas.draw()
        fig.canvas.flush_events()


#!/usr/bin/env python
import rospy
from sensor_msgs.msg import NavSatFix
from std_msgs.msg import Float64
import math

#function to calculate distance in meters from gps lat-long
#Source: http://www.johndcook.com/blog/python_longitude_latitude/
def coordinate_distance(lat1, long1, lat2, long2):
    degrees_to_radians = math.pi/180.0
    phi1 = (90.0 - lat1)*degrees_to_radians
    phi2 = (90.0 - lat2)*degrees_to_radians
    theta1 = long1*degrees_to_radians
    theta2 = long2*degrees_to_radians
    cos = (math.sin(phi1)*math.sin(phi2)*math.cos(theta1 - theta2) + math.cos(phi1)*math.cos(phi2))
    arc = math.acos( cos )
    return arc * 6373 * 1000

#global variables to store latest gps positions
gps1 = NavSatFix()
gps2 = NavSatFix()

pub = rospy.Publisher('gps_distance', Float64, queue_size=10)

def gpscb2(data):
    global gps1
    gps1 = data

def gpscb1(data):
    global gps2, pub
    gps2 = data
    curr_distance = coordinate_distance(gps1.latitude, gps1.longitude, gps2.latitude, gps2.longitude)
    #rospy.loginfo("%f", curr_distance)
    pub.publish(curr_distance)

def listener():
    rospy.init_node('gps_listener', anonymous=True)
    rospy.Subscriber("gps1", NavSatFix, gpscb1)
    rospy.Subscriber("gps2", NavSatFix, gpscb2)
    rospy.spin()

if __name__ == '__main__':
    listener()


import keras
from keras.datasets import mnist

pixel_width = 28
pixel_height = 28
pixel_depth = 1

# Load the dataset
(features_train, labels_train), (features_test, labels_test) = mnist.load_data()

# Reshape data to fit model
# Add pixel depth (grayscale)
features_train = features_train.reshape(features_train.shape[0], pixel_width, pixel_height, pixel_depth)
features_test = features_test.reshape(features_test.shape[0], pixel_width, pixel_height, pixel_depth)

input_shape = (pixel_width, pixel_height, pixel_depth)

# Convert data to float32
features_train = features_train.astype('float32')
features_test = features_test.astype('float32')

# Normalize data
features_train /= 255
features_test /= 255

# Convert labels to categorical
labels_train = keras.utils.to_categorical(labels_train, 10)
labels_test = keras.utils.to_categorical(labels_test, 10)

###################################################
###################################################

from multiprocessing import Process, Manager
import time as time
import datetime as datetime
import tensorflow as tf

# tf.device("/gpu:0")

# Create a queue to store the results

start_time = time.time()

def fit(batch_size, epochs, r, verbose=False):
		model = keras.models.Sequential()
		model.add(keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))
		model.add(keras.layers.Conv2D(64, (3, 3), activation='relu'))
		model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))
		model.add(keras.layers.Dropout(0.25))
		model.add(keras.layers.Flatten())
		model.add(keras.layers.Dense(128, activation='relu'))
		model.add(keras.layers.Dropout(0.5))
		model.add(keras.layers.Dense(10, activation='softmax'))
		model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])

		verbose = 1 if verbose else 0

		print(f"Batch size: {batch_size}, Epochs: {epochs} - Training...")
		model.fit(features_train, labels_train, batch_size=batch_size, epochs=epochs, verbose=verbose, validation_data=(features_test, labels_test))

		print(f"Batch size: {batch_size}, Epochs: {epochs} - Evaluating...")
		score = model.evaluate(features_test, labels_test, verbose=verbose)

		# Print the time taken since start_time in a nice format
		total_time = time.time() - start_time
		seconds = total_time % 60
		minutes = (total_time / 60) % 60
		hours = (total_time / 3600) % 24
		print(f"Batch size: {batch_size}, Epochs: {epochs} - Time taken: {int(hours)}:{int(minutes)}:{int(seconds)}")

		r.append((batch_size, epochs, score[0], score[1]))

def run_parallel(batch_size, epochs, r, verbose=False):
		# Create a new process
		p = Process(target=fit, args=(batch_size, epochs, r, verbose))

		# Start the process
		p.start()

		# Return the process
		return p

def main():
	# with tf.device("/gpu:0"):
		with Manager() as manager:
			# Create a queue to store the results
			results = manager.list()

			# Print the start_time in a nice format
			print(f"Start time: {datetime.datetime.fromtimestamp(start_time).strftime('%Y-%m-%d %H:%M:%S')}")

			p0 = run_parallel(32, 100, results, True)

			# fit(32, 20000, results, 1)
			# fit(32, 20, results)

			# Wait for the processes to finish
			p0.join()

			# Print the results
			print("RESULTS")
			print(results)
			if not len(results) == 0:
				for result in results:
					print(f"Batch size: {result[0]}, Epochs: {result[1]} - Loss: {result[2]}, Accuracy: {result[3]}")

if __name__ == '__main__':
		main()


from setuptools import setup, find_packages
import sys

platform_specific_packages = {
    "darwin": ["tensorflow-macos", "tensorflow-metal"],
    "linux": ["tensorflow>=2.4"],
    "cywin": ["tensorflow>=2.4"],
    "win3D": ["tensorflow>=2.4"],
}

setup(
    name='tf_layers',
    author='Gabriel Van Zandycke',
    author_email="gabriel.vanzandycke@hotmail.com",
    url="https://github.com/gabriel-vanzandycke/tf_layers",
    licence="LGPL",
    python_requires='>=3.6',
    description="Some tensorflow layers",
    version='0.2.0',
    packages=find_packages(),
    install_requires=[
        *platform_specific_packages[sys.platform],
        "numpy"
    ],
)


#!/usr/bin/python3
"""Displays the Github ID of a given user.
"""
import requests
import sys

if __name__ == '__main__':
    url = 'https://api.github.com/user'
    r = requests.get(url, auth=(sys.argv[1], sys.argv[2]))
    try:
        json = r.json()
        print(json.get('id'))
    except requests.exceptions.JSONDecodeError:
        print('Not a valid JSON')


# Задайте список из нескольких чисел. Напишите программу, которая найдёт сумму элементов списка, стоящих на нечётной позиции.

# Пример:

# - [2, 3, 5, 9, 3] -> на нечётных позициях элементы 3 и 9, ответ: 12

my_list = [1, 2.2, 3, 4, 5.1, 6, 7, 8, 9, 0]
summ = [my_list[i] for i in range(len(my_list)) if not i % 2]
print(summ)
print(sum(summ))

exit()
sum = 0
my_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]
for digit in my_list[1::2]:
    if digit != 0:
        print(digit, end=" + ")
    else:
        print(digit, end="")
    sum += digit
print(" =", sum)

# ^ Второй вариант
my_list = [2, 3, 5, 9, 3]
odd = my_list[1::2]
print(odd, end=" ")
print("=", sum(odd))

Task_3/1_Сумма чисел на нечетной позиции.py

1_Сумма чисел на нечетной позиции.py


#!/afs/cern.ch/user/p/pyhdtl/public/anaconda/bin/python
from __future__ import division
import os
import numpy as np
import scipy as sci
import h5py
import matplotlib.pyplot as plt
import glob as glob
import re
import mystyle as ms

filln = 4927
tag = ''
beam = 'B1'

output_path = '/afs/cern.ch/work/l/lcarver/public/Instability_Data/{:d}'.format(filln)

sus_filename = '{:s}/{:s}_Sussix_Tunes.h5'.format(output_path,beam)

f = h5py.File(sus_filename,'r')
turn_split = f.attrs['Turn_Split']
sus_h = f['Horizontal']
sus_v = f['Vertical']
time = f.attrs['Start_Time']
tune_h = sus_h['Peak_0']
tune_v = sus_v['Peak_0']

tune_h2 = np.zeros((len(tune_h),2))
tune_v2 = np.zeros((len(tune_v),2))

turns = turn_split*np.arange(0,len(tune_h)) + turn_split/2

#crossing_turn = 125000
#crossing_turn = 150000

#for j in np.arange(0,len(tune_h),1):
#  for i in np.arange(1,18,1):
#    peak = 'Peak_{:d}'.format(i)
#    if ((sus_h[peak][j][0] > 0.275) & (sus_h[peak][j][0] < 0.295)):
#        tune_h2[j] = sus_h[peak][j]
#    if ((sus_v[peak][j][0] > 0.29) & (sus_v[peak][j][0] < 0.5)):
#        tune_v2[j] = sus_v[peak][j]

#    else:
#        print 'No tune in window'

#f1001 = 0.5*np.arctan(np.sqrt((tune_h2[:,1]*tune_v2[:,1])/(tune_h[:,1]*tune_v[:,1])))
#Cminus = 4*np.abs(f1001)*np.abs(tune_v[:,0]-tune_h[:,0])

fig = plt.figure(figsize=(8,6))
rect = fig.patch
rect.set_facecolor('white')
ax1 = fig.add_subplot(111)
ax2 = ax1.twinx()

#ax1.plot(0.28,'g-',label=r'$C^{{-}}$')
ax1.plot(turns,tune_h[:,0],'ro',label=r'$Q_{{h}}$')
ax1.plot(turns,tune_v[:,0],'bo',label=r'$Q_{{v}}$')

def calc_cminus(tune_h,tune_v,mid_point):
  arr_1 = tune_h > mid_point
  arr_2 = tune_v > mid_point
  pk_1 = np.amin(np.concatenate((tune_h[arr_1],tune_v[arr_2])))

  arr_1 = tune_h < mid_point
  arr_2 = tune_v < mid_point
  pk_2 = np.amax(np.concatenate((tune_h[arr_1],tune_v[arr_2])))

  return [pk_1,pk_2,np.abs(pk_2 - pk_1)]

#[pk_1, pk_2, cminus] = calc_cminus(tune_h[:,0], tune_v[:,0], 0.295)
#ax1.axhline(y=pk_1,color='g',linestyle='dashed',label=r'$|C^{{-}}|={:g}$'.format(cminus))
#ax1.axhline(y=pk_2,color='g',linestyle='dashed')

#ax1.set_title(r'$C^{{-}}={:g}$'.format(cminus),fontsize=18)
ax1.set_title('Tunes for {:s} from {:s}'.format(beam, time),fontsize=14)

ax1.set_xlabel('Turns')
ax1.set_ylabel('Fractional Tune')
ax1.legend(loc=1)
#turns_mask = turns > crossing_turn
#ax2.plot(turns[turns_mask],Cminus[turns_mask],'g-',label=r'$C^{{-}}$')
#plt.savefig('{:s}/Crossing.png'.format(output_path))

plt.show()

f.close()


import os

import torchvision.models as models
import pytorch_lightning as pl
from feature_net import *
from losses import *
from datasets import *
from textures import MapDensePoseTexModule
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader
import cv2
from facenet_pytorch import MTCNN, InceptionResnetV1
import torch
from datetime import datetime
import torchvision.transforms as transforms
import shutil

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class HumanRendering(pl.LightningModule):

    def __init__(self, data_path, batch_size=64):
        super(HumanRendering, self).__init__()
        self.save_hyperparameters()
        self.data_path = data_path
        self.batch_size = batch_size
        self.texture_mapper = MapDensePoseTexModule(256).eval()
        self.feature_net = FeatureNet(3, 16).to(device)
        self.render_net = RenderNet(16, 3).to(device)
        self.discriminators_feature = [PatchDiscriminator(3).to(device), PatchDiscriminator(3).to(device),
                                       PatchDiscriminator(3).to(device)]
        self.discriminators_render = [PatchDiscriminator(3).to(device), PatchDiscriminator(3).to(device),
                                      PatchDiscriminator(3).to(device)]
        self.vgg19 = models.vgg19(pretrained=True).to(device).eval()
        self.face_detector = MTCNN(image_size=160, margin=15, device='cuda').eval()
        self.face_rec = InceptionResnetV1(pretrained='vggface2').eval().cuda()
        self.hook = SaveOutput()
        self.vgg19_normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        # self.contextual_loss = cl.ContextualLoss()
        hook_handles = []
        for idx, layer in enumerate(self.vgg19.features):
            if isinstance(layer, nn.ReLU):
                handle = layer.register_forward_hook(self.hook)
                hook_handles.append(handle)
        self.total_loss = {
            'generator': 0,
            'discriminator': 0
        }
        self.losses = {
            'generator': [],
            'discriminator': []
        }

    def forward(self, train_batch):
        feature_out = self.feature_net(train_batch['sample']['texture'])
        feature_out_tex = self.apply_texture(feature_out, train_batch['target']['instances'],
                                             train_batch['target']['uv'])
        render_out = self.render_net(feature_out_tex)
        return feature_out, feature_out_tex, render_out

    def training_step(self, train_batch, batch_nb, optimizer_idx):
        feature_out, feature_out_tex, render_out = self.forward(train_batch)
        target_applied_texture = self.apply_texture(train_batch['target']['texture'],
                                                    train_batch['target']['instances'], train_batch['target']['uv'])

        generated_image = target_applied_texture[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'images/target_applied_texture.jpg', generated_image)

        # train generator
        if optimizer_idx == 0:
            loss_inpainting = inpainting_loss(feature_out, train_batch['sample']['texture'],
                                              train_batch['target']['texture'])
            loss_adversarial_feature = adversarial_loss(self.discriminators_feature, target_applied_texture,
                                                        feature_out_tex, is_discriminator=False, is_feature=True)
            loss_adversarial_render = adversarial_loss(self.discriminators_render, train_batch['target']['image'],
                                                       render_out, is_discriminator=False, is_feature=False)
            target_vgg = self.vgg19_normalize(torch.clone(train_batch['target']['image']).add(1).true_divide(2))
            generated_vgg = self.vgg19_normalize(torch.clone(render_out).add(1).true_divide(2))
            self.vgg19(target_vgg)
            ground_truth_activations = self.hook.outputs
            self.hook.clear()
            self.vgg19(generated_vgg)
            generated_activations = self.hook.outputs
            self.hook.clear()
            loss_perceptual = perceptual_loss(ground_truth_activations, generated_activations)
            loss_identity = face_identity_loss(self.face_rec, train_batch['sample']['image'],
                                               train_batch['sample']['instances'], train_batch['target']['image'],
                                               train_batch['target']['instances'], render_out, self.face_detector)
            # contextual_loss = self.contextual_loss(train_batch['sample']['image'], train_batch['target']['image'])
            fft_loss = fourier_loss(render_out, train_batch['target']['image'])
            total_loss = (loss_adversarial_render + loss_adversarial_feature + loss_inpainting
                          + 5 * loss_perceptual + 2 * loss_identity + 3 * fft_loss) / 6
            self.total_loss['generator'] = total_loss
            self.log('gen_adversarial_render', loss_adversarial_render)
            self.log('gen_adversarial_feature', loss_adversarial_feature)
            return total_loss

        # train discriminator
        if optimizer_idx == 1:
            loss_adversarial_feature = adversarial_loss(self.discriminators_feature, target_applied_texture,
                                                        feature_out_tex, is_discriminator=True, is_feature=True)
            self.total_loss['discriminator'] += loss_adversarial_feature
            self.log('disc_adversarial_feature', loss_adversarial_feature)
            return loss_adversarial_feature

        if optimizer_idx == 2:
            loss_adversarial_render = adversarial_loss(self.discriminators_render, train_batch['target']['image'],
                                                       render_out, is_discriminator=True, is_feature=False)
            self.total_loss['discriminator'] += loss_adversarial_render
            self.log('disc_adversarial_render', loss_adversarial_render)
            self.from_batch_generate_image(train_batch, False)
            return loss_adversarial_render

    def test_step(self, train_batch, batch_nb):
        feature_out, feature_out_tex, render_out = self.forward(train_batch)
        for idx in range(feature_out.shape[0]):
            path = f'test/pair{batch_nb * 4 + idx}'
            if not os.path.exists(path):
                os.makedirs(path)
            self.save_img(render_out[idx], path + '/rendered.jpg')
            self.save_img(train_batch['sample']['image'][idx], path + '/source.jpg')
            self.save_img(train_batch['sample']['texture'][idx], path + '/source_texture.jpg')
            self.save_uv(train_batch['sample']['uv'][idx], path + '/source_pose.jpg')
            self.save_img(train_batch['target']['image'][idx], path + '/target.jpg')
            self.save_uv(train_batch['target']['uv'][idx], path + '/target_pose.jpg')
            self.save_img(feature_out[idx], path + '/generated_texture.jpg', False)
            self.save_img(feature_out_tex[idx], path + '/texture_applied.jpg', False)
        # for idx, (source, target, feature, feature_tex, render) in enumerate(zip(train_batch['sample']['image'],
        #                                                                          train_batch['target']['image'],
        #                                                                          feature_out, feature_out_tex,
        #                                                                          render_out)):
        #     path = f'test/pair{batch_nb * 4 + idx}'
        #     if not os.path.exists(path):
        #         os.makedirs(path)
        #     self.save_img(render, path + '/rendered.jpg')
        #     self.save_img(source, path + '/source.jpg')
        #     self.save_img(target, path + '/target.jpg')
            # self.save_img(feature, path + '/texture.jpg')
            # self.save_img(feature_tex, path + '/texture_applied.jpg')

    def configure_optimizers(self):
        lr = 0.0002
        b1 = 0.5
        b2 = 0.9

        opt_gen = torch.optim.Adam(list(self.feature_net.parameters()) + list(self.render_net.parameters()),
                                   lr=lr, betas=(b1, b2))
        opt_disc_feature = torch.optim.Adam(list(self.discriminators_feature[0].parameters()) +
                                            list(self.discriminators_feature[1].parameters()) +
                                            list(self.discriminators_feature[2].parameters()), lr=lr, betas=(b1, b2))
        opt_disc_render = torch.optim.Adam(list(self.discriminators_render[0].parameters()) +
                                           list(self.discriminators_render[1].parameters()) +
                                           list(self.discriminators_render[2].parameters()), lr=lr, betas=(b1, b2))
        lr_lambda = lambda epoch: 0.999
        scheduler_gen = torch.optim.lr_scheduler.MultiplicativeLR(opt_gen, lr_lambda)
        scheduler_disc_feature = torch.optim.lr_scheduler.MultiplicativeLR(opt_disc_feature, lr_lambda)
        scheduler_disc_render = torch.optim.lr_scheduler.MultiplicativeLR(opt_disc_render, lr_lambda)
        return [opt_gen, opt_disc_feature, opt_disc_render], [scheduler_gen, scheduler_disc_feature,
                                                              scheduler_disc_render]

    def train_dataloader(self):
        dataset = DeepFashionPairedDataset(self.data_path)
        return DataLoader(dataset, batch_size=self.batch_size, num_workers=4)

    def test_dataloader(self):
        dataset = DeepFashionPairedTestDataset('test_pairs.txt')
        return DataLoader(dataset, batch_size=self.batch_size, num_workers=4)

    def on_training_step_end(self):
        self.losses['generator'].append(self.total_loss['generator'])
        self.losses['discriminator'].append(self.total_loss['discriminator'])
        self.total_loss['discriminator'] = 0
        plt.clf()
        plt.title('Losses')
        plt.plot(self.losses['generator'], label='Generator loss')
        plt.plot(self.losses['discriminator'], label='Discriminator loss')
        plt.legend()
        plt.savefig('images/losses.jpg')

    def save_img(self, tensor, savepath, more_channels=True):
        img = tensor.detach().permute((1, 2, 0)).add(1).true_divide(2).mul(255).cpu().numpy().astype(np.uint8) if \
            more_channels else tensor.detach().permute((1, 2, 0)).add(1).true_divide(2).mul(255).cpu().numpy().astype(np.uint8)[:, :, :3]
        cv2.imwrite(savepath, img)

    def save_uv(self, tensor, savepath):
        u, v = tensor.detach().cpu().numpy()
        img = np.stack((u, v, (u + v) * 0.5), axis=-1).astype(np.uint8)
        cv2.imwrite(savepath, img)

    def from_batch_generate_image(self, batch, save_every=False):
        save_images_path = 'saved_images'
        if save_every:
            now = datetime.now()
            os.makedirs(f'{save_images_path}/{now}')
        savepath = f'{save_images_path}/{now}/' if save_every else 'images/'
        self.set_models_mode('eval', False)
        textures, textures_applied, rendered = self.forward(batch)

        generated_image = textures[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(255).cpu().numpy(). \
            astype(np.uint8)
        cv2.imwrite(f'{savepath}/texture.jpg', generated_image[:, :, :3])

        generated_image = textures_applied[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(255).cpu().numpy(). \
            astype(np.uint8)
        cv2.imwrite(f'{savepath}/genrated_texture.jpg', generated_image[:, :, :3])

        original_im = batch['sample']['image']
        generated_image = original_im[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'{savepath}/original.jpg', generated_image)

        original_im = batch['sample']['texture']
        generated_image = original_im[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'{savepath}/original_texture.jpg', generated_image)

        original_im = batch['target']['image']
        generated_image = original_im[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'{savepath}/target.jpg', generated_image)

        original_im = batch['target']['texture']
        generated_image = original_im[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'{savepath}/target_texture.jpg', generated_image)

        generated_image = rendered[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(255).cpu().numpy(). \
            astype(np.uint8)
        cv2.imwrite(f'{savepath}/rendered.jpg', generated_image)

        target_applied_texture = self.apply_texture(batch['target']['texture'], batch['target']['instances'], batch['target']['uv'])
        generated_image = target_applied_texture[0].detach().permute((1, 2, 0)).add(1).true_divide(2).mul(
            255).cpu().numpy().astype(np.uint8)
        cv2.imwrite(f'{savepath}/target_applied_texture.jpg', generated_image)

        self.set_models_mode('train', False)

    def apply_texture(self, generated_textures, target_instances, target_uv_maps):
        uv_tensor = target_uv_maps.byte()
        instances_tensor = target_instances.byte()
        iuv_tensor = torch.cat((instances_tensor[:, None, :, :], uv_tensor), dim=1)
        output = self.texture_mapper(generated_textures, iuv_tensor)
        return output

    def set_models_mode(self, mode, only_discriminators):
        if mode == 'eval':
            if not only_discriminators:
                self.feature_net.eval()
                self.render_net.eval()
            for disc_feature, disc_render in zip(self.discriminators_feature, self.discriminators_render):
                disc_feature.eval()
                disc_render.eval()
        if mode == 'train':
            if not only_discriminators:
                self.feature_net.train()
                self.render_net.train()
            for disc_feature, disc_render in zip(self.discriminators_feature, self.discriminators_render):
                disc_feature.train()
                disc_render.train()

model = HumanRendering('/home/pkowaleczko/datasets/deepfashion/deepfashion_paired', batch_size=4)

trainer = pl.Trainer(gpus=1, auto_select_gpus=True, max_epochs=105, resume_from_checkpoint='/home/pkowaleczko/projects/human_rendering/lightning_logs/basic_last/checkpoints/epoch=102.ckpt')
trainer.test(model)

# trainer.fit(model)


import operator
w=input("Please enter a string: ")
def most_frequent(string):
    d=dict()
    for key in string:
        if key not in d:
            d[key]=1
        else:
            d[key]+=1
    x=dict(sorted(d.items(),key=operator.itemgetter(1),reverse=True))
    for i in x:
        print(i,"=",str(0)+str(x[i]))
a=(most_frequent(w))


# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
from datetime import datetime

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

project = 'CfA Git Users'
now = datetime.now()
author = 'Center for Astrophysics | Harvard & Smithsonian'
copyright = "{}, {}".format(now.year, author)
version = '1.0'
release = version

# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

extensions = ['myst_parser']

source_suffix = {
    '.rst': 'restructuredtext',
    '.txt': 'restructuredtext',
    '.md': 'markdown',
}

templates_path = ['_templates']
exclude_patterns = ['requirements.txt']

# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

html_theme = 'sphinx_rtd_theme'
html_title = "CfA Git Users Documentation"
html_static_path = ['_static']
html_css_files = ['css/mod_theme.css']

# -- Options for LaTeX PDF output --------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output
latex_elements = {
  'extraclassoptions': 'openany,oneside'
}


from django.conf.urls.defaults import *
from os import path as os_path
from django.conf import settings
import economico.views
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    (r'^admin/(.*)', admin.site.root),
    (r'^', include('economico.urls')),
    (r'^seguridad-alimentaria/', include('seguridad_alimentaria.urls')),
    (r'^demografico/', include('demografico.urls')),
    (r'^ajax/municipio/(?P<departamento>\d+)/$', 'lugar.views.get_municipios'),
    (r'^ajax/comunidad/(?P<municipio>\d+)/$', 'lugar.views.get_comunidad'),
    (r'^ajax/entrevista/(?P<comunidad>\d+)/$', 'sequias.views.get_entrevista'),
    (r'sequia/', include('sequias.urls')),
)

if settings.DEBUG:
    urlpatterns += patterns('',
            (r'^css/(.*)$', 'django.views.static.serve',
                {'document_root': os_path.join(settings.MEDIA_ROOT + '/css')}),
            (r'^js/(.*)$', 'django.views.static.serve',
                {'document_root': os_path.join(settings.MEDIA_ROOT + '/js')}),
            (r'^imagen/(.*)$', 'django.views.static.serve',
                {'document_root': os_path.join(settings.MEDIA_ROOT + '/imagen')}),
)


rules = {"S" : {"aB", "bA", "A"}, "A" : {"B", "Sa", "bBA", "b"},
         "B" : {"b", "bS", "aD", "$"}, "D":{"AA"}, "C":{"Ba"}}

keys = list(rules.keys())

def check_for_epsilon(rules):
    is_epsilon = False
    epsilon = 0
    for key in keys:

        if "$" in rules[key]:
            is_epsilon = True
            epsilon = key
            rules[key].remove("$")

    return is_epsilon, epsilon

def count_of_eps(string, epsilon):
    count = 0
    for ch in string:
        if ch == epsilon:
            count += 1
    return count

def check_for_unit(rules):
    are_units = False
    unit_add = ''
    unit_from = ''
    for key in keys:
        for node in rules[key]:
            if len(node) == 1 and node.isupper():
                are_units = True
                unit_add = key
                unit_from = node
                rules[key].remove(node)
                return are_units, unit_add, unit_from
    return are_units, unit_add, unit_from

def is_unaccessible(rules, variable):
    unaccessible = True
    for check_in in keys:
        for node in rules[check_in]:
            if variable in node:
                unaccessible = False

    return unaccessible

is_eps, eps = check_for_epsilon(rules)
while is_eps:
    for k in keys:
        for node in rules[k].copy():
            if len(node) > 1 and eps in node and count_of_eps(node, eps) == 1:
                rules[k].add(node.replace(eps, ''))
            elif len(node) > 1 and eps in node and count_of_eps(node, eps) == 2:
                rules[k].add(node.replace(eps, '', 1))
                rules[k].add(node[::-1].replace(eps, '', 1)[::-1])
            elif len(node) == 1 and eps in node:
                rules[k].add(node.replace(eps, '$'))
                rules[k].remove(node)

    is_eps, eps = check_for_epsilon(rules)

are_units, unit_add, unit_from = check_for_unit(rules)
while are_units:
    for node in rules[unit_from]:
        rules[unit_add].add(node)

    are_units, unit_add, unit_from = check_for_unit(rules)

for k in keys:
    if is_unaccessible(rules, k):
        rules.pop(k, None)

keys = list(rules.keys())
substitution = {"a": "X1", "b": "X2", "X2B": "Y1"}
subs_keys = list(substitution.keys())

for key in keys:
    for node in rules[key]:
        if len(node) == 2 and not node.isupper() and not node.islower():
            for ch in node:
                if ch.islower():
                    new_node = node.replace(ch, substitution[ch])
                    rules[key].remove(node)
                    rules[key].add(new_node)
                    break
        elif len(node) == 3 and node[:2] not in substitution.values() and node[:2] in subs_keys:
            new_node = node.replace(node[:2], substitution[node[:2]])
            rules[key].remove(node)
            rules[key].add(new_node)

        elif len(node) == 3 and node[:2] not in substitution.values() and node[:2] not in subs_keys:
            new_node = node.replace(node[:2], substitution[node[0]] + node[1])
            new_node1 = new_node.replace(new_node[:3], substitution[new_node[:3]])
            rules[key].remove(node)
            rules[key].add(new_node1)

for k in substitution.keys():
    if k not in rules:
        rules[k] = {substitution.get(k)}
    elif k in rules:
        rules[k].add(substitution.get(k))

print(rules)


import asyncio
import random

from prosumer import Prosumer

class PrintNumbersProducer(Prosumer):
    """print numbers asynchronously"""

    def __init__(self, data: int, context):
        super().__init__(data, context)

    async def produce(self):
        self.logger.debug('%s producer starting...', self.name)
        for i in range(self.data):
            self.add_task(i)
        await asyncio.gather(*self.tasks)
        self.logger.debug('[%s] finished', self.name)
        self.context.prosumers.remove(self)

    async def fill_queue(self):
        for i in range(1, self.data):
            await self.queue.put(i)

    async def work(self, number):
        sleep_time = random.randint(1, 3)
        await asyncio.sleep(sleep_time)
        self.logger.info(number)
        self.append()

    @property
    def name(self):
        return 'print_number'


'''
给定一个非负整数 numRows，生成杨辉三角的前 numRows 行。

在杨辉三角中，每个数是它左上方和右上方的数的和。

示例:

输入: 5
输出:
[
     [1],
    [1,1],
   [1,2,1],
  [1,3,3,1],
 [1,4,6,4,1]
]
在真实的面试中遇到过这道题？

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/pascals-triangle
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
'''

class Solution:
    def generate(self, numRows: int) -> List[List[int]]:
        if numRows==0: return []
        triangle = []
        triangle.append([1])
        for i in range(1, numRows):
            temp = [1 for _ in range(i+1)]
            for j in range(1, i):
                temp[j] = triangle[i-1][j-1] + triangle[i-1][j]
            triangle.append(temp)
        return triangle