#!/usr/bin/env python

"""Docstring"""
import csv
import gzip
import math
import random
import sys
import zlib
from collections import defaultdict
from pathlib import Path

import numpy
import pandas
import requests
from pyfaidx import Fasta
from requests.adapters import HTTPAdapter
from sklearn import metrics
from sklearn.metrics import auc, precision_recall_curve
from urllib3 import Retry

# DNA alphabet is A/C/G/T
DNA_ALPHABET = ["C", "G", "T", "A"]
# CG proportion of 40.87% per Piovesan et al (2019), BMC Res. Notes)
HSA_DNA_PROBABILITIES = [0.20435, 0.20435, 0.29565, 0.29565]
# AT content of intergenic regions of P falciparum per doi:10.1093/nar/gkw1259
PFA_DNA_PROBABILITIES = [0.068, 0.068, 0.432, 0.432]
# GC content of S. coelicolor per https://doi.org/10.1038/417141a
SCO_DNA_PROBABILITIES = [0.3606, 0.3606, 0.1394, 0.1394]
# Even split nucleotides
EVEN_DNA_PROBABILITIES = [0.25, 0.25, 0.25, 0.25]

SPECIES_NAME_DICT = {"hsa":"Homo sapiens", "pfa":"Plasmodium falciparum",
                     "sco":"Streptomyces coelicolor", "even":"50% GC content"}
RAND_SEQ_CLASS_LABEL = 0
RAND_IN_READ_STR_LABEL = 1
REPEAT_READ_LABEL = 2
FLANKING_READ_LABEL = 3

def compress(string):
    data = bytes(string, 'utf-8')
    # 1=Z_BEST_SPEED, -1=Z_DEFAULT_COMPRESSION, 9=Z_BEST_COMPRESSION
    s_out = zlib.compress(data, level=1)
    compression = (len(s_out)/len(data))
    return compression

def get_sequence_by_coord_local(selected_chromo, selected_start_pos,
                                selected_end_pos, selected_strand, reader):
    # TODO: strand selection
    return reader[selected_chromo][selected_start_pos:selected_end_pos].seq

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

def get_sequence_by_coord(selected_chromo, selected_start_pos, selected_end_pos, selected_strand):
    server = "http://rest.ensembl.org"
    ext = "/sequence/region/human/"+selected_chromo+":"+str(selected_start_pos)+".."+str(selected_end_pos)+":"+ ("1" if selected_strand == "+" else "-1")+ "?coord_system_version=GRCh37;mask=soft"
    s = requests.Session()
    r = requests_retry_session(session=s).get(server + ext, headers={"Content-Type": "text/plain"})
    if not r.ok:
        print(r)
        r.raise_for_status()
        print("ERROR! Getter failed.")
        sys.exit(-2)
    else:
        return r.text.upper()

def get_random_DNA_string(genome_dict, read_len=150, reader=None):
    ''' Sample a random DNA sequence from hg19.

    :param read_len: the length of the DNA sample to return
    :return:
    '''
    sample_chr = random.choice(genome_dict)
    start_location = random.randint(1, int(sample_chr['length'])-read_len-1)
    strand = random.choice(["+", "-"])
    location_string = sample_chr['name']+":"+str(start_location)+"-"+ \
                      str(start_location+read_len)+":"+strand
    if reader:
        sample_seq = get_sequence_by_coord_local(sample_chr['name'], start_location,
                                                 (start_location+read_len), strand,
                                                 reader)
        if len(sample_seq) != read_len:
            print(len(sample_seq))
            print(location_string)
        assert len(sample_seq) == read_len
    else:
        sample_seq = get_sequence_by_coord(sample_chr['name'], start_location,
                                           (start_location+read_len-1), strand)
    return location_string, sample_seq

def get_random_pseudoDNA_string(probabilities, read_len=150):
    ''' Generate a random DNA-like string that has the same base distribution
    as human DNA (ie, CG proportion of 40.87% per Piovesan et al (2019), BMC
    Res. Notes).
    :param read_len: the length of the DNA-like string to return
    :return:
    '''
    random_seq = random.choices(DNA_ALPHABET, weights=probabilities,
                                k=read_len)
    return "".join(random_seq)

def get_repeat_string(repeat_length, repeat_motif=None):
    ''' Get a repeat length '''
    repeat_string = repeat_motif * int(repeat_length / len(repeat_motif))
    repeat_string += repeat_motif[0:repeat_length % len(repeat_motif)]
    return repeat_string

def get_DNA_random_samples(read_len=150, samplelimit=100000):
    if REFERENCE_FILE_PATH != "ENSEMBL":
        genome_dict_path = Path(REFERENCE_FILE_PATH).with_suffix(".superSTR.dict")
        if not genome_dict_path.exists():
            print("Generating contig length dictionary. This may take some time.")
            len_dict = defaultdict(int)
            with open(REFERENCE_FILE_PATH, 'rt') as in_ref:
                current_header = in_ref.readline().strip().replace(">", "")
                for line in in_ref:
                    if ">" in line:
                        current_header = line.strip().replace(">", "")
                    else:
                        len_dict[current_header] += len(line.strip())
            with open(genome_dict_path, 'wt') as genome_file:
                for key in len_dict:
                    genome_file.write(str(key)+"\t"+str(len_dict[key])+"\n")
        with open(genome_dict_path, 'rt') as genome_file:
            genome_dict = list(csv.DictReader(genome_file, fieldnames=["name", "length"], delimiter='\t'))
        reader = Fasta(REFERENCE_FILE_PATH)
    else:
        print("You have selected ENSEMBL download of human sequence. Depending on your internet connection this can take significant time; you may want to download this sequence and use the --reference_path option instead.")
        if not Path("hg19.chrom.sizes").exists():
            print("Downloading UCSC contig length dictionary...")
            url = "http://hgdownload.cse.ucsc.edu/goldenpath/hg19/bigZips/hg19.chrom.sizes"
            response = requests.get(url)
            open("hg19.chrom.sizes", "wb").write(response.content)
        with open("hg19.chrom.sizes", 'rt') as genome_file:
            genome_dict = csv.DictReader(genome_file, fieldnames=["name", "length"], delimiter='\t')
            sublist = []
            for entry in genome_dict:
                key = entry["name"]
                if "chrUn" in key or "gl" in key or "hap" in key or "ctg" in key: continue
                if key == "chrM":
                    entry["name"] = "chrMT"
                sublist.append(entry)
            genome_dict = sublist
        reader = None
    compression_ratios = []
    sample_count = 0
    sublist = []
    for g_dict in genome_dict:
        if int(g_dict["length"]) <= read_len:
            print(g_dict["name"], "< read length of ", str(read_len), "dropping")
        else:
            sublist.append(g_dict)
    genome_dict = sublist
    with open("genome_data.txt", 'wt', buffering=1) as genome_dump:
        print("Obtaining sequence...")
        while sample_count < samplelimit:
            location_string, sample_seq = get_random_DNA_string(genome_dict,
                                                                read_len=read_len,
                                                                reader=reader)
            if sample_seq.count("N") <= 10:
                compression_ratios.append(compress(sample_seq))
                genome_dump.write(location_string)
                genome_dump.write("\t")
                genome_dump.write(sample_seq)
                genome_dump.write("\n")
                sample_count += 1
    return compression_ratios

def get_repeat_read(distribution, read_len=150, repeat_motif=None, max_repeat_period=15):
    '''Generate a read that contains only repetitive sequence.

    :param read_len (int): The read length; 150nt default.
    :param repeat_motif (str): Allows manual specification of repeat motif.
    :param max_repeat_period (int): Limit length of generated repeat.

    :return:
    '''
    # Get the period (ie, the k for the repeat k-mer) length.
    repeat_period = random.randint(1, max_repeat_period)
    # Generate a random repeat motif.
    if not repeat_motif:
        repeat_motif = "".join(random.choices(DNA_ALPHABET, weights=distribution, k=repeat_period))
    read = get_repeat_string(read_len, repeat_motif)
    descriptor_string = repeat_motif + "_" + str(read_len) \
                        + "bp_in_repeat_read"
    return descriptor_string, read

def get_random_in_read_str(distribution, read_len=150, repeat_motif="ACG",
                           max_repeat_period=None, min_number_repeats=25):
    '''Generate an in-read repeat. This is a short repetitive sequence flanked
    by random sequence on both sides within the read; ie, RAND-REPEAT-RAND.
    Such repeats are relatively short.

    :param read_len (int): The read length; 150nt default.
    :param repeat_motif (str): Allows manual specification of repeat motif.
    :param max_repeat_period (int): Limit length of generated repeat.
    :param min_number_repeats (int) : The minimum number of repeats of a motif.
    :param min_rand_seq_length (int): The minimum length of flanking random
        sequence

    :return:
    '''
    # The maximum amount of the in-read repeat that can be repetitive is 80%
    maximum_repeat_size = math.floor(read_len*0.8)
    if not max_repeat_period:
        max_repeat_period = math.floor(maximum_repeat_size/min_number_repeats)
    #print(str(maximum_repeat_size)+"/"+str(min_number_repeats) + " = " + str(max_repeat_period))
    # Get the period (ie, the k for the repeat k-mer) length.
    repeat_period = random.randint(2, max_repeat_period)
    # Generate a random repeat motif.
    if not repeat_motif:
        repeat_motif = "".join(random.choices(DNA_ALPHABET,
                                              weights=distribution,
                                              k=repeat_period))
    # Generate minimum and maximum repeat substring sizes
    minimum_repeat_size = repeat_period*min_number_repeats
    # Repeat length is random within the [minimum_repeat_size,
    # maximum_repeat_size] interval
    repeat_length = random.randint(minimum_repeat_size, maximum_repeat_size)
    avail_length = read_len-repeat_length
    flank_1_length = random.randint(3, avail_length)
    flank_2_length = avail_length-flank_1_length
    # # Pick a start position for the repeat.
    # startpos = random.randint(0, read_len - repeat_length)
    # Generate the lead substring; random sequence
    read = get_random_pseudoDNA_string(distribution, read_len=flank_1_length)
    # Generate the repeat substring; repeated STRs
    read += get_repeat_string(repeat_length, repeat_motif)
    # Generate the trailing substring; random sequence
    read += get_random_pseudoDNA_string(distribution, read_len=flank_2_length)
    # Generate descriptor string.
    descriptor_string = repeat_motif + "_" + str(repeat_length) \
                        + "repeat_in_read"
    return descriptor_string, read

def get_flanking_read_str(distribution, read_len=150, repeat_motif="ACG", left_assign=None,
                          max_repeat_period=None, min_number_repeats=10):
    '''Generate a flanking read. This is a short repetitive sequence joined to
    a random sequence; either RAND-REPEAT or REPEAT-RAND.

    :param read_len (int): The read length; 150nt default.
    :param repeat_motif (str): Allows manual specification of repeat motif.
    :param left_assign (bool): Allows manual specification of where the repeat
        is joined to the random sequence.
    :param max_repeat_period (int): Limit length of generated repeat.

    :return:
    '''
    maximum_repeat_size = math.floor(read_len * 0.8)
    # Get the period (ie, the k for the repeat k-mer) length.
    if not max_repeat_period:
        max_repeat_period = math.floor(maximum_repeat_size/min_number_repeats)
    repeat_period = random.randint(2, max_repeat_period)
    minimum_repeat_size = repeat_period*min_number_repeats
    # Get length of repetitive sequence.
    repeat_length = random.randint(minimum_repeat_size, maximum_repeat_size)
    nonrepeat_length = read_len - repeat_length
    # Generate a random repeat motif.
    if not repeat_motif:
        repeat_motif = "".join(random.choices(DNA_ALPHABET,
                                              weights=distribution,
                                              k=repeat_period))
    # Get flanking sequence string.
    seq_str = get_random_pseudoDNA_string(distribution, nonrepeat_length)
    # Get repeat sequence string.
    repeat_str = get_repeat_string(repeat_length, repeat_motif)
    # If we're not manually specifying the ordering, we randomly choose.
    if left_assign is None:
        left_assign = random.choice((True, False))
    if left_assign:
        # Read is sequence-repeat
        read = seq_str + repeat_str
        descriptor_string = repeat_motif + "_" + str(repeat_length) \
                            + "_seq_repeat_read"
    else:
        # Read is repeat-sequence
        read = repeat_str + seq_str
        descriptor_string = repeat_motif + "_" + str(repeat_length) \
                            + "_repeat_seq_read"
    return descriptor_string, read

def generate_pandas_sampleset(distribution, read_len=150):
    ratio_list = []
    label_list = []
    retain_list = []
    for i in range(0, 50000):
        if read_len > 25*3:
            ratio_list.append(
                compress(get_random_in_read_str(distribution, read_len)[1]))
            label_list.append(RAND_IN_READ_STR_LABEL)
            retain_list.append(1)
        if read_len > 10*3:
            ratio_list.append(
                compress(get_flanking_read_str(distribution, read_len)[1]))
            label_list.append(FLANKING_READ_LABEL)
            retain_list.append(1)
        ratio_list.append(compress(get_random_pseudoDNA_string(distribution, read_len)))
        label_list.append(RAND_SEQ_CLASS_LABEL)
        retain_list.append(0)
        ratio_list.append(compress(get_repeat_read(distribution, read_len)[1]))
        label_list.append(REPEAT_READ_LABEL)
        retain_list.append(1)
    for i in range(0, 50000*4):
        ratio_list.append(compress(get_random_pseudoDNA_string(distribution, read_len)))
        label_list.append(RAND_SEQ_CLASS_LABEL)
        retain_list.append(0)
    d = {"ratio":ratio_list, "class":label_list, "retain":retain_list}
    return pandas.DataFrame(d)

def render_distributions(random_sequence_ratios, repeat_in_read_ratios,
                         read_in_repeat_ratios, flanking_read_ratios, species,
                         dna_samples=None, read_len=150, threshold=0.55,
                         render_output_path="plots/"):
    import matplotlib.pyplot as plt
    import seaborn as sns
    sns.distplot(repeat_in_read_ratios, hist=False, rug=False,
                 label="containing read")
    sns.distplot(read_in_repeat_ratios, hist=False, rug=False,
                 label="repeat read")
    sns.distplot(flanking_read_ratios, hist=False, rug=False,
                 label="flanking read")
    sns.distplot(random_sequence_ratios, hist=False, rug=False,
                 label="pseudorandom DNA/random read", color="black")
    if dna_samples:
        sns.distplot(dna_samples, hist=False, rug=False, label="DNA samples",
                     color="red")
    plt.title(""+str(read_len)+"nt read length")

    plt.xlabel("Compression ratio (compressed/uncompressed)")
    plt.ylabel("Density")
    plt.axvline(x=threshold, linestyle='--', color="grey")
    plt.savefig(render_output_path.joinpath(species+"_compressibility_classes_"+str(read_len)+"nt.png"), bbox_inches="tight")
    plt.close()

def get_random_quality_string(read_len):
    string_list = []
    for i in range(0, read_len):
        string_list.append(str(chr(random.randint(34, 125))))
    return "".join(string_list)

def generate_sampleset(distribution, read_len=150):
    if read_len > 30*3:
        repeat_in_read_ratios = [compress(
            get_random_in_read_str(distribution, read_len=read_len)[1])
                                 for i in range(0, 100000)]
    random_sequence_ratios = [compress(get_random_pseudoDNA_string(distribution, read_len=read_len))
                              for i in range(0, 100000)]
    read_in_repeat_ratios = [compress(get_repeat_read(distribution, read_len=read_len)[1])
                             for i in range(0, 100000)]
    flanking_read_ratios = [compress(get_flanking_read_str(distribution, read_len=read_len)[1])
                            for i in range(0, 100000)]
    return random_sequence_ratios, repeat_in_read_ratios, \
           read_in_repeat_ratios, flanking_read_ratios

def write_choice(choice_str, read_idx, read_len, file_handler, distribution):
    if choice_str == "random":
        file_handler.write("@rand_" + str(read_idx) + "\n" + get_random_pseudoDNA_string(distribution, read_len=read_len))
        file_handler.write("\n+\n"+get_random_quality_string(read_len)+"\n")
    elif choice_str == "rir":
        file_handler.write("@randir_" + str(read_idx) + "\n" + get_random_in_read_str(distribution, read_len=read_len)[1])
        file_handler.write("\n+\n" + get_random_quality_string(read_len) + "\n")
    elif choice_str == "pure":
        file_handler.write("@rir_" + str(read_idx) + "\n" + get_repeat_read(distribution, read_len=read_len)[1])
        file_handler.write("\n+\n" + get_random_quality_string(read_len) + "\n")
    elif choice_str == "flanking":
        file_handler.write("@flank_" + str(read_idx) + "\n" + get_flanking_read_str(distribution, read_len=read_len)[1])
        file_handler.write("\n+\n" + get_random_quality_string(read_len) + "\n")
    else:
        print("ERROR")
        import sys
        sys.exit(-1)

def generate_and_save_simulated_samples(distribution, output_root, species):
    out_root_folder = output_root.joinpath("simulations/")
    out_root_folder.mkdir(parents=True, exist_ok=True)
    out_random_path = out_root_folder.joinpath(species+"_random_seq.fa.gz")
    random_sequence_ratios = []
    with gzip.open(out_random_path, 'wt') as random_seq_file:
        for i in range(0, sample_size):
            seq = get_random_pseudoDNA_string(distribution,
                                              read_len=master_read_len)
            random_seq_file.write(">rand_" + str(i) + "\n")
            random_seq_file.write(seq)
            random_seq_file.write("\n")
            random_sequence_ratios.append(compress(seq))
    repeat_in_read_ratios = []
    out_repeat_in_read_path = out_root_folder.joinpath(species+"_repeat_in_read.fa.gz")
    with gzip.open(out_repeat_in_read_path, 'wt') as repeat_seq_file:
        for i in range(0, sample_size):
            detail_string, seq = get_random_in_read_str(distribution,
                                                        read_len=master_read_len)
            repeat_seq_file.write(
                ">repeat_in_read_" + str(i) + "_" + detail_string + "\n")
            repeat_seq_file.write(seq)
            repeat_seq_file.write("\n")
            repeat_in_read_ratios.append(compress(seq))
    read_in_repeat_ratios = []
    out_read_in_repeat_path = out_root_folder.joinpath(species+"_read_in_repeat.fa.gz")
    with gzip.open(out_read_in_repeat_path, 'wt') as repeatread_seq_file:
        for i in range(0, sample_size):
            detail_string, seq = get_repeat_read(distribution,
                                                 read_len=master_read_len)
            repeatread_seq_file.write(
                ">read_in_repeat_" + str(i) + "_" + detail_string + "\n")
            repeatread_seq_file.write(seq)
            repeatread_seq_file.write("\n")
            read_in_repeat_ratios.append(compress(seq))
    flanking_read_ratios = []
    out_flank_path = out_root_folder.joinpath(species+"_flanking.fa.gz")
    with gzip.open(out_flank_path, 'wt') as flank_seq_file:
        for i in range(0, sample_size):
            detail_string, seq = get_flanking_read_str(distribution,
                                                       read_len=master_read_len)
            flank_seq_file.write(
                ">flanking_" + str(i) + "_" + detail_string + "\n")
            flank_seq_file.write(seq)
            flank_seq_file.write("\n")
            flanking_read_ratios.append(compress(seq))

if __name__ == "__main__":
    # Argparser for command line arguments.
    import argparse
    parser = argparse.ArgumentParser()
    mode_group = parser.add_argument_group(title="Mode selection (one argument required)")
    mode_group.add_argument("--simulation", dest="simulation",
                            action="store_true",
                            help="Generate (and analyse) simulated data.")
    mode_group.add_argument("--performance", action="store_true",
                            dest="performance", help="Generate PRC and ROC "
                                                     "curves for the "
                                                     "compressor.")
    required_group = parser.add_argument_group(title="Required arguments")
    required_group.add_argument("--output", action="store", required=True, dest="output_path",
                        help="Output path for files.")
    parser.add_argument("--sample", action="store_true", dest="sample",
                        help="Use sampled DNA sequence, not pseudorandom. Requires either --reference_path or --ensembl flags set; the latter requires an internet connection.")
    parser.add_argument("-r", action="store", dest="read_len",
                        help="Read length for simulation (default: 150nt)")
    parser.add_argument("-t", action="store", dest="threshold",
                        help="Compressibility threshold.")
    parser.add_argument("--reference_path", action="store", dest="reference_path",
                        help="Path to genome reference for sampling; must be uncompressed FASTA")
    parser.add_argument("--write_sims", action="store_true", dest="write_sims",
                        help="Write simulated reads to disk; requires output "
                             "path.")

    parser.add_argument("--sample_size", action="store", dest="sample_size",
                        help="Number of sequences to generate/sample.")
    parser.add_argument("--hsa", action="store_true", dest="hsa",
                        default=False,
                        help="Simulate with H. sapiens genomic GC content.")
    parser.add_argument("--sco", action="store_true", dest="sco",
                        default=False,
                        help="Simulate with S. coelicolor genomic GC content.")
    parser.add_argument("--pfa", action="store_true", dest="pfa",
                        default=False,
                        help="Simulate with P. falciparum genomic GC content.")
    parser.add_argument("--even", action="store_true", dest="even",
                        default=False,
                        help="Simulate with even probabilities for each "
                             "nucleotide")
    parser.add_argument("--all", action="store_true", dest="all", default=False,
                        help="Simulate for all four GC contents.")

    parser.add_argument("--ensembl", action="store_true", dest="ensembl", default=False,
                        help="Use randomly selected ENSMEBL hg19 sequence instead of pseudorandom genomic sequence for background.")
    if len(sys.argv) == 1:
        print("ERROR: No options specified.", file=sys.stderr)
        parser.print_help(sys.stderr)
        sys.exit(1)
    args = parser.parse_args()
    if not (args.simulation or args.performance):
        print("ERROR: Select at least one of --simulation and --performance.", file=sys.stderr)
        parser.print_help(sys.stderr)
        sys.exit(1)
    # Parse variables from input, otherwise use sensible defaults.
    master_read_len = args.read_len if args.read_len else 150
    sample_size = args.sample_size if args.sample_size else 150000
    if args.all:
        species_rundict = {"hsa":True, "sco":True, "pfa":True, "even":True}
    else:
        species_rundict = {"hsa": args.hsa, "sco": args.sco, "pfa": args.pfa,
                           "even": args.even}
    dist_dict = {"hsa":HSA_DNA_PROBABILITIES, "sco":SCO_DNA_PROBABILITIES,
                 "pfa":PFA_DNA_PROBABILITIES, "even":EVEN_DNA_PROBABILITIES}
    if args.simulation:
        if not args.threshold:
            print("ERROR: --simulation mode requires -t to be set.", file=sys.stderr)
            parser.print_help(sys.stderr)
            sys.exit(1)
        threshold = float(args.threshold)
        for species in species_rundict:
            dist = dist_dict[species]
            if not species_rundict[species]:
                continue
            # Generate simulated files:
            if args.write_sims:
                random_sequence_ratios, repeat_in_read_ratios, \
                read_in_repeat_ratios, flanking_read_ratios = \
                    generate_and_save_simulated_samples(dist,
                                                        Path(args.output_path),
                                                        species)
            else:
                random_sequence_ratios, repeat_in_read_ratios, \
                read_in_repeat_ratios, flanking_read_ratios = \
                    generate_sampleset(dist, read_len=master_read_len)
            if args.sample:
                if args.reference_path:
                    REFERENCE_FILE_PATH = args.reference_path
                elif args.ensembl:
                    REFERENCE_FILE_PATH = "ENSEMBL"
                else:
                    print("ERROR: Sampling mode requires one of --reference_path or --ensembl to be set.", file=sys.stderr)
                    parser.print_help(sys.stderr)
                    sys.exit(1)
                random_dna_ratios = get_DNA_random_samples(read_len=master_read_len)
                render_distributions(random_dna_ratios, repeat_in_read_ratios,
                                     read_in_repeat_ratios, flanking_read_ratios,
                                     species, read_len=master_read_len,
                                     threshold=threshold,
                                     render_output_path=Path(args.output_path))
            else:
                render_distributions(random_sequence_ratios,
                                     repeat_in_read_ratios,
                                     read_in_repeat_ratios,
                                     flanking_read_ratios, species,
                                     read_len=master_read_len,
                                     threshold=threshold,
                                     render_output_path=Path(args.output_path))
    elif args.performance:
        for species in species_rundict:
            if not species_rundict[species]:
                continue
            print(SPECIES_NAME_DICT[species])
            print("-"*80)
            print("Read length\tROC\tPrecision+Recall")
            if args.read_len and str.isdigit(args.read_len):
                proc_list = [int(args.read_len)]
            else:
                proc_list = [16, 36, 75, 150, 200, 250, 300]
            for rlen in proc_list:
                data_frame = generate_pandas_sampleset(dist_dict[species], read_len=rlen)
                retain_nparray = data_frame['retain'].to_numpy()
                ratio_nparray = data_frame['ratio'].to_numpy()
                fpr, tpr, thresholds = metrics.roc_curve(retain_nparray, ratio_nparray,
                                                         pos_label=0)
                optimal = numpy.argmax(tpr-fpr)
                roc_threshold = thresholds[optimal]
                roc_auc = auc(fpr, tpr)
                import matplotlib.pyplot as plt
                plt.figure()
                lw = 2
                plt.plot(fpr, tpr, color='darkorange',
                         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
                plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
                plt.xlim([0.0, 1.0])
                plt.ylim([0.0, 1.05])
                plt.xlabel('False Positive Rate')
                plt.ylabel('True Positive Rate')
                plt.title('ROC - 400,000 simulated reads\n50% retain; min ')
                plt.legend(loc="lower right")
                render_path = Path(args.output_path)
                plt.savefig(render_path.joinpath(args.output_path, species+"_ROC"+str(rlen)+"nt.png"), bbox_inches="tight")
                plt.close()
                precision, recall, thresholds = precision_recall_curve(retain_nparray,
                                                                       ratio_nparray,
                                                                       pos_label=0)
                optimal = numpy.argmax(precision + recall)
                prc_threshold = thresholds[optimal]
                print(str(rlen) + "\t" + str(roc_threshold) + "\t" + str(prc_threshold))
                plt.figure()
                lw = 2
                plt.plot(recall, precision, color='darkorange',
                         lw=lw, label='PRC curve (asrea = %0.2f)' % roc_auc)
                plt.xlim([0.0, 1.0])
                plt.ylim([0.0, 1.05])
                plt.ylabel('Precision')
                plt.xlabel('Recall')
                plt.title(
                    'PRC - gzip compression of 400,000 simulated reads\n(200,000 '
                    'retain, 200,000 reject)')
                plt.legend(loc="lower right")
                plt.savefig(Path(args.output_path).joinpath(args.output_path, species+"_PRC_"+str(rlen)+"nt.png"), bbox_inches="tight")
                plt.close()


# The problem: https://app.codility.com/programmers/trainings/4/first_unique/
from collections import Counter

def first_unique(A):
    # Counting the items, keep only those which have a count of 1:
    all_unique = [k for k, v in Counter(A).items() if v == 1]
    if len(all_unique) == 0:
        return -1
    return all_unique[0]

if __name__ == '__main__':
    A = [2, 2, 3, 4, 5, 5]
    print(first_unique(A))


#gifbot.py
import random

#sends a reaction gif of God of War
async def GetGoWGif(ctx):
    with open('resources/gow.txt') as f:
        sony_gif = f.read().splitlines()

    response = random.choice(sony_gif)
    await ctx.send(response)

#sends a reaction gif of The Last of Us 2
async def GetLou2Gif(ctx):
    with open('resources/lou2.txt') as f:
        sony_gif = f.read().splitlines()

    response = random.choice(sony_gif)
    await ctx.send(response)

#sends a reaction gif of B99
async def GetB99Gif(ctx):
    with open('resources/99.txt') as f:
        gif = f.read().splitlines()

    response = random.choice(gif)
    await ctx.send(response)

#sends a random ayt image/gif
async def GetAYT(ctx):
    with open('resources/ayt.txt') as f:
        gif = f.read().splitlines()

    response = random.choice(gif)
    await ctx.send(response)


#!python3
import helper
import argparse
import os

def extractRow(line, sep, col):
    row = line.split(sep)
    values = []
    for i in range(len(row)):
        v = row[i].strip()
        if i == col:
            key = v
        else:
            values.append(v)

    return key, values

def merge(t1, t2, sep1, sep2, id_col1, id_col2, h1, h2, outfile):

    tb1 = {}
    with open(t1,'r') as f:
        if h1:
            key_col1, hd1 = extractRow(next(f),sep1,id_col1)
        for line in f:
            if line.strip()=='':
                continue
            k, v = extractRow(line,sep1,id_col1)
            tb1[k] = v

    if not h1 and h2:
        hd1 = [ f"col:{x}" for x in range(len(v)) ] # use length of last column as headers

    with open(t2,'r') as f:
        if h2:
            key_col2, hd2 = extractRow(next(f),sep2,id_col2)
        for line in f:
            if line.strip()=='':
                continue
            k, v = extractRow(line,sep2,id_col2)
            if k in tb1:
                tb1[k] += v # extend the list

    if not h2 and h1:
        hd2 = [ f"col:{x}" for x in range(len(v)-len(hd1)) ]

    if not h1 and h2: # use the column 2 key header for the table
        key_col1 = key_col2

    if not h1 and not h2:
        hd1 = [ f"col:{x}" for x in range(len(tb1[list(tb1.keys())[0]])) ]
        hd2 = []
        key_col1 = "key"

    with open(outfile, 'w') as fo:
        # decide headers
        fo.write(f"{key_col1}{sep1}{sep1.join(hd1+hd2)}\n")
        for k, v in tb1.items():
            fo.write(f"{k}{sep1}{sep1.join(v)}\n")

if __name__=="__main__":
    parser = argparse.ArgumentParser(description="Takes two tables and merge them based on a key column.")
    parser.add_argument('table1', type=str, help='Input table 1. This is the MAIN table, only items that are found in this table will be returned in the final results.')
    parser.add_argument('table2', type=str, help='Input table 2. Entries that match the key column of table 1 will be appended to table 1.')
    parser.add_argument('outlabel', type=str, help='Output file for the merged table.')

    parser.add_argument('-id_col1', '-i1', metavar='NUM', type=int, default=0, help = 'The position number of the column to use as a key for table 1, 0 by default. Zero-indexed.')
    parser.add_argument('-id_col2', '-i2', metavar='NUM', type=int, default=0, help = 'The position number of the column to use as a key for table 2, 0 by default. Zero-indexed.')
    parser.add_argument('-sep1', '-s1', metavar='s', type=str, default='\t', help = 'The character used to separate columns in table 1. This will also be the separator format of the merged output table. TAB by default.')
    parser.add_argument('-sep2', '-s2', metavar='s', type=str, default='\t', help = 'The character used to separate columns in table 2. TAB by default.')
    parser.add_argument('-h1', action='store_true', help = 'Flag. Indicates that table 1 has a header.')
    parser.add_argument('-h2', action='store_true', help = 'Flag. Indicates that table 2 has a header.')
    args = parser.parse_args()

    merge(args.table1, args.table2, args.sep1, args.sep2, args.id_col1, args.id_col2, args.h1, args.h2, args.outlabel)


from unit_test_common import execute_csv2_request, initialize_csv2_request, ut_id, sanity_requests
from sys import argv

# lno: VV - error code identifier.

def main(gvar):
    if not gvar:
        gvar = {}
        if len(argv) > 1:
            initialize_csv2_request(gvar, selections=argv[1])
        else:
            initialize_csv2_request(gvar)

    # 01 - 05
    sanity_requests(gvar, '/vm/list/', ut_id(gvar, 'vtg1'), ut_id(gvar, 'vtu1'), ut_id(gvar, 'vtg2'), ut_id(gvar, 'vtu2'))

    # 06
    execute_csv2_request(
        gvar, 1, 'VV', 'request contained a bad parameter "invalid_unit_test".',
        '/vm/list/', group=ut_id(gvar, 'vtg1'), form_data={'invalid_unit_test': 'invalid-unit-test'},
        server_user=ut_id(gvar, 'vtu1')
    )

    # 07
    execute_csv2_request(
        gvar, 0, None, None,
        '/vm/list/', group=ut_id(gvar, 'vtg1'),
        server_user=ut_id(gvar, 'vtu1')
    )

if __name__ == "__main__":
    main(None)


from django.shortcuts import render, redirect, HttpResponse
from rest_framework import viewsets
from store.serializers import *
from datetime import datetime, timezone, timedelta
import django.utils.timezone as timezones
from store import models as Store_models
from store import forms
from org import models as Org_models
from pdm import models as Pdm_models
from general import models as General_models
from django.contrib.auth.decorators import login_required
import general.common_variable
import general.functions
import general.views
import json
import uuid

# Create your views here.

class PartLiveViewSet(viewsets.ModelViewSet):
    queryset = PartLive.objects.all()
    serializer_class = PartLiveSerializer

class StoreViewSet(viewsets.ModelViewSet):
    queryset = Store.objects.all()
    serializer_class = StoreSerializer

def leaf_node(Head_id, NO):
    '''
    所有叶子仓库uuid
    :param Head_id:
    :return:
    '''
    temp = []
    StoreCon_objs = Store_models.StoreCon.objects.filter(Last__Status__NO=NO)

    def tree_traversal(Head_id):
        # 寻找下一级仓库
        val = StoreCon_objs.filter(Head_id=Head_id).values('Last_id')
        # 循环遍历下一级仓库queryset
        for row in val:
            # 判断下一级仓库是在否还存在下一级仓库
            if StoreCon_objs.filter(Head_id=row['Last_id']):
                val = StoreCon_objs.filter(Head_id=row['Last_id']).values('Last_id')
                # 还存下一级仓库则进行递归
                tree_traversal(row['Last_id'])
            else:
                # 不存在下一级仓库则为leaf_node将其存入列表中
                if row['Last_id']:
                    temp.append(row['Last_id'])

    tree_traversal(Head_id)
    return temp

def all_node(Head_id, NO):
    '''
    所有仓库uuid
    :param Head_id:
    :return:
    '''
    temp = []
    StoreCon_objs = Store_models.StoreCon.objects.filter(Last__Status__NO=NO)

    def tree_traversal(Head_id):

        # 讲该节点存入列表中
        temp.append(Head_id)
        # 寻找下一级仓库
        val = StoreCon_objs.filter(Head_id=Head_id).values('Last_id')
        # 循环遍历下一级仓库queryset
        for row in val:
            # 判断下一级仓库是在否还存在下一级仓库
            if StoreCon_objs.filter(Head_id=row['Last_id']):
                val = StoreCon_objs.filter(Head_id=row['Last_id']).values('Last_id')
                # 还存下一级仓库则进行递归
                tree_traversal(row['Last_id'])
            else:
                # 不存在下一级仓库则为leaf_node将其存入列表中
                if row['Last_id']:
                    temp.append(row['Last_id'])

    tree_traversal(Head_id)
    return temp

# 存储操作名称和函数直接对应关系
_registered_actions = {}

# 装饰器, 将操作名称存入dict
def actions(name):
    def decorator(f):
        _registered_actions[name] = f
        return f

    return decorator

# 存储查询操作对不同表的操作函数和操作名称的对应关系
_registered_tables = {}

# 装饰器, 将表名存入dict
def table_chose(name):
    def decorator(f):
        _registered_tables[name] = f
        return f

    return decorator

tzutc_8 = timezone(timedelta(hours=8))

# 处理uuid对象和datetime不能直接被序列号
class Encoder(json.JSONEncoder):
    def default(self, obj):

        # 处理uuid对象
        if isinstance(obj, uuid.UUID):
            return str(obj)
        # 处理datetime对象
        if isinstance(obj, datetime):
            return obj.astimezone(tzutc_8).strftime('%Y-%m-%d %H:%M:%S')
        return json.JSONEncoder.default(self, obj)

# 展示store模块各页面
@login_required(login_url='/general/login/')
def Store(request):
    if request.method == 'GET':
        Company_objs = Org_models.Company.objects.values(
            'id', 'Name'
        )
        Status_objs = Store_models.Store_Status.objects.values(
            'id', 'Name'
        )

        return render(request, 'store_Store.html', {
            'Company_objs': Company_objs,
            'Status_objs': Status_objs

        })

@login_required(login_url='/general/login/')
def PartLive(request):
    if request.method == 'GET':
        Part_obj = Pdm_models.Part.objects.values(
            'id', 'Name'
        )
        return render(request, 'store_PartLive.html', {
            'Part_objs': Part_obj,
        })

@login_required(login_url='/general/login/')
def StoreConPartLive(request):
    if request.method == 'GET':
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'Part__Name'
        )

        Store_objs = Store_models.Store.objects.values(
            'id', 'Status__Name', 'Name'
        )
        return render(request, 'store_StoreConPartLive.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs
        })

@login_required(login_url='/general/login/')
def StoreCon(request):
    if request.method == 'GET':
        StoreCon_objs = Store_models.StoreCon.objects.values(
            'Head_id', 'Head__Name', 'Last_id', 'Last__Name'
        )
        Store_objs = Store_models.Store.objects.values(
            'id', 'Status__Name', 'Name'
        )

        return render(request, 'store_StoreCon.html', {
            'StoreCon_objs': StoreCon_objs,
            'Store_objs': Store_objs

        })

@login_required(login_url='/general/login/')
def Store_Status(request):
    if request.method == 'GET':
        return render(request, 'store_Store_Status.html')

@login_required(login_url='/general/login/')
def Form_Status(request):
    if request.method == 'GET':
        return render(request, 'store_Form_Status.html', {})

@login_required(login_url='/general/login/')
def StoreForm(request):
    if request.method == 'GET':
        Form_Status_objs = Store_models.Form_Status.objects.values(
            'id', 'NO', 'Name'
        )

        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )

        return render(request, 'store_StoreForm.html', {
            'Form_Status_objs': Form_Status_objs,
            'Store_objs': Store_objs,

        })

@login_required(login_url='/general/login/')
def StoreFormConPartLive(request):
    if request.method == 'GET':
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_StoreFormConPartLive.html', {
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_Interface(request):
    if request.method == 'GET':
        return render(request, 'store_Interface.html')

@login_required(login_url='/general/login/')
def Store_CRUD_Interface(request):
    if request.method == 'GET':
        return render(request, 'store_CRUD_Interface.html')

@login_required(login_url='/general/login/')
def Store_PutIn(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_PutIn.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_PutInTemp(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_PutInTemp.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_Stock(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        return render(request, 'store_Store_Stock.html', {
            'Store_objs': Store_objs,
        })

@login_required(login_url='/general/login/')
def Store_StockTemp(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        return render(request, 'store_Store_StockTemp.html', {
            'Store_objs': Store_objs
        })

def edit_PartLive(request):
    if request.method == 'GET':
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'Part__Name'
        )

        Store_objs = Store_models.Store.objects.values(
            'id', 'Status__Name', 'Name'
        )
        if request.session['Account_Type'] == '2':
            Store_objs = Store_models.Store.objects.filter(
                Company__id=request.session['Company_id']
            ).values(
                'id', 'Status__Name', 'Name'
            )

        return render(request, 'edit_PartLive.html', {
            'PartLive_objs': PartLive_objs,
            'Store_objs': Store_objs

        })

@login_required(login_url='/general/login/')
def Store_ReceiveTemp(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_ReceiveTemp.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_RefuseTemp(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_RefuseTemp.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_Receive(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_Receive.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

@login_required(login_url='/general/login/')
def Store_Remove(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )

        return render(request, 'store_Store_Remove.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

# 送货单
@login_required(login_url='/general/login/')
def Store_001Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_001Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs

        })

# 收货单
@login_required(login_url='/general/login/')
def Store_002Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_002Order.html', {
            'Store_objs': Store_objs,
            'PartLive_objs': PartLive_objs

        })

# 领货单
@login_required(login_url='/general/login/')
def Store_003Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_003Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs

        })

# 退货单
@login_required(login_url='/general/login/')
def Store_004Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_004Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

# 送料单
@login_required(login_url='/general/login/')
def Store_005Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_005Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

# 入库单
@login_required(login_url='/general/login/')
def Store_006Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_006Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

# 领料单
@login_required(login_url='/general/login/')
def Store_007Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_007Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

# 出库单
@login_required(login_url='/general/login/')
def Store_008Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_008Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

# 盘库单
@login_required(login_url='/general/login/')
def Store_009Order(request):
    if request.method == 'GET':
        Store_objs = Store_models.Store.objects.filter(Status__NO='002').values(
            'id', 'Status__Name', 'Name'
        )
        StoreForm_objs = Store_models.StoreForm.objects.values(
            'id', 'Name', 'Status__Name', 'OriginStore__Name'
        )
        PartLive_objs = Store_models.PartLive.objects.values(
            'id', 'QR_Code', 'Part__Name'
        )
        return render(request, 'store_Store_009Order.html', {
            'Store_objs': Store_objs,
            'StoreForm_objs': StoreForm_objs,
            'PartLive_objs': PartLive_objs
        })

@table_chose('Store')
def table_Store(ret, content, *args, **kwargs):
    '''
    获取Store表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        Store = Store_models.Store.objects.filter(
            Tell__contains=content.get('Tell', '').strip(),
            Name__contains=content.get('Name', '').strip(),
            Address__contains=content.get('Address', '').strip(),
        )

        if content.get('Company_id', ''):
            Store = Store.filter(
                Company_id=content.get('Company_id', '')

            )
        if content.get('Status_id', ''):
            Store = Store.filter(
                Status_id=content.get('Status_id', '')

            )

        objs = Store.values_list(
            'id', 'Status__Name', 'Status_id', 'Name', 'Tell', 'Address', 'Company__Name', 'Company_id'
        )

        header_title = ['id', '状态', 'Status_id', '名称', '电话', '地址', '公司', 'Company_id']

        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'Store')

    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('PartLive')
def table_PartLive(ret, content, *args, **kwargs):
    '''
    获取PartLive表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:

        PartLive = Store_models.PartLive.objects.filter(

            QR_Code__contains=content.get('QR_Code', '').strip(),

        )
        if content.get('Part_id', ''):
            PartLive = PartLive.filter(
                Part_id=content.get('Part_id', '')
            )

        objs = PartLive.values_list(
            'id', 'QR_Code', 'Part__ItemNO', 'Part__Name', 'Part__Material__Name', 'Part__Company__Name',
            'Part__Rev', 'Part__id'
        )
        header_title = ['id', '二维码', '物号', '名称', '材料', '公司', '版本号', 'Part__id']

        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'PartLive')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('StoreConPartLive')
def table_StoreConPartLive(ret, content, Store_id, *args, **kwargs):
    '''
    获取StoreConPartLive表数据
    :param ret:
    :param content:
    :param type:
    :param Store_id:
    :param args:
    :param kwargs:
    :return:
    '''
    type = content.get('store_type', None)
    try:

        if Store_id:
            obj = Store_models.Store.objects.filter(id=Store_id).first()
            NO = obj.Status.NO
            from django.db.models import Q
            nodes_list = []
            if type == 'node_store':
                nodes_list = [Store_id, ]
            elif type == 'all_store':
                nodes_list = all_node(Store_id, NO)
            elif type == 'all_leaf_node_store':
                nodes_list = leaf_node(Store_id, NO)

            q1 = Q()
            q1.connector = 'OR'
            if nodes_list:
                for row in nodes_list:
                    q1.children.append(('Store_id', row))
            else:
                q1.children.append(('Store_id', None))

            temp_objs = Store_models.StoreConPartLive.objects.filter(q1)
        else:
            temp_objs = Store_models.StoreConPartLive.objects.all()

        StoreConPartLive = temp_objs.filter(
            PartLive__QR_Code__contains=content.get('PartLive__QR_Code', '').strip(),
            PartLive__Part__ItemNO__contains=content.get('PartLive__Part__ItemNO', '').strip(),
            PartLive__Part__Name__contains=content.get('PartLive__Part__Name', '').strip()
        )

        objs = StoreConPartLive.values_list(
            'id', 'PartLive_id', 'PartLive__QR_Code', 'PartLive__Part__ItemNO', 'PartLive__Part__Name',
            'PartLive__Part__Material__Name', 'Store_id',
            'Store__Name', 'Store__Status__Name', 'PartLive__Part__Rev', 'Number'
        )

        header_title = ['id', 'PartLive_id', '二维码', '物号', '名称', '材料', 'Store_id', ' 所在仓库', '状态', '版本号', '数量']

        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'StoreConPartLive')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('StoreCon')
def table_StoreCon(ret, content, *args, **kwargs):
    '''
    获取StoreCon表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        objs = Store_models.StoreCon.objects.values_list(
            'id', 'Head__Status__Name', 'Head__Name', 'Last__Status__Name', 'Last__Name'
        )
        header_title = ['id', '状态', '上一级仓库', '状态', '下一级仓库', ]
        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'StoreCon')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('Store_Status')
def table_Store_Status(ret, content, *args, **kwargs):
    '''
    获取Store_Status表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        objs = Store_models.Store_Status.objects.values_list(
            'id', 'NO', 'Name'
        )
        header_title = ['id', '代码', '名称']
        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'Store_Status')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('Form_Status')
def table_Form_Status(ret, content, *args, **kwargs):
    '''
    获取Store_Status表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        objs = Store_models.Form_Status.objects.values_list(
            'id', 'NO', 'Name'
        )
        header_title = ['id', '代码', '名称']
        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'Form_Status')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('StoreForm')
def table_StoreForm(ret, content, *args, **kwargs):
    '''
    获取StoreCon表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''

    try:
        if content.get('Store_id', ''):
            temp_objs = Store_models.StoreForm.objects.filter(
                OriginStore_id=content.get('Store_id', '')
            )
        else:
            temp_objs = Store_models.StoreForm.objects

        if content.get('Status_id', ''):
            temp_objs = temp_objs.filter(
                Status_id=content.get('Status_id', '')

            )
        objs = temp_objs.values_list(
            'id', 'Name', 'Status_id', 'Status__Name', 'OriginStore_id', 'OriginStore__Name', 'TargetStore_id',
            'TargetStore__Name', 'created_date', 'modified_date'
        )
        header_title = ['id', '名称', 'Status_id', '类型', 'OriginStore_id', '仓库', 'TargetStore_id', '目标仓库', '创建时间',
                        '最后修改时间']
        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'StoreForm')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)
    return ret

@table_chose('StoreFormConPartLive')
def table_StoreFormConPartLive(ret, content, *args, **kwargs):
    '''
    获取StoreCon表数据
    :param ret:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        temp_objs = Store_models.StoreFormConPartLive.objects
        if content.get('StoreForm_id', ''):
            temp_objs = Store_models.StoreFormConPartLive.objects.filter(
                StoreForm_id=content.get('StoreForm_id', '')

            )
        objs = temp_objs.values_list(
            'id', 'StoreForm_id', 'StoreForm__Name', 'StoreForm__Status__Name', 'StoreForm__OriginStore__Name',
            'PartLive_id', 'PartLive__QR_Code', 'PartLive__Part__Name', 'Number'
        )

        header_title = ['id', 'StoreForm_id', '表单名称', '表单类型', '所属仓库', 'PartLive_id', '二维码', '物料', '数量']
        ret['content']['title'] = header_title
        ret['content']['value'] = list(objs)
        ret['status'] = True
        ret['message'] = '{0}{1}'.format('获取', 'StoreFormConPartLive')
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)

    return ret

@table_chose('ModuleInterface')
def table_ModuleInterface(ret, content, *args, **kwargs):
    content['Module_id'] = General_models.Module.objects.filter(APPName='store').first().id
    try:
        ret = general.views.table_ModuleInterface(ret, 'ModuleInterface', content)
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)
    return ret

@actions(4000)
def action_get(ret, table, content, Store_id, *args, **kwargs):
    '''
    获取数据 get
    :param ret:
    :param table:
    :param action:
    :param content:
    :param type:
    :param Store_id:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        f = _registered_tables[table]

    except KeyError as e:
        ret['status'] = False
        ret['message'] = str(e)
        return ret
    else:
        return f(ret, content, Store_id, type)

@actions(3000)
def action_put(ret, table, content, *args, **kwargs):
    '''
    修改数据 put
    :param ret:
    :param table:
    :param action:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    objs = getattr(forms, '{0}{1}'.format(table, 'Form'))(content)
    if objs.is_valid():
        try:
            uuid = content.pop('uuid', None)

            getattr(Store_models, table).objects.filter(id=uuid).update(**content)

            # 当修改单据时跟新存储的最后修改时间
            if table == 'StoreForm':
                getattr(Store_models, table).objects.filter(id=uuid).update(modified_date=timezones.now())

            # 当修改单据内物料时跟新存储的最后修改时间
            if table == 'StoreFormConPartLive':
                Store_models.StoreForm.objects.filter(id=content.get('StoreForm_id', None)).update(
                    modified_date=datetime.now())

            ret['status'] = True
            ret['message'] = '{0}{1}'.format('修改数据', table)
        except Exception as e:
            ret['status'] = False
            ret['message'] = str(e)
    else:
        ret['status'] = False
        print(objs.errors.get_json_data())
        ret['errors'] = objs.errors.get_json_data()

    return ret

@actions(2000)
def action_delete(ret, table, content, *args, **kwargs):
    '''
    删数据 delete
    :param ret:
    :param table:
    :param action:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''
    try:
        uuid = content.get('uuid', None)

        # 当删除单据内物料时跟新存储的最后修改时间
        if table == 'StoreFormConPartLive':
            StoreForm_id = Store_models.StoreFormConPartLive.objects.filter(id=uuid).first().StoreForm_id
            Store_models.StoreForm.objects.filter(id=StoreForm_id).update(modified_date=timezones.now())

        getattr(Store_models, table).objects.filter(id=uuid).delete()

        ret['status'] = True
        ret['message'] = '{0}{1}'.format('删除数据', table)
    except Exception as e:
        ret['status'] = False
        ret['message'] = str(e)
    return ret

@actions(1000)
def action_add(ret, table, content, *args, **kwargs):
    '''
    新增数据 add
    :param ret:
    :param table:
    :param action:
    :param content:
    :param args:
    :param kwargs:
    :return:
    '''

    objs = getattr(forms, '{0}{1}'.format(table, 'Form'))(content)
    if objs.is_valid():
        try:
            import uuid
            content['id'] = uuid.uuid4()
            getattr(Store_models, table).objects.create(**content)

            # 当新增单据内物料时跟新存储的最后修改时间
            if table == 'StoreFormConPartLive':
                Store_models.StoreForm.objects.filter(id=content.get('StoreForm_id', None)).update(
                    modified_date=timezones.now())

            ret['status'] = True
            ret['message'] = '{0}{1}'.format('添加数据', table)
        except Exception as e:
            ret['status'] = False
            ret['message'] = str(e)
    else:
        ret['status'] = False
        print(objs.errors.get_json_data())
        ret['errors'] = objs.errors.get_json_data()

    return ret

def action_store_json(request):
    '''

    :param request:
    :return:
    '''
    if request.method == 'GET':
        pass
    if request.method == 'POST':
        ret = {'status': None, 'message': None, 'errors': None, 'content': {}}
        val = json.loads(request.body.decode('utf8'))
        content = val.get('content', None)
        action = val.get('action', None)
        table = val.get('table', None)
        Store_id = content.get('Store_id', None)

        # 进行权限判断
        if general.functions.general_has_authority(request, '/store/'):
            try:
                f = _registered_actions[action]
            except KeyError as e:
                ret['status'] = False
                ret['message'] = str(e)
            else:

                ret = f(ret, table, content, Store_id)
        else:
            try:
                ret['message'] = General_models.Message.objects.filter(NO='005').first().Content
            except Exception as e:
                ret['message'] = str(e)

        return HttpResponse(json.dumps(ret, cls=Encoder), content_type='application/json')


import streamlit as st
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
import plotly.express as px
from PIL import Image

st.set_page_config(
    page_title="Supermarket Sales",
    page_icon="ðŸ“ˆ",
    layout="centered",
    initial_sidebar_state="expanded",
    menu_items={
        'Get Help': 'https://www.google.com',
        'Report a bug': "https://github.com/Andreean99?tab=repositories",
        'About': " Visualisasi dan Hipotesis Testing Supermarket Sales Milestone 1"
    }
)

st.sidebar.title('Select a Page')
selected = st.sidebar.selectbox('Page:', ['Data Visualisasi', 'Hipotesis Testing'])

if selected == 'Data Visualisasi':
    st.title('Data Visualisasi')

    PAYMENT_COLUMN = 'payment'
    PRODUCT_COLUMN = 'product line'
    TOTAL_COLUMN = 'total'
    GENDER_COLUMN = 'gender'
    GROSS_INCOME_COLUMN = 'gross income'
    CITY_COLUMN = 'city'
    DATE_COLUMN = 'date'
    TIME_COLUMN = 'time'
    CUSTOMER_TYPE_COLUMN = 'customer type'
    DATA_URL = ('supermarket_sales - Sheet1.csv')

    @st.cache_data
    def load_data(nrows):
        data = pd.read_csv(DATA_URL, nrows=nrows)
        lowercase = lambda x: str(x).lower()
        data.rename(lowercase, axis='columns', inplace=True)
        data[DATE_COLUMN] = pd.to_datetime(data[DATE_COLUMN])
        return data

    data_load_state = st.text('Loading data...')
    data = load_data(10000)
    data_load_state.text("Done! (using st.cache)")

    if st.checkbox('Show raw data'):
        st.subheader('Raw data')
        st.write(data)

    st.subheader('Sebaran data gross income tiap kota')
    selected_city=st.selectbox('Pilih Kota', options=('Yangon', 'Naypyitaw', 'Mandalay'))
    fig1, ax1= plt.subplots(figsize=[8,5])
    sns.histplot(data[data[CITY_COLUMN]==selected_city][GROSS_INCOME_COLUMN])
    st.pyplot(fig1)

    average_yangon = data[data[CITY_COLUMN]=='Yangon'][[DATE_COLUMN,'gross income']].groupby(DATE_COLUMN).sum()
    average_naypyitaw = data[data[CITY_COLUMN]=='Naypyitaw'][[DATE_COLUMN,'gross income']].groupby(DATE_COLUMN).sum()
    average_mandalay = data[data[CITY_COLUMN]=='Mandalay'][[DATE_COLUMN,'gross income']].groupby(DATE_COLUMN).sum()

    st.subheader('Rata - rata Gross Income harian Yangon')
    st.line_chart(average_yangon)
    st.subheader('Rata - rata Gross Income harian Naypyitaw')
    st.line_chart(average_naypyitaw)
    st.subheader('Rata - rata Gross Income harian Mandalay')
    st.line_chart(average_mandalay)

    st.subheader('Total Pendapatan tiap kota')
    tipe = data.groupby(CITY_COLUMN)[TOTAL_COLUMN].sum()
    st.bar_chart(tipe)

    st.subheader('Jumlah pelanggan berdasarkan Gender')
    gen = data[GENDER_COLUMN].value_counts()
    st.bar_chart(gen)

    st.subheader('Jumlah tipe pelanggan berdasarkan gender')
    selected_type = st.radio(label='Pilih Gender', options=data[GENDER_COLUMN].unique())
    st.write('Female = Member : 261, Normal : 240')
    st.write('Male = Normal : 259, Member : 240 ')
    fig2, ax2= plt.subplots(figsize=[8,5])
    sns.histplot(data[data[GENDER_COLUMN]==selected_type][CUSTOMER_TYPE_COLUMN])
    st.pyplot(fig2)

    st.subheader('Total pembayaran berdasarkan Gender')
    st.write('Total pembayaran Female 167882, sedangkan Male 155083')
    gender_total = data.groupby(GENDER_COLUMN)[TOTAL_COLUMN].sum()
    fig3, ax3 = plt.subplots(figsize=[8,5])
    sns.barplot(x=gender_total.index,y=gender_total,orient='v')
    st.pyplot(fig3)

else :
    st.title('Hipotesis Testing')

    st.write('Berdasarkan hasil pengujian Hipotesis Testing, didapatkan p-value sebesar 0.5762514561129986 yang mana lebih besar dari alpha 0.05. Maka disimpulkan dengan rentang kepercayaan 95%, rata-rata gross income harian Yangon secara signifikan hampir sama dengan rata-rata gross income harian Naypyitaw, maka di simpulkan bahwa H0 gagal di tolak.')

    img = Image.open('hipotesis.png')

    st.subheader('H0 : Î¼ Gross Income Yangon   =  Gross Income Mandalay')
    st.subheader('H1 : Î¼ Gross Income Yangon  !=  Gross Income Mandalay')

    st.header('Plot Hipotesis testing')
    st.image(img, caption='Hipotesis Testing', width=400, use_column_width=True)

    st.write('Untuk pembuktian hipotesis testing pada gambar di atas bisa dilihat garis confidence interval berada diluar garis alternate hypotesis maka bisa di katakan bahwa H0 gagal di tolak dan bisa dikatakan Î¼ gross income harian Yangon secara signifikan hampir sama dengan Î¼ gross income harian Naypyitaw ')


# https://leetcode.com/problems/climbing-stairs/
# Runtime: 29 ms, faster than 95.46% of Python3 online submissions for Climbing Stairs.
# Memory Usage: 13.8 MB, less than 96.23% of Python3 online submissions for Climbing Stairs.

# 18 minutes

# Time Complexity: O(n)
# Space Complexity: O(1)

class Solution:
    def climbStairs(self, n: int) -> int:
        num_prev = 1
        num_prev_prev = 1
        num = 1

        for i in range(2,n+1):
            num = num_prev + num_prev_prev
            num_prev_prev = num_prev
            num_prev = num

        return num


def solution(num_list):
    evenNum = ''
    oddNum = ''
    for i in num_list:
        if i % 2 == 1:
            oddNum += str(i)
        else:
            evenNum += str(i)
    return int(oddNum) + int(evenNum)

def solution2(num_list):
    oddNum = ''.join(str(i) for i in num_list if i % 2 == 1)
    evenNum = ''.join(str(i) for i in num_list if i % 2 == 0)

    return int(oddNum) + int(evenNum)


from PIL import Image

# 24비트 BMP 이미지를 엽니다.
input_image = Image.open('infer/Wing/101201_01.bmp')
# input_image = Image.open('infer/Wing/104545_01.bmp')
# input_image = Image.open('infer/Wing/132344_01.bmp')

# 이미지를 16색 비트맵 형식으로 변환합니다.
# dither 매개변수를 사용하여 색상 감지를 활성화합니다.
# output_image = input_image.convert("P", palette=Image.ADAPTIVE, colors=10,dither=1)
quantized_image = input_image.quantize(colors=8)
# 16색 비트맵 이미지를 저장합니다.
quantized_image.save("101201_01.bmp")

print("이미지 변환이 완료되었습니다.")

# import cv2
# import numpy as np

# # 이미지 로드
# # image = cv2.imread('infer/Wing/101201_01.bmp')
# # image = cv2.imread('infer/Wing/104545_01.bmp')
# image = cv2.imread('infer/Wing/132344_01.bmp')

# # 이미지 양자화
# quantized_image = cv2.convertScaleAbs((image // 80) * 75)

# # 결과 이미지 저장
# cv2.imwrite('132344_01.bmp', quantized_image)


# from xgboost import XGBRegressor  # noqa: F401
# from catboost import CatBoostRegressor  # noqa: F401
# import lightgbm as lgb  # noqa: F401
# from sklearn.linear_model import LinearRegression  # noqa: F401
# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor, StackingRegressor  # noqa: F401

import datetime

import joblib
from django.http import HttpResponse  # noqa: F401
from django.shortcuts import redirect, render
from sklearn.preprocessing import MinMaxScaler  # noqa: F401

from .base_functions import lasted_deals_street  # noqa: E402
from .utils.calc_utils import create_list, table_to_df
from .utils.plots import *

def asset_value_page(request):

    Search = False
    neighborhoods = create_list('neighborhood')
    streets = create_list('street')

    return render(request, "asset_value.html", {
        'neighborhoods': neighborhoods,
        'Search': Search,
        'streets': streets
    })

def calc_asset_value(request):

    neighborhoods = create_list('neighborhood')
    streets = create_list('street')
    params = {}
    Search = False
    if request.method == 'GET':
        Search = True
        params['City'] = str(request.GET.get('city')).strip()
        params['Neighborhood'] = str(request.GET.get('neighborhood')).strip()
        params['Street'] = str(request.GET.get('street')).strip()
        params['Home_number'] = int(request.GET.get('home-number'))
        params['Rooms'] = int(request.GET.get('rooms'))
        params['Size'] = int(request.GET.get('size'))
        params['Floor'] = int(request.GET.get('floor'))
        params['New'] = int(request.GET.get('condition'))
        params['Year'] = int(datetime.datetime.now().year)
        df_a = table_to_df('address')

        params = calc_missing_values(df_a, params)

        models = joblib.load('core/static/saved_models.pkl')
        model = models['stacking']

        predicted_price = predict_apt_price(params, model)
        predicted_price = "₪{:,}".format(predicted_price)
        df = table_to_df('nadlan')
        neighborhood_plt = Price_Increases_Neighborhood(
            df, params['Neighborhood'])
        city_plt = Price_Change_City(df)
        last_deals = lasted_deals_street(params['Street'])
        return render(
            request, "asset_value.html", {
                'predicted_price': predicted_price,
                'neighborhoods': neighborhoods,
                'streets': streets,
                'params': params,
                'Search': Search,
                'neighborhood_plt': neighborhood_plt,
                'city_plt': city_plt,
                'last_deals': last_deals
            })

    return render(request, "asset_value.html", {
        'neighborhoods': neighborhoods,
        'Search': Search
    })

def add_neighborhood(search_street):
    df = table_to_df('nadlan')
    neighborhoods = {}
    for index, row in df.iterrows():
        neighborhood = row['neighborhood']
        street = row['street']
        if neighborhood not in neighborhoods:
            neighborhoods[neighborhood] = set()
        neighborhoods[neighborhood].add(street)

    for neighborhood, streets in neighborhoods.items():
        if search_street in streets:
            return neighborhood

    return None

def check_for_match(df, params, number, target, col_1, col_2=None):
    number = int(number)
    number_offsets = [
        number, number - 1, number + 1, number + 2, number - 2, number + 3,
        number - 3, number + 4, number - 4, number + 5, number - 5
    ]
    for number in number_offsets:
        if col_2:
            try:
                match = df.loc[(df[col_1] == params[col_1]) &
                               (df[col_2] == number), target].values[0]
                return match
            except:
                pass
        else:
            try:
                match = df.loc[(df[col_1] == params[col_1]), target].values[0]
                return match
            except:
                pass

    return int(df[target].mean())

def calc_missing_values(df, params):

    # params['lat'] = check_for_match(df, params, params['home_number'], 'lat',
    #                                 'street', 'home_number')
    # params['long'] = check_for_match(df, params, params['home_number'], 'long',
    #                                  'street', 'home_number')

    params['Gush'] = check_for_match(df, params, params['Home_number'], 'gush',
                                     'street', 'home_number')
    params['Helka'] = check_for_match(df, params, params['Home_number'],
                                      'helka', 'street', 'home_number')

    # params['distance_sea'] = calc_distance_from_the_see_TLV(
    #     params['long'], params['lat'])
    params['Neighborhood'] = add_neighborhood(params['Street'])

    # params['Train'] = calc_distance_from_train_station(params['Long'],
    #                                                    params['Lat'])

    nadlan_df = table_to_df('nadlan')
    params['Build_year'] = check_for_match(nadlan_df, params, params['Helka'],
                                           'build_year', 'gush', 'helka')

    params['Floors'] = check_for_match(nadlan_df, params, params['Helka'],
                                       'floors', 'gush', 'helka')

    year = int(datetime.datetime.now().year)
    params['Age'] = year - params['Build_year']
    params['Street_rank'] = nadlan_df.loc[(
        nadlan_df['street'] == params['Street']), 'street_rank'].max()

    params['Gush_rank'] = nadlan_df.loc[(nadlan_df['gush'] == params['Gush']),
                                        'gush_rank'].max()

    params['Helka_rank'] = check_for_match(nadlan_df, params, params['Helka'],
                                           'helka_rank', 'gush', 'helka')

    params['Neighborhood_rank'] = nadlan_df.loc[(
        nadlan_df['neighborhood'] == params['Neighborhood']),
                                                'neighborhood_rank'].max()

    for param in params:
        if params[param] is np.nan:
            params[param] = int(nadlan_df[param].mean())

    return params

def predict_apt_price(params, model):
    df = pd.DataFrame([params])
    df = df.reindex(columns=[
        "Rooms", "Floor", "Size", "Build_year", "Floors", "Year", 'Age',
        'Neighborhood_rank', 'Street_rank', 'Gush_rank', 'Helka_rank', 'New'
    ])

    scaler = joblib.load('core/static/scaler.pkl')
    X_scaled = scaler.transform(df)
    y_pred = model.predict(X_scaled)
    appraised_price = int(y_pred[0])
    return appraised_price


#-*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.support.ui import Select
import time
import csv
import re
import datetime
import sys
if(len(sys.argv) > 1):
    ASIN = sys.argv[1]
else:
    ASIN = 'B07WDF1YPL'
print (ASIN, 'asin')
driver = webdriver.Chrome()
driver.get('https://members.helium10.com/cerebro')

# class test1(driver):
#     def isElementExist(self, element):
#         flag = True
#         browser = self.driver
#         try:
#             browser.find_element_by_xpath(element)
#             return flag
#         except:
#             flag = False
#             return flag

driver.find_element_by_id("loginform-email").send_keys("hi920@139.com")
driver.find_element_by_id("loginform-password").send_keys("GDjhd698VV+")
driver.find_element_by_css_selector('#login-form > button').click()
time.sleep(5)
#input
driver.find_element_by_xpath('//*[@id="wrapper"]/div[3]/div/div/div[2]/div/div/div[1]/span/span[1]/span/ul/li/input').send_keys(ASIN)
driver.find_element_by_xpath('//*[@id="search-form"]/a[1]').click()
time.sleep(10)
# result = driver.find_element_by_xpath('/html/body/div[17]/div[7]').is_displayed()
# print (result)   /html/body/div[20]/div[7]/div/button
# /html/body/div[17]/div[7]
# /html/body/div[17]/div[7]
# /html/body/div[17]/div[7]/button
# //*[@id="w3"]/div/table/thead/tr
# //*[@id="w3"]/div/table/thead/tr/th[1]
# //*[@id="w3"]/div/table/thead/tr/th[3]

# flags = False
# try:
#     # driver.find_element_by_xpath('/html/body/div[17]/div[7]/button')
#     driver.find_element_by_class_name('confirm')
#     flags = True
# except:
#     flags = False
# print (flags)
# if(flags):
#     print('sleep 5 second')
#     time.sleep(5)
#     # driver.find_element_by_xpath('/html/body/div[17]/div[7]/button').click()
#     driver.find_element_by_class_name('confirm').click()
#     time.sleep(5)

# thNum = driver.find_element_by_xpath('//*[@id="w3"]/div/table/thead/tr').find_elements_by_tag_name('th')
# headerList = []

for num in range(1, len(thNum)+1):
    xpath = '//*[@id="w3"]/div/table/thead/tr' + '/th[' + str(num) +']'
    xpathText = driver.find_element_by_xpath(xpath).text
    if (num == 2):
        headerList.append('Amazon Choise')
    elif(num == 9):
        headerList.append('Amazon Recommended')
        headerList.append('Sponsored')
        headerList.append('Organic')
        headerList.append(xpathText)
    else:
        headerList.append(xpathText)

time.sleep(7)
Select(driver.find_element_by_xpath('//*[@id="w3"]/select')).select_by_value('150')

# driver.quit()
# print (headerList)


students = ["Nick", "John", "Keith", "Brett"]

is_student = False

print("Are your a student? Lets find out...")
question = input("Are you a student?\n(yes/no): ").strip()
while question == "" or question[0] != "y" and question[0] != "n":
    question = input("Are you a student?\n(yes/no): ")
if question[0] == "y":
    name = input("What is your name?\nname: ").strip()
    for student_name in students:
        if name == student_name:
            is_student = True
    if is_student:
        print("Welcome to class.")
    else:
        print("You're not supposed to be here.")
else:
    print("Bye!")
    exit()


import os
import sys
import re

rootdir='.';
pat2=re.compile('''(?<=['"])/todo22''');

for root, subFolders, files in os.walk(rootdir):
    for filename in files:
        if filename.find(".html")!=-1 or filename.find(".js")!=-1 or filename.find(".jsp")!=-1:
            path=root+'/'+filename;
            with open(path,'r') as f:
                lines=f.readlines();

            with open(path,'w') as f:
                for line in lines:
                    line=pat2.sub(r'''/todo''',line);
                    f.write(line);


# Author:Zhang Yuan
import MyPackage
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import seaborn as sns
import statsmodels.api as sm
from scipy import stats

#------------------------------------------------------------
__mypath__ = MyPackage.MyClass_Path.MyClass_Path("\\量化投资以Python为工具")  #路径类
myfile = MyPackage.MyClass_File.MyClass_File()  #文件操作类
myplt = MyPackage.MyClass_Plot.MyClass_Plot()  #直接绘图类(单个图窗)
myfig = MyPackage.MyClass_Plot.MyClass_Figure()  #对象式绘图类(可多个图窗)
mypltpro = MyPackage.MyClass_PlotPro.MyClass_PlotPro()  #Plot高级图系列
myfigpro = MyPackage.MyClass_PlotPro.MyClass_FigurePro()  #Figure高级图系列
mynp = MyPackage.MyClass_Array.MyClass_NumPy()  #多维数组类(整合Numpy)
mypd = MyPackage.MyClass_Array.MyClass_Pandas()  #矩阵数组类(整合Pandas)
mypdpro = MyPackage.MyClass_ArrayPro.MyClass_PandasPro()  #高级矩阵数组类
mytime = MyPackage.MyClass_Time.MyClass_Time()  #时间类
myDA = MyPackage.MyClass_DataAnalysis.MyClass_DataAnalysis()  #数据分析类
#MyPackage.MyClass_ToDefault.DefaultMatplotlibBackend()       #恢复默认设置(仅main主界面)
#------------------------------------------------------------
Path="C:\\Users\\i2011\\OneDrive\\Book_Code&Data\\量化投资以python为工具\\数据及源代码\\021"
Path2="C:\\Users\\i2011\\OneDrive\\Book_Code&Data\\量化投资以python为工具\\习题解答"

import pandas as pd
import pandas_datareader.data as web
import datetime as dt
import statsmodels.formula.api as smf
from statsmodels.api import add_constant
import statsmodels.api as sm
# 1.
wanke = web.DataReader('000002.SZ', 'yahoo', dt.datetime(2015, 1, 1), dt.datetime(2015, 12, 31))
gldc = web.DataReader('600185.SS', 'yahoo', dt.datetime(2015, 1, 1), dt.datetime(2015, 12, 31))
price = pd.concat([wanke.Close, gldc.Close], 1)
price.columns = ['wanke', 'geli']
ret = (price - price.shift(1)) / price.shift(1)
model = smf.ols('wanke~geli', data=ret).fit()
print(model.summary())

# 5.
zyhy = pd.read_table(Path2+'/Part3/004/problem21.txt',sep='\t', usecols=['zyhy', 'Date'], index_col='Date')
zyhy.index = pd.to_datetime(zyhy.index)

ret = (zyhy - zyhy.shift(1)) / zyhy.shift(1)

ThreeFactors = pd.read_table(Path2+'/Part3/004/ThreeFactors.txt',sep='\t', index_col='TradingDate')
ThreeFactors.index = pd.to_datetime(ThreeFactors.index)
ThrFac = ThreeFactors['2014']
ThrFac.columns
ThrFac = ThrFac.iloc[:, [2, 4, 6]]
dat = pd.concat([ret, ThrFac], 1).dropna()
dat = pd.concat([ret, ThrFac], 1)
model = smf.ols('zyhy~RiskPremium2+SMB2+HML2', data=dat).fit()
print(model.summary())
myDA.three_factors(dat.zyhy,dat.RiskPremium2,dat.SMB2,dat.HML2,False)

# 6.
zhongxin = pd.read_table(Path2+'/Part3/004/problem21.txt',sep='\t', usecols=['zhongxin', 'Date'], index_col='Date')
zhongxin.index = pd.to_datetime(zhongxin.index)
ret = (zhongxin - zhongxin.shift(1)) / zhongxin.shift(1)

ThreeFactors = pd.read_table(Path2+'/Part3/004/ThreeFactors.txt',sep='\t', index_col='TradingDate')
ThreeFactors.index = pd.to_datetime(ThreeFactors.index)
ThrFac = ThreeFactors['2014']
ThrFac = ThrFac.iloc[:, [2, 4, 6]]
dat = pd.concat([ret, ThrFac], 1)

model = smf.ols('zhongxin~RiskPremium2', data=dat).fit()
print(model.summary())

model2 = smf.ols('zhongxin~RiskPremium2+SMB2+HML2', data=dat).fit()
print(model2.summary())

ThrFac = ThreeFactors['2015-01']
preCAPM = model.predict(add_constant(ThrFac.RiskPremium2),transform=False)
preFactors = model2.predict(add_constant(ThrFac[['RiskPremium2', 'SMB2', 'HML2']]),transform=False)
preCAPM.plot()
preFactors.plot()
plt.show()

# 7.
codes = pd.read_csv(Path2+'/Part3/004/codes.csv', header=None, dtype=str)

ThreeFactors = pd.read_table(Path2+'/Part3/004/ThreeFactors.txt', sep='\t', index_col='TradingDate')

ThreeFactors.index = pd.to_datetime(ThreeFactors.index)

ThrFac = ThreeFactors['2014']

ThrFac = ThrFac.iloc[:, [2, 4, 6]]

def create_func(model):
    def cal_alpha(code, model_name=model):
        price = web.DataReader(code, 'yahoo', dt.datetime(2014, 1, 1), dt.datetime(2014, 12, 31)).Close
        ret = (price - price.shift(1)) / price.shift(1)
        ret.name = 'ret'
        dat = pd.concat([ret, ThrFac], 1)
        if model_name == 'CAPM':
            model = smf.ols('ret~RiskPremium2', data=dat).fit()
        elif model_name == 'factors':
            model = smf.ols('ret~RiskPremium2+SMB2+HML2', data=dat).fit()
        return (model.params[0])
    return (cal_alpha)

alpha_CAPM = list(map(create_func('CAPM'), codes[0].values))
alpha_CAPM2 = pd.Series(alpha_CAPM).sort(ascending=False, inplace=False)
alpha_CAPM2[:3]

alpha_factors = list(map(create_func('factors'), codes[0]))
alpha_factors2 = pd.Series(alpha_factors).sort(ascending=False, inplace=False)
alpha_factors2[:3]


import sys
import os
from xml.etree.ElementTree import ElementTree as ET
from xml.etree.ElementTree import Element, SubElement, Comment, tostring
import xml.etree

import YoUtil

class MmcResource:
	def __init__(self,path):
		self.path = path
		self.res_id  = None
		self.node_list=list()
		self.io_list = list()
		self.general_list = list()
		self.virtual_list = list()
		self.gateway_list = list()

	def load(self):
		YoUtil.debug_print('loading ->', self.path)
		self.tree = ET()
		self.tree.parse(self.path)
		root = self.tree.getroot()
		xml_list = root.findall('GLOBAL_PARAMS/NAME')
		if xml_list != None:
			#YoUtil.debug_print('xml_list=',xml_list)
			for xml_name in xml_list:
				res_id =  xml_name.get('RES_ID')
				#YoUtil.debug_print('Res_ID=',res_id)
				if res_id != None:
					self.res_id =res_id
		xml_list = root.findall('MOTION_DEVICES/NODE')
		if xml_list != None:
			#YoUtil.debug_print('Nodes xml_list=',xml_list)
			for xml_node in xml_list:
				self.load_node(xml_node)

		xml_list = root.findall('IO_DEVICES/NODE')
		if xml_list != None:
			#YoUtil.debug_print('Nodes xml_list=',xml_list)
			for xml_node in xml_list:
				self.load_io(xml_node)

		xml_list = root.findall('GENERAL_DEVICES/NODE')
		if xml_list != None:
			#YoUtil.debug_print('Nodes xml_list=',xml_list)
			for xml_node in xml_list:
				self.load_general(xml_node)

		xml_list = root.findall('VIRTUAL_OBJECTS/VIRTUAL_OBJECT')
		if xml_list != None:
			#YoUtil.debug_print('Nodes xml_list=',xml_list)
			for xml_node in xml_list:
				self.load_virtual(xml_node)

	def load_node(self,xml_node):
		#YoUtil.debug_print('node->',xml_node)
		if xml_node != None:
			node = MmcNode(xml_node)
			self.node_list.append(node)

	def load_io(self,xml_node):
		#YoUtil.debug_print('node->',xml_node)
		if xml_node != None:
			node = MmcNode(xml_node)
			self.io_list.append(node)

	def load_general(self,xml_node):
		#YoUtil.debug_print('node->',xml_node)
		if xml_node != None:
			node = MmcNode(xml_node)
			self.general_list.append(node)

	def load_virtual(self,xml_node):
		#YoUtil.debug_print('node->',xml_node)
		if xml_node != None:
			node = virtual_object(xml_node)
			self.virtual_list.append(node)

	def print(self):
		print('MMCResource ->',self.path)
		print('RES_ID=',self.res_id)
		print('MOTION_DEVICES:')
		for node in self.node_list:
			print('\t',node.to_string())
		print('IO_DEVICES:')
		for node in self.io_list:
			print('\t',node.to_string())
		print('GENERAL_DEVICES:')
		for node in self.general_list:
			print('\t',node.to_string())
		print('VIRTUAL_OBJECTS:')
		for node in self.virtual_list:
			print('\t',node.to_string())
		print('SUBGATEWAY:')
		for node in self.gateway_list:
			print('\t',node.to_string())

class MmcNode:
	def __init__(self,xml_node):
		self.xml_node = xml_node
		self.name=None
		self.type=None
		self.id=None
		self.node_name = None
		self.node_offset=None
		self.subtype = None
		self.drive_type=None
		self.load()

	def load(self):
		pass
		self.name=self.xml_node.get('NAME')
		self.type=self.xml_node.get('TYPE')
		self.id=self.xml_node.get('ID')
		self.node_name = self.xml_node.get('NODE_NAME')
		self.node_offset = self.xml_node.get('NODE_OFFSET')
		self.subtype = self.xml_node.get('SUBTYPE')
		self.drive_type = self.xml_node.get('DRIVE_TYPE')

	def to_string(self):
		ret= 'Node Name='+self.name+'\tID='+self.id
		if self.node_name != None:
			ret = ret+'\tNode_Name='+self.node_name
		if self.node_offset != None:
			ret = ret+'\tNode_Offset='+self.node_offset
		if self.subtype != None:
			ret = ret+'\tSubtype='+self.subtype
		if self.drive_type != None:
			ret = ret+'\tDriveType='+self.drive_type
		return ret

class virtual_object:
	def __init__(self,xml_node):
		self.xml_node = xml_node
		self.name=None
		self.type=None
		self.id=None
		self.member_list = list()
		self.load()

	def load(self):
		pass
		self.name=self.xml_node.get('NAME')
		self.type=self.xml_node.get('TYPE')
		self.id=self.xml_node.get('ID')
		xml_list = self.xml_node.findall('MEMBER')
		if xml_list!= None:
			for xml_node in xml_list:
				member = virtual_object_member(xml_node)
				self.member_list.append(member)

	def to_string(self):
		ret= 'Virtual Name='+self.name+' ID='+self.id
		if self.type != None:
			ret = ret+'\ttype='+self.type
		if len(self.member_list) > 0:
			for member in self.member_list:
				ret+= '\n\t\t'+member.to_string()
		return ret

class virtual_object_member:
	def __init__(self,xml_node):
		self.xml_node = xml_node
		self.name=None
		self.type=None
		self.id=None
		self.index_in_group = None
		self.member_list = list()
		self.load()

	def load(self):
		pass
		self.name=self.xml_node.get('NAME')
		self.type=self.xml_node.get('TYPE')
		self.id=self.xml_node.get('ID')
		self.index_in_group = self.xml_node.get('INDEX_IN_GROUP')

	def to_string(self):
		ret= 'Member Name='+self.name
		if self.id != None:
			ret+=' ID='+self.id
		if self.type != None:
			ret = ret+'\ttype='+self.type
		if self.index_in_group != None:
			ret = ret+'\tIndex_in_group='+self.index_in_group

		return ret


import json

def add_countries(apps, schema_editor):
    """
    Read geojson file and add all countries to Country model
    """
    Country = apps.get_model("api", "Country")
    geojson_path = '../frontend/src/assets/countries.json'

    with open(geojson_path) as json_file:
        data = json.load(json_file)
        for feature in data['features']:
            country = feature['properties']['ADMIN']
            entry = Country(name=country)
            entry.save()


def fact(n,r):
    f1=1
    f2=1
    for i in range(1,n+1,1):
        f1=f1*i
    for i in range(1,r+1,1):
        f2=f2*i
    value =f1/f2
    print('the value=',value)
n=int(input('enter the value of n='))
r=int(input('enter the value of r='))
fact(n,r)


import sys
import os
import json
import torch
import numpy as np
import pandas as pd

path = "\\".join(os.path.abspath(__file__).split("\\")[:-2])
sys.path.insert(0, path)
from ModelAuthorConvolution.author_matrices import concatenate_abstracts
from models import MLP, BERT
from utils import check_running_file

# change according to the actual node embedding dimension
NODE_EMBEDDING_DIM = 64

def make_inputs_targets_list(graph_file="node_embeddings.json", targets_file="train.csv"):
    """ read author BERT embeddings and graph embeddings, concatenate them and
        return them as list, meanwhile, fill a list with targets so that inputs
        and targets are correctly aligned

        also fill a dictionnary with the author who are not in the train list
    """

    bert = BERT()
    bert_sentences = bert.load_sentences()      # dictionnary ob author abstracts embeddings
    print("loading node embeddings...")
    with open(graph_file, "r") as f:
        node_embeddings = json.load(f)
    print("done.")
    node_embeddings = {int(k):v for k, v in node_embeddings.items()}
    target_df = pd.read_csv(targets_file).set_index("authorID")
    test_dic = {}

    full_embeddings = []
    targets_list = []
    print("pulling embeddings...")
    for i, auth in enumerate(bert_sentences.keys()):
        if i % 10 == 0 and i > 0:
            print("{} authors processed".format(i))
        auth = int(auth)
        bert_embedding = bert.to_embedding(bert_sentences[auth])
        try:
            h_index = target_df.loc[auth]['h_index']
        except KeyError:
            try:
                test_dic[auth] = torch.tensor(bert_embedding + node_embeddings[auth])
                continue
            except KeyError:
                continue
        try:
            node_embedding = node_embeddings[auth]
        except KeyError:
            continue
        full_embeddings.append(torch.tensor(bert_embedding + node_embedding))
        targets_list.append(h_index)

    return (full_embeddings, targets_list), test_dic

def make_predictions(
    model,
    test_dic,
    input_file="test.csv",
    output_file="test_predictions.csv"
):
    """ create a csv file containing the author indexes and the h-index predictions associated """

    path_to_predictions = "ModelBert\\data"
    try:
        os.mkdir(path_to_predictions)
    except FileExistsError:
        pass

    preds = []
    df_test = pd.read_csv(input_file)
    for i, auth in enumerate(df_test['authorID']):
        if i % 20000 == 0 and i > 0:
            print("{} authors processed".format(i))
        emb = test_dic[auth]
        preds.append(model.predict(emb))
    preds = np.array(preds)
    df_test['h_index_pred'].update(pd.Series(np.round_(preds, decimals=3)))
    df_test.loc[:,["authorID","h_index_pred"]].to_csv(
        os.path.join(path_to_predictions, output_file), index=False
    )

if __name__ == "__main__":

    # whether to train the model or not
    TRAIN = True

    # whether to make predictions or not
    MAKE_PREDICTIONS = True

    # should run from "altegrad-2020"
    TO_RUN_FROM = "code"
    if not check_running_file(TO_RUN_FROM):
        raise OSError("the file should run from \"{}/\" folder".format(TO_RUN_FROM))

    # check if author_abstracts.txt file exists, else create it
    if not os.path.exists("author_abstracts.txt"):
        concatenate_abstracts()

    bert = BERT()

    node_embedding_dim = NODE_EMBEDDING_DIM
    bert_embedding_dim = bert.dimension
    embedding_dim = node_embedding_dim + bert_embedding_dim

    # launch network
    model = MLP(embedding_dim=embedding_dim, path_to_checkpoints="ModelBert\\checkpoints")
    if not model.existing_model():
        TRAIN = True

    # train model
    if TRAIN:
        (inputs_list, targets_list), test_dic = make_inputs_targets_list()
        print("fitting model")
        model.train()
        CHECKPOINT_FILE = "BERT_checkpoint_1.pt"
        model.fit(inputs_list, targets_list, n_epochs=40, batch_size=32, save_file=CHECKPOINT_FILE)

    else:
        TO_LOAD = "BERT_checkpoint_1.pt"
        model.load_model(load_file=TO_LOAD)
        print("successfully loaded {}".format(TO_LOAD))

    # build up the predictions and put it in a csv file
    CSV_OUTPUT = "test_predictions_1.csv"
    if MAKE_PREDICTIONS:
        model.eval()
        make_predictions(model, test_dic, output_file=CSV_OUTPUT)

    FILES_TO_TEST = [
        "test_predictions_1.csv"
    ]