import collections
import tqdm
import math
import numpy as np
import json
import sys
from sympy import simplify
import torch.utils.data as torchdata

from utils.general_utils import *

RawResult = collections.namedtuple("RawResult", "unique_id logits loss")
all_ops = [
    "add", "subtract", "multiply", "divide", "exp", "greater", "table_max",
    "table_min", "table_sum", "table_average"
]

def read_txt(input_path):
    """Read a txt file into a list."""
    with open(input_path) as input_file:
        input_data = input_file.readlines()
    items = []
    for line in input_data:
        items.append(line.strip())
    return items

class Examples_dataset(torchdata.Dataset):

    def __init__(self, input_data, tokenizer, retrieve_mode, dataset_type,
                 max_seq_length, program_mode):
        self.input_data = input_data
        self.tokenizer = tokenizer
        self.retrieve_mode = retrieve_mode
        self.dataset_type = dataset_type
        self.max_seq_length = max_seq_length
        self.program_mode = program_mode

    def __getitem__(self, index: int):
        entry = self.input_data[index]
        example = read_mathqa_entry(entry, self.tokenizer, self.retrieve_mode,
                                    self.dataset_type, self.max_seq_length,
                                    self.program_mode)

        return entry, example

    def __len__(self):
        return len(self.input_data)

    @staticmethod
    def collect_fn():

        def sub_collect_fn(features):
            return [feature[0] for feature in features
                    ], [feature[1] for feature in features]

        return sub_collect_fn

def read_examples(input_path, tokenizer, op_list, const_list, retrieve_mode,
                  dataset_type, max_seq_length, program_mode):
    """Read a json file into a list of examples."""
    with open(input_path) as input_file:
        input_data = json.load(input_file)

    d = Examples_dataset(input_data, tokenizer, retrieve_mode, dataset_type,
                         max_seq_length, program_mode)
    train_dataloader = torchdata.DataLoader(d,
                                            batch_size=50,
                                            num_workers=20,
                                            collate_fn=d.collect_fn())
    new_input_data = []
    new_examples = []
    for step, batch in enumerate(tqdm(train_dataloader)):
        new_input_data.extend(batch[0])
        new_examples.extend(batch[1])
    return new_input_data, new_examples, op_list, const_list

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self,
                 unique_id,
                 example_index,
                 tokens,
                 question,
                 input_ids,
                 input_mask,
                 option_mask,
                 segment_ids,
                 options,
                 answer=None,
                 program=None,
                 program_ids=None,
                 program_weight=None,
                 program_mask=None,
                 example=None):
        self.unique_id = unique_id
        self.example_index = example_index
        self.tokens = tokens
        self.question = question
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.option_mask = option_mask
        self.segment_ids = segment_ids
        self.options = options
        self.answer = answer
        self.program = program
        self.program_ids = program_ids
        self.program_weight = program_weight
        self.program_mask = program_mask
        self.example = example

class MathQAExample(
        collections.namedtuple(
            "MathQAExample",
            "id original_question question_tokens options answer \
        numbers number_indices original_program program table")):

    def convert_single_example(self, *args, **kwargs):
        return convert_single_mathqa_example(self, *args, **kwargs)

def read_mathqa_entry(entry, tokenizer, retrieve_mode, dataset_type,
                      max_seq_length, program_mode):
    if dataset_type == 'finqa':
        question = entry["qa"]["question"]
        this_id = entry["id"]
        context = ""

        if retrieve_mode == "single":
            for ind, each_sent in entry["model_input"]:
                context += each_sent
                context += " "
        elif retrieve_mode == "gold":
            for each_con in entry["qa"]["gold_inds"]:
                context += entry["qa"]["gold_inds"][each_con]
                context += " "
        elif retrieve_mode == "none":
            # no retriever, use longformer
            table = entry["table"]
            table_text = ""
            for row in table[1:]:
                this_sent = table_row_to_text(table[0], row)
                table_text += this_sent

            context = " ".join(
                entry["pre_text"]) + " " + table_text + " " + " ".join(
                    entry["post_text"])

        context = context.strip()
        # process "." and "*" in text
        context = context.replace(". . . . . .", "")
        context = context.replace("* * * * * *", "")
        original_question = question + " " + tokenizer.sep_token + " " + context.strip(
        )

        if "exe_ans" in entry["qa"]:
            options = entry["qa"]["exe_ans"]
        else:
            options = None

        table = entry['table']

        original_question_tokens = original_question.split(' ')
        numbers = []
        number_indices = []
        question_tokens = []
        for i, tok in enumerate(original_question_tokens):
            num = str_to_num(tok)
            if num is not None:
                numbers.append(tok)
                number_indices.append(len(question_tokens))
                if tok[0] == '.':
                    numbers.append(str(str_to_num(tok[1:])))
                    number_indices.append(len(question_tokens) + 1)

            tok_proc = tokenize(tokenizer, tok, dataset_type)
            question_tokens.extend(tok_proc)
            if len(question_tokens) > max_seq_length:
                break

        if "exe_ans" in entry["qa"]:
            answer = entry["qa"]["exe_ans"]
        else:
            answer = None

        # table headers
        for row in entry["table"]:
            tok = row[0]
            if tok and tok in original_question:
                numbers.append(tok)
                tok_index = original_question.index(tok)
                prev_tokens = original_question[:tok_index]
                number_indices.append(
                    len(tokenize(tokenizer, prev_tokens, dataset_type)) + 1)

        if program_mode == "seq":
            if 'program' in entry["qa"]:
                original_program = entry["qa"]['program']
                program = program_tokenization(original_program)
            else:
                program = None
                original_program = None

        elif program_mode == "nest":
            if 'program_re' in entry["qa"]:
                original_program = entry["qa"]['program_re']
                program = program_tokenization(original_program)
            else:
                program = None
                original_program = None
        else:
            program = None
            original_program = None
    else:
        question = " ".join(entry["annotation"]["cur_dial"])
        this_id = entry["id"]
        context = ""

        if retrieve_mode == "single":
            for ind, each_sent in entry["annotation"]["model_input"]:
                context += each_sent
                context += " "
        elif retrieve_mode == "gold":
            for each_con in entry["annotation"]["gold_ind"]:
                context += entry["annotation"]["gold_ind"][each_con]
                context += " "

        elif retrieve_mode == "none":
            # no retriever, use longformer
            table = entry["table"]
            table_text = ""
            for row in table[1:]:
                this_sent = table_row_to_text(table[0], row)
                table_text += this_sent

            context = " ".join(entry["pre_text"]) + " " + \
                " ".join(entry["post_text"]) + " " + table_text

        context = context.strip()
        # process "." and "*" in text
        context = context.replace(". . . . . .", "")
        context = context.replace("* * * * * *", "")

        original_question = question + " " + tokenizer.sep_token + " " + context.strip(
        )

        if "exe_ans" in entry["annotation"]:
            options = entry["annotation"]["exe_ans"]
        else:
            options = None

        original_question_tokens = original_question.split(' ')

        numbers = []
        number_indices = []
        question_tokens = []
        for i, tok in enumerate(original_question_tokens):
            num = str_to_num(tok)
            if num is not None:
                numbers.append(tok)
                number_indices.append(len(question_tokens))
                if tok[0] == '.':
                    numbers.append(str(str_to_num(tok[1:])))
                    number_indices.append(len(question_tokens) + 1)
            tok_proc = tokenize(tokenizer, tok, dataset_type)
            question_tokens.extend(tok_proc)

        if "exe_ans" in entry["annotation"]:
            answer = entry["annotation"]["exe_ans"]
        else:
            answer = None

        # table headers
        for row in entry["table"]:
            tok = row[0]
            if tok and tok in original_question:
                numbers.append(tok)
                tok_index = original_question.index(tok)
                prev_tokens = original_question[:tok_index]
                number_indices.append(
                    len(tokenize(tokenizer, prev_tokens, dataset_type)) + 1)

        if program_mode == "seq":
            if 'cur_program' in entry["annotation"]:
                original_program = entry["annotation"]['cur_program']
                program = program_tokenization(original_program)
            else:
                program = None
                original_program = None
        elif program_mode == "nest":
            if 'program_re' in entry["annotation"]:
                original_program = entry["annotation"]['cur_program_re']
                program = program_tokenization(original_program)
            else:
                program = None
                original_program = None
        else:
            program = None
    return MathQAExample(id=this_id,
                         original_question=original_question,
                         question_tokens=question_tokens,
                         options=options,
                         answer=answer,
                         numbers=numbers,
                         number_indices=number_indices,
                         original_program=original_program,
                         program=program,
                         table=table)

def prog_token_to_indices(prog, numbers, number_indices, max_seq_length,
                          op_list, op_list_size, const_list, const_list_size):
    prog_indices = []
    for i, token in enumerate(prog):
        if token in op_list:
            prog_indices.append(op_list.index(token))
        elif token in const_list:
            prog_indices.append(op_list_size + const_list.index(token))
        else:
            if token in numbers:
                cur_num_idx = numbers.index(token)
            else:
                cur_num_idx = -1
                for num_idx, num in enumerate(numbers):
                    if str_to_num(num) == str_to_num(token):
                        cur_num_idx = num_idx
                        break
            # print(prog)
            # print(token)
            # print(const_list)
            # print(numbers)
            if cur_num_idx == -1:
                print(prog)
                print(numbers)
                print(number_indices)
            # assert cur_num_idx != -1
            prog_indices.append(op_list_size + const_list_size +
                                number_indices[cur_num_idx])
    return prog_indices

def convert_single_mathqa_example(example, is_training, tokenizer,
                                  max_seq_length, max_program_length, op_list,
                                  op_list_size, const_list, const_list_size,
                                  cls_token, sep_token):
    """Converts a single MathQAExample into an InputFeature."""
    features = []
    question_tokens = example.question_tokens
    if len(question_tokens) > max_seq_length - 2:
        print("too long")
        question_tokens = question_tokens[:max_seq_length - 2]
    tokens = [cls_token] + question_tokens + [sep_token]
    query_seq_pos = tokens.index(sep_token)
    segment_ids = [0] * len(tokens)
    # segment_ids[0:query_seq_pos + 1] = [1] * (query_seq_pos + 1)
    input_ids = tokenizer.convert_tokens_to_ids(tokens)

    input_mask = [1] * len(input_ids)
    for ind, offset in enumerate(example.number_indices):
        if offset < len(input_mask):
            input_mask[offset] = 2
        else:
            if is_training == True:
                # invalid example, drop for training
                return features

            # assert is_training == False

    padding = [tokenizer.pad_token_id] * (max_seq_length - len(input_ids))
    padding_mask = [0] * (max_seq_length - len(input_ids))
    input_ids.extend(padding)
    input_mask.extend(padding_mask)
    segment_ids.extend(padding_mask)

    # print(len(input_ids))
    assert len(input_ids) == max_seq_length
    assert len(input_mask) == max_seq_length
    assert len(segment_ids) == max_seq_length

    number_mask = [tmp - 1 for tmp in input_mask]
    for ind in range(len(number_mask)):
        if number_mask[ind] < 0:
            number_mask[ind] = 0
    option_mask = [1, 0, 0, 1] + [1] * (len(op_list) + len(const_list) - 4)
    option_mask = option_mask + number_mask
    option_mask = [float(tmp) for tmp in option_mask]

    for ind in range(len(input_mask)):
        if input_mask[ind] > 1:
            input_mask[ind] = 1

    numbers = example.numbers
    number_indices = example.number_indices
    program = example.program
    if program is not None and is_training:
        program_ids = prog_token_to_indices(program, numbers, number_indices,
                                            max_seq_length, op_list,
                                            op_list_size, const_list,
                                            const_list_size)
        program_mask = [1] * len(program_ids)
        program_ids = program_ids[:max_program_length]
        program_mask = program_mask[:max_program_length]
        if len(program_ids) < max_program_length:
            padding = [0] * (max_program_length - len(program_ids))
            program_ids.extend(padding)
            program_mask.extend(padding)
    else:
        program = ""
        program_ids = [0] * max_program_length
        program_mask = [0] * max_program_length
    assert len(program_ids) == max_program_length
    assert len(program_mask) == max_program_length
    features.append(
        InputFeatures(unique_id=-1,
                      example_index=-1,
                      tokens=tokens,
                      question=example.original_question,
                      input_ids=input_ids,
                      input_mask=input_mask,
                      option_mask=option_mask,
                      segment_ids=segment_ids,
                      options=example.options,
                      answer=example.answer,
                      program=program,
                      program_ids=program_ids,
                      program_weight=1.0,
                      program_mask=program_mask,
                      example=example))
    return features

class Feature_dataset(torchdata.Dataset):

    def __init__(self, examples, is_training, tokenizer, max_seq_length,
                 max_program_length, op_list, op_list_size, const_list,
                 const_list_size, cls_token, sep_token):
        self.examples = examples
        self.is_training = is_training
        self.tokenizer = tokenizer
        self.max_seq_length = max_seq_length
        self.max_program_length = max_program_length
        self.op_list = op_list
        self.op_list_size = op_list_size
        self.const_list = const_list
        self.const_list_size = const_list_size
        self.cls_token = cls_token
        self.sep_token = sep_token

    def __getitem__(self, index: int):
        example = self.examples[index]
        features = example.convert_single_example(
            is_training=self.is_training,
            tokenizer=self.tokenizer,
            max_seq_length=self.max_seq_length,
            max_program_length=self.max_program_length,
            op_list=self.op_list,
            op_list_size=self.op_list_size,
            const_list=self.const_list,
            const_list_size=self.const_list_size,
            cls_token=self.tokenizer.cls_token,
            sep_token=self.tokenizer.sep_token)
        res = []
        unique_id = 1000000000 + index
        for feature in features:
            feature.unique_id = unique_id
            feature.example_index = index
            res.append(feature)
        return res

    def __len__(self):
        return len(self.examples)

    @staticmethod
    def collect_fn():

        def sub_collect_fn(features):
            res = []
            for feature in features:
                res.extend(feature)
            return res

        return sub_collect_fn

def convert_examples_to_features(examples,
                                 tokenizer,
                                 max_seq_length,
                                 max_program_length,
                                 is_training,
                                 op_list,
                                 op_list_size,
                                 const_list,
                                 const_list_size,
                                 verbose=True):
    """Converts a list of DropExamples into InputFeatures."""
    feature_dataset = Feature_dataset(examples=examples,
                                      is_training=is_training,
                                      tokenizer=tokenizer,
                                      max_seq_length=max_seq_length,
                                      max_program_length=max_program_length,
                                      op_list=op_list,
                                      op_list_size=op_list_size,
                                      const_list=const_list,
                                      const_list_size=const_list_size,
                                      cls_token=tokenizer.cls_token,
                                      sep_token=tokenizer.sep_token)
    dataloader = torchdata.DataLoader(feature_dataset,
                                      batch_size=50,
                                      num_workers=20,
                                      collate_fn=feature_dataset.collect_fn())
    res = []
    for step, batch in enumerate(tqdm(dataloader)):
        res.extend(batch)
    return res

def write_predictions(all_predictions, output_prediction_file):
    """Writes final predictions in json format."""

    with open(output_prediction_file, "w") as writer:
        writer.write(json.dumps(all_predictions, indent=4) + "\n")

def _compute_softmax(scores):
    """Compute softmax probability over raw logits."""
    if not scores:
        return []

    max_score = None
    for score in scores:
        if max_score is None or score > max_score:
            max_score = score

    exp_scores = []
    total_sum = 0.0
    for score in scores:
        x = math.exp(score - max_score)
        exp_scores.append(x)
        total_sum += x

    probs = []
    for score in exp_scores:
        probs.append(score / total_sum)
    return probs

def compute_prog_from_logits(logits,
                             max_program_length,
                             example,
                             template=None):
    pred_prog_ids = []
    op_stack = []
    loss = 0
    for cur_step in range(max_program_length):
        cur_logits = logits[cur_step]
        cur_pred_softmax = _compute_softmax(cur_logits)
        cur_pred_token = np.argmax(cur_logits)
        loss -= np.log(cur_pred_softmax[cur_pred_token])
        pred_prog_ids.append(cur_pred_token)
        if cur_pred_token == 0:
            break
    return pred_prog_ids, loss

def indices_to_prog(program_indices, numbers, number_indices, max_seq_length,
                    op_list, op_list_size, const_list, const_list_size):
    prog = []
    for i, prog_id in enumerate(program_indices):
        if prog_id < op_list_size:
            prog.append(op_list[prog_id])
        elif prog_id < op_list_size + const_list_size:
            prog.append(const_list[prog_id - op_list_size])
        else:
            prog.append(numbers[number_indices.index(prog_id - op_list_size -
                                                     const_list_size)])
    return prog

def compute_predictions(all_examples, all_features, all_results, n_best_size,
                        max_program_length, tokenizer, op_list, op_list_size,
                        const_list, const_list_size):
    """Computes final predictions based on logits."""
    example_index_to_features = collections.defaultdict(list)
    for feature in all_features:
        example_index_to_features[feature.example_index].append(feature)

    unique_id_to_result = {}
    for result in all_results:
        unique_id_to_result[result.unique_id] = result

    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
        "PrelimPrediction", ["feature_index", "logits"])

    all_predictions = collections.OrderedDict()
    all_predictions["pred_programs"] = collections.OrderedDict()
    all_predictions["ref_programs"] = collections.OrderedDict()
    all_nbest = collections.OrderedDict()
    for (example_index, example) in tqdm(enumerate(all_examples)):
        features = example_index_to_features[example_index]
        prelim_predictions = []
        for (feature_index, feature) in enumerate(features):
            result = unique_id_to_result[feature.unique_id]
            logits = result.logits
            prelim_predictions.append(
                _PrelimPrediction(feature_index=feature_index, logits=logits))

        _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
            "NbestPrediction", "options answer program_ids program")

        nbest = []
        for pred in prelim_predictions:
            if len(nbest) >= n_best_size:
                break
            program = example.program
            pred_prog_ids, loss = compute_prog_from_logits(
                pred.logits, max_program_length, example)
            pred_prog = indices_to_prog(pred_prog_ids, example.numbers,
                                        example.number_indices,
                                        conf.max_seq_length, op_list,
                                        op_list_size, const_list,
                                        const_list_size)
            nbest.append(
                _NbestPrediction(options=example.options,
                                 answer=example.answer,
                                 program_ids=pred_prog_ids,
                                 program=pred_prog))

        assert len(nbest) >= 1

        nbest_json = []
        for (i, entry) in enumerate(nbest):
            output = collections.OrderedDict()
            output["id"] = example.id
            output["options"] = entry.options
            output["ref_answer"] = entry.answer
            output["pred_prog"] = [str(prog) for prog in entry.program]
            output["ref_prog"] = example.program
            output["question_tokens"] = example.question_tokens
            output["numbers"] = example.numbers
            output["number_indices"] = example.number_indices
            nbest_json.append(output)

        assert len(nbest_json) >= 1

        all_predictions["pred_programs"][example_index] = nbest_json[0][
            "pred_prog"]
        all_predictions["ref_programs"][example_index] = nbest_json[0][
            "ref_prog"]
        all_nbest[example_index] = nbest_json

    return all_predictions, all_nbest


import requests
from bs4 import BeautifulSoup

#product = input("Enter product name:")
#product = product.replace('+', '').strip()

#url = 'https://www.ebay.com/sch/i.html?_nkw='+product+'&_pgn=1'

titles = []
prices = []
solds = []
link = []

def job_data(url,items):

    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')

    links = soup.find_all('a', class_='s-item__link')
    urls = [item.get('href') for item in links]
    titles.clear()
    prices.clear()
    solds.clear()
    link.clear()
    #print(titles,prices,solds,link)
    for i in urls:
    	response = requests.get(i)
    	soup = BeautifulSoup(response.text, 'lxml')

    	try:
            title = soup.find('h1', id='itemTitle')
            #print(title)
    	except:
           title = ''

    	try:
           price = soup.find('span', id='prcIsum').text.strip()
           #print(price)
    	except:
           price = ''

    	try:
           sold = soup.find('span', class_='vi-qtyS-hot-red').text.strip()
           #print(sold)
    	except:
           sold = ''

    	titles.append(title.text.strip())
    	prices.append(price)
    	solds.append(sold)
    	#link.append(urls)
    #print(prices)
    #print(titles)
    #print(solds)
    #print(urls)

    return titles,prices,solds,urls


from django.dispatch import receiver
from django.db.models.signals import post_save
from .models import User, UserProfile

# Decorator for connecting to signal, PREFERRED METHOD
# This method requires: from django.dispatch import receiver; from django.db.models.signals import post_save
# Alternative to decorator is lower in file
@receiver(post_save, sender=User)
def post_save_create_profile_receiver(sender, instance, created, **kwargs):
    if created:
        UserProfile.objects.create(user=instance)
    else:
        try:
            profile = UserProfile.objects.get(user=instance)
            profile.save
        except:
            # Create user profile if one doesn't exist
            UserProfile.objects.create(user=instance)

# Alternative way to connect signal, however, decorator is preferred
# The method below requires from django.db.models.signals import post_save
# post_save.connect(post_save_create_profile_receiver, sender=User)


# importing modules
from flask import Flask, render_template, escape, flash, redirect
from flask_wtf.csrf import CSRFProtect
from flask_bootstrap import Bootstrap
import shops
import forms
from threading import Thread
import argparse

# declaring app name
csrf = CSRFProtect()
app = Flask(__name__)
csrf.init_app(app)
Bootstrap(app)
app.config['SECRET_KEY'] = "datraadjenooitpannekoek"

@app.route("/", methods=['GET', 'POST'])
def root():
    form = forms.SearchForm()
    searchterm = form.searchterm.data
    shopname = form.shopname.data
    minPrice = form.minPrice.data
    maxPrice = form.maxPrice.data
    return searchshoppage(searchterm, shopname, minPrice, maxPrice)

@app.route("/<searchterm>", methods=['GET', 'POST'])
def searchshoppage(searchterm, shopname=None, minPrice=None, maxPrice=None):
    form = forms.SearchForm()
    shopresult = shop.get_shop_results(
        searchterm, shopname, minPrice, maxPrice)
    return render_template(
        "index.html",
        shoplist=shop.shoplist,
        shopresult=shopresult,
        searchterm=searchterm,
        form=form
    )

# start listening
def run(ip):
    app.run(debug=True, use_reloader=False, port="3000", host=ip)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog='whyzard')
    parser.add_argument('--ip', nargs='?',
                        help='Ip adres to serve on', default="0.0.0.0")
    args = parser.parse_args()

    t = Thread(target=run, kwargs=dict(ip=args.ip))
    t.start()


#! /NLPShare/nlpenv/bin/python3

from common import *

def catNotes(notes, noteTypes, destDir):
    ntype = noteTypes.lower()
    cnt = 0
    err = 0
    print('Writing categorized notes to %s' % destDir)

    for mrn in notes:
        # access list of note entries for each patient
        data = notes[mrn]
        if not len(data[-1]) == 15: continue  # no audit data
        audit = data.pop()  # remove and return the audit data
        fDestDir = destDir + audit[-2].lower() + '/'
        ensureDirs(fDestDir)

        for note in data:
            # access each entry for a patient

            try:

                if note[-2].lower() in ntype:
                    # check the note type field
                    fName = fDestDir + mrn + '.txt'
                    if not os.path.exists(fName): cnt += 1

                    with open(fName, 'a') as f:
                        f.write(note[-1])

            except Exception as e:
                err += 1
                writeLog('Error at %s in %s: %s' % (currentTime(), sys.argv[0], str(e)), False)
    print('Total notes written: %d, errors: %d' % (cnt, err))
    return

if __name__ == '__main__':
    catNotes(loadJson(allNotesWithDiags), loadText(ancNoteTypes), dataDir + 'anc_notes/')
    catNotes(loadJson(allNotesWithDiags), loadText(labNoteTypes), dataDir + 'lab_notes/')
    catNotes(loadJson(allNotesWithDiags), loadText(otherNoteTypes), dataDir + 'other_notes/')


from django.urls import path, include
from . import views

app_name = 'portfolio'

urlpatterns = [
    path('', views.index, name='index'),
    path('blogs/', views.BlogListView.as_view(), name='blogs'),
    path('blog_detail/<int:pk>', views.BlogDetailView.as_view(), name='blog_detail'),
    # path('contact/', views.contact, name='contact'),
    path('contact/', views.ContactUs.as_view(), name='contact'),
    path('portfolio/', views.PortfolioListView.as_view() , name='portfolio'),
    path('project/', views.ProjectListView.as_view(), name='project'),
    path('services/', views.services, name='services'),
    path('about/', views.about, name='about'),
]


def solution(wallpaper):
    # answer = [51, 51, 0, 0]
    # lux, luy, rdx, rdy= 0, 1, 2, 3

    # for i, elements in enumerate(wallpaper):
    #     print(i,elements)
    #     for j, element in enumerate(elements):
    #         if element == '#':
    #             answer[lux] = min(answer[lux], i)
    #             answer[luy] = min(answer[luy], j)
    #             answer[rdx] = max(answer[rdx], i+1)
    #             answer[rdy] = max(answer[rdy], j+1)

    # return answer
    answer = []
    min_location_y = 60
    min_location_x = 60
    max_location_y = 0
    max_location_x = 0
    for i, paper in enumerate(wallpaper):
        for j, p in enumerate(paper):
            if(p == '#'):
                    if(i<min_location_y):
                        min_location_y = i
                    if( j<min_location_x):
                        min_location_x = j

                    if(i+1 > max_location_y):
                        max_location_y = i+1
                    if( j+1 > max_location_x):
                        max_location_x = j+1
    answer.append([min_location_y,min_location_x,max_location_y,max_location_x])

    return answer

wallpaper = []
wallpaper.append([".#...", "..#..", "...#."])
wallpaper.append(["..........", ".....#....", "......##..", "...##.....", "....#....."])
wallpaper.append([".##...##.", "#..#.#..#", "#...#...#", ".#.....#.", "..#...#..", "...#.#...", "....#...."])
wallpaper.append(["..", "#."])
print(solution(wallpaper))


from bs4 import BeautifulSoup
from selenium.webdriver import Chrome
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
import selenium.webdriver
from selenium.webdriver import Chrome
import time
import requests
import os
import pandas as pd
import json
from queue import Queue
import threading
import logging

logging.basicConfig(level=logging.INFO,
    format='%(asctime)s - %(levelname)s : %(message)s',
    datefmt='%Y%m%dT%H%M%S',filename='mylog.txt')
logging.debug('debug')
logging.info('info')
logging.warning('warning')
logging.error('error')
logging.critical('critical')

q = Queue()

# 抓所有車網址
def crawl_page():
    # 讓driver等頁面內容5秒，超過5秒則跳過
    # element = WebDriverWait(driver,5).until(expected_conditions.presence_of_element_located((By.CLASS_NAME,'abc-container')))
    options = selenium.webdriver.ChromeOptions()
    options.add_argument('--headless')
    driver = selenium.webdriver.Chrome(chrome_options=options, executable_path='./chromedriver')
    url='https://www.abccar.com.tw/search?tab=1&SearchType=1'
    driver.get(url)
    url_dict = {}

    for page in range(1,11): # 設定爬取10頁
        try:
            # 若網頁資料5秒後還未出現，便跳過此頁
            element = WebDriverWait(driver,5).until(expected_conditions.presence_of_element_located((By.XPATH,'/html/body/main/form/section[3]/section/div[2]/div[3]/div[3]/div[3]/button')))
            time.sleep(3)

            soup = BeautifulSoup(driver.page_source,'html.parser')
            url_list = soup.select('div[class="row row--10 ng-scope"]')[0].select('a[itemprop="mainEntityOfPage"]')

            for i in range(len(url_list)):
                url = url_list[i]['href']
                id = url[30:37]
                if id in url_dict:
                    pass
                else:
                    url_dict[id]=[url]
            # 點擊下一頁，降低被網站重複推薦同物件
            driver.find_element_by_xpath('/html/body/main/form/section[3]/section/div[2]/div[3]/div[3]/div[3]/button').click()
        except Exception as e:
            print(e,page)
        except :
            print("系統error",page)
        print(f"第{page}頁")
    print(len(url_dict))
    js = json.dumps(url_dict)
    with open(r'./abcCar_url.txt', 'w', encoding='utf-8') as f:
        f.write(js)
def crawl_img(url):
    options = selenium.webdriver.ChromeOptions()
    options.add_argument('--headless')
    driver = selenium.webdriver.Chrome(chrome_options=options, executable_path='./chromedriver')

    try:
        driver.get(url)
        time.sleep(6)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        id = url[30:37]
        article = soup.select('section[class="abc-container"]')[3].select('div[class="col col-auto"]')
        brand = soup.select('span[itemprop="name"]')[2].text
        model = soup.select('span[itemprop="name"]')[3].text
        year = article[0].text.replace("\n", "")[4:8]
        if not os.path.exists(f'./group_work/filter_all'):
            os.makedirs(f'./group_work/filter_all') # 如果該資料夾不存在則新增
        img_list = [i['src'] for i in soup.select('div[class="slick-track"]')[0].select('img[class="img-responsive"]')]
        print(brand,model,year)
        k = 0
        for i in img_list:
            k += 1 # 給圖片編號
            res_img = requests.get(url=i)
            img_content = res_img.content
            with open(f'./group_work/filter_all/' + f'{brand}_{model}_{year}_{k}_{id}_m.jpg',
                      'wb') as f:  # 依廠牌 車型 年分存入，wb二進制文字檔寫入圖片
                f.write(img_content)

    except ConnectionResetError as c:
        print(c)
        options = selenium.webdriver.ChromeOptions()
        options.add_argument('--headless')
        driver = selenium.webdriver.Chrome(chrome_options=options, executable_path='./chromedriver')
    except Exception as e:
        print(e, url)
        with open(r'./abcCar_img_error.txt', 'a', encoding='utf-8') as f:
            f.write(f'{url}\n')

def crawl_img_thread():
    with open(r'./abcCar_url.txt', "r", encoding='utf-8')as file:
        js = file.read()
    url_dict = json.loads(js)
    list = [i[0] for i in url_dict.values()]
    for j in range(len(list)):
        current_url = list[j]
        q.put(current_url) # 將網址放在Queue
    threads = []
    for t in range(3): # 建3個執行緒
        t = Parser("t"+str(t))
        t.start()
        threads.append(t)
    for thread in threads:
        thread.join() # 主線程必須等到所有threads執行完畢才繼續執行

class Parser(threading.Thread): # 此為python繼承語法

    def __init__(self, name): # 接受 name 參數
        threading.Thread.__init__(self) # initialize class
        self.name = name # 每條Parser的名子

    def run(self): # thread啟動後執行函數
        while q.empty() is False:  #檢查titles queue不為空的話，獲取URL後parse
            url = q.get() # 從Queue取網址
            crawl_img(url)

if __name__ == '__main__':
    tStart = time.time() # 起始時間
    logging.debug('debug')
    logging.info('info')
    logging.warning('warning')
    logging.error('error')
    logging.critical('critical')
    print("start crawling url and content")
    crawl_page()
    crawl_img_thread()
    tEnd = time.time() # 結束時間
    print('Cost %d seconds' % (tEnd - tStart)) # 完成花費時間


#!/usr/bin/python2.7
"""check_solr_cloud.py.

Nagios plugin for checking the status of a SolrCloud instance by
examining the clusterstate.json file on zookeeper.

Usage:

check_solr_cloud.py --zkhosts=<zookeeper hosts> --collection=<collection>

    zkhosts: Comma-delimited list of zookeeper hosts with port numbers
    collection: Solr collection to check

NOTE: aggiunto "cose" da Lorenzo Cocchi <lorenzo.cocchi@softecspa.it>

"""

import argparse
import json
import kazoo.client
import sys
import time

# At least 1 active node per shard
# Only 1 active leader per shard
# Leader is active

# Config
ZK_HOSTS = 'localhost:9001'
SOLR_INDEX = 'index'

# Exit codes
STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3

STATE_STR_OK = 'SOLR CLOUD OK'
STATE_STR_WARNING = 'SOLR CLOUD WARNING'
STATE_STR_CRITICAL = 'SOLR CLOUD CRITICAL'
STATE_STR_UNKNOWN = 'SOLR CLOUD UNKNOWN'

def exit_ok(message=None):
    if message:
        print('%s: %s' % (STATE_STR_OK, message))
    else:
        print(STATE_STR_OK)
    sys.exit(STATE_OK)

def exit_warning(message):
    print('%s: %s' % (STATE_STR_WARNING, message))
    sys.exit(STATE_WARNING)

def exit_critical(message):
    print('%s: %s' % (STATE_STR_CRITICAL, message))
    sys.exit(STATE_CRITICAL)

def main(zk_host, collections, verbose=False):
    collections = collections

    try:
        if verbose:
            print('Connect to: %s' % zk_host)
        zk_client = kazoo.client.KazooClient(hosts=zk_host, read_only=True)
        zk_client.start(timeout=5)
    except kazoo.handlers.threading.TimeoutError:
        exit_critical('Unable to connect to zookeeper hosts')

    try:
        cluster_state_str = zk_client.get('/clusterstate.json')[0]
    except kazoo.exceptions.NoNodeError:
        exit_critical('clusterstate.json file missing from zookeeper')

    try:
        if not collections:
            collections = zk_client.get_children('/collections')
        # workaround for threading bug
        time.sleep(0.001)
    except (kazoo.exceptions.NoNodeError,
            kazoo.exceptions.ZookeeperError) as e:
        exit_critical('get_children(/collections): %s' % e)

    if not collections:
        exit_critical('collections is empty')

    # cluster state
    c_state = json.loads(cluster_state_str)

    for collection in collections:
        if verbose:
            print('Check %s' % collection)
        for shard, shard_data in c_state[collection]['shards'].iteritems():
            at_least_one_active_node = False
            one_leader = False
            # replica
            for rep, rep_data in shard_data['replicas'].iteritems():
                if rep_data['state'] == 'active':
                    at_least_one_active_node = True
                if 'leader' in rep_data and rep_data['leader'] == 'true':
                    if one_leader:
                        exit_critical(
                            'More than one leader for shard: %s' % shard)
                    else:
                        one_leader = True
            if not at_least_one_active_node:
                zk_client.stop()
                exit_critical('No active nodes for shard: %s' % shard)
            if not one_leader:
                zk_client.stop()
                exit_critical('No leader for shard: %s' % shard)

    zk_client.stop()
    exit_ok()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-z', '--zkhosts',
                        help='Comma delimited list of zookeeper hosts with '
                        'port numbers', required=True)
    parser.add_argument('-c', '--collection', help='Solr colletion, can be '
                        'repeated', action='append', default=[])
    parser.add_argument('-v', '--verbose', help='verbose output',
                        action='store_true')
    args = parser.parse_args()
    main(args.zkhosts, args.collection, args.verbose)


def read_input():
    with open('input', 'r') as f:
        return int(f.read().strip())

def is_open(x, y, number):
    if x < 0 or y < 0:
        return False
    s = x * x + 3 * x + 2 * x * y + y + y * y + number
    b = str(bin(s))[2:]
    return b.count('1') % 2 == 0

def neighbours(vertex, openf):
    nghs = set()
    if openf(vertex[0] - 1, vertex[1]):
        nghs.add((vertex[0] - 1, vertex[1]))
    if openf(vertex[0] + 1, vertex[1]):
        nghs.add((vertex[0] + 1, vertex[1]))
    if openf(vertex[0], vertex[1] - 1):
        nghs.add((vertex[0], vertex[1] - 1))
    if openf(vertex[0], vertex[1] + 1):
        nghs.add((vertex[0], vertex[1] + 1))
    return nghs

visited = set()
distances = {}

def bfs(v, openf):
    Q = []
    visited.add(v)
    Q.append(v)
    while len(Q) > 0:
        vert = Q.pop(0)
        if distances[vert] < 50:
            for n in neighbours(vert, openf):
                if n not in visited:
                    visited.add(n)
                    distances[n] = distances[vert] + 1
                    Q.append(n)

def count_visited(source, openf):
    distances[source] = 0
    bfs(source, openf)
    return len(visited)

favourite_number = read_input()
open_function = lambda x, y: is_open(x, y, favourite_number)
target = (31, 39)
initial = (1, 1)
result = count_visited(initial, open_function)
print(result)


# you can write to stdout for debugging purposes, e.g.
# print "this is a debug message"

def isListSorted(lst):
    return sorted(lst) == lst or sorted(lst, reverse=True) == lst

def solution(A):
    # write your code in Python 2.7
    global indexA
    global indexB
    i = 0
    while(i < len(A)-1):
        if(A[i] > A[i+1]):
            indexA = i
            break
        else:
            indexA = -999999
        i += 1

    if (indexA == -999999):
        return True
    else:
        j = len(A)-1
        while(j > indexA):
            if(A[j] < A[indexA]):
                indexB = j
                break
            else:
                indexB = -999999
            j -= 1

        for k,num in enumerate(A):
            if num == A[indexA]:
                indexA = k
                break
        if(indexB != -999999):
            A[indexA],A[indexB]  = A[indexB],A[indexA]
            if isListSorted(A):
                return True
            else:
                return False

def main():
    A = [1,2,2,4,4,3]
    # A = [1,5,3,3,7]
    # A = [1,3,5,3,4]
    # A = [1,3,5]
    solution(A)

if __name__=='__main__':
    main()


# -*- coding: utf-8 -*-
# files in the project
import networks.curve_models as mynet
# pytorch
import torch
import torch.optim

# torchvision
from torchvision import transforms
from torch.utils.tensorboard import SummaryWriter
from torch.autograd import Variable
from torchvision.utils import save_image

# utils packages
import os
import datetime
import time
from glob import glob
import numpy as np

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

from PIL import Image
from torchsummaryX import summary
# other files in the project
from networks.model_utils import weight_init_normal
import myloss
from metrics import MetricAll
from datapipeline.data_augmentation import DataAugmentation

class Model(object):
    def __init__(self, opt=None):
        super(Model, self).__init__()
        assert opt != None, "opt is required !"
        # print('\n'.join(["%s: %s" % (key, value) for key, value in vars(opt).items()]))
        self.opt = opt
        self.cuda = True if torch.cuda.is_available() else False
        self.Tensor = torch.cuda.FloatTensor if self.cuda else torch.Tensor
        # network

        if opt.model=="ChebyAll3DAB":
            self.curvenet = mynet.ChebyAll3DAB(num_blocks=opt.num_orders, require_a=self.opt.require_a,num_feature=self.opt.num_fea, adb_type=self.opt.adb_type)

        if self.cuda:
            self.curvenet = self.curvenet.cuda()
        summary(self.curvenet, torch.rand((1, 3, 600, 400)).cuda())
        # data augmentation
        self.data_aug = DataAugmentation(self.opt.image_size)
        # creterion
        self.criterion_mse = torch.nn.MSELoss()
        self.criterion_l1 = torch.nn.L1Loss()
        self.criterion_ssim = myloss.SSIM()
        # self.criterion_tv = myloss.TVLoss(TVLoss_weight=2.0)
        # metrics
        self.metric_all = MetricAll()
        # tensorboard log
        tb_log_dir = os.path.join(opt.exp_path,opt.exp_name,"tb_runs")
        self.writer = SummaryWriter(log_dir=tb_log_dir, comment=opt.exp_name)

    def train(self, train_dataloader, eval_dataloader, logger=None):
        # model
        if self.opt.start_epoch != 0:
            checkpoint = self.load_checkpoint(ckp_name="cheby_%d" % self.opt.start_epoch)  # 不需要加.pth
            self.curvenet.load_state_dict(checkpoint["net"])
            optimizer = torch.optim.Adam(self.curvenet.parameters(), lr=self.opt.lr,
                                         betas=(self.opt.b1, self.opt.b2),
                                         weight_decay=self.opt.weight_decay)
            optimizer.load_state_dict(checkpoint["optimizer"])
            if self.opt.lr_scheduler=="exp":
                scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=self.opt.lr_gamma)
            elif self.opt.lr_scheduler=="step":
                scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                                                            step_size=self.opt.lr_step_size,
                                                            gamma=self.opt.lr_gamma)
            elif self.opt.lr_scheduler=="multistep":
                scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
                                                                 milestones=[200, 300, 320, 340, 200],
                                                                 gamma=0.8)
            scheduler.load_state_dict(checkpoint["scheduler"])
            self.opt.start_epoch += 1
        else:
            self.curvenet.apply(weight_init_normal)
            # optim
            optimizer = torch.optim.Adam(self.curvenet.parameters(), lr=self.opt.lr,
                                         betas=(self.opt.b1, self.opt.b2),
                                         weight_decay=self.opt.weight_decay)
            # optimizer
            if self.opt.lr_scheduler=="exp":
                scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=self.opt.lr_gamma)
            elif self.opt.lr_scheduler=="step":
                scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                                                            step_size=self.opt.lr_step_size,
                                                            gamma=self.opt.lr_gamma)
            elif self.opt.lr_scheduler=="multistep":
                scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
                                                                 milestones=[200, 300, 320, 340, 200],
                                                                 gamma=0.8)
        vgg_loss = myloss.PerceptualLoss(self.opt)
        vgg_loss.cuda()
        vgg16 = myloss.load_vgg16('./pretrained_model')
        # vgg16.to(torch.device("cuda"))
        vgg16.eval()
        for param in vgg16.parameters():
            param.requires_grad = False

        prev_time = time.time()
        loss_dict_list = {"loss_ssim": [],
                          #"loss_tv": [],
                          "loss_vgg": [],
                          "loss_mse": [],
                          "loss_l1": [],
                          "loss_total": []
                          }
        metrics_dict_list = {"psnr": []}
        # start train
        best_psnr = 0.0
        for epoch in range(self.opt.start_epoch, self.opt.num_epoch):
            for i, sample in enumerate(train_dataloader):
                # data augmentation
                sample = self.data_aug(sample)
                input = Variable(sample["l"].type(self.Tensor))
                gt = Variable(sample["h"].type(self.Tensor))
                out = self.curvenet(input)

                loss_ssim = self.opt.w_ssim*(1 - self.criterion_ssim(out, gt))
                loss_vgg = self.opt.w_vgg * vgg_loss(vgg16, out, gt)
                loss_mse = self.opt.w_mse*self.criterion_mse(out,gt)
                loss_l1 = self.opt.w_l1*self.criterion_l1(out,gt)
                loss = loss_ssim + loss_mse + loss_vgg + loss_l1

                optimizer.zero_grad()
                loss.backward()
                # torch.nn.utils.clip_grad_norm_(net.parameters(), opt.grad_clip_norm)
                optimizer.step()

                # Determine approximate time left
                batches_done = epoch * len(train_dataloader) + i
                batches_left = self.opt.num_epoch * len(train_dataloader) - batches_done
                time_left = datetime.timedelta(seconds=batches_left * (time.time() - prev_time))
                prev_time = time.time()

                # tensorboard  log
                loss_dict = {"loss_ssim": loss_ssim.item(),
                             #"loss_tv": loss_tv.item(),
                             "loss_vgg": loss_vgg.item(),
                             "loss_mse": loss_mse.item(),
                             "loss_l1": loss_l1.item(),
                             "loss_total": loss.item()
                             }
                self.writer.add_scalar('train_loss/total_loss', loss.item(), epoch)
                self.writer.add_scalars('train_loss/every_single_loss', loss_dict, epoch)
                loss_dict["loss_total"] = loss.item()
                for key in loss_dict.keys():
                    loss_dict_list[key].append(loss_dict[key])

                logger.info(
                    "\r[Epoch %d/%d] [Batch %d/%d] [Lr %f] [loss: %f] [loss_ssim: %f] [loss_mse: %f] [loss_l1: %f] [loss_vgg: %f] ETA: %s"
                    % (epoch+1, self.opt.num_epoch, i+1, len(train_dataloader),
                       optimizer.state_dict()['param_groups'][0]['lr'],
                       loss.item(), loss_ssim.item(), loss_mse.item(), loss_l1.item(), loss_vgg.item(), time_left))
            psnr = self.evaluate(eval_dataloader, epoch,logger=logger)
            if psnr > best_psnr:
                checkpoint_ = {
                    "net": self.curvenet.state_dict(),
                    "optimizer": optimizer.state_dict(),
                    "scheduler": scheduler.state_dict(),
                    "epoch": epoch
                }
                best_psnr = psnr
                self.save_checkpoint(checkpoint_, "cheby_best_%d_%0.3f"%(epoch,best_psnr))
            metrics_dict_list['psnr'].append(psnr)
            scheduler.step()

            if self.opt.ckp_freq != -1 and epoch % self.opt.ckp_freq == 0:
                # Save model checkpoints
                checkpoint_ = {
                    "net": self.curvenet.state_dict(),
                    "optimizer": optimizer.state_dict(),
                    "scheduler": scheduler.state_dict(),
                    "epoch": epoch
                }
                self.save_checkpoint(checkpoint_, "cheby_%d" % epoch)
        self.writer.close()

        checkpoint_ = {
            "net": self.curvenet.state_dict(),
            "optimizer": optimizer.state_dict(),
            "scheduler": scheduler.state_dict(),
            "epoch": epoch
        }
        self.save_checkpoint(checkpoint_, "cheby_latest")
        # Train finished
        self.plotloss(loss_dict_list)
        self.plotloss(metrics_dict_list,info='psnr')
        logger.info('\n[ * ]  Train finished ~ ')#, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        self.test(logger=logger)
        # self.testtime(logger=logger)
        self.test(logger=logger,best_psnr=best_psnr)

    def evaluate(self, eval_dataloader, epoch, logger=None):
        images_path = os.path.join(self.opt.exp_path, self.opt.exp_name,"result_eval")
        os.makedirs(images_path, exist_ok=True)
        logger.info('[ * ]  Evaluating %d images ' % (len(eval_dataloader) * self.opt.eval_bsz))#,time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        # error_list = []
        self.curvenet.eval()
        time_consuming = 0
        for i, sample in enumerate(eval_dataloader):
            # to cuda tensor
            l_eval = Variable(sample["l"].type(self.Tensor), requires_grad=False)
            h_eval = Variable(sample["h"].type(self.Tensor), requires_grad=False)

            with torch.no_grad():
                ts = time.time()
                out = self.curvenet(l_eval)
                te = time.time()
                # error = self.criterion_ssim(out, h_eval)
                # error_list.append(error)

            img_sample = torch.cat((l_eval.data, out.data, h_eval.data), 0)
            save_image(img_sample, "%s/rst_%d_%d.png" % (images_path, epoch, i), nrow=self.opt.eval_bsz,
                       normalize=True)
            time_consuming += (te - ts)
            self.metric_all.compute_metrics(h_eval, out)
            torch.cuda.empty_cache()
        self.curvenet.train()
        time_consuming = time_consuming / len(eval_dataloader) / self.opt.eval_bsz
        # tensorboard  log
        # self.writer.add_scalar('eval/ssim', error_list, epoch)
        self.metric_all.compute_statics()
        # logger.info("\r psnr for each test image:",self.metric_all.psnr_list)
        self.writer.add_scalars('eval/psnr', self.metric_all.psnr_statics, epoch)
        self.writer.add_scalar('eval/time comsuming', time_consuming, epoch)
        logger.info("\r evaluate finished [PSNR %f] [Average time per image: %fs]"
              % (self.metric_all.psnr_statics["mean"], time_consuming))
        psnr = self.metric_all.psnr_statics["mean"]
        self.metric_all.clear_all()

        # return np.mean(error_list)
        return psnr

    def test(self, logger=None, best_psnr=None):

        image_list = []
        for ext in ('.bmp', '.png', '.jpg', '.JPG', '.jpeg'):
            image_list.extend(glob(self.opt.test_path + "/*/*" + ext))
            # image_list.extend(glob(test_path + "/*" + ext))
        logger.info("[*] Test data from %s " % (self.opt.test_path + "/*/*"))
        imgs_num = len(image_list)
        logger.info("[*] Got %d images for test" % imgs_num)
        # transforms
        trans_list = [
            transforms.ToTensor()
        ]
        if self.opt.test_epoch==-1 and best_psnr is None:
            checkpoint = self.load_checkpoint(ckp_name="cheby_latest")
            result_path = os.path.join(self.opt.exp_path, self.opt.exp_name,"result_test_latest")
        elif best_psnr is None:
            checkpoint = self.load_checkpoint(ckp_name="cheby_%d" % self.opt.test_epoch)
            result_path =  os.path.join(self.opt.exp_path, self.opt.exp_name,"result_test_%d"%self.opt.test_epoch)
        elif best_psnr is not None:
            checkpoint = self.load_checkpoint(ckp_name="cheby_best_%0.3f"%best_psnr)
            result_path = os.path.join(self.opt.exp_path, self.opt.exp_name, "result_test_best_%0.3f"%best_psnr)
        self.curvenet.load_state_dict(checkpoint["net"])
        for d in glob(os.path.join(self.opt.test_path,"*")):
            savepath = os.path.join(result_path, d.split('/')[-1])
            os.makedirs(savepath, exist_ok=True)
        logger.info("[*] RESULT will be saved in %s " % result_path)
        # curve_net.eval()
        for param in self.curvenet.parameters():
            param.requires_grad = False
        logger.info("[*] Model is READY.")
        self.curvenet.eval()
        prev_time = time.time()
        for i, img_path in enumerate(image_list):
            img_path_spilt = img_path.split("/")
            dataset_name, img_name = img_path_spilt[-2], img_path_spilt[-1]
            img_PIL = Image.open(img_path)
            img_PIL = ((np.asarray(img_PIL) / 255.0) - 0.5) / 0.5
            img_tensor = torch.from_numpy(img_PIL).float()
            img_tensor = img_tensor.permute(2, 0, 1)
            img_tensor = img_tensor.cuda().unsqueeze(0)

            if self.opt.require_a:
                Y, A, I = self.curvenet(img_tensor)
                out = Y[-1]
                A = self.normalizeto01(A)
                save_image(out, "%s/%s/%s" % (result_path, dataset_name, img_name))
                x = torch.cat(Y, 0)
                a = torch.cat(A, 0)
                ai = torch.cat(I, 0)
                save_image(x, "%s/%s/Y_%s" % (result_path, dataset_name, img_name))
                save_image(a, "%s/%s/A_%s" % (result_path, dataset_name, img_name))
                save_image(ai, "%s/%s/I_%s" % (result_path, dataset_name, img_name))
                del Y, A, I
            else:
                out = self.curvenet(img_tensor)
                save_image(out, "%s/%s/%s" % (result_path, dataset_name, img_name))
            torch.cuda.empty_cache()
            # Determine approximate time left
            num_left = imgs_num - i
            time_left = datetime.timedelta(seconds=num_left * (time.time() - prev_time))
            prev_time = time.time()
            logger.info(
                "\r[Testing %d/%d]  %s %s ETA: %s"% (i, imgs_num, dataset_name, img_name, time_left)
            )

        logger.info("[*] Test Finished ~ \n")

    def save_checkpoint(self, checkpoint, ckp_name):
        """
        :param checkpoint: dict include net params, optimizer, epoch
        :return: None
        """
        print("[ * ] Svaing checkpoint : %s"%ckp_name)
        save_path = os.path.join(self.opt.exp_path, self.opt.exp_name, "checkpoint/%s.pth" % ckp_name)
        os.makedirs(os.path.dirname(save_path), exist_ok=True)
        torch.save(checkpoint, save_path)

    def load_checkpoint(self, ckp_name):
        """
        :param ckp_name:
        :param epoch:
        :return:
        """
        load_name = os.path.join(self.opt.exp_path, self.opt.exp_name, "checkpoint/%s.pth" % ckp_name)
        return torch.load(load_name)

    def plotloss(self, loss_dict_list, info=None):
        for key in loss_dict_list.keys():
            plt.plot(range(len(loss_dict_list[key])), loss_dict_list[key], linewidth=1, label=key)
        plt.legend()
        plt.xlabel("iterations")

        save_path = os.path.join(self.opt.exp_path, self.opt.exp_name)
        os.makedirs(save_path, exist_ok=True)
        if info is not None:
            savename = '%s/loss_%s_%s.svg' % (save_path, self.opt.exp_name,info)
        else:
            savename = '%s/loss_%s.svg' % (save_path, self.opt.exp_name)
        plt.savefig(savename, dpi=1200, format='svg')
        plt.close()

    def normalizeto01(self, tesnsor_lst):
        out_lst = []
        for i, x in enumerate(tesnsor_lst):
            out_lst.append(x * 0.5 + 0.5)
        return out_lst

    def testtime(self, logger=None):

        testdata_path = '../LOLRealow100/*'
        logger.info("[*] Test data from %s " % testdata_path)
        image_list = glob(testdata_path)
        imgs_num = len(image_list)
        logger.info("[*] Got %d images for test" % imgs_num)
        # 定义transforms
        if self.opt.test_epoch==-1:
            checkpoint = self.load_checkpoint(ckp_name="cheby_latest")
            result_path = os.path.join(self.opt.exp_path, self.opt.exp_name,"result_test_latest")
        else:
            checkpoint = self.load_checkpoint(ckp_name="cheby_%d" % self.opt.test_epoch)
            result_path =  os.path.join(self.opt.exp_path, self.opt.exp_name,"result_test_%d"%self.opt.test_epoch)
        self.curvenet.load_state_dict(checkpoint["net"])

        savepath = os.path.join(result_path, 'fivek_test')
        os.makedirs(savepath, exist_ok=True)
        logger.info("[*] RESULT will be saved in %s " % savepath)
        # curve_net.eval()
        for param in self.curvenet.parameters():
            param.requires_grad = False

        logger.info("[*] Model is READY.")

        total_time = 0.0
        for i, img_path in enumerate(image_list):
            img_path_spilt = img_path.split("/")
            img_name = img_path_spilt[-1]
            img_PIL = Image.open(img_path)
            img_PIL = ((np.asarray(img_PIL) / 255.0) - 0.5) / 0.5
            img_tensor = torch.from_numpy(img_PIL).float()
            img_tensor = img_tensor.permute(2, 0, 1)
            img_tensor = img_tensor.cuda().unsqueeze(0)
            start_time = time.time()
            out = self.curvenet(img_tensor)
            end_time = time.time()
            logger.info("{} {} s".format(img_name, end_time - start_time))
            total_time += end_time - start_time
            save_image(out, "%s/%s" % (result_path, img_name))
            torch.cuda.empty_cache()

        average_time = total_time / 100.0
        logger.info("total: {} s, average: {} s".format(total_time, average_time))


from itertools import combinations
from Heuristicas_energia import *

class Nodo:
    def __init__(self, estado, pai=None, acao=None, custo=0, distancia=1000):
        self.estado = estado
        self.pai = pai
        self.acao = acao
        self.sucessores = []
        self.custo = custo
        self.distancia = distancia
        self.custo_total = self.custo + self.distancia

    def addSucessores(self, meta):
        #Gera os sucessores do nodo
        for i,j in list(combinations([0,1,2,3,4], 2)):
            self.meta = meta

            #  Elevadores subindo
            novos_estados = list(self.estado)
            if novos_estados[i] < 42 and novos_estados[j] < 42:
                novos_estados[i] += 8
                novos_estados[j] += 8
                heuristica = Heuristica2_03(novos_estados, self.meta)
                self.sucessores.append(Nodo(estado = tuple(novos_estados), pai = Nodo(self.estado, self.pai, self.acao), acao = "Subir os elevadores {} e {}".format(i+1, j+1), custo = self.custo + 800, distancia = heuristica))

            #  Elevadores descendo
            novos_estados = list(self.estado)
            if novos_estados[i] > 12 and novos_estados[j] > 12:
                novos_estados[i] -= 13
                novos_estados[j] -= 13
                heuristica = Heuristica2_03(novos_estados, self.meta)
                self.sucessores.append(Nodo(estado = tuple(novos_estados), pai = Nodo(self.estado, self.pai, self.acao), acao = "Descer os elevadores {} e {}".format(i+1, j+1), custo = self.custo + 32.5, distancia = heuristica))

    def __eq__(self, other):
        return isinstance(other, Nodo) and self.estado == other.estado

    def __lt__(self, other):
        return self.custo_total < other.custo_total

    def __hash__(self):
        return hash(self.estado)


import base64, sys, traceback
from infraxys.communicator import Communicator
from infraxys.logger import Logger
from .server_answer import *

class JsonWindow(object):

    _instance = None

    @staticmethod
    def get_instance():
        if not JsonWindow._instance:
            JsonWindow._instance = JsonWindow()

        return JsonWindow._instance

    @staticmethod
    def is_open():
        return self._instance and self._instance.window_is_open

    def __init__(self):
        self.css_file = None
        self.canceled = False
        self.auto_close = False
        self.window_is_open = False
        self.logger = Logger.get_logger(self.__class__.__name__)

    def show(self, css_file=None):
        self.css_file = css_file
        json = {
            "requestType": "UI",
            "subType": "WINDOW",
        }

        if css_file:
            file = open(css_file, 'r')
            css = file.read()
            base64_css = base64.b64encode(css.encode("utf-8"))
            json["css"] = base64_css.decode("utf-8")

        Communicator.get_instance().send_synchronous(json=json, callback=self.answer_received,
                                                     return_on_first_answer=False)
        self.window_is_open = True

    def set_form(self, form, form_loaded_listener=None, auto_close=False, status=""):
        self.active_form = form
        self.auto_close = auto_close
        self.form_loaded_listener = form_loaded_listener
        json = form.generate_json()
        if status == "":
            self.set_status(status)
        Communicator.get_instance().send_synchronous(json=json, callback=self.answer_received)
        # return from this method is only when the form is closed or replaced

    def answer_received(self, json_object):
        try:
            if json_object == "":
                self.logger.info("Empty answer received from server.")
                Communicator.get_instance().wait_for_server(callback=self.answer_received)
                return

            #print("===", flush=True)
            #print(json_object, flush=True)
            #print("===", flush=True)
            schema = ServerAnswerSchema()
            event_data = schema.load(json_object)
            if event_data.event_type == "FORM_LOADED":
                if self.form_loaded_listener:
                    self.form_loaded_listener(event_data)

                Communicator.get_instance().wait_for_server(callback=self.answer_received)
                return

            elif event_data.event_type == "BUTTON_CLICK":
                if event_data.event_details == "CANCEL":
                    self.canceled = True
                    self.close()
                    return

            if self.active_form.event(event_data):
                if self.auto_close and event_data.event_details == "OK":
                    self.close()
            else:
                Communicator.get_instance().wait_for_server(callback=self.answer_received)

        except Exception as e:
            print("Exception handling event", flush=True)
            traceback.print_exc()
            print(e, flush=True)
            raise e

    def close(self):
        if not self.window_is_open:
            return
        json = {
            "requestType": "UI",
            "subType": "ACTION",
            "action": "close"
        }

        Communicator.get_instance().send_synchronous(json=json)
        Communicator.get_instance().stop_listening()
        self.window_is_open = False

    def close_with_error(self, message):
        self.logger.error(message)
        self.close()
        sys.exit(1)

    def set_status(self, message):
        Communicator.set_status(message)


#!/usr/bin/env python
from setuptools import setup

version = '0.0.0.1'

try:
    hash = (
        subprocess
        .check_output(shlex.split('git rev-parse --short HEAD'))
        .rstrip()
        .decode('ASCII')
    )
    commit = (
        subprocess
        .check_output(shlex.split('git rev-list --count HEAD'))
        .rstrip()
        .decode('ASCII')
    )
except:
    pass
else:
    version = '{}.dev{}+{}'.format(version, commit, hash)

setup(
    name='ultrastar-scripts',
    version=version,
    description='A collection of scripts to aid developing songs in Ultrastar format.',
    author='barbeque',
    author_email='barbeque.squared@gmail.com',
    url='https://github.com/barbeque-squared/ultrastar-scripts',
    packages=[
        'ultrastar_scripts'
    ],
    package_dir={'': 'src'},
    license='MIT',
    entry_points={
        'console_scripts': [
            'ultrastar-check = ultrastar_scripts.check:main',
            'ultrastar-fix-linebreaks = ultrastar_scripts.fix_linebreaks:main',
            'ultrastar-fix-whitespace = ultrastar_scripts.fix_whitespace:main',
            'ultrastar-multiply-bpm = ultrastar_scripts.multiply_bpm:main',
            'ultrastar-split-lyrics = ultrastar_scripts.split_lyrics:main',
            'ultrastar-songlist-csv = ultrastar_scripts.songlist_csv:main',
            'ultrastar-songlist-json = ultrastar_scripts.songlist_json:main'
        ]
    }
)


import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
from scipy.integrate import cumtrapz
from scipy.optimize import minimize
def simulation(t, x_0, param):
    lambda_m = param[0]
    lambda_e = param[1]
    omega_d = param[2]
    m = param[3]

    omega_0 = np.sqrt((lambda_e+lambda_m)**2 + omega_d**2)
    xi_m = lambda_m/omega_0
    xi_e = lambda_e/omega_0
    xi = xi_e + xi_m

    k = omega_0**2*m
    R_coil = 300
    R_load = 1000
    L = 0.01
    k_t = np.sqrt(lambda_e*(R_coil + R_load)*2*m)

    args = (omega_0, xi, k_t, R_coil+R_load, L)

    x = odeint(calc_derivatives, x_0, t, args=args)

    pos = x[:, 0]
    vel = x[:, 1]
    current = x[:, 2]
    emf = k_t*vel

    E_m = cumtrapz(calc_mechanical_dissipation(emf, t, k_t, xi_m, omega_0), t)
    E_c = cumtrapz(calc_coil_dissipation(current, R_coil), t)
    E_L = cumtrapz(calc_load_dissipation(current, R_load), t)

    E_kin = m*vel**2/2
    E_pot = k*pos**2/2

    '''
    fig_motion = plt.figure()
    ax_pos = fig_motion.add_subplot(311)
    ax_pos.plot(t, pos)
    ax_vel = fig_motion.add_subplot(312)
    ax_vel.plot(t, vel)
    ax_current = fig_motion.add_subplot(313)
    ax_current.plot(t, current)

    '''
    return(pos, vel, current, emf, E_pot, E_kin, E_m, E_c, E_L)

def calc_derivatives(x, t, omega_0, xi, k_t, R, L):
    A = np.array([[0, 1, 0],
                  [-omega_0**2, -2*omega_0*xi, 0],
                  [0, -k_t/L, -R/L]])

    return np.matmul(A,x)

def calc_mechanical_dissipation(emf, t, k_t, xi_m, omega_0):
    return (xi_m*omega_0/k_t**2)*emf**2

def calc_coil_dissipation(current, R_coil):
    return current**2/R_coil

def calc_load_dissipation(current, R_load):
    return current**2/R_load

def iterate_mass(m, args):
    t = args[0]
    x_0 = args[1]
    param = args[2]
    param = np.array([param[0], param[1], param[2], m])

    (pos, vel, current, emf, E_pot, E_kin, E_m, E_c, E_L) = simulation(t, x_0, param)

    E_diss = E_m + E_c + E_L
    E_diff = E_pot[0] - E_diss[-1]

    return E_diff**2

if __name__ == '__main__':
    '''
    This file contains a self adjusting loop which calculates the mass given
    the damped frequency and the damping factor lambda.
    '''

    ''' Meassured parameters '''
    with open('data/damping_model_D.txt', 'r') as textfile:
        next(textfile)
        lambda_m = float(next(textfile))
        lambda_e = float(next(textfile))
        omega_d = float(next(textfile))
    param = [lambda_m, lambda_e, omega_d]

    ''' Simulation parameters '''
    t_0 = 0
    t_f = 0.5
    dt = 0.0001
    t = np.arange(t_0, t_f, dt)

    ''' Initial conditions '''
    pos_0 = 1
    vel_0 = 0
    current_0 = 0
    x_0 = np.array([pos_0, vel_0, current_0])

    m_0 = 0.001 # Mass, this is not a free variable! I just did not manage to solve it yet
    args = [t, x_0, param]
    res = minimize(iterate_mass, m_0, args=args)
    m = res.x[0]

    param.append(m)
    (pos, vel, current, emf, E_pot, E_kin, E_m, E_c, E_L) = simulation(t, x_0, param)

    E_diss = E_m + E_c + E_L
    E_tot = E_diss + E_kin[1:] + E_pot[1:]
    E_internal = E_kin[1:] + E_pot[1:]

    with open('simulation_param.txt', 'w') as textfile:
        textfile.write('lambda_m, lambda_e, omega_d, mass\n')
        text = ''
        for el in param:
            text = text + str(el) + '\n'
        text = text[:-1]
        textfile.write(text)

    fig_energy = plt.figure()
    ax_energy = fig_energy.add_subplot(111)
    ax_energy.plot(t[1:], E_diss, label='Dissipated energy', color='red')
    ax_energy.plot(t[1:], E_internal, label='Energy in spring', color='blue')
    ax_energy.plot(t[1:], E_tot, label='Total energy', color='black', linestyle='--')
    ax_energy.legend()
    fig_energy.savefig('figures/energy_conservation_mass_iteration.png', bbox_inches='tight')
    plt.show()


import typing
from itertools import chain
from .logic import global_langs, translation_models_list

def setup_inlines(*args):
    """
    Method to compose Admin inliners with each other
    :param args:
    :return:
    """
    inlines_admins = list(chain(*args))
    return inlines_admins

class TranslateAdminInlines:
    """
    Class iterator. Compose admin inliners, model into new type
    """
    def __init__(self, base_translating_model, inliner_type,
                 include_langs=(),
                 exclude_langs=(),
                 **kwargs):
        assert base_translating_model in translation_models_list, 'This model is not registered as translatable'
        self.base_model = base_translating_model
        self.inliner = inliner_type
        self.kwargs = kwargs
        if not self.kwargs.get('extra'):
            self.kwargs['extra'] = 0
        self.include_langs = [include_lang.lower() for include_lang in include_langs]
        self.exclude_langs = [exclude_lang.lower() for exclude_lang in exclude_langs]
        self.langs = global_langs

        self._prepare_langs()

    def _prepare_langs(self) -> typing.NoReturn:
        """ Method deal with global_langs stuff """
        if self.include_langs and self.exclude_langs:
            raise ValueError("Not possible include and exclude languages")
        elif self.include_langs:
            self.langs = filter(lambda x: x in self.langs, self.include_langs)
        elif self.exclude_langs:
            self.langs = iter([lang for lang in self.langs if lang not in self.exclude_langs])
        else:
            self.langs = iter(self.langs)

    def _prepare_item(self):
        lang = next(self.langs)  # make langs iterator
        name = self.base_model.get_translate_model_name(lang)  # getting our model created name
        base = (self.inliner,)  # set up inliner class
        dct = {'model': self.base_model.get_connected_translated_model_class(lang), **self.kwargs}  # creating dict
        return type(name, base, dct)  # returning class

    def __iter__(self):
        return self

    def __next__(self):
        return self._prepare_item()


"""Person models"""
import random
import string

from django.contrib.sites.models import Site
from django.db import models
from django.contrib.auth.models import User

class Person(User):
    """Person model with sorting by the last name."""

    def __str__(self):
        if self.first_name and self.last_name:
            return '{0} {1}'.format(self.first_name, self.last_name)
        else:
            return self.username

    class Meta:
        proxy = True
        ordering = ('last_name', 'first_name')

def generate_invitation_key(length: int = 40) -> str:
    characters = (
        string.ascii_letters
        + string.digits
        + '-._~'
    )
    return ''.join(random.choices(characters, k=length))

class Invitation(models.Model):
    invitation_key = models.CharField(max_length=100, null=True, blank=True)
    deadline = models.DateTimeField()

    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def save(
        self,
        force_insert=False,
        force_update=False,
        using=None,
        update_fields=None,
    ):
        if self.invitation_key is None:
            self.invitation_key = generate_invitation_key()
        super().save(force_insert, force_update, using, update_fields)

    @property
    def link(self):
        current_site = Site.objects.get_current()
        return 'https://{domen}/electives/users/invite/?key={key}'.format(
            domen=current_site,
            key=self.invitation_key,
        )


from ._version import get_versions

__version__ = get_versions()["version"]
del get_versions
from . import ticu

def initialize(user_ns, broker_name, reduced_cat_name):
    """
    Initialize an interactive name space for use.

    Parameters
    ----------
    user_ns : dict
        The nampespace to populate.  Typically ``get_ipython().user_ns``

    broker_name : str
        The name of the databroker to insert into

    reduced_cat_name : str
        The catalog that has the reduced TiCu data available
    """
    import nslsii
    import databroker

    ret = nslsii.configure_base(
        user_ns, broker_name, configure_logging=False, ipython_logging=False
    )
    user_ns["db"] = user_ns["db"].v2
    ticu_sim = ticu.make_sim_devices(databroker.catalog[reduced_cat_name])
    if set(user_ns).intersection(ticu_sim):
        overlap = set(user_ns).intersection(ret)
        raise ValueError(f"there are overlapping names {overlap}")

    user_ns.update(ticu_sim)

    return ret + list(ticu_sim)


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.0
#   kernelspec:
#     display_name: Python [conda env:ds]
#     language: python
#     name: conda-env-ds-py
# ---

# + slideshow={"slide_type": "skip"}
#import pyforest

# + slideshow={"slide_type": "skip"}
#pyforest.active_imports()

# + slideshow={"slide_type": "skip"}
import altair as alt
alt.data_transformers.disable_max_rows()
# -

# ## Get Data

from pathlib import Path

# +
#https://npgeo-corona-npgeo-de.hub.arcgis.com/datasets/dd4580c810204019a7b8eb3e0b329dd6_0

file = Path("data/RKI_COVID19.csv")
import pandas as pd
date_parser = lambda x : pd.to_datetime(x.replace("Uhr", ""), errors="coerce")

date_cols = ["Meldedatum","Datenstand","Refdatum"]
categories = ["Bundesland", "Landkreis","Altergruppe", "Geschlecht","IdLandkreis"]
cat_dict = dict(zip(categories, ["category"]*len(categories)))

if not file.exists():
    pd.read_csv("https://opendata.arcgis.com/datasets/dd4580c810204019a7b8eb3e0b329dd6_0.csv").to_csv(file)
rki_data = pd.read_csv(file, dtype=cat_dict)

for col in date_cols:
    rki_data[col] = pd.to_datetime(rki_data[col].str.replace("Uhr",""))

ddr = ["Mecklenburg","Brandenburg","Sachsen-Anhalt","Sachsen","Thüringen"]
rki_data["ddr"] = rki_data.Bundesland.isin(ddr)
# -

print(rki_data.Meldedatum.max())

data_landkreise = pd.read_csv("data/RKI_Corona_Landkreise.csv")
data_bundesländer = pd.read_csv("data/RKI_Corona_Bundesländer.csv")

rki_data = rki_data.join(data_landkreise.set_index("county")[["GEN","cases7_bl_per_100k","cases7_bl","death7_bl","cases7_lk","death7_lk","cases7_per_100k"]], on="Landkreis")

rki_data["fälle_100_kreis"] = rki_data.join(data_landkreise.set_index("county").EWZ, on="Landkreis").pipe(lambda df: df.AnzahlFall / (df.EWZ / 100000))
rki_data["fälle_100_land"] = rki_data.join(data_bundesländer.set_index("LAN_ew_GEN").LAN_ew_EWZ, on="Bundesland").pipe(lambda df: df.AnzahlFall / (df.LAN_ew_EWZ / 100000))

# + [markdown] slideshow={"slide_type": "slide"}
# # Grammar of Graphics
#
# * [Buch](https://www.springer.com/de/book/9780387245447) von Wilkinson, Leland aus 2005
#
# ![](https://miro.medium.com/max/1800/1*mcLnnVdHNg-ikDbHJfHDNA.png)

# + [markdown] slideshow={"slide_type": "subslide"}
# * Data => [Chart](https://altair-viz.github.io/user_guide/data.html) + [Transformations](https://altair-viz.github.io/user_guide/transform/index.html)
# * Aesthetics => [encodings](https://altair-viz.github.io/user_guide/encoding.html)
# * Scale => [Scales](https://altair-viz.github.io/user_guide/scale_resolve.html)
# * Geometric objects => [Marks](https://altair-viz.github.io/user_guide/marks.html)
# * Statistics => [BuildIn vega-lite Aggregations](https://vega.github.io/vega-lite/docs/aggregate.html#ops)
# * Facets => Compound Charts (https://altair-viz.github.io/user_guide/compound_charts.html)
# * Coordinate system => ? [Map](https://altair-viz.github.io/gallery/index.html#maps)
# -

# # Basic Examples

# ## Pandas API

rki_data.groupby("Meldedatum").AnzahlFall.sum().rolling(7).mean().plot.line()

pd.__version__

fig = rki_data.groupby("Meldedatum").AnzahlFall.sum().rolling(7).mean().plot.line(backend="altair")
fig

pd.Series(rki_data.Landkreis.unique()).apply(lambda s: s.split(" ")[0]).value_counts().plot.barh(backend="altair")

rki_data.groupby(["Bundesland", "Geschlecht"]).AnzahlFall.sum().unstack().plot.barh(stacked=True, backend="altair")

# ## Altair syntax
#
# * Chart: Which Data Source
# * Marks. How to represent (Chart Type)
# * Encodings:
#     * [Channels](https://altair-viz.github.io/user_guide/encoding.html#encoding-channels): X-Axis, Y-Axis, Color, Size
#     * [Encode Data Type](https://altair-viz.github.io/user_guide/encoding.html#encoding-data-types): Quantiative, Ordinal, Nominal

# +
source = rki_data.groupby("Meldedatum").AnzahlFall.sum().reset_index()

alt.Chart(source).mark_line().encode(
    x="Meldedatum:T",
    y="AnzahlFall:Q"
).properties(width=1000).interactive()
# -

# ## Custom Order oder Colors

# +
fälle_je_tag_bundesland = rki_data.query("Meldedatum > '2020-08-15'").groupby(["Altersgruppe","Meldedatum"])["fälle_100_land"].sum().fillna(0).reset_index()
rolling_fall_bundesland = fälle_je_tag_bundesland.groupby(["Altersgruppe"]).rolling(7, on="Meldedatum")["fälle_100_land"].mean().fillna(0).reset_index()

ages = ['unbekannt','A00-A04', 'A05-A14', 'A15-A34','A35-A59', 'A60-A79', 'A80+'][::-1]

timeline_age = alt.Chart(rolling_fall_bundesland).mark_area().encode(
    x="Meldedatum:T",
    y="fälle_100_land:Q",
    tooltip="Altersgruppe",
    color=alt.Color('Altersgruppe', scale=alt.Scale(scheme='spectral'), sort=ages)
)
timeline_age

# +
fälle_je_tag_bundesland = rki_data.query("Meldedatum > '2020-08-15'").groupby(["Geschlecht","Meldedatum"])["AnzahlFall"].sum().fillna(0).reset_index()
rolling_fall_bundesland = fälle_je_tag_bundesland.groupby(["Geschlecht"]).rolling(7, on="Meldedatum")["AnzahlFall"].mean().fillna(0).reset_index()

timeline_sex= alt.Chart(rolling_fall_bundesland).mark_line().encode(
    x="Meldedatum:T", y="AnzahlFall:Q", color="Geschlecht"
)
timeline_sex
# -

# # Aggregations

group = ["Altersgruppe","Geschlecht","Bundesland"]
fälle_je_tag_bundesland = rki_data.query("Meldedatum > '2020-05-15'").groupby(group + ["Meldedatum"])["AnzahlFall"].sum().fillna(0).reset_index()
rolling_fall_bundesland = fälle_je_tag_bundesland.groupby(group).rolling(7, on="Meldedatum")["AnzahlFall"].mean().fillna(0).reset_index()

rolling_fall_bundesland

agg_timeline = alt.Chart(rolling_fall_bundesland).mark_area().encode(
    x="Meldedatum:T",
    y="sum(AnzahlFall):Q",
    tooltip="Altersgruppe:N",
    color=alt.Color('Altersgruppe:N', sort=ages)
)
agg_timeline

agg_timeline.encode(
    column='Geschlecht',
    row='Bundesland'
)

# # Combining Charts

# +
# Pandas Syntax
mean = rki_data.groupby("Meldedatum").AnzahlFall.sum().plot.line(backend="altair")
rolling = rki_data.groupby("Meldedatum").AnzahlFall.sum().rolling(7).mean().plot.line(backend="altair")

comb = (mean + rolling).properties(width=1000)
#for l in comb.layer:
#    l.selection = alt.Undefined
comb

# +
summed = rki_data.groupby("Meldedatum").AnzahlFall.sum()

actual = alt.Chart(summed.reset_index()).mark_line()
rolling = alt.Chart(summed.rolling(7).mean().reset_index()).mark_line().encode(
    color=alt.value("orange")
)

comb = (actual + rolling).encode(
    x="Meldedatum:T",
    y="AnzahlFall:Q"
).properties(width=1000)
comb
# -

timeline_age | timeline_age.encode(y=alt.Y("fälle_100_land:Q", stack="normalize"))

timeline_sex | timeline_sex.mark_area()

# +
multi_select = alt.selection_multi(encodings=['color'])

(timeline_age.add_selection(multi_select) | timeline_age.transform_filter(multi_select).add_selection(multi_select)).resolve_scale(y='shared')
# -

# # Interaction & Scales

data = data_landkreise[["GEN", "BL", "BEZ","EWZ", "cases7_lk","death7_lk","cases7_per_100k","cases7_bl_per_100k"]]
alt.Chart(data).mark_circle().encode(
    x=alt.X("EWZ", scale=alt.Scale(type="log", zero=False)),
    y="cases7_per_100k",
    color="BL",
    tooltip=['GEN', 'BL', 'cases7_lk', 'cases7_per_100k']
).interactive()

# +
brush = alt.selection(type='interval', encodings=['x'])

upper = comb.encode(
    alt.X('Meldedatum:T', scale=alt.Scale(domain=brush))
)

lower = comb.properties(
    height=60
).add_selection(brush)

upper & lower
# -

# # Geo-Data

# ## Get Data

import json
from urllib.request import urlopen
map_url = "https://raw.githubusercontent.com/AliceWi/TopoJSON-Germany/master/germany.json"
json_data = json.loads(urlopen(map_url).read())
states = [c.get("properties").get("name") for c in json_data.get("objects").get("states").get("geometries")]
json_data.get("objects").get("states").get("geometries")[0].keys()

# Missing matched
conainted = pd.Series({
    e.get("properties").get("name"): e.get("properties").get("name") in data_landkreise.GEN.to_list()
    for e in json_data.get("objects").get("counties").get("geometries")
})
conainted.loc[conainted==False]

json_data.get("objects").get("counties").get("geometries")[0].get("properties")

# ## Simple Map

counties = alt.topo_feature(map_url, 'states')
alt.Chart(counties).mark_geoshape().encode(
    text='properties.name:N',
    tooltip='properties.name:N'
)

# +
highlight = alt.selection_single(on='mouseover', fields=['Bundesland'], empty='none')
bundesland_cnt = rki_data.groupby("Bundesland").size().to_frame("cnt").reset_index()

counties = alt.topo_feature(map_url, 'states')
county_map = alt.Chart(counties).mark_geoshape().encode(
    color=alt.condition(highlight, alt.value('red'), 'Bundesland:N'),
    text='name:N'
).transform_lookup(
    lookup='properties.name',
    from_=alt.LookupData(bundesland_cnt, 'Bundesland', ['Bundesland','cnt'])
)

county_map.add_selection(highlight)
# -

fälle_je_tag_bundesland = rki_data.query("Meldedatum > '2020-10-15'").groupby(["Bundesland","Meldedatum"])["fälle_100_land"].sum().fillna(0).reset_index()
rolling_fall_bundesland = fälle_je_tag_bundesland.groupby(["Bundesland"]).rolling(7, on="Meldedatum")["fälle_100_land"].mean().fillna(0).reset_index()
line = alt.Chart(rolling_fall_bundesland).mark_line().encode(
    x="Meldedatum:T",
    y="fälle_100_land:Q",
    color="Bundesland",
    tooltip=['Bundesland', 'fälle_100_land']
)
line

agg_timeline

# +
sel_neg = alt.selection_multi(fields=["Bundesland"], empty='none')
sel = alt.selection_multi(fields=["Bundesland"])

fig = county_map.encode(
    color=alt.condition(sel_neg, alt.value('red'), 'cnt:Q'),
).add_selection(sel).add_selection(sel_neg)

fig | (line.transform_filter(sel) & \
       agg_timeline.transform_filter(sel)
      ).resolve_scale(color='independent')
# -

# # Landkreise

# +
data = data_landkreise[["GEN", "BL", "BEZ", "cases7_lk","death7_lk","cases7_per_100k","cases7_bl_per_100k"]]

brush = alt.selection_interval(empty='none', zoom=False)
brush = alt.selection_interval(fields=['GEN:N'], empty='none', zoom=False)

color = alt.condition(sel_neg,
                      alt.Color('BL:N', legend=None),
                      alt.value('lightgray'))

scatter = alt.Chart(data).mark_circle().encode(
    x=alt.X("cases7_lk", scale=alt.Scale(type="log", zero=False)),
    y="cases7_per_100k",
    color=color,
    tooltip=['GEN', 'BL', 'cases7_lk', 'cases7_per_100k']
).add_selection(sel_neg)#.interactive()

# +
#category = "Bundesland"
category = "Landkreis"
if category == "Bundesland":
    map_data = alt.topo_feature(map_url, 'states')
if category == "Landkreis":
    map_data = alt.topo_feature(map_url, 'counties')

#cnt_data = rki_data.groupby(["GEN","Bundesland"]).size().to_frame("cnt")
#cnt_data = rki_data.loc[rki_data.Meldedatum == rki_data.Meldedatum.max()].groupby(["GEN"]).cases7_lk.mean().to_frame("cnt").reset_index()
#cnt_data = rki_data.groupby(["GEN","Meldedatum"]).AnzahlFall.sum().reset_index().pipe(lambda df: df.loc[df.groupby("GEN").Meldedatum.idxmax()])

cnt_data = data_landkreise[["GEN", "BL", "BEZ", "cases7_lk","death7_lk","cases7_per_100k","cases7_bl_per_100k"]]

sel_neg = alt.selection_multi(fields=["GEN"], init=[{'GEN': 'Köln'}, {'GEN': 'Frankfurt am Main'}], empty='none')
sel = alt.selection_multi(fields=["GEN"], init=[{'GEN': 'Köln'}, {'GEN': 'Frankfurt am Main'}])

fälle_je_tag = rki_data.query("Meldedatum > '2020-10-15'").groupby(["GEN","Meldedatum"])["fälle_100_kreis"].sum().fillna(0).reset_index()
rolling_fall = fälle_je_tag.groupby(["GEN"]).rolling(7, on="Meldedatum")["fälle_100_kreis"].mean().fillna(0).reset_index()

timeline = alt.Chart(rolling_fall).mark_line().encode(
    x="Meldedatum:T",
    y="fälle_100_kreis:Q",
    color="GEN",
    tooltip=["GEN"]
)

county_map = alt.Chart(map_data).mark_geoshape().encode(
    color=alt.condition(sel_neg, alt.value('red'), 'cases7_per_100k:Q'),
    tooltip='GEN:N',
).transform_lookup(
    lookup='properties.name',
    from_=alt.LookupData(cnt_data, "GEN", ["GEN",'cases7_per_100k'])
).add_selection(sel).add_selection(sel_neg)

county_map.properties(width=500, height=800) | timeline.transform_filter(sel) & scatter
# -

# # Conclusion
#
# ## Strengths
#
# * Modular, Logical Concept
# * Interactions!
# * Support on Geo-Data
# * Handling of [many data](https://altair-viz.github.io/user_guide/faq.html#maxrowserror-how-can-i-plot-large-datasets)
# * [Flexibility](https://www.youtube.com/watch?v=SNaWwk_HzK0)
#
# ## Weaknesses
#
# * Cumbersome definitions
# * Sometimes unclear behaivor
# * Still some bugs and missing support for elementary things ([See my issues](https://github.com/altair-viz/altair/issues?q=author%3Akrlng+))
#
# ## When do I use what?
#
# * Typical static Chart: Seaborn
# * Complex static Chart: Matplotlib
# * Typical interactive Chart: Plotly Express
# * Complex interactive Chart: Altair

# # Sonstiges

# +
brush = alt.selection_interval()

legend_selection = alt.selection_multi(fields=['set'], bind='legend')

#
chart = alt.Chart(data.query("(training_points <65000) & (rel<2)")).mark_circle().encode(
    alt.X('training_points', scale=alt.Scale(zero=False)),
    alt.Y("rel"),
    color=alt.condition(brush | legend_selection, 'set:N', alt.value('lightgray'), sort=sorter)
).add_selection(brush).add_selection(legend_selection).properties(width=800)

bar =  alt.Chart(data, width=800).mark_bar().encode(y=alt.Y("set",sort=sorter)).transform_filter( brush ).properties(width=300)

err_bars = bar.encode(x=alt.X("average(rel)", scale=alt.Scale(zero=False)), color=alt.condition(legend_selection, "stdev(rel):Q", alt.value('lightgray'), sort=sorter))
est_bars = bar.encode(x=alt.X("average(params_n_estimators_)", scale=alt.Scale(domain=(100,1000))), color=alt.condition(legend_selection, "stdev(params_n_estimators_):Q", alt.value('lightgray'), sort=sorter))
fit_bars = bar.encode(x=alt.X("average(fit_time)", scale=alt.Scale(zero=False)), color=alt.condition(legend_selection, "stdev(fit_time):Q", alt.value('lightgray'), sort=sorter))
cnt_bars = bar.encode(x="count(fit_time)")
chart &  (((err_bars | est_bars) & (fit_bars | cnt_bars)).resolve_scale(color = 'independent') )

# +
source = data.set_index("set").rel.clip(0,2).reset_index()

best_set = source.groupby("set").rel.mean().idxmin()
selection = alt.selection_single(fields=["set"], init={'set': best_set})
ticks = alt.Chart(source, width=800, height=200).mark_tick().encode(
    y="set:N",
    x="rel:Q",
    color=alt.condition(selection, alt.value('navy'), alt.value('lightgray'))
).add_selection(selection)

hist = alt.Chart(source).mark_bar().encode(
    alt.X('rel:Q', bin=alt.Bin(1, step=0.05)),
    alt.Y('count():Q'),
)

density = alt.Chart(source).transform_bin(
    "binned_rel", "rel", bin=alt.Bin(1, step=0.05)
).transform_density(
    'binned_rel',
    as_=['rel', 'density'],
    counts=True,
).mark_area().encode(
    alt.X('rel:Q'),
    alt.Y('density:Q'),
    opacity=alt.value(0.5)
)

ticks & (hist + density).resolve_scale(
    y='independent'
).transform_filter(
    selection
).properties(width=800, height=200)


from django.http import HttpResponseRedirect
from django.views import generic
from viewflow.views import FormLayoutMixin
from viewflow.workflow.flow.views import (
    SuccessMessageMixin,
    TaskSuccessUrlMixin,
    TaskViewTemplateNames,
    UpdateArtifactView,
)
from .forms import ArticleSelectForm
from .models import Article

class CreateArticleView(
    FormLayoutMixin,
    SuccessMessageMixin,
    TaskSuccessUrlMixin,
    TaskViewTemplateNames,
    generic.CreateView,
):
    model = Article
    fields = ["content"]
    template_name = "gptflow/create_article.html"

    def form_valid(self, form):
        article = form.save(commit=False)
        article.summary = self.request.activation.process.artifact
        article.model = "Claude"
        article.save()
        self.request.activation.execute()
        return HttpResponseRedirect(self.get_success_url())

class SelectArticleView(UpdateArtifactView):
    form_class = ArticleSelectForm

    def get_form_kwargs(self):
        kwargs = super().get_form_kwargs()
        kwargs["summary"] = self.request.activation.process.artifact
        return kwargs


import os
from datetime import datetime
from pathlib import Path

import pandas as pd

import networkx as nx
import osmnx as ox

import matplotlib.pyplot as plt
import imageio as iio

def get_shortest_paths(G, origin_nodes: list, destination_nodes: list, weight: str):
    """
    This function calculates the shortest paths based on weight parameter.
    ---
    Args:
        origin_nodes (list): list of origin nodes
        destination_nodes (list): list of destination nodes
        weight (str): graph property for shortest path calculation
    """
    route_list = []
    nan_index_list = []

    for index, node in enumerate(origin_nodes):
        origin = origin_nodes[index]
        destination = destination_nodes[index]
        route = ox.shortest_path(G, origin, destination, weight=weight)

        if route and isinstance(route, list):
            route_list.append(route)
        else:
            nan_index_list.append(index)

    print(f'Generated {len(route_list)} routes from {len(origin_nodes)} origins and {len(destination_nodes)} destinations.')
    print(f'{len(nan_index_list)} routes returned NaN.')

    return route_list, nan_index_list

def create_g(df):
    """
    """
    centroid = [60.1699, 24.9384]
    mode = "drive"
    G = ox.graph_from_point(centroid, dist=2000, simplify=True, network_type=mode)

    origin_nodes = ox.distance.nearest_nodes(G, X=df['VENUE_LONG'], Y=df['VENUE_LAT'])
    destination_nodes = ox.distance.nearest_nodes(G, X=df['USER_LONG'], Y=df['USER_LAT'])

    return G, origin_nodes, destination_nodes

def plot_routes(G: nx.Graph, df: pd.DataFrame, route_list: list, start, end, hour, day):
    """
    This function Plots routes within a specified date range.
    ---
    Args:
        G (nx.Graph): network x Graphs
        df (pd.DataFrame):
        routes (list): routes as a list
        start (datetime): index for filtering the routes
        end (datetime): index for filtering the routes

    Returns: None
    """

    window = df[(df['TIMESTAMP'] >= start) & (df['TIMESTAMP'] <= end)]
    routes_to_plot = route_list[0: len(window)]

    fig, ax = ox.plot_graph_routes(G,
                                   routes_to_plot,
                                   route_colors='#3cb4dc',
                                   route_linewidth=3,
                                   route_alpha=0.9,
                                   save=True,
                                   filepath=f'images/day_{day}_hour_{hour}.png',
                                   node_size=7)

def generate_images(G, df, route_list):
    """
    """
    year = 2020
    month = 9
    day_range = range(1, 14)
    hour_range = range(7, 19)

    for day in day_range:
        for hour in hour_range:
            next_hour = hour + 1
            start = datetime(year, month, day, hour)
            end = datetime(year, month, day, next_hour)

            plot_routes(G, df, route_list, start, end, hour, day)

def main_images(path, filenames):
    images = list()
    for file in Path(path).iterdir():
        im = iio.imread(file)
        images.append(im)

    duration = 0.25
    iio.mimsave(f'wolt_{duration}.gif', images, duration=duration)

def main():
    path = '../images/'
    filenames = os.listdir(path)

    if len(filenames) == 0:
        df = pd.read_csv('data/orders.csv')
        df['TIMESTAMP'] = pd.to_datetime(df['TIMESTAMP'])
        print(df.head())

        G, origin_nodes, destination_nodes = create_g(df)

        weight = 'travel_time'
        route_list, nan_index = get_shortest_paths(G, origin_nodes, destination_nodes, weight)
        df = df.drop(df.index[nan_index])
        print(route_list)

        generate_images(G, df, route_list)

    main_images(path, filenames)
    print('done')

if __name__ == '__main__':
    main()