#! /usr/bin/env python
""" Create job files and run jobs to see effects of mismatching timescales at
tutor and student.
"""

import sys
import os
import time

import numpy as np
import cPickle as pickle

try:
  from argparse import ArgumentParser
  from subprocess import check_output
except:
  sys.exit("Need Python version 2.7. Make sure the right module is loaded.")

if __name__ == '__main__':
  version = sys.version_info
  if version.major != 2 or version.minor < 7:
    sys.exit("Need Python version 2.7. Make sure the right module is loaded.")

  parser = ArgumentParser(description="Create and run job files for testing " +
      " the effect of timescale mismatch in the spiking simulations.")

  parser.add_argument('--defaults', help="pickle containing default parameters",
      required=True)
  parser.add_argument('--nreps', help="number of steps per simulation",
      default=1500)
  parser.add_argument('--relaxation', help="simulation relaxation time",
      default=400.0)
  parser.add_argument('--relaxation_conductor', help="conductor relaxation",
      default=25.0)
  parser.add_argument('--ntaus', help="number of tau values",
      default=8)

  args = parser.parse_args()

  taus = 10*2**np.arange(int(args.ntaus))

  with open(args.defaults, 'rb') as inp:
    defaults = pickle.load(inp)

  tau1, tau2 = defaults['plasticity_taus']

  for (i, tau_student) in enumerate(taus):
    alpha = float(tau_student - tau2)/(tau1 - tau2)
    beta = alpha - 1

    run_name = "job/job_bird_tscale_{}.sh".format(i)
    job_name = "bird_t_{}".format(i)

    job_script = """#!/bin/bash
#!/bin/bash
#PBS -q production
#PBS -N {job_name}
#PBS -l select=1:ncpus=1
#PBS -l walltime=24:00:00
#PBS -l place=free
#PBS -V

date
cd $PBS_O_WORKDIR

SCRIPT_PATH=songbird/batch
SCRIPT_NAME=run_once.py
SCRIPT_FULL="$SCRIPT_PATH/$SCRIPT_NAME"
""".format(job_name=job_name)

    for (j, tau_tutor) in enumerate(taus):
      base_name = "songspike_tscale_batch_{}.{}".format(i, j) + "." + \
        time.strftime("%y%m%d.%H%M")

      # start setting up the command line options
      cmdline_opts = ("--plasticity_params=\"({},{})\" --tutor_rule_tau={}".
          format(alpha, beta, tau_tutor))
      cmdline_opts += " --nreps={} --defaults={}".format(args.nreps,
          args.defaults)
      cmdline_opts += " --relaxation={} --relaxation_conductor={}".format(
          args.relaxation, args.relaxation_conductor)

      log_name = "log/" + base_name + "_out.txt"

      cmdline_opts += " --out=out/" + base_name + ".pkl"
      cmdline_opts += " --log=" + log_name

      job_name = "bird_t_{}.{}".format(i, j)

      job_script += "$SCRIPT_FULL {cmdline_opts} 2>&1\n\n".format(
          cmdline_opts=cmdline_opts)

    job_script += "\ndate\n"

    with open(run_name, 'wt') as out:
      out.write(job_script)

    # and now submit them
    qsub_opts = run_name
    print("About to call qsub " + qsub_opts + "...")
    out = check_output(['qsub'] + qsub_opts.split())
    print("Answer: " + out)


import datetime
import json
import requests
import sys
import os.path
import pprint

CLARIFAI_BASE_URL = "https://api.clarifai.com/v1/"
CLARIFAI_TAG_URL = "tag/"
CLARIFAI_TOKEN_URL = "token/"

class ClarifaiApi:

    def __init__(self, app_id, app_secret):
        self.app_id = app_id
        self.app_secret = app_secret
        self.clarifai_access_token = None

    def fetch_token_response(self):
        post_data = {"client_id": self.app_id, "client_secret": self.app_secret, "grant_type": "client_credentials"}
        try:
            return requests.post(CLARIFAI_BASE_URL + CLARIFAI_TOKEN_URL, data=post_data)
        except requests.exceptions.RequestException as e:
            print("Something went wrong! \n" + "Error Message: \n" + str(e))
            return None

    def set_token(self):
        token_response = self.fetch_token_response()
        if token_response.status_code != 200:
            return {"response": token_response.text, "status": "error"}
        json_response = json.loads(token_response.text)
        self.clarifai_access_token =  json_response["access_token"]
        return {"status": "ok"}

    def get_image_data(self, image_file_path):
        try:
            return open(image_file_path, 'rb')
        except (OSError, IOError) as e:
            print("Something went wrong! \n" + "Error Message: \n" + str(e))
            return None

    def get_tagging_response_and_time(self, image_data):
        try:
            post_data = {"encoded_data": image_data}
            headers = {"Authorization": "Bearer " + self.clarifai_access_token}
            start_time = datetime.datetime.now()
            response = requests.post(CLARIFAI_BASE_URL + CLARIFAI_TAG_URL, headers=headers, files=post_data)
            end_time = datetime.datetime.now()
            return {"response": response, "time": (end_time-start_time).microseconds}
        except requests.exceptions.RequestException as e:
            print("Something went wrong! Exiting. \n" + "Error Message: \n" + str(e))
            return None

    def get_tag_score_list(self, json_response):
        tag_list = json_response['results'][0]['result']['tag']['classes']
        prob_list = json_response['results'][0]['result']['tag']['probs']
        return list(zip(tag_list, prob_list))

    def parse_response(self, response_and_time):
        status_code = response_and_time["response"].status_code
        if status_code != 200:
            return {"response": json.loads(response_and_time["response"].text), "status": "error"}
        json_response = json.loads(response_and_time['response'].text)
        tag_list_with_score = self.get_tag_score_list(json_response)
        return {"tag_list": tag_list_with_score, "time": response_and_time['time'], "status": "ok"}

    def get_tag_scores_and_time(self, image_file_path):
        if self.clarifai_access_token is None:
            token_status = self.set_token()
            if token_status["status"] == "error":
                return token_status
        image_data = self.get_image_data(image_file_path)
        if image_data is None:
            return None
        response_and_time = self.get_tagging_response_and_time(image_data)
        if response_and_time is None:
            return None
        return self.parse_response(response_and_time)

if __name__ == '__main__':
    if len(sys.argv) != 4:
        print("Wrong number of arguments! \n Usage: python clarifai_api.py <file path to image> <Clarifai app id> "
              "<Clarifai app secret>")
    else:
        if os.path.isfile(sys.argv[1]):
            clarifai_api = ClarifaiApi(sys.argv[2], sys.argv[3])
            pprint.pprint(clarifai_api.get_tag_scores_and_time(sys.argv[1]))
        else:
            print("File not found: " + sys.argv[1])


from datetime import datetime

# Importing the BookHelper class from books_helper.py, and the required functions and classes from other modules
from helper.books_helper import BookHelper
from initializer import get_database_connection
from model.book import Book, tuple_to_book
from model.borrowed_book import tuple_to_borrowed_book

class BorrowsHelper:

    # Static method to borrow a book for a user
    @staticmethod
    def borrow_book(user_id, book: Book):
        # Get the current date
        borrow_date = datetime.now().strftime("%Y-%m-%d")
        # Get a connection to the database
        with get_database_connection() as conn:
            cursor = conn.cursor()
            # Remove any reservations for the book by the user
            cursor.execute(
                "DELETE FROM reserved_books WHERE bookID = :book_id AND userID = :user_id",
                {"user_id": user_id, "book_id": book.bookID})
            # Add a new entry to the borrowed_books table for the user and book
            cursor.execute(
                "INSERT INTO borrowed_books (userID, bookID, borrow_date, status) "
                "VALUES (:user_id, :book_id, :borrow_date, 'borrowed')",
                {"user_id": user_id, "book_id": book.bookID, "borrow_date": borrow_date})
            # Commit the changes to the database
            conn.commit()
            # Decrement the available count of the book, and update the book in the database
            book.available = book.available - 1
            BookHelper.update_book(book)

    # Static method to return a borrowed book
    @staticmethod
    def return_book(borrow_id):
        # Get the current date
        return_date = datetime.now().strftime("%Y-%m-%d")
        # Get a connection to the database
        with get_database_connection() as conn:
            cursor = conn.cursor()
            # Update the borrowed_books entry for the given borrow_id, setting the return date and status to "returned"
            cursor.execute(
                "UPDATE borrowed_books SET return_date = :return_date, status = 'returned' WHERE borrowID = :borrow_id",
                {"return_date": return_date, "borrow_id": borrow_id})
            # Commit the changes to the database
            conn.commit()
            # Get the book ID from the borrowed_books entry
            cursor.execute(
                "SELECT * FROM borrowed_books WHERE borrowID = :borrow_id",
                {"borrow_id": borrow_id})
            borrow_tuple = cursor.fetchone()
            book_id = borrow_tuple[2]
            # Get the book information from the books table
            cursor.execute(
                "SELECT * FROM books WHERE bookID = :book_id",
                {"book_id": book_id})
            book_tuple = cursor.fetchone()
            book = tuple_to_book(book_tuple)
            # Increment the available count of the book, and update the book in the database
            book.available = book.available + 1
            BookHelper.update_book(book)

    # Static method to get a list of borrowed books for a given user
    @staticmethod
    def get_borrowed_books_for_user(user_id):
        with get_database_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM borrowed_books WHERE userID = :user_id AND status = 'borrowed'",
                {"user_id": user_id})
            borrows_tuples = cursor.fetchall()

            borrows = []
            for borrow_tuple in borrows_tuples:
                book_id = borrow_tuple[2]
                cursor.execute(
                    "SELECT * FROM books WHERE bookID = :book_id",
                    {"book_id": book_id})
                book_tuple = cursor.fetchone()
                book = tuple_to_book(book_tuple)
                borrowed_book = tuple_to_borrowed_book(book, borrow_tuple)
                borrows.append(borrowed_book)
        return borrows

    @staticmethod
    def get_books(status: str):
        with get_database_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM borrowed_books WHERE status = :status",
                {"status": status}
            )
            borrows_tuples = cursor.fetchall()
            borrows = []
            for borrow_tuple in borrows_tuples:
                book_id = borrow_tuple[2]
                cursor.execute(
                    "SELECT * FROM books WHERE bookID = :book_id",
                    {"book_id": book_id})
                book_tuple = cursor.fetchone()
                book = tuple_to_book(book_tuple)
                borrowed_book = tuple_to_borrowed_book(book, borrow_tuple)
                borrows.append(borrowed_book)
            return borrows

    @staticmethod
    def get_books_within_period(start_date: str, end_date: str):
        with get_database_connection() as conn: cursor = conn.cursor()
        cursor.execute("SELECT * FROM borrowed_books WHERE borrow_date >= :start_date and borrow_date <= :end_date",
                       {"start_date": start_date, "end_date": end_date})
        borrows_tuples = cursor.fetchall()
        borrows = []
        for borrow_tuple in borrows_tuples:
            book_id = borrow_tuple[2]
            cursor.execute("SELECT * FROM books WHERE bookID = :book_id",{"book_id": book_id})
            book_tuple = cursor.fetchone()
            book = tuple_to_book(book_tuple)
            borrowed_book = tuple_to_borrowed_book(book, borrow_tuple)
            borrows.append(borrowed_book)
            return borrows


import argparse
import importlib.util as import_util
import os
import pathlib
import subprocess
import sys
from typing import List, Optional, Sequence, Union

VENV_NAME = ".venv"
CWD = pathlib.PurePath(os.getcwd())

class VenvError(Exception):
    pass

def parse_args(argv: Sequence[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--requirements",
        action="append",
        default=[],
        help="Install additional dependencies into the virtual environment.",
    )

    parser.add_argument(
        "--toml",
        action="store",
        default=None,
        help="Install additional dependencies from sources like `pyproject.toml` into the virtual environment.",
    )
    parser.add_argument(
        "--extras",
        action="append",
        default=[],
        help="Install specific package groups from `pyproject.toml` into the virtual environment.",
    )

    parser.add_argument(
        "--git-ignore",
        action="store_true",
        default=False,
        help="Add .gitignore to the newly created virtual environment.",
    )
    parser.add_argument(
        "--name",
        default=VENV_NAME,
        type=str,
        help="Name of the virtual environment.",
        metavar="NAME",
        action="store",
    )
    return parser.parse_args(argv)

def is_installed(module: str) -> bool:
    return import_util.find_spec(module) is not None

def file_exists(path: Union[str, pathlib.PurePath]) -> bool:
    return os.path.exists(path)

def venv_exists(name: str) -> bool:
    return os.path.exists(CWD / name) and file_exists(get_venv_path(name))

def run_process(args: Sequence[str], error_message: str) -> None:
    try:
        print("Running: " + " ".join(args))
        subprocess.run(args, cwd=os.getcwd(), check=True)
    except subprocess.CalledProcessError:
        raise VenvError(error_message)

def get_venv_path(name: str) -> str:
    # See `venv` doc here for more details on binary location:
    # https://docs.python.org/3/library/venv.html#creating-virtual-environments
    if sys.platform == "win32":
        return os.fspath(CWD / name / "Scripts" / "python.exe")
    else:
        return os.fspath(CWD / name / "bin" / "python")

def install_requirements(venv_path: str, requirements: List[str]) -> None:
    if not requirements:
        return

    print(f"VENV_INSTALLING_REQUIREMENTS: {requirements}")
    args = []
    for requirement in requirements:
        args += ["-r", requirement]
    run_process(
        [venv_path, "-m", "pip", "install"] + args,
        "CREATE_VENV.PIP_FAILED_INSTALL_REQUIREMENTS",
    )
    print("CREATE_VENV.PIP_INSTALLED_REQUIREMENTS")

def install_toml(venv_path: str, extras: List[str]) -> None:
    args = "." if len(extras) == 0 else f".[{','.join(extras)}]"
    run_process(
        [venv_path, "-m", "pip", "install", "-e", args],
        "CREATE_VENV.PIP_FAILED_INSTALL_PYPROJECT",
    )
    print("CREATE_VENV.PIP_INSTALLED_PYPROJECT")

def upgrade_pip(venv_path: str) -> None:
    run_process(
        [venv_path, "-m", "pip", "install", "--upgrade", "pip"],
        "CREATE_VENV.PIP_UPGRADE_FAILED",
    )

def add_gitignore(name: str) -> None:
    git_ignore = CWD / name / ".gitignore"
    if not file_exists(git_ignore):
        print("Creating: " + os.fspath(git_ignore))
        with open(git_ignore, "w") as f:
            f.write("*")

def main(argv: Optional[Sequence[str]] = None) -> None:
    if argv is None:
        argv = []
    args = parse_args(argv)

    if not is_installed("venv"):
        raise VenvError("CREATE_VENV.VENV_NOT_FOUND")

    pip_installed = is_installed("pip")
    deps_needed = args.requirements or args.extras or args.toml
    if deps_needed and not pip_installed:
        raise VenvError("CREATE_VENV.PIP_NOT_FOUND")

    if venv_exists(args.name):
        venv_path = get_venv_path(args.name)
        print(f"EXISTING_VENV:{venv_path}")
    else:
        run_process(
            [sys.executable, "-m", "venv", args.name],
            "CREATE_VENV.VENV_FAILED_CREATION",
        )
        venv_path = get_venv_path(args.name)
        print(f"CREATED_VENV:{venv_path}")
        if args.git_ignore:
            add_gitignore(args.name)

    if pip_installed:
        upgrade_pip(venv_path)

    if args.toml:
        print(f"VENV_INSTALLING_PYPROJECT: {args.toml}")
        install_toml(venv_path, args.extras)

    if args.requirements:
        print(f"VENV_INSTALLING_REQUIREMENTS: {args.requirements}")
        install_requirements(venv_path, args.requirements)

if __name__ == "__main__":
    main(sys.argv[1:])


import torch
from torch import nn
from torch.nn import functional as F
from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence

DEVICE = torch.device('cuda')

class model(nn.Module):
  # initialize model
  def __init__(self,args):
    super(model, self).__init__()
    outtoks, embsize, hsize = args.ntoks, args.esz, args.hsz
    self.outemb = nn.Embedding(outtoks,args.esz)
    self.posemb = nn.Embedding(args.posembs,args.esz)
    self.qemb = nn.Embedding(args.rtoks,args.edgesz)
    self.vtoks = args.vtoks
    self.dlstm = nn.LSTMCell(args.esz*2 + args.edgesz, hsize)
    self.outlin = nn.Linear(hsize, outtoks)
    self.entemb_drop = nn.Dropout(args.drop)

    self.sigmoid = nn.Sigmoid()
    self.tanh = nn.Tanh()
    self.maxlen = args.max
    self.outtoks = outtoks
    self.endtok = args.endtok
    self.edgesz = args.edgesz
    self.embsize = embsize
    self.hsize = hsize

  # lstm decoder
  def decode_single(self, k, h, c):
    h, c = self.dlstm(k,(h,c))
    decoded = self.outlin(h.contiguous())
    return decoded, h, c
  # language model
  def langModel(self, outp, pos, q, h=None, c=None):

    outp, outlen = outp
    pos, poslen = pos
    q, qlen = q

    outp = self.outemb(outp)
    outp = self.entemb_drop(outp)
    pos = self.posemb(pos)
    pos = self.entemb_drop(pos)

    q = self.qemb(q)
    q = self.entemb_drop(q)

    decin = torch.cat([outp,pos,q],dim=-1)
    decin = decin.transpose(0,1)

    h  = torch.zeros(outp.size(0), self.hsize).to(DEVICE)
    c = torch.zeros(outp.size(0), self.hsize).to(DEVICE)

    outputs = []
    for k in decin:
        o, h, c = self.decode_single(k, h, c)
        outputs.append(o)
    o = torch.stack(outputs,1)
    return o

  def forward(self,outp,pos,q):

    o = self.langModel(outp,pos,q)

    return o


import serial
from flask import Flask
from migrations.setup import SetupMigration
from flask import jsonify, send_file
import sqlite3
from data_reader import DataReader

# Initialize the flask app
app = Flask(__name__)
ser = serial.Serial('/dev/ttyS1', 19200, timeout=1)

@app.route("/")
def default():
    return send_file("./chartdisplay.html")

@app.route("/api/getMeasurementData")
def getMeasurementData():
    conn = sqlite3.connect("database.db")
    cur = conn.cursor()
    results = cur.execute("SELECT * FROM `measurement_data`").fetchall()
    data = []
    for el in results:
        data.append({
            "ID": el[0],
            "timestamp": el[1],
            "dimmFactor": el[2],
            "voltage": el[3],
            "angle": el[4]
        })
    return jsonify({
        "results": data
    })

@app.route("/api/updateValues/<dimm>/<angle>")
def updateAngle(dimm, angle):
    dimmValue = int(dimm)
    angleValue = int(angle)
    ser.write(f"{dimmValue};{angleValue}\n")
    return ""

if __name__ == '__main__':
    conn = sqlite3.connect("database.db")
    SetupMigration(conn)
    conn.close()
    ser.open()
    reader = DataReader(serial=ser)
    reader.start()
    app.run(host="0.0.0.0", port=8080)
    reader.join()


import os
import pickle

import pandas as pd
import pytest
from sklearn.model_selection import train_test_split

from ml.data import process_data
from ml.model import compute_model_metrics, inference, train_model

@pytest.fixture
def root():
    return os.getcwd()

@pytest.fixture
def files(root):
    data = pd.read_csv("../data/census_cleaned.csv")

    model = os.path.join(root, "../model/gbclassifier.pkl")
    with open(model, "rb") as f:
        model = pickle.load(f)

    encoder = os.path.join(root, "../model/encoder.pkl")
    with open(encoder, "rb") as f:
        encoder = pickle.load(f)

    lb = os.path.join(root, "../model/lb.pkl")
    with open(lb, "rb") as f:
        lb = pickle.load(f)

    return data, model, encoder, lb

@pytest.fixture
def train_test_data(files):
    data, model, encoder, lb = files
    train, test = train_test_split(data, test_size=0.20, random_state=42)
    return train, test

def test_train_model(files, root):
    data, model, encoder, lb = files
    train, test = train_test_split(data, test_size=0.20)
    cat_features = [
        "workclass",
        "education",
        "marital-status",
        "occupation",
        "relationship",
        "race",
        "sex",
        "native-country",
    ]
    X_train, y_train, encoder, lb = process_data(
        train, categorical_features=cat_features, label="salary", training=True
    )
    filepath = os.path.join(root, "../model/gbclassifier_test.pkl")
    model = train_model(X_train, y_train, filepath=filepath)

    assert os.path.exists(filepath)
    return X_train, y_train, model, encoder, lb

def test_compute_model_metrics(files, train_test_data):
    _, model, encoder, lb = files
    train, test = train_test_data
    cat_features = [
        "workclass",
        "education",
        "marital-status",
        "occupation",
        "relationship",
        "race",
        "sex",
        "native-country",
    ]

    X_train, y_train, _, _ = process_data(
        train,
        categorical_features=cat_features,
        label="salary",
        training=True,
        encoder=encoder,
        lb=lb,
    )

    X_test, y_test, _, _ = process_data(
        test,
        categorical_features=cat_features,
        label="salary",
        training=False,
        encoder=encoder,
        lb=lb,
    )

    y_train_pred = inference(model, X_train)
    y_test_pred = inference(model, X_test)

    precision_train, recall_train, fbeta_train = compute_model_metrics(
        y_train, y_train_pred
    )
    precision_test, recall_test, fbeta_test = compute_model_metrics(
        y_test, y_test_pred
    )

    assert isinstance(precision_train, float)
    assert isinstance(precision_test, float)
    assert isinstance(recall_train, float)
    assert isinstance(recall_test, float)
    assert isinstance(fbeta_train, float)
    assert isinstance(fbeta_test, float)

def test_inference(train_test_data, files):
    _, model, encoder, lb = files
    train, test = train_test_data

    cat_features = [
        "workclass",
        "education",
        "marital-status",
        "occupation",
        "relationship",
        "race",
        "sex",
        "native-country",
    ]
    X_train, y_train, _, _ = process_data(
        train,
        categorical_features=cat_features,
        label="salary",
        training=True,
        encoder=encoder,
        lb=lb,
    )
    X_test, y_test, _, _ = process_data(
        test,
        categorical_features=cat_features,
        label="salary",
        training=False,
        encoder=encoder,
        lb=lb,
    )

    y_train_pred = inference(model, X_train)
    assert len(y_train_pred) == X_train.shape[0]
    assert len(y_train_pred) > 0

    y_test_pred = inference(model, X_test)
    assert len(y_test_pred) == X_test.shape[0]
    assert len(y_test_pred) > 0


import warnings
import requests
import os
import twill
import time
import io

import numpy as np
import pandas as pd

from memilio.epidata import defaultDict as dd
from memilio.epidata import geoModificationGermany as geoger
from memilio.epidata import getDataIntoPandasDataFrame as gd

# activate CoW for more predictable behaviour of pandas DataFrames
pd.options.mode.copy_on_write = True

def read_population_data(username, password, read_data, directory):
    '''! Reads Population data either from regionalstatistik.de or from directory

    A request is made using the twill package. Username and Password are required to
    sign in on regionalstatistik.de. After the sign twill navigates to the file to download.

    @param username Username to sign in at regionalstatistik.de.
    @param password Password to sign in at regionalstatistik.de.
    @param read_data False or True. Defines if data is read from file or downloaded.
    @param directory Path to folder where data is read from.
    @return DataFrame
    '''

    filename = '12411-02-03-4'
    if not read_data:
        sign_in_url = 'https://www.regionalstatistik.de/genesis/online?Menu=Anmeldung'

        # sign in to regionalstatistik.de with given username and password
        twill.browser.user_agent = requests.utils.default_headers()[
            'User-Agent']
        twill.commands.go(sign_in_url)
        twill.commands.fv('3', 'KENNUNG', username)
        twill.commands.fv('3', 'PASSWORT', password)
        twill.commands.submit('login', '3')
        # navigate to file as in documentation
        twill.commands.follow('Themen')
        twill.commands.follow(filename[:2])
        # wait 2 seconds to prevent error
        # page needs some time to load
        time.sleep(2)
        twill.commands.follow(filename.split('-')[0])
        twill.commands.follow(filename)
        # start 'Werteabruf'
        twill.commands.submit('45', '3')
        # read csv file (1,4 for xlsx)
        twill.commands.submit('1', '5')

        df_pop_raw = pd.read_csv(io.StringIO(
            twill.browser.html), sep=';', header=6)

    else:
        data_file = os.path.join(directory, filename)
        if os.path.isfile(data_file+'.xlsx'):
            df_pop_raw = pd.read_excel(
                data_file+'.xlsx', engine='openpyxl', sheet_name=filename, header=4)
        elif os.path.isfile(data_file+'.csv'):
            df_pop_raw = pd.read_excel(data_file+'.csv', sep=';', header=6)
        else:
            raise FileNotFoundError(
                'Data file '+filename+' was not found in out_folder/Germany')

    return df_pop_raw

def export_population_dataframe(df_pop, directory, file_format, merge_eisenach):
    '''! Writes population dataframe into directory with new column names and age groups

    @param df_pop Population data DataFrame to be exported
    @param directory Directory where data is written to.
    @param file_format File format which is used for writing the data.
    @param merge_eisenach Defines whether the counties 'Wartburgkreis'
        and 'Eisenach' are listed separately or
        combined as one entity 'Wartburgkreis'.
    @return exported DataFrame
    '''

    new_cols = [
        dd.EngEng['idCounty'],
        dd.EngEng['population'],
        '<3 years', '3-5 years', '6-14 years', '15-17 years',
        '18-24 years', '25-29 years', '30-39 years', '40-49 years',
        '50-64 years', '65-74 years', '>74 years']
    df_pop_export = pd.DataFrame(columns=new_cols)
    df_pop_export[new_cols[0]] = df_pop[dd.EngEng['idCounty']]
    # <3 and 3-5
    df_pop_export[new_cols[2:4]] = df_pop[df_pop.columns[2:4]]
    # 6-14
    df_pop_export[new_cols[4]] = \
        df_pop[df_pop.columns[4:6]].sum(axis=1)
    # 15-17
    df_pop_export[new_cols[5]] = df_pop[df_pop.columns[6]]
    # 18-24
    df_pop_export[new_cols[6]] = \
        df_pop[df_pop.columns[7:9]].sum(axis=1)
    # 25-29
    df_pop_export[new_cols[7]] = df_pop[df_pop.columns[9]]
    # 30-39
    df_pop_export[new_cols[8]] = \
        df_pop[df_pop.columns[10:12]].sum(axis=1)
    # 40-49
    df_pop_export[new_cols[9]] = \
        df_pop[df_pop.columns[12:14]].sum(axis=1)
    # 50-64
    df_pop_export[new_cols[10]] = \
        df_pop[df_pop.columns[14:17]].sum(axis=1)
    # 65-74
    df_pop_export[new_cols[11]] = df_pop[df_pop.columns[17]]
    # >74
    df_pop_export[new_cols[12]] = df_pop[df_pop.columns[18]]

    df_pop_export[dd.EngEng['population']
                  ] = df_pop_export.iloc[:, 2:].sum(axis=1)

    # merge eisenach if no data available
    if '16056' in df_pop_export[dd.EngEng['idCounty']].values:
        if df_pop_export[df_pop_export[dd.EngEng['idCounty']] == '16056'][dd.EngEng['population']].values[0] == 0:
            df_pop_export = geoger.merge_df_counties_all(
                df_pop_export, sorting=[dd.EngEng["idCounty"]],
                columns=dd.EngEng["idCounty"])

    gd.check_dir(directory)

    if len(df_pop_export) == 401:
        filename = 'county_current_population_dim401'
        gd.write_dataframe(df_pop_export, directory, filename, file_format)

    if len(df_pop_export) == 400 or merge_eisenach:
        filename = 'county_current_population'

        # Merge Eisenach and Wartburgkreis
        df_pop_export = geoger.merge_df_counties_all(
            df_pop_export, sorting=[dd.EngEng["idCounty"]],
            columns=dd.EngEng["idCounty"])

        gd.write_dataframe(df_pop_export, directory, filename, file_format)

    return df_pop_export

def assign_population_data(df_pop_raw, counties, age_cols, idCounty_idx):
    '''! Assigns population data of all counties of old dataframe in new created dataframe

    In df_pop_raw there might be additional information like federal states,
    governing regions etc. which is not necessary for the dataframe.
    Also checks for incomplete data.

    @param df_pop_raw Raw Population DataFrame read from regionalstatistik.de
    @param counties List of counties to be assigned in new DataFrame
    @param age_cols Age groups in old DataFrame
    @param idCountyidx indexes in old DataFrame where data of corresponding county starts
    @return new DataFrame
    '''

    new_cols = {dd.EngEng['idCounty']: counties[:, 1],
                dd.EngEng['county']: counties[:, 0]}

    # number of age_cols
    num_age_groups = len(age_cols)

    # add age_cols with zero initialization to new_cols
    new_cols.update({age: 0 for age in age_cols})

    df_pop = pd.DataFrame(new_cols)

    empty_data = ['.', '-']
    for start_idx in idCounty_idx:

        county_id = df_pop_raw.loc[start_idx, dd.EngEng['idCounty']]

        # check for empty rows
        if df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1,
                          dd.EngEng['number']].values.any() in empty_data:
            if not df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1,
                                  dd.EngEng['number']].values.all() in empty_data:
                raise gd.DataError(
                    'Error. Partially incomplete data for county ' +
                    county_id)

        # county information needed
        elif county_id in counties[:, 1]:
            # direct assignment of population data found
            df_pop.loc[df_pop[dd.EngEng['idCounty']] == df_pop_raw.loc
                       [start_idx, dd.EngEng['idCounty']],
                       age_cols] = df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1, dd.EngEng
                                                  ['number']].values.astype(int)
        # Berlin and Hamburg
        elif county_id + '000' in counties[:, 1]:
            # direct assignment of population data found
            df_pop.loc[df_pop[dd.EngEng['idCounty']] == df_pop_raw.loc
                       [start_idx, dd.EngEng['idCounty']] + '000',
                       age_cols] = df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1, dd.EngEng
                                                  ['number']].values.astype(int)

        # additional information for local entities not needed
        elif county_id in ['03241001', '05334002', '10041100']:
            pass
        # Germany, federal states, and governing regions
        elif len(county_id) < 5:
            pass
        else:
            print('no data for ' + df_pop_raw.loc
                  [start_idx, dd.EngEng['idCounty']])
            raise gd.DataError(
                'Error. County ID in input population data '
                'found which could not be assigned.')

    return df_pop

def test_total_population(df_pop, age_cols):
    """! Tests if total population matches expectation
    @param df_pop Population Dataframe with all counties
    @param age_cols All age groups in DataFrame"""

    total_sum_2020 = 83155031
    total_sum_2021 = 83237124

    if df_pop[age_cols].sum().sum() != total_sum_2021:
        if df_pop[age_cols].sum().sum() == total_sum_2020:
            warnings.warn('Using data of 2020. Newer data is available.')
        else:
            raise gd.DataError('Total Population does not match expectation.')

def get_population_data(read_data=dd.defaultDict['read_data'],
                        file_format=dd.defaultDict['file_format'],
                        out_folder=dd.defaultDict['out_folder'],
                        no_raw=dd.defaultDict['no_raw'],
                        merge_eisenach=True,
                        username='',
                        password=''):
    """! Download age-stratified population data for the German counties.

    The data we use is:
    Official 'Bevölkerungsfortschreibung' 12411-02-03-4:
    'Bevölkerung nach Geschlecht und Altersgruppen (17)'
    of regionalstatistik.de.
    ATTENTION: The raw file cannot be downloaded
    automatically by our scripts without an Genesis Online account. In order to
    work on this dataset, please enter your username and password or manually download it from:

    https://www.regionalstatistik.de/genesis/online -> "1: Gebiet, Bevölkerung,
    Arbeitsmarkt, Wahlen" -> "12: Bevölkerung" -> "12411 Fortschreibung des
    Bevölkerungsstandes" ->  "12411-02-03-4: Bevölkerung nach Geschlecht und
    Altersgruppen (17) - Stichtag 31.12. - regionale Tiefe: Kreise und
    krfr. Städte".

    Download the xlsx or csv file and put it under dd.defaultDict['out_folder'],
    this normally is Memilio/data/pydata/Germany.
    The folders 'pydata/Germany' have to be created if they do not exist yet.
    Then this script can be run.

    @param read_data False or True. Defines if data is read from file or
        downloaded. Default defined in defaultDict.
    @param file_format File format which is used for writing the data.
        Default defined in defaultDict.
    @param out_folder Path to folder where data is written in folder
        out_folder/Germany. Default defined in defaultDict.
    @param no_raw True or False. Defines if unchanged raw data is written or
        not. Default defined in defaultDict. Currently not used.
    @param merge_eisenach [Default: True] or False. Defines whether the
        counties 'Wartburgkreis' and 'Eisenach' are listed separately or
        combined as one entity 'Wartburgkreis'.
    @param username Username to sign in at regionalstatistik.de.
    @param password Password to sign in at regionalstatistik.de.
    @return DataFrame with adjusted population data for all ages to current level.
    """

    directory = os.path.join(out_folder, 'Germany')
    gd.check_dir(directory)

    df_pop_raw = read_population_data(username, password, read_data, directory)

    column_names = list(df_pop_raw.columns)
    # rename columns
    rename_columns = {
        column_names[0]: dd.EngEng['idCounty'],
        column_names[1]: dd.EngEng['county'],
        column_names[2]: dd.EngEng['ageRKI'],
        column_names[3]: dd.EngEng['number'],
        column_names[4]: dd.EngEng['male'],
        column_names[5]: dd.EngEng['female']
    }
    df_pop_raw.rename(columns=rename_columns, inplace=True)
    # remove date and explanation rows at end of table
    df_pop_raw = df_pop_raw[:np.where(df_pop_raw[dd.EngEng['idCounty']].str.contains(
        '__') == True)[0][0]].reset_index(drop=True)
    # get indices of counties first lines
    idCounty_idx = df_pop_raw.groupby(
        dd.EngEng['idCounty']).head(1).index.tolist()

    # read county list and create output data frame
    counties = np.array(geoger.get_county_names_and_ids(
        merge_berlin=True, merge_eisenach=merge_eisenach, zfill=True))
    age_cols = df_pop_raw.loc[
        idCounty_idx[0]: idCounty_idx[1] - 2,
        dd.EngEng['ageRKI']].values.copy()
    for i in range(len(age_cols)):
        if i == 0:
            upper_bound = str(int(age_cols[i][
                age_cols[i].index('unter ')+6:].split(' ')[0])-1)
            age_cols[i] = '0-' + upper_bound
        elif i == len(age_cols)-1:
            lower_bound = age_cols[i].split(' ')[0]
            age_cols[i] = lower_bound + '-99'
        else:
            lower_bound = age_cols[i].split(' ')[0]
            upper_bound = str(int(age_cols[i][
                age_cols[i].index('unter ')+6:].split(' ')[0])-1)
            age_cols[i] = lower_bound + '-' + upper_bound

    df_pop = assign_population_data(
        df_pop_raw, counties, age_cols, idCounty_idx)

    test_total_population(df_pop, age_cols)

    df_pop_export = export_population_dataframe(
        df_pop, directory, file_format, merge_eisenach)

    return df_pop_export

def main():
    """! Main program entry."""

    arg_dict = gd.cli("population")
    get_population_data(**arg_dict)

if __name__ == "__main__":
    main()


import json

import numpy as np
from matplotlib import pyplot as plt

from Metrics import Metrics
from evaluation import Evaluation
import seaborn as sns

class visualization():
    def plt_fault_type_rauc(self, srtd_fault_list, fault_type_num):
        fault_set = set()
        Y = []
        for x in srtd_fault_list:
            if x[0] == x[1]:
                Y.append(len(fault_set))
            else:
                fault_set.add(x)
                Y.append(len(fault_set))
        X = [_ for _ in range(len(srtd_fault_list))]
        Tro = [_ if _ < fault_type_num else fault_type_num for _ in range(len(srtd_fault_list))]
        return X, Y, Tro

    def draw_distribution(self, score_throd, modeltype, datatype, sel_num):
        with open('./data/' + modeltype + '_' + datatype + score_throd + '.json') as f1:
            pre_list = json.load(f1)
        with open('./data/gaus_' + modeltype + '_' + datatype + score_throd + '.json') as f1:
            dif_list = json.load(f1)
        with open('./data/' + modeltype + '_' + datatype + 'image.json') as f2:
            imgpre_list = json.load(f2)

        eva = Evaluation()
        score_list, iou_list, tp_list = eva.get_score_iou(pre_list)
        result = {}
        metrics = Metrics(pre_list=pre_list, dif_list=dif_list, imgpre_list=imgpre_list)
        # result['ours(difference1)'] = metrics.difference_1()
        result['ours(difference)'] = metrics.difference()
        result['ours(receptive field)'] = metrics.Dis_p()
        result['$1-pall_{max}$(instance)'] = metrics.one_minus_pmax(part="all")
        # result['1vs2'] = metrics.one_vs_two(0)
        # result['entropy'] = metrics.entropy_image(0)
        # result['Gini'] = metrics.gini(0)
        # result['random'] = metrics.random_srt()
        metric_names = [_[0] for _ in result.items()]
        col = ['red', 'green', 'blue']
        for i in range(len(result)):
            metric = result[metric_names[i]]
            zipped = zip(metric, score_list, iou_list, tp_list)
            sort_zipped = sorted(zipped, key=lambda x: (x[0], x[1]))
            srt_ = zip(*sort_zipped)
            srtd_metric_list, srtd_score_list, srtd_iou_list, srtd_tp_list = [list(x) for x in srt_]
            srtd_score_list = srtd_score_list[::-1]
            srtd_iou_list = srtd_iou_list[::-1]
            srtd_tp_list = srtd_tp_list[::-1]
            print(sum(srtd_tp_list[:1000])/1000)
            data = {'x': srtd_iou_list[:1000], 'y': srtd_score_list[:1000]}
            sns.jointplot(data=data, x='x', y='y', ylim=(0, 1))
            plt.show()

if __name__ == "__main__":
    vis = visualization()
    vis.draw_distribution(score_throd='030', modeltype='SSD', datatype='COCO', sel_num=50)


#进程池pool

import multiprocessing
import time

def Process_work(msg):
    print("process start:",msg)
    time.sleep(3)
    print("process end:",msg)

if __name__ == "__main__":
    """设置进程池中可容纳3个子进程"""
    p =multiprocessing.Pool(3)
    for i in range(1,7):
        msg = "Process{0}".format(i)
        """异步阻塞方法，3个一组，完成一个进一个"""
        p.apply_async(Process_work,(msg,))
        """同步阻塞方法，一个完成执行下一个"""
        #p.apply(Process_work,(msg,))
    p.close()
    p.join()


#%%
import pandas as pd
import pickle
import numpy as np

import gensim
import sklearn.metrics

#%%
#--Read in data
D2V_WOstop = gensim.models.Doc2Vec.load(r'..\data\process\D2V_WOstop')
df = pd.read_csv(r'..\data\df_cb_main_combined.csv', index_col=0, encoding='utf-8', error_bad_lines=False).dropna(subset=['Review']).drop_duplicates(['Author', 'Game'])

#Acquire feature and key word lists
D2VFeatures = D2V_WOstop.vocab.keys()

#Setup target keyword group cluster
numOfGroups = [10, 30, 100, 300, 1000]
numOfGroups = [300]

#%%
for n in numOfGroups:
    numOfGroup = n
    #--Read in corresponding keyword group file
    df_keyword = pd.read_csv(r'..\data\output\keywordGroup_hierarchy_' + str(numOfGroup) + '.csv', encoding='utf-8')

    #--Make into keyword groups
    keyWords = []
    for g in range(numOfGroup):
        key = df_keyword.query('cluster == @g + 1').keyword.tolist()
        keyWords.append(key)

    keyGroups = {
        'groupNo': np.arange(numOfGroup) + 1,
        'keyWords': keyWords,
        'keyWords_filtered': None,
        'centroid': None
    }

    df_keyGroups = pd.DataFrame(keyGroups)

    keyWords_filtered = []
    centroids = []
    for index, row in df_keyGroups.iterrows():
        #Remove key words outside the D2V space
        filtered = [w for w in row['keyWords'] if w in D2VFeatures]
        keyWords_filtered.append(filtered)

        #Compute centroid vector of each group
        try: centroid = np.mean(D2V_WOstop[filtered], axis=0)
        except: centroid = None
        centroids.append(centroid)

    df_keyGroups['keyWords_filtered'] = keyWords_filtered
    df_keyGroups['centroid'] = centroids

    #Leave only non-null groups
    df_keyGroups = df_keyGroups[pd.notnull(df_keyGroups['centroid'])]

    #%%
    #--Acquire score of each key word group
    #By the average distance of the docvec and each key word group
    #Cosine similarity here: -1 ~ 1; -1=opposite; 1=same
    for g in range(len(df_keyGroups)):
        scores = []
        try:
            gvec = df_keyGroups['centroid'][g].reshape(1, -1)
            for index, row in df.iterrows():
                dvec = D2V_WOstop.docvecs['id_' + str(index)].reshape(1, -1)
                score = sklearn.metrics.pairwise.cosine_similarity(gvec, dvec)[0, 0]

                scores.append(score)

            df['group' + str(g + 1)] = scores
        #Some groups contain to word
        except: pass

    #--Review and save
    # df.drop('Review', axis=1).to_csv(r'..\data\process\score_' + str(numOfGroup) + '_doc2vec.csv')
    pickle.dump(df, open(r'..\data\process\score_' + str(numOfGroup) + '_doc2vec.p', 'wb'))
    df.query('CoreID > 0')

#--Print top games of each keygroup
#Also the score must be the highest among all scores of the game to be chosen
if False:
    CORE_GAMES = pd.read_csv(r'..\data\core_games.csv', encoding='utf-8', header=None)[0].tolist()
    for i in np.arange(numOfGroup) + 1:
        print('Games with top scores in group' + str(i))
        maxScore = df[df['group' + str(i)] == np.max(df.filter(regex='^group\d+$'), axis=1)]
        topScoreGames = df.sort_values(by='group' + str(i))[:100]
        top_notcore = topScoreGames[topScoreGames.Core == 0]['Game']
        print(top_notcore)


''' Classes used to represent the concept of a parameter in a .c3d file.
'''
import struct
import numpy as np
from .utils import DEC_to_IEEE, DEC_to_IEEE_BYTES

class ParamData(object):
    '''A class representing a single named parameter from a C3D file.

    Attributes
    ----------
    name : str
        Name of this parameter.
    dtype: DataTypes
        Reference to the DataTypes object associated with the file.
    desc : str
        Brief description of this parameter.
    bytes_per_element : int, optional
        For array data, this describes the size of each element of data. For
        string data (including arrays of strings), this should be -1.
    dimensions : list of int
        For array data, this describes the dimensions of the array, stored in
        column-major (Fortran) order. For arrays of strings, the dimensions here will be
        the number of columns (length of each string) followed by the number of
        rows (number of strings).
    bytes : str
        Raw data for this parameter.
    '''

    def __init__(self,
                 name,
                 dtype,
                 desc='',
                 bytes_per_element=1,
                 dimensions=None,
                 bytes=b'',
                 handle=None):
        '''Set up a new parameter, only the name is required.'''
        self.name = name
        self.dtypes = dtype
        self.desc = desc
        self.bytes_per_element = bytes_per_element
        self.dimensions = dimensions or []
        self.bytes = bytes
        if handle:
            self.read(handle)

    def __repr__(self):
        return '<Param: {}>'.format(self.desc)

    @property
    def num_elements(self) -> int:
        '''Return the number of elements in this parameter's array value.'''
        e = 1
        for d in self.dimensions:
            e *= d
        return e

    @property
    def total_bytes(self) -> int:
        '''Return the number of bytes used for storing this parameter's data.'''
        return self.num_elements * abs(self.bytes_per_element)

    @property
    def binary_size(self) -> int:
        '''Return the number of bytes needed to store this parameter.'''
        return (
            1 +  # group_id
            2 +  # next offset marker
            1 + len(self.name.encode('utf-8')) +  # size of name and name bytes
            1 +  # data size
            # size of dimensions and dimension bytes
            1 + len(self.dimensions) +
            self.total_bytes +  # data
            1 + len(self.desc.encode('utf-8'))  # size of desc and desc bytes
        )

    def write(self, group_id, handle):
        '''Write binary data for this parameter to a file handle.

        Parameters
        ----------
        group_id : int
            The numerical ID of the group that holds this parameter.
        handle : file handle
            An open, writable, binary file handle.
        '''
        name = self.name.encode('utf-8')
        handle.write(struct.pack('bb', len(name), group_id))
        handle.write(name)
        handle.write(struct.pack('<h', self.binary_size - 2 - len(name)))
        handle.write(struct.pack('b', self.bytes_per_element))
        handle.write(struct.pack('B', len(self.dimensions)))
        handle.write(struct.pack('B' * len(self.dimensions), *self.dimensions))
        if self.bytes is not None and len(self.bytes) > 0:
            handle.write(self.bytes)
        desc = self.desc.encode('utf-8')
        handle.write(struct.pack('B', len(desc)))
        handle.write(desc)

    def read(self, handle):
        '''Read binary data for this parameter from a file handle.

        This reads exactly enough data from the current position in the file to
        initialize the parameter.
        '''
        self.bytes_per_element, = struct.unpack('b', handle.read(1))
        dims, = struct.unpack('B', handle.read(1))
        self.dimensions = [struct.unpack('B', handle.read(1))[
            0] for _ in range(dims)]
        self.bytes = b''
        if self.total_bytes:
            self.bytes = handle.read(self.total_bytes)
        desc_size, = struct.unpack('B', handle.read(1))
        self.desc = desc_size and self.dtypes.decode_string(handle.read(desc_size)) or ''

    def _as(self, dtype):
        '''Unpack the raw bytes of this param using the given struct format.'''
        return np.frombuffer(self.bytes, count=1, dtype=dtype)[0]

    def _as_array(self, dtype, copy=True):
        '''Unpack the raw bytes of this param using the given data format.'''
        if not self.dimensions:
            return [self._as(dtype)]
        elems = np.frombuffer(self.bytes, dtype=dtype)
        # Reverse shape as the shape is defined in fortran format
        view = elems.reshape(self.dimensions[::-1])
        if copy:
            return view.copy()
        return view

class ParamReadonly(object):
    ''' Wrapper exposing readonly attributes of a `c3d.parameter.ParamData` entry.
    '''

    def __init__(self, data):
        self._data = data

    def __eq__(self, other):
        return self._data is other._data

    @property
    def name(self) -> str:
        ''' Get the parameter name. '''
        return self._data.name

    @property
    def desc(self) -> str:
        ''' Get the parameter descriptor. '''
        return self._data.desc

    @property
    def dtypes(self):
        ''' Convenience accessor to the `c3d.dtypes.DataTypes` instance associated with the parameter. '''
        return self._data.dtypes

    @property
    def dimensions(self) -> (int, ...):
        ''' Shape of the parameter data (Fortran format). '''
        return self._data.dimensions

    @property
    def num_elements(self) -> int:
        '''Return the number of elements in this parameter's array value.'''
        return self._data.num_elements

    @property
    def bytes_per_element(self) -> int:
        '''Return the number of bytes used to store each data element.'''
        return self._data.bytes_per_element

    @property
    def total_bytes(self) -> int:
        '''Return the number of bytes used for storing this parameter's data.'''
        return self._data.total_bytes

    @property
    def binary_size(self) -> int:
        '''Return the number of bytes needed to store this parameter.'''
        return self._data.binary_size

    @property
    def int8_value(self):
        '''Get the parameter data as an 8-bit signed integer.'''
        return self._data._as(self.dtypes.int8)

    @property
    def uint8_value(self):
        '''Get the parameter data as an 8-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint8)

    @property
    def int16_value(self):
        '''Get the parameter data as a 16-bit signed integer.'''
        return self._data._as(self.dtypes.int16)

    @property
    def uint16_value(self):
        '''Get the parameter data as a 16-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint16)

    @property
    def int32_value(self):
        '''Get the parameter data as a 32-bit signed integer.'''
        return self._data._as(self.dtypes.int32)

    @property
    def uint32_value(self):
        '''Get the parameter data as a 32-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint32)

    @property
    def uint_value(self):
        ''' Get the parameter data as a unsigned integer of appropriate type. '''
        if self.bytes_per_element >= 4:
            return self.uint32_value
        elif self.bytes_per_element >= 2:
            return self.uint16_value
        else:
            return self.uint8_value

    @property
    def int_value(self):
        ''' Get the parameter data as a signed integer of appropriate type. '''
        if self.bytes_per_element >= 4:
            return self.int32_value
        elif self.bytes_per_element >= 2:
            return self.int16_value
        else:
            return self.int8_value

    @property
    def float_value(self):
        '''Get the parameter data as a floating point value of appropriate type.'''
        if self.bytes_per_element > 4:
            if self.dtypes.is_dec:
                raise AttributeError("64 bit DEC floating point is not supported.")
            # 64-bit floating point is not a standard
            return self._data._as(self.dtypes.float64)
        elif self.bytes_per_element == 4:
            if self.dtypes.is_dec:
                return DEC_to_IEEE(self._data._as(np.uint32))
            else:  # is_mips or is_ieee
                return self._data._as(self.dtypes.float32)
        else:
            raise AttributeError("Only 32 and 64 bit floating point is supported.")

    @property
    def bytes_value(self) -> bytes:
        '''Get the raw byte string.'''
        return self._data.bytes

    @property
    def string_value(self):
        '''Get the parameter data as a unicode string.'''
        return self.dtypes.decode_string(self._data.bytes)

    @property
    def int8_array(self):
        '''Get the parameter data as an array of 8-bit signed integers.'''
        return self._data._as_array(self.dtypes.int8)

    @property
    def uint8_array(self):
        '''Get the parameter data as an array of 8-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint8)

    @property
    def int16_array(self):
        '''Get the parameter data as an array of 16-bit signed integers.'''
        return self._data._as_array(self.dtypes.int16)

    @property
    def uint16_array(self):
        '''Get the parameter data as an array of 16-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint16)

    @property
    def int32_array(self):
        '''Get the parameter data as an array of 32-bit signed integers.'''
        return self._data._as_array(self.dtypes.int32)

    @property
    def uint32_array(self):
        '''Get the parameter data as an array of 32-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint32)

    @property
    def int64_array(self):
        '''Get the parameter data as an array of 32-bit signed integers.'''
        return self._data._as_array(self.dtypes.int64)

    @property
    def uint64_array(self):
        '''Get the parameter data as an array of 32-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint64)

    @property
    def float32_array(self):
        '''Get the parameter data as an array of 32-bit floats.'''
        # Convert float data if not IEEE processor
        if self.dtypes.is_dec:
            # _as_array but for DEC
            if not self.dimensions:
                return [self.float_value]
            return DEC_to_IEEE_BYTES(self._data.bytes).reshape(self.dimensions[::-1])  # Reverse fortran format
        else:  # is_ieee or is_mips
            return self._data._as_array(self.dtypes.float32)

    @property
    def float64_array(self):
        '''Get the parameter data as an array of 64-bit floats.'''
        # Convert float data if not IEEE processor
        if self.dtypes.is_dec:
            raise ValueError('Unable to convert bytes encoded in a 64 bit floating point DEC format.')
        else:  # is_ieee or is_mips
            return self._data._as_array(self.dtypes.float64)

    @property
    def float_array(self):
        '''Get the parameter data as an array of 32 or 64 bit floats.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 4:
            return self.float32_array
        elif self.bytes_per_element == 8:
            return self.float64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit " % self.bytes_per_element +
                            "floating-point precission is not unsupported.")

    @property
    def int_array(self):
        '''Get the parameter data as an array of integer values.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 1:
            return self.int8_array
        elif self.bytes_per_element == 2:
            return self.int16_array
        elif self.bytes_per_element == 4:
            return self.int32_array
        elif self.bytes_per_element == 8:
            return self.int64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit integer values is not unsupported." %
                            self.bytes_per_element)

    @property
    def uint_array(self):
        '''Get the parameter data as an array of integer values.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 1:
            return self.uint8_array
        elif self.bytes_per_element == 2:
            return self.uint16_array
        elif self.bytes_per_element == 4:
            return self.uint32_array
        elif self.bytes_per_element == 8:
            return self.uint64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit integer values is not unsupported." %
                            self.bytes_per_element)

    @property
    def bytes_array(self):
        '''Get the parameter data as an array of raw byte strings.'''
        # Decode different dimensions
        if len(self.dimensions) == 0:
            return np.array([])
        elif len(self.dimensions) == 1:
            return np.array(self._data.bytes)
        else:
            # Convert Fortran shape (data in memory is identical, shape is transposed)
            word_len = self.dimensions[0]
            dims = self.dimensions[1:][::-1]  # Identical to: [:0:-1]
            byte_steps = np.cumprod(self.dimensions[:-1])[::-1]
            # Generate mult-dimensional array and parse byte words
            byte_arr = np.empty(dims, dtype=object)
            for i in np.ndindex(*dims):
                # Calculate byte offset as sum of each array index times the byte step of each dimension.
                off = np.sum(np.multiply(i, byte_steps))
                byte_arr[i] = self._data.bytes[off:off+word_len]
            return byte_arr

    @property
    def string_array(self):
        '''Get the parameter data as a python array of unicode strings.'''
        # Decode different dimensions
        if len(self.dimensions) == 0:
            return np.array([])
        elif len(self.dimensions) == 1:
            return np.array([self.string_value])
        else:
            # Parse byte sequences
            byte_arr = self.bytes_array
            # Decode sequences
            for i in np.ndindex(byte_arr.shape):
                byte_arr[i] = self.dtypes.decode_string(byte_arr[i])
            return byte_arr

    @property
    def any_value(self):
        ''' Get the parameter data as a value of 'traditional type'.

        Traditional types are defined in the Parameter section in the [user manual].

        Returns
        -------
        value : int, float, or str
            Depending on the `bytes_per_element` field, a traditional type can
            be a either a signed byte, signed short, 32-bit float, or a string.

        [user manual]: https://www.c3d.org/docs/C3D_User_Guide.pdf
        '''
        if self.bytes_per_element >= 4:
            return self.float_value
        elif self.bytes_per_element >= 2:
            return self.int16_value
        elif self.bytes_per_element == -1:
            return self.string_value
        else:
            return self.int8_value

    @property
    def any_array(self):
        ''' Get the parameter data as an array of 'traditional type'.

        Traditional types are defined in the Parameter section in the [user manual].

        Returns
        -------
        value : array
            Depending on the `bytes_per_element` field, a traditional type can
            be a either a signed byte, signed short, 32-bit float, or a string.

        [user manual]: https://www.c3d.org/docs/C3D_User_Guide.pdf
        '''
        if self.bytes_per_element >= 4:
            return self.float_array
        elif self.bytes_per_element >= 2:
            return self.int16_array
        elif self.bytes_per_element == -1:
            return self.string_array
        else:
            return self.int8_array

    @property
    def _as_any_uint(self):
        ''' Attempt to parse the parameter data as any unsigned integer format.
            Checks if the integer is stored as a floating point value.

            Can be used to read 'POINT:FRAMES' or 'POINT:LONG_FRAMES'
            when not accessed through `c3d.manager.Manager.last_frame`.
        '''
        if self.bytes_per_element >= 4:
            # Check if float value representation is an integer
            value = self.float_value
            if float(value).is_integer():
                return int(value)
            return self.uint32_value
        elif self.bytes_per_element >= 2:
            return self.uint16_value
        else:
            return self.uint8_value

class Param(ParamReadonly):
    ''' Wrapper exposing both readable and writable attributes of a `c3d.parameter.ParamData` entry.
    '''
    def __init__(self, data):
        super(Param, self).__init__(data)

    def readonly(self):
        ''' Returns a readonly `c3d.parameter.ParamReadonly` instance. '''
        return ParamReadonly(self._data)

    @property
    def bytes(self) -> bytes:
        ''' Get or set the parameter bytes. '''
        return self._data.bytes

    @bytes.setter
    def bytes(self, value):
        self._data.bytes = value


# JN 2016-01-12
"""
manage spikes for data viewer
"""
import os
from .tools import debug

class SpikeManager(object):
    """
    Represent spikes for data viewer
    """
    def __init__(self, sign, label):
        self.fnames = {}
        self.openfiles = {}
        self.times = {}
        self.spikes = {}
        self.beg_end = {}
        self.sortednames = {}
        self.sortedfiles = {}
        self.label = label
        self.sign = sign

    def __del__(self):
        for fid in self.openfiles.values():
            fid.close()

    def check_add(self, key, entname):
        sppath = os.path.join(key, 'data_{}.h5'.format(key))
        sorted_path = os.path.join(key, self.label)
        if os.path.isdir(sorted_path):
            self.sortednames[entname] = sppath

        if os.path.exists(sppath):
            self.fnames[entname] = sppath

        debug(self.sortednames)

    def h5f_by_key(self, key):
        if key in self.sortednames:
            print('loading sorted spikes for ' + key)
            self.sortedfiles[key] = Combinato(self.sortednames[key],
                                              self.sign, self.label)

        if key not in self.fnames:
            debug('No spike file for ' + key)
            return

        if key not in self.openfiles:
            fid = tables.open_file(self.fnames[key], 'r')
            self.openfiles[key] = fid
            # warning! times get loaded as copies
            # because we have to search them
            print('Loading times for ' + key)
            t = fid.get_node('/' + self.sign + '/times')[:]
            print('Done')
            self.times[key] = t
            s = fid.get_node('/' + self.sign + '/spikes')
            self.spikes[key] = s

        else:
            t = self.times[key]
            s = self.spikes[key]

        return t, s

    def set_beg_end(self, key, start, stop, sign='pos'):

        ret = self.h5f_by_key(key, sign)
        if ret is None:
            return

        beg = ret[0]
        end = ret[1]

        t = self.times[key]
        beg = max(t.searchsorted(start) - 1, 0)
        end = min(t.searchsorted(stop) + 1, t.shape[0])

        self.beg_end[key] = (beg, end)
        print(self.beg_end[key])

    def get_sorted_data(self, key, start, stop):
        ret = {}
        ea = np.array([])
        if key in self.sortedfiles:
            print(start, stop)
            clu = self.sortedfiles[key].get_groups_joined()
            for c in clu:
                times = clu[c]['times']
                idx = (times >= start) & (times <= stop)
                print(times[0], times[-1], start, stop)
                print(c, idx.sum())
                if idx.any():
                    t = times[idx]
                else:
                    t = ea
                ret[c] = {'times': t}
                print(c, idx.sum())

            return ret

        else:
            return np.array([])

    def get_sp_data(self, key, which='times'):
        key = str(key)

        if key in self.beg_end:
            beg = self.beg_end[key][0]
            end = self.beg_end[key][1]
        else:
            return np.array([])

        if which == 'times':
            retv = self.times[key]
        elif which == 'spikes':
            retv = self.spikes[key]

        return retv[beg:end]


"""matplotlib
plotで線の色と形状を一括で変更する方法

[説明ページ]
https://tech.nkhn37.net/matplotlib-color-linestyle/#i-3
"""
import matplotlib.pyplot as plt
import numpy as np

plt.style.use('seaborn-whitegrid')

x = np.linspace(0, 10, 100)

# ===== 腺の色と形状を一括で指定する
# 通常の線 + 青
plt.plot(x, x, '-b')
# 点線 + マゼンタ
plt.plot(x, x+1, '--m')
# 点線ドット + 黒
plt.plot(x, x+2, '-.k')
# ドット + 緑
plt.plot(x, x+3, ':g')

plt.show()


class Solution(object):
    def lengthOfLongestSubstring(self, s):
        """
        :type s: str
        :rtype: int
        """

        longest, start, visited = 0, 0, [False for _ in range(256)]
        for ind, val in enumerate(s):
            if not visited[ord(val)]:
                visited[ord(val)] = True
            else:
                while val != s[start]:
                    visited[ord(s[start])] = False
                    start += 1
                start += 1
            longest = max(longest, ind - start + 1)
        return longest


from django.shortcuts import render,redirect
from .serializers import InscritoSerializer
from .models import Inscrito
from rest_framework.response import Response
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.views import APIView
from django.http import Http404
from seminarioApp.models import Inscrito
from seminarioApp.forms import FormReserva
from django.http import JsonResponse

##########
#API REST#
##########
def verinscritosapi(request):
    inscritos = Inscrito.objects.all()
    data = {'inscritos': list(inscritos.values('nombre', 'telefono', 'fecha', 'institucion', 'hora', 'estado', 'observacion'))}
    return JsonResponse(data)

######
#CRUD#
######
def index (request):
    return render(request, 'index.html')

def reservas (request):
    reservas = Inscrito.objects.all()
    data = {'reservas': reservas}
    return render(request, 'reservas.html', data)

def agregarreserva(request):
    form = FormReserva()
    if request.method == 'POST':
        form = FormReserva(request.POST)
        if (form.is_valid()):
            form.save()
        return index(request)
    data = {'form': form}
    return render(request, 'agregar.html', data)

def editarreserva(request, id):
    reserva = Inscrito.objects.get(id=id)
    form = FormReserva(instance=reserva)
    if request.method == "POST":
        form = FormReserva(request.POST, instance=reserva)
        if form.is_valid():
            try:
                form.save()
                model = form.instance
                return redirect('/reservas')
            except Exception as e:
                pass
    return render(request,'agregar.html',{'form':form})

def eliminarreserva(request, id):
    reserva = Inscrito.objects.get(id = id)
    try:
        reserva.delete()
    except:
        pass
    return redirect('/reservas')

#################
#CLASS BASE VIEW#
#################

class ListaInscrito(APIView):
    def get(self, request):
        inscrito = Inscrito.objects.all()
        serial = InscritoSerializer(inscrito, many=True)
        return Response(serial.data)

    def post(self, request):
        serial = InscritoSerializer(data = request.data)
        if serial.is_valid():
            serial.save()
            return Response(serial.data, status=status.HTTP_201_CREATED)
        return Response(serial.errors, status=status.HTTP_400_BAD_REQUEST)

class DetalleInscrito(APIView):
    def get_object(self, pk):
        try:
            return Inscrito.objects.get(id=pk)
        except Inscrito.DoesNotExist:
            return Http404

    def get(self, request, pk):
        inscrito = self.get_object(pk)
        serial = InscritoSerializer(inscrito)
        return Response(serial.data)

    def put(self, request, pk):
        inscrito = self.get_object(pk)
        serial = InscritoSerializer(inscrito, data=request.data)
        if serial.is_valid():
            serial.save()
            return Response(serial.data)
        return Response(serial.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, pk):
        inscrito = self.get_object(pk)
        inscrito.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

####################
#FUNCTION BASE VIEW#
####################

@api_view(['GET', 'POST'])
def listar_inscritos(request):
    if request.method == 'GET':
        persona = Inscrito.objects.all()
        serial = InscritoSerializer(persona, many=True)
        return Response(serial.data)

    if request.method == 'POST':
        serial = InscritoSerializer(data = request.data)
        if serial.is_valid() :
            serial.save()
            return Response(serial.data, status=status.HTTP_201_CREATED)
        return Response(serial.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
def inscrito_detalle(request, pk):
    try:
        inscrito = Inscrito.objects.get(pk=pk)
    except Inscrito.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serial = InscritoSerializer(inscrito)
        return Response(serial.data)

    if request.method == 'PUT':
        serial = InscritoSerializer(inscrito, data=request.data)
        if serial.is_valid():
            serial.save()
            return Response(serial.data)
        return Response(serial.errors, status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'DELETE':
        inscrito.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


import json
import os
from datetime import datetime
from subprocess import run

import evaluation.util.hash as hash

EXPERIMENT_LOG_HASHES = 'experiment_log_hashes.csv'

config = json.load(open('config.json', 'r'))

def _get_log_file(results_dir, filepath, execution_name):
    now = int(datetime.now().timestamp())

    if execution_name is None:
        execution_name = os.path.basename(filepath)

    return open(os.path.join(results_dir, 'log', f'{now}_{execution_name}.log'), 'w')

def run_r_script(results_dir, file, execution_name):
    cmd = ['Rscript', file]
    logfile = _get_log_file(results_dir, file, execution_name)
    run(cmd, cwd=config['git']['repopath'], stdout=logfile, stderr=logfile)
    logfile.close()
    log_hash = hash.calculate_hash_of_file(logfile.name)
    with open(os.path.join(results_dir, 'csv', EXPERIMENT_LOG_HASHES), 'a') as hash_file:
        hash_file.writelines(f"{os.path.basename(logfile.name)};{log_hash}\n")

    return log_hash

def run_bash_script(results_dir, file, execution_name):
    cmd = ['/bin/bash', file]
    logfile = _get_log_file(results_dir, file, execution_name)

    run(cmd, cwd=config['git']['repopath'], stdout=logfile, stderr=logfile)

    logfile.close()
    log_hash = hash.calculate_hash_of_file(logfile.name)

    with open(os.path.join(results_dir, 'csv', EXPERIMENT_LOG_HASHES), 'a') as hash_file:
        hash_file.writelines(f"{os.path.basename(logfile.name)};{hash.calculate_hash_of_file(logfile.name)}\n")

    return log_hash


import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import numpy as np
import operator
from queue import PriorityQueue

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
MAX_LENGTH = 10
SOS_token = 0
EOS_token = 1
EOS_char_token = 0

class DecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size):
        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        output = self.embedding(input).view(1, 1, -1)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.softmax(self.out(output[0]))
        return output, hidden

class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.max_length = max_length

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)
        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0), encoder_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)

        return output, hidden, attn_weights

    def init_hidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

class EncoderRNN(nn.Module):
    def __init__(self, n_chars, n_words, hidden_size, char_embedding_dim=25, char_representation_dim=25):
        """
        Input parameters:
            n_chars = Number of unique characters in input language
            n_words = Number of unique words in input language
            hidden_size = Dimension of GRU input and output.
            char_embedding_dim = Dimension of the character embeddings
            char_representation_dim = Output dimension from the CNN encoder for character
        """
        super(EncoderRNN, self).__init__()

        # Parameters
        self.n_chars = n_chars
        self.char_embedding_dim = char_embedding_dim
        self.char_representation_dim = char_representation_dim
        self.n_words = n_words
        self.hidden_size = hidden_size

        # Character-level encoder
        self.char_embedding_layer = nn.Embedding(n_chars, char_embedding_dim)
        self.char_cnn3_layer = nn.Conv2d(in_channels=1, out_channels=char_representation_dim, kernel_size=(3, char_embedding_dim), padding=(2, 0))

        # Word embedding layer (Dimension of the word embeddings is automatically derived as hidden_size - char_representation_dim)
        self.word_embedding_layer = nn.Embedding(n_words, hidden_size - char_representation_dim)

        # GRU layer
        self.gru = nn.GRU(hidden_size, hidden_size)

        # Weights
        self.init_embedding(self.char_embedding_layer.weight)

    def forward(self, char_indexes, word_index, hidden):
        # Get char representation
        char_embedding = self.char_embedding_layer(char_indexes).unsqueeze(1)
        char_cnn3 = self.char_cnn3_layer(char_embedding).squeeze(-1).unsqueeze(1)
        char_representation = F.max_pool2d(char_cnn3, kernel_size=(1, char_cnn3.size(-1))).squeeze(-1)

        # Get word embedding
        word_embedding = self.word_embedding_layer(word_index).view(1, 1, -1)

        # Concatenate char representation with word embedding
        combined = torch.cat((char_representation, word_embedding), dim=2)

        # Feed combined and hidden to GRU
        output, hidden = self.gru(combined, hidden)

        return output, hidden

    def init_hidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

    @staticmethod
    def init_embedding(input_embedding):
        """
        Initialize embedding
        """
        bias = np.sqrt(3.0 / input_embedding.size(1))
        nn.init.uniform_(input_embedding, -bias, bias)

class BeamSearchNode(object):
    def __init__(self, hidden_state, previous_node, word_id, log_prob, length):
        self.hidden_state = hidden_state
        self.prev_node = previous_node
        self.word_id = word_id
        self.log_p = log_prob
        self.length = length

    def eval(self, alpha=1.0):
        reward = 0
        # Add here a function for shaping a reward
        return self.log_p / float(self.length - 1 + 1e-6) + alpha * reward

def beam_decode(decoder_hiddens, decoder, encoder_outputs=None):

    beam_width = 5
    top_k = 1  # how many sentences do you want to generate
    decoded_batch = []

    # print(target_tensor.shape)
    # print(decoder_hiddens.shape)
    # print(encoder_outputs.shape)

    # decoding goes sentence by sentence
    for idx in range(decoder_hiddens.size(1)):
        decoder_hidden = decoder_hiddens[:, idx, :]
        encoder_output = encoder_outputs[:, idx, :]

        # Start with the start of the sentence token
        decoder_input = torch.LongTensor([[SOS_token]]).to(device)

        # Number of sentence to generate
        endnodes = []
        number_required = min((top_k + 1), top_k - len(endnodes))

        # Starting node - hidden vector, previous node, word id, logp, length
        node = BeamSearchNode(decoder_hidden, None, decoder_input, 0, 1)
        nodes = PriorityQueue()

        # Start the queue
        nodes.put((-node.eval(), node))
        qsize = 1

        # Start beam search
        while True:

            # Give up when decoding takes too long
            if qsize > 2000:
                break

            # Fetch the best node
            score, n = nodes.get()
            decoder_input = n.word_id
            decoder_hidden = n.hidden_state

            if n.word_id.item() == EOS_token and n.prev_node is not None:
                endnodes.append((score, n))
                # If we reached maximum # of sentences required
                if len(endnodes) >= number_required:
                    break
                else:
                    continue

            # Decode for one step using decoder
            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_output)

            # PUT HERE REAL BEAM SEARCH OF TOP
            log_prob, indexes = torch.topk(decoder_output, beam_width)
            # print(indexes)
            next_nodes = []

            for new_k in range(beam_width):
                decoded_t = indexes[0][new_k].view(1, -1)
                log_p = log_prob[0][new_k].item()

                node = BeamSearchNode(decoder_hidden, n, decoded_t, n.log_p + log_p, n.length + 1)
                score = -node.eval()
                next_nodes.append((score, node))

            # Put them into queue
            for i in range(len(next_nodes)):
                score, next_node = next_nodes[i]
                nodes.put((score, next_node))
                # increase qsize
            qsize += len(next_nodes) - 1

        # Choose nbest paths, back trace them
        if len(endnodes) == 0:
            endnodes = [nodes.get() for _ in range(top_k)]

        utterances = []
        for score, n in sorted(endnodes, key=operator.itemgetter(0)):
            utterance = []
            utterance.append(n.word_id)
            # back trace
            while n.prev_node is not None:
                n = n.prev_node
                utterance.append(n.word_id)

            utterance = utterance[::-1]
            utterances.append(utterance)

        decoded_batch.append(utterances)

    return decoded_batch


INPUT = "input/03.in"

def read_input():
    with open(INPUT) as f:
        return [line.split(",") for line in f]

def directions_to_coords(line):
    points = set()
    distances = dict()
    current_pos = (0, 0)
    current_dist = 0
    for dir in line:
        orientation = dir[0]
        amount = int(dir[1:])

        if orientation == "U":
            for i in range(amount):
                coord = (current_pos[0], current_pos[1] - i - 1)
                if not coord in distances:
                    distances[coord] = current_dist + i + 1
                points.add(coord)
            current_pos = (current_pos[0], current_pos[1] - amount)
        elif orientation == "R":
            for i in range(amount):
                coord = (current_pos[0] + i + 1, current_pos[1])
                if not coord in distances:
                    distances[coord] = current_dist + i + 1
                points.add(coord)
            current_pos = (current_pos[0] + amount, current_pos[1])
        elif orientation == "D":
            for i in range(amount):
                coord = (current_pos[0], current_pos[1] + i + 1)
                if not coord in distances:
                    distances[coord] = current_dist + i + 1
                points.add(coord)
            current_pos = (current_pos[0], current_pos[1] + amount)
        elif orientation == "L":
            for i in range(amount):
                coord = (current_pos[0] - i - 1, current_pos[1])
                if not coord in distances:
                    distances[coord] = current_dist + i + 1
                points.add(coord)
            current_pos = (current_pos[0] - amount, current_pos[1])

        current_dist += amount
    return (points, distances)

lines = read_input()
(first_coords, first_distances) = directions_to_coords(lines[0])
(second_coords, second_distances) = directions_to_coords(lines[1])
intersect = first_coords.intersection(second_coords)

# PART 1
min_intersect = min(intersect, key = lambda coord: abs(coord[0]) + abs(coord[1]))
print(f"Manhattan distance of closest intersection: {abs(min_intersect[0]) + abs(min_intersect[1])}")

# PART 2
min_intersect = min(intersect, key = lambda coord: first_distances[coord] + second_distances[coord])
print(f"Fewest combined steps for an intersection: {first_distances[min_intersect] + second_distances[min_intersect]}")


from flask import Flask, request
from operations import add, sub, mult, div

app = Flask(__name__)

# http://127.0.0.1:5000/add?a=1&b=2

@app.route("/add")
def get_sum():
    a = request.args["a"]
    b = request.args["b"]

    return f"
Sum: {add(int(a),int(b))}
"

@app.route("/sub")
def get_difference():
    a = request.args["a"]
    b = request.args["b"]

    return f"
Difference: {sub(int(a),int(b))}
"

@app.route("/mult")
def get_product():
    a = request.args["a"]
    b = request.args["b"]

    return f"
Product: {mult(int(a),int(b))}
"

@app.route("/div")
def get_quotient():
    a = request.args["a"]
    b = request.args["b"]

    return f"
Quotient: {div(int(a),int(b))}
"

#Part 2

OPERATIONS = {
    "add": add,
    "sub": sub,
    "mult": mult,
    "div": div
}

# http://127.0.0.1:5000/math/add?a=1&b=2

@app.route('/math/<operation>')
def math_operation(operation):
    a = request.args["a"]
    b = request.args["b"]
    result = OPERATIONS[operation](int(a), int(b))

    return f"""

        Values a: {a} b: {b}
Produced the result of: {result}
Using the {operation.capitalize()} operation

        """


def ngram(text, num):
    res = []
    slen = len(text) - num +1
    for i in range(slen):
        ss = text[i:i+num]
        res.append(ss)
    return res

def diff_ngram(texta, textb, num):
    a = ngram(texta, num)
    b = ngram(textb, num)
    r = []
    cnt = 0
    for i in a:
        for j in b :
            if i == j:
                cnt +=1
                r.append(i)
    return cnt / len(a), r

text1 = "오늘 강남에서 맛있는 스파게티를 먹었다."
text2 = "강남에서 먹었던 오늘의 스파게티는 맛있었다."
# 2-gram
r2, word2 = diff_ngram(text1, text2, 2)
print("2-gram:", r2, word2)

# 3-gram
r3, word3 = diff_ngram(text1, text2, 3)
print("3-gram:", r3, word3)


import pandas as pd
import sys
import json
import requests
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
from pandas.io.json import json_normalize

# api-endpoint
URL = "http://35.228.102.158:3000/api/v1/Restaurant/ratings"

# location given here
# sending get request and saving the response as response object
r = requests.get(url = URL)

# extracting data in json format
data = r.json()

df = pd.DataFrame(data['data'], columns= ['id','rating','meal_id','user_id'])

#
#count=df.groupby(['meal_id','restaurant_name'])['rating'].count()
mean=df.groupby(['meal_id'])['rating'].mean()

df_meal_features = df.pivot_table(index ='user_id',
              columns =['meal_id'], values ='rating').fillna(0)

mat_meal_features = csr_matrix(df_meal_features.values)

#print(mat_meal_features)

#### model and fit

model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
# fit the dataset
model_knn.fit(mat_meal_features)

def find_similar_user(model_knn, data, user):
    n_recommendations=5
    # fit
    model_knn.fit(data)
    distances, indices = model_knn.kneighbors(data[user], n_neighbors=n_recommendations+1)

    #raw_recommends = \
    #    sorted(list(zip(df_meal_features.index[indices.squeeze().tolist()], distances.squeeze().tolist())), key=lambda x: x[1])[:0:-1]
    # get reverse mapper
    arr =[]
    for i in df_meal_features.index[indices.squeeze().tolist()] :
        arr.append(i)
    # print recommendations
    arr.pop(0)
    return arr

def make_recommendation(users):
    dp_user = df.pivot_table(index ='meal_id',
              columns ='user_id', values ='rating').fillna(0)

    recommend = dp_user[users[0]]
    recommend = pd.merge(recommend,dp_user[users[1]],on=['meal_id'])
    recommend = pd.merge(recommend, dp_user[users[2]], on=['meal_id'])

    return recommend.mean(axis=1).sort_values(ascending = False).head(20)

x=list(make_recommendation(find_similar_user(model_knn,mat_meal_features,df_meal_features.index.get_loc(sys.argv[1]))).index)

y = json.dumps(x)

print(y)
#print(df.loc[df["user_id"]==array[0]])


from flask import Flask, request, render_template
import os
app = Flask(__name__)

@app.route('/')
def home():
	topadvs = os.listdir('static/images/topadvs')
	slideshow = os.listdir('static/images/slideshow')
	#top_factor = len(topadvs)
	return render_template('home.html',
		topadvs=topadvs,
		slideshow = slideshow,
		#top_factor=top_factor,

		)

if __name__ == "__main__":
	app.run(debug=True)


from random import choice

from backend.database.db_schema import Medicine, db
from sqlalchemy.sql import func

def get_medicine_detail(medicine_id: int):
    medicine_entity: Medicine = db.get_or_404(Medicine, medicine_id)
    return {"id": medicine_entity.id,
            "name": medicine_entity.name,
            "sukl_code": medicine_entity.sukl_code,
            "contraindications": medicine_entity.contraindications,
            "adverse_effects": medicine_entity.adverse_effects,
            "active_ingredients": [{"id": ingredient_with_dosage.active_ingredient.id,
                                    "name": ingredient_with_dosage.active_ingredient.name,
                                    "dosage": ingredient_with_dosage.dosage,
                                    "units": ingredient_with_dosage.units
                                    } for ingredient_with_dosage
                                   in medicine_entity.active_ingredients_with_dosage],
            "negative_interactions": [{"id": medicine.id,
                                       "name": medicine.name} for medicine
                                      in get_negative_interactions_for_medicine(medicine_entity)],
            "alternative_medicines": [{"id": medicine.id,
                                       "name": medicine.name,
                                       "sukl_code": medicine.sukl_code,
                                       "contraindications": medicine.contraindications,
                                       "adverse_effects": medicine.adverse_effects,
                                       "active_ingredients": [
                                           {"id": ingredient_with_dosage.active_ingredient.id,
                                            "name": ingredient_with_dosage.active_ingredient.name,
                                            "dosage": ingredient_with_dosage.dosage,
                                            "units": ingredient_with_dosage.units
                                            } for ingredient_with_dosage
                                           in medicine.active_ingredients_with_dosage],
                                       "pdf_dropbox_link": medicine.pdf_dropbox_link
                                       } for medicine
                                      in get_alternative_medicines(medicine_entity)],
            "pdf_dropbox_link": medicine_entity.pdf_dropbox_link
            }

def get_negative_interactions_for_medicine(medicine: Medicine):
    return list(medicine.negative_interactions_1) + list(medicine.negative_interactions_2)

def get_alternative_medicines(medicine: Medicine):
    active_ingredients_with_dosage = medicine.active_ingredients_with_dosage
    return [medicine_with_dosage.medicine for active_ingredients_with_dosage in
            active_ingredients_with_dosage
            for medicine_with_dosage in
            list(active_ingredients_with_dosage.active_ingredient.medicines_with_dosage)
            if medicine_with_dosage.medicine != medicine]

def get_random_medicine_and_adverse_effect():
    random_medicine = Medicine.query.order_by(func.random()).first()
    random_adverse_effect = choice(random_medicine.adverse_effects)
    return {"name": random_medicine.name,
            "random_adverse_effect": random_adverse_effect}


import pandas as pd
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, mean_squared_error, mean_absolute_error
from sklearn import preprocessing
from sklearn.model_selection import train_test_split

dataset = pd.read_csv("listings.csv", delimiter=",")
dataset["reviews_per_month"] = dataset["reviews_per_month"].replace(",", ".", regex=True)
dataset = dataset.dropna()
dataset = dataset.astype({"price" : "int32", "minimum_nights" : "int32", "number_of_reviews" : "int32", "calculated_host_listings_count" : "int32", "availability_365" : "int32", "reviews_per_month" : "float64"})
dataset["last_review"] = pd.to_datetime(dataset["last_review"], format= "%Y-%m-%d")
print(dataset["room_type"].unique())
dataset["room_type"] = pd.Categorical(dataset["room_type"], dataset["room_type"].unique())
dataset["room_type"] = dataset["room_type"].cat.rename_categories([1,2,3])
# print(dataset.dtypes)
# print(dataset.isna().values.any())
# print(dataset.head())
# print(dataset[["room_type"]])
# print(len(dataset))
type1 = 0
type2 = 0
type3 = 0
# print(dataset.iloc[1]["price"])
for i in range(len(dataset)):
    if dataset.iloc[i]["room_type"] == 1:
        type1 = type1+1
    if dataset.iloc[i]["room_type"] == 2:
        type2 = type2+1
    if dataset.iloc[i]["room_type"] == 3:
        type3 = type3+1
    # print(i)
# print(type1, type2, type3)
probability1 = type1/len(dataset)
probability2 = type2/len(dataset)
probability3 = type3/len(dataset)

mean = 1 * probability1 + 2 * probability2 + 3 * probability3
print("Mean of room_type : " + str(mean))
print("Variance of reviews_per_month : " + str(dataset["reviews_per_month"].var()))


import os

import numpy as np
from PIL import Image
import skimage
import skimage.io
from torch.utils.data import Dataset
from tqdm import tqdm

def read_rgb_image(path_to_image):
    image = skimage.img_as_ubyte(skimage.io.imread(path_to_image))
    if len(image.shape) == 2:
        image = image[:, :, np.newaxis]
    if image.shape[-1] == 1:
        image = image * np.ones((1,1,3), dtype=np.uint8)
    return image

class KITTIStereoRAM(Dataset):
    def __init__(self, root, train=True, transforms=None):
        super(KITTIStereoRAM, self).__init__()

        self.root = root
        if train:
            path_to_dataset = os.path.join(root, 'train')
        else:
            path_to_dataset = os.path.join(root, 'val')

        self.transforms = transforms
        self.left_images = []
        self.right_images = []
        self.targets = []
        self.valid_pixels_masks = []
        for img_name in tqdm(os.listdir(os.path.join(path_to_dataset, 'colored_0'))):
            left_img_path = os.path.join(path_to_dataset, 'colored_0', img_name)
            right_img_path = os.path.join(path_to_dataset, 'colored_1', img_name)
            disp_img_path = os.path.join(path_to_dataset, 'disp_noc', img_name)

            left_img = read_rgb_image(left_img_path)
            right_img = read_rgb_image(right_img_path)

            # disparity normalization
            disp_img = skimage.io.imread(disp_img_path)[:,:,np.newaxis]
            valid_pixels_mask = disp_img > 0
            disp_img = disp_img.astype(np.float32) / 256.

            self.left_images.append(Image.fromarray(left_img))
            self.right_images.append(Image.fromarray(right_img))
            self.targets.append(disp_img)
            self.valid_pixels_masks.append(valid_pixels_mask)

    def __len__(self):
        return len(self.left_images)

    def __getitem__(self, index):
        left_img = self.left_images[index]
        right_img = self.right_images[index]
        disp = self.targets[index]
        valid_pixels_mask = self.valid_pixels_masks[index]

        if self.transforms is not None:
            left_img, right_img, disp, valid_pixels_mask = self.transforms(
                    left_img, right_img, disp, valid_pixels_mask)
        return left_img, right_img, disp, valid_pixels_mask


from flask import Flask, render_template
from models import Lingkaran

app = Flask(__name__)

@app.route('/')
def index():
    str_var = 'Pengembangan Aplikasi Web lanjut'
    int_var = 20
    float_var = 24.1
    list_var = [1,2,3]
    dict_var = {'satu': 1, 'dua': 2, 'tiga': 3}
    model = Lingkaran(30.0)

    a=20
    b=30
    c=20
    d = -5

    return render_template("p10-jinja-1.html",
                            str_var=str_var,
                            int_var=int_var,
                            float_var=float_var,
                            list_var=list_var,
                            dict_var=dict_var,
                            model=model,
                            a=a,
                            b=b,
                            c=c,
                            d=d)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Note: To use the 'upload' functionality of this file, you must:
#   $ pip install twine

import io
import os
import sys
from shutil import rmtree

from setuptools import setup, Command

# Package meta-data.
NAME = "yam-py"
DESCRIPTION = "HTML Parsing for Humans."
URL = "https://github.com/plugarut/yam-py"
EMAIL = "plugaru.tudor@gmail.com"
AUTHOR = "Tudor Plugaru"
VERSION = "0.1.0"

# What packages are required for this module to be executed?
REQUIRED = ["requests", "requests-html", "click", "colorama"]

# The rest you shouldn't have to touch too much :)
# ------------------------------------------------
# Except, perhaps the License and Trove Classifiers!
# If you do change the License, remember to change the Trove Classifier for that!

here = os.path.abspath(os.path.dirname(__file__))

# Import the README and use it as the long-description.
# Note: this will only work if 'README.rst' is present in your MANIFEST.in file!
with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
    long_description = "\n" + f.read()

# Where the magic happens:
setup(
    name=NAME,
    version=VERSION,
    description=DESCRIPTION,
    long_description=long_description,
    author=AUTHOR,
    author_email=EMAIL,
    url=URL,
    python_requires=">=3.6.0",
    # If your package is a single module, use this instead of 'packages':
    py_modules=["yam_py"],
    install_requires=REQUIRED,
    include_package_data=True,
    license="MIT",
    classifiers=[
        # Trove classifiers
        # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: Implementation :: CPython",
        "Programming Language :: Python :: Implementation :: PyPy",
    ],
    entry_points={"console_scripts": ["yam-py=yam_py:cli"]},
)