import pandas as pd
import numpy as np
import os
import csv
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from math import sqrt
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
metric = {}
met = 'MCC'
threshold = 970000
# for house_num in range(14,16):
# devices = [4322480268,3424383659,1130799754,921227642,684659082]
devices = [3424383659,921227642]
sum = 0
ratio = {}
for device in devices:
    sum = sum + device
print(sum)
for i in range(0,2):
    ratio[i] = devices[i]/sum
print(ratio)

# models = ['FHMM']
models= ["CO",'RNN','WindowGRU','DAE','Seq2Seq','Mean','FHMM']
input_path = '/aul/homes/qli027/projects/disaggregation/final_version/prepare_for_figures/different_approaches/data/VPN_/'
output_path = '/aul/homes/qli027/projects/disaggregation/final_version/prepare_for_figures/different_approaches/data/VPN_/F1.csv'
input_path_2 = '/aul/homes/qli027/projects/disaggregation/final_version/prepare_for_figures/different_approaches/data/VPN/'

for model in models:
    sum_ = 0
    # for i in [2,3,4,5,6]:
    for i in [3,5]:
        if i !=6 :
            groundtruth = pd.read_csv(input_path + 'new_API_groundtruth.csv')
            prediction = pd.read_csv(input_path +  str(model) + '.csv')

        #         header_device.append(i)
            gt_ = np.array(groundtruth.iloc[:,[i]])
            gt_ = np.where(gt_<= threshold,0,1)
            pred_ = np.array(prediction.iloc[:,[i]])
            pred_ = np.where(pred_<= threshold,0,1)
    #         if i == 2:
    #             sum_ = sum_ + matthews_corrcoef(gt_,pred_,) *ratio[0]
    #         if i == 3:
    #             sum_ = sum_ + matthews_corrcoef(gt_,pred_) *ratio[1]
    #         if i == 4:
    #             sum_ = sum_ + matthews_corrcoef(gt_,pred_) *ratio[2]
    #         if i == 5:
    #             sum_ = sum_ + matthews_corrcoef(gt_,pred_) *ratio[3]

#             if i == 3:
#                 sum_ = sum_ + matthews_corrcoef(gt_,pred_,average='micro') *ratio[0]
#             if i == 5:
#                 sum_ = sum_ + matthews_corrcoef(gt_,pred_,average='micro') *ratio[1]
            if i == 3:
                sum_ = sum_ + f1_score(gt_,pred_,average='micro') *ratio[0]
            if i == 5:
                sum_ = sum_ + f1_score(gt_,pred_,average='micro') *ratio[1]
    #     if i == 6:
    #         groundtruth = pd.read_csv(input_path_2 + 'groundtruth.csv')
    #         prediction = pd.read_csv(input_path_2 +  'FHMM.csv')
    #         gt_ = np.array(groundtruth.iloc[:,[5]])
    #         gt_ = np.where(gt_<= threshold,0,1)
    #         pred_ = np.array(prediction.iloc[:,[5]])
    #         pred_ = np.where(pred_<= threshold,0,1)
    #         sum_ = sum_ + matthews_corrcoef(gt_,pred_) *ratio[1]
    with open(output_path , 'a') as f:
        writer = csv.writer(f)
        writer.writerow([model,sum_])


"""Testbench for the model class."""
import logging
import unittest
import traceback
import numpy as np
import pandas as pd
import tensorflow as tf
from config_tb import *
from src.model import Model
from src.utilizer import Utilizer
from src.visualizer import Visualizer
from src.indicators import Indicators
from src.preprocessor import Preprocessor
from src.data_aquirer import Data_Aquirer
from src.model import Architecture, Branch, Output

class Test_Model(unittest.TestCase):
    """Integration test for the Model class.

    @remarks This test is not a unit test, but an integration test. It tests the
                preprocessor and model class together.
    """

    def synchronize_dataframes(dataframes, column_to_sync="t"):
        # Create a list to hold all synchronized dataframes
        synced_dataframes = []

        # Start with the first dataframe
        synced_df = dataframes[0]

        # Loop through the rest of the dataframes
        for df in dataframes[1:]:
            # Merge on the column_to_sync with an inner join
            synced_df = pd.merge(synced_df, df, how="inner", on=column_to_sync)

        # Now we have a dataframe that contains rows with 't' values
        # that appear in all original dataframes. Next, we need to
        # create a synchronized version of each original dataframe.

        # Loop through the original dataframes again
        for df in dataframes:
            # For each dataframe, keep only the rows that exist in synced_df
            synced_dataframes.append(
                df[df[column_to_sync].isin(synced_df[column_to_sync])]
            )

        return synced_dataframes

    def test_compile_fit_predict(self):
        """Test the compile, fit and predict method with data from the preprocessor."""
        pair = os.environ.get("START_PAIR")
        from_saved_file = os.environ.get("FROM_SAVED_FILE")
        use_multiple_gpus = os.environ.get("USE_MULTIPLE_GPUS")
        utilize_model = os.environ.get("UTILIZE_MODEL")
        # Data
        CORR_PAIRS = ["C:USDCHF", "C:EURCAD", "C:GBPJPY", "C:GBPCAD", "C:USDJPY"]
        pairs = []
        # First get the target pair
        aquirer = Data_Aquirer(PATH_PAIRS, API_KEY, api_type="full")
        target_pair = aquirer.get(
            pair,
            MINUTES_TRAIN,
            start=START,
            end=END,
            save=True,
            from_file=from_saved_file,
        )
        indicators = Indicators(PATH_INDICATORS, pair, target_pair, TEST_INDICATORS)
        indicator_data = indicators.calculate(save=True)
        pairs.append(indicator_data)
        # Get correlated pairs
        for corr_pair in CORR_PAIRS:
            api_data_corr = aquirer.get(
                corr_pair,
                MINUTES_TRAIN,
                start=START,
                end=END,
                save=True,
                from_file=from_saved_file,
            )
            # Apply indicator to correlated pair
            indicators = Indicators(
                PATH_INDICATORS, corr_pair, api_data_corr, TEST_INDICATORS
            )
            indicator_data_corr = indicators.calculate(save=False)
            # rename all columns except 't'
            indicator_data_corr.columns = [
                f"{col}_{corr_pair}" if col != "t" else col
                for col in indicator_data_corr.columns
            ]
            # Preprocess the correlated pair
            pairs.append(indicator_data_corr)
            # Append to the list of correlated pairs
        # Synchronize the dataframes
        synced_df = Test_Model.synchronize_dataframes(pairs)
        # Target pair is first dataframe of the list
        target_pair = synced_df[0]
        # Correlated pairs are the rest of the dataframes
        correlated_pairs = synced_df[1:]
        # Preprocessor for target pair
        target_pair = Preprocessor(
            target_pair,
            time_steps_in=TEST_TIME_STEPS_IN,
            time_steps_out=TEST_TIME_STEPS_OUT,
            test_length=TEST_LENGTH,
            target=TARGET,
            scale=TEST_SCALE,
            shift=TEST_SHIFT,
        )
        # Preprocessor for correlated pairs
        corr_pairs = []
        for corr_pair in correlated_pairs:
            corr_pair = Preprocessor(
                corr_pair,
                time_steps_in=TEST_TIME_STEPS_IN,
                time_steps_out=TEST_TIME_STEPS_OUT,
                test_length=TEST_LENGTH,
                scale=TEST_SCALE,
                shift=TEST_SHIFT,
            )
            corr_pairs.append(corr_pair)
        corr_pairs.append(target_pair)

        # Model
        model = Model(
            MODEL_PATH,
            MODEL_NAME,
            target_pair.y_train,
        )
        if utilize_model != "True":
            branches = []
            for corr_pair in corr_pairs:
                if isinstance(corr_pair.x_train, np.ndarray):
                    branches.append(
                        Branch(
                            corr_pair.x_train,
                            [256, 128, 128],
                            [128, 64, 64],
                            [256, 128, 128],
                            [4, 4, 4],
                            [0.3, 0.3, 0.3],
                        )
                    )
            main_branch = Branch(
                target_pair.x_train,
                [128, 128, 128],
                [128, 64, 64],
                [128, 128, 128],
                [4, 4, 4],
                [0.3, 0.3, 0.3],
            )
            output = Output([96, 96, 96], [0.2, 0.2, 0.2])
            architecture = Architecture(
                branches, main_branch, output
            )  # Run for testing
            if use_multiple_gpus:
                strategy = tf.distribute.MirroredStrategy()
            model.compile(
                architecture,
                learning_rate=TEST_LEARNING_RATE,
                strategy=strategy if use_multiple_gpus == "True" else None,
            )
            model.fit(
                epochs=TEST_EPOCHS,
                batch_size=TEST_BATCH_SIZE,
                validation_split=TEST_VALIDATION_SPLIT,
                patience=TEST_PATIENCE,
                strategy=strategy if use_multiple_gpus == "True" else None,
            )
        else:
            utilizer = Utilizer(model, corr_pairs)
            y_hat = utilizer.predict(box_pts=TEST_BOX_PTS)
            visualizer = Visualizer(pair)
            path = os.path.join(MODEL_PATH, "model_predictions")
            x_target = utilizer.x_target
            visualizer.plot_prediction(
                path, y_hat, test_predict=x_target, time_base=aquirer.time_base
            )

if __name__ == "__main__":
    # get API_KEY from environment variable
    API_KEY = os.environ.get("API_KEY")
    try:
        unittest.main()
    except Exception as e:
        traceback.print_exc()
        logging.error(traceback.format_exc())


from flask import session
from flask_login import current_user
import pandas as pd
import azure.cosmos.exceptions as exceptions
import os
import azure.cosmos.cosmos_client as cosmos_client
from dotenv import load_dotenv
load_dotenv()

def get_cosmos_db():
    settings = {
        'host': os.environ.get('ACCOUNT_HOST', 'https://emergencycosmos.documents.azure.com:443/'),
        'master_key': os.getenv('COSMOS_KEY'),
        'database_id': os.environ.get('COSMOS_DATABASE', 'ReliefApp'),
        'container_id': os.environ.get('COSMOS_CONTAINER', 'Beneficiaries'),
    }
    client = cosmos_client.CosmosClient(settings['host'],
                                        {'masterKey': settings['master_key']},
                                        user_agent="ReliefApp",
                                        user_agent_overwrite=True)
    return client.get_database_client(settings['database_id'])

cosmos_db = get_cosmos_db()

def get_local_data_path(user_email, distrib_id):
    data_dir = 'instance'
    os.makedirs(data_dir, exist_ok=True)
    return f"{data_dir}/user_{user_email}_distrib_{distrib_id}_data.csv"

def delete_beneficiary_data(user_email, distrib_id):
    if os.getenv("MODE") == "online":
        cosmos_container = cosmos_db.get_container_client('Beneficiaries')
        item_list = query_items_by_partition_key(cosmos_container, user_email)
        item_list = filter_by_distribution(item_list, distrib_id)
        for item in item_list:
            cosmos_container.delete_item(item=item['id'], partition_key=user_email)
    elif os.getenv("MODE") == "offline":
        database = get_local_data_path(user_email, distrib_id)
        if os.path.exists(database):
            os.remove(database)

def get_beneficiary_data(user_email, distrib_id):
    df = pd.DataFrame()

    if os.getenv("MODE") == "online":
        cosmos_container = cosmos_db.get_container_client('Beneficiaries')
        item_list = query_items_by_partition_key(cosmos_container, user_email)
        item_list = filter_by_distribution(item_list, distrib_id)
        df = pd.DataFrame(item_list)
    elif os.getenv("MODE") == "offline":
        database = get_local_data_path(user_email, distrib_id)
        if os.path.exists(database):
            df = pd.read_csv(database, index_col=0, sep=';')
        else:
            return None
    if df.empty:
        return None
    else:
        df = df.drop(columns=['partitionKey', 'id', 'distrib_id'], errors='ignore')
        df = df.drop(columns=[c for c in df.columns if c.startswith('_')])
        return df

def get_beneficiary_entry(beneficiary_id, user_email, distrib_id):
    if os.getenv("MODE") == "online":
        cosmos_container = cosmos_db.get_container_client('Beneficiaries')
        try:
            data = cosmos_container.read_item(item=beneficiary_id,
                                              partition_key=user_email)
            for internal_field in [k for k in data.keys() if k.startswith('_')]:
                data.pop(internal_field)
            return data
        except exceptions.CosmosResourceNotFoundError:
            return "not_found"
    elif os.getenv("MODE") == "offline":
        database = get_local_data_path(user_email, distrib_id)
        if os.path.exists(database):
            df = pd.read_csv(database, sep=';', dtype={'id': str}).set_index('id')
            if beneficiary_id in df.index:
                return df.loc[beneficiary_id].to_dict()
            else:
                return "not_found"
        else:
            return "no_data"

def update_beneficiary_entry(beneficiary_id, user_email, distrib_id, replace_body):
    if os.getenv("MODE") == "online":
        try:
            cosmos_container = cosmos_db.get_container_client('Beneficiaries')
            read_item = cosmos_container.read_item(item=beneficiary_id, partition_key=user_email)
            for key in replace_body.keys():
                read_item[key] = replace_body[key]
            cosmos_container.replace_item(item=read_item, body=read_item)
            return "success"
        except exceptions.CosmosResourceNotFoundError:
            return "not_found"
    elif os.getenv("MODE") == "offline":
        database = get_local_data_path(user_email, distrib_id)
        if os.path.exists(database):
            df = pd.read_csv(database, sep=';', dtype={'id': str}).set_index('id')
            if beneficiary_id in df.index:
                for key in replace_body.keys():
                    df.at[beneficiary_id, key] = replace_body[key]
                df.to_csv(database, sep=';')
                return "success"
            else:
                return "not_found"
        else:
            return "no_data"

def save_beneficiary_data(data, distrib_id, user_email):
    data_to_save = []
    for ix, row in data.iterrows():
        body = {
            'id': str(distrib_id) + str(row['code']),
            'partitionKey': user_email,
            'distrib_id': str(distrib_id)
        }
        for key in row.keys():
            if key != 'id' and key != 'partitionKey' and key != 'distrib_id':
                body[key] = str(row[key])
        data_to_save.append(body)

    print(data_to_save)

    if os.getenv("MODE") == "online":
        cosmos_container = cosmos_db.get_container_client('Beneficiaries')
        # save to cosmos db
        for entry in data_to_save:
            cosmos_container.create_item(body=entry)
    elif os.getenv("MODE") == "offline":
        database = get_local_data_path(user_email, distrib_id)
        df = pd.DataFrame.from_records(data_to_save)
        df.index = df['id']
        df = df.drop(columns=['id'])
        df.to_csv(database, sep=';')

def query_items_by_partition_key(container, key):
    # Including the partition key value of account_number in the WHERE filter results in a more efficient query
    items = list(container.query_items(
        query="SELECT * FROM r WHERE r.partitionKey=@key",
        parameters=[
            {"name": "@key", "value": key}
        ]
    ))
    return items

def filter_by_distribution(item_list, distrib_id):
    for item in item_list[:]:
        if 'distrib_id' in item.keys():
            if str(item['distrib_id']) != str(distrib_id):
                item_list.remove(item)
    return item_list

def pandas_to_html(df, replace_values={}, replace_columns={}, titlecase=False):
    df = df.replace(replace_values)
    df = df.rename(columns=replace_columns)
    if titlecase:
        df.columns = [x.title() for x in df.columns]
    columns = df.columns
    rows = []
    for ix, row in df.iterrows():
        rows.append(row.to_dict())
    return columns, rows


import sys
import json
import urllib.request as HttpUtils
from bs4 import BeautifulSoup
import py
import json

urls = []

def preurl(provinces):
    for province in provinces:
        path = py.getFirsLetter(province)
        urls.append({
            'name':
            province,
            'path':
            path,
            'url':
            'http://www.sciencehr.net/uploads/ptgdxxmd/{0}.html'.format(path)
        })
    print(len(urls))
    return len(urls) > 0

def main(provinces):
    if (preurl(provinces)):
        list = []
        for item in urls:
            url = item['url']
            province=item['name']
            html = HttpUtils.urlopen(url).read()
            soup = BeautifulSoup(html, "lxml", from_encoding="gb18030")
            trs = soup.select('.wenzi > table > tr')

            for index, row in enumerate(trs):
                if (index > 1):
                    list.append({
                        'province':
                        province,
                        'code':
                        row.find_all("td")[2].get_text(),
                        'name':
                        row.find_all("td")[1].get_text(),
                        'level':
                        row.find_all("td")[5].get_text(),
                        'city':
                        row.find_all("td")[4].get_text(),
                        'isclosed':
                        False,
                        "remark":
                        "".join(row.find_all("td")[6].get_text().split())
                    })
        writeTxt(list)

def writeTxt(list):
    with open("data.json", "w", encoding='utf-8') as f:
        f.write(json.dumps(list, ensure_ascii=False))

if __name__ == '__main__':
    p = [
        "北京", "上海", "天津", "重庆", "河北", "山西", "内蒙古", "安徽", "山东", "江苏", "福建",
        "江西", "浙江", "广东", "广西", "海南", "辽宁", "吉林", "黑龙江", "河南", "湖南", "湖北",
        "陕西", "宁夏", "甘肃", "青海", "新疆", "四川", "贵州", "云南", "西藏"
    ]
    main(p)


#!/usr/bin/env python

"""
Author: Sam Overduin
Student nr: 920208637010
Script to: find eulerian paths in spectrum of l-mers
"""
# Import statements
import random

def count_inout_degree(graph_dct):
    """Returns a dct of vertices as values and (in_degree,out_degree) as values

    :param graph_dct: directed graph as key with lst of keys
    :return: dct of vertice keys and (in_degree, out_degree) as values.
    """
    out_dct = {key:0 for key in graph_dct.keys()}
    in_dct = {}
    for out_node, in_lst in graph_dct.items():
        out_dct[out_node] = len(in_lst)
        for in_node in in_lst:
            try:
                in_dct[in_node] += 1
            except:
                in_dct[in_node] = 1

    inout_dct = {}
    in_keys = list(in_dct.keys())
    out_keys = list(out_dct.keys())
    nodes = set(list(in_keys+out_keys))
    for node in nodes:
        # every node represented in in_dct and out_dct
        if node not in in_keys:
            in_dct[node] = 0
        if node not in out_keys:
            out_dct[node] = 0

        inout_dct[node] = (in_dct[node], out_dct[node])
    return inout_dct

def is_eulerian(graph_dct):
    """Returns a bool for if graph is eulerian

    :param graph_dct: directed graph as key with lst of keys
    :return: True/False if all vertices are balanced
    """
    inout_dct = count_inout_degree(graph_dct)
    for key, tup in inout_dct.items():
        in_cnt, out_cnt = tup
        if in_cnt != out_cnt:
            return False
    return True

def has_eulerian_path(graph_dct, return_start_end_nodes=False):
    """Returns a bool for if graph has a eulerian path

    :param graph_dct: directed graph as key with lst of keys
    :param return_start_end_nodes: bool to return start+end nodes
    :return: True/False if max 2 vertices have |in_deg-out_deg| = 1,
        node start (vertice with 1 less in than out),
        node end (vertice with 1 more in than out)

    """
    deg_diffs = 0
    inout_dct = count_inout_degree(graph_dct)
    start_node = None
    end_node = None
    has_path = True

    for key, tup in inout_dct.items():
        in_cnt, out_cnt = tup
        if in_cnt - out_cnt == 1:
            if not end_node:
                end_node = key
            else:
                has_path, end_node = (False, None)
                break
        elif in_cnt - out_cnt == -1:
            if not start_node:
                start_node = key
            else:
                has_path, start_node = (False, None)
                break
        elif abs(in_cnt - out_cnt) > 1:
             has_path, start_node, end_node = (False, None, None)

    if return_start_end_nodes:
        return has_path, start_node, end_node
    else:
        return has_path

def make_euler_graph(spect):
    """ Creates a (l-1)-mer graph where the edges correspond to every l-mer.

    :param spect: a lst of l-mer sequences (spectrum)
    :return: directed graph as key with lst of keys
    """
    assert(len(set([len(x) for x in spect])) == 1),\
        "spect elements must be same length"

    graph_dct = {}
    for seq in spect:
        try:
            graph_dct[seq[:-1]].append(seq[1:])
        except KeyError:
            graph_dct[seq[:-1]] = [seq[1:]]
    return graph_dct

def find_eulerian_cycle(graph_dct, start_point=None):
    """ Returns a eulerian cycle containing all edges of the graph

    :param graph_dct: directed graph as key with lst of keys
    :param start_point: optional key in graph_dct to start from
    :return: list of vertices in order.
    """
    edges_left = 0
    for key, to_lst in graph_dct.items():
        edges_left += len(to_lst)

    if not start_point:
        start_point = random.choice(list(graph_dct.keys()))

    path = []
    left_paths = {}
    cur_point = start_point
    new_point = None

    while edges_left != 0 and new_point != start_point:
        # randomized edge to follow
        new_point = graph_dct[cur_point].pop(
            random.randint(0, len(graph_dct[cur_point])-1))
        path.append((cur_point, new_point))
        left_paths[cur_point] = len(graph_dct[cur_point])
        cur_point = new_point
        edges_left -= 1
    # Check unused edges
    for key, v in left_paths.items():
        if v > 0:
            extra_path = find_eulerian_cycle(graph_dct, start_point=key)
            # find where to cut open and place extra path in between
            i = [start for start, end in path].index(key)
            path = path[:i] + extra_path + path[i:]
    return path

def find_eulerian_path(graph):
    """ Returns a eulerian path as lst of keys from graph_dct
        by visiting all edges once

    :param graph: directed graph as key with lst of keys
    :return: lst of keys of eulerian path.
    """
    # copy due to aliasing and popping of stuff in graph
    graph_dct = {k:v[:] for k, v in graph.items()}

    if not is_eulerian(graph_dct):
        has_path, start, end = has_eulerian_path(graph_dct,
                                                 return_start_end_nodes=True)
        if not has_path:
            print("graph_dct does not contain Eulerian path.")
            return None
        # make euler cycle:
        try:
            graph_dct[end].append(start)
        except KeyError:
            graph_dct[end] = [start]

        path_lst = find_eulerian_cycle(graph_dct, start_point=start)
        # remove last one because not actually a cycle
        path_lst = path_lst[:-1]
    else:
        path_lst = find_eulerian_cycle(graph_dct)

    assert len(path_lst) == len(set(path_lst)), "duplicate edges in path"

    # Take all first nodes from edges + add final end node (2nd line)
    node_lst = [start for start, end in path_lst]
    node_lst.append(path_lst[-1][1])

    return node_lst

def main():
    # Eulerian graph
    graph_822 = {'J': ['D'], 'D': ['C'], 'I': ['H'], 'H': ['F'], 'F': ['G', 'E'],
                 'C': ['I', 'A'], 'G': ['J'], 'E': ['A'], 'A': ['F', 'B'], 'B': ['C']}
    # 3-mers created from sequence (not in order)
    s = ['ATG', 'TGG', 'TGC', 'GTG', 'GGC', 'GCA', 'GCG', 'CGT']

    print('Eulerian path through 822:')
    print(find_eulerian_path(graph_822))

    print("\nCreate graph from k-mers s & find path through the graph")
    s_graph = make_euler_graph(s)
    seq_path = find_eulerian_path(s_graph)
    print('path:', seq_path)
    tot_seq = seq_path[0]
    for i in range(1, len(seq_path)):
        tot_seq += seq_path[i][-1:]
    print('sequence:', tot_seq)

if __name__ == "__main__":
    main()


import matplotlib.pyplot as plt
import pandas as pd
import os
import re
import geopy.distance
import contextily
import rasterio
import sys
import errno
import geopandas as gpd
from shapely.geometry import Point
import datatypes as dt
from pyproj import CRS
from pyproj.aoi import AreaOfInterest
from pyproj.database import query_utm_crs_info
import cv2
import numpy as np
from PIL import Image

##################################################
# This file contains helper functions
# Author: Tobias Ranft & Kimberly Mason
# Date: 2022-08-27
##################################################

# Converts a image as png and the coordinate this image covers to a geotif. This is a file with geo information which can be
# for example be imported to QGIS
# input:
# pathToImage  path to the png image
# coord        coordinates of the image
def convertToGeoTif(pathToImage, coord):
    dataset = rasterio.open(pathToImage, 'r')
    bands = [1,2,3]
    data = dataset.read(bands)
    transform = rasterio.transform.from_bounds(coord.west, coord.south, coord.east,coord.north, data.shape[1], data.shape[2])
    crs = {'init': 'epsg:4326'}

    writePath=os.path.dirname(pathToImage)

    _, width, height = data.shape
    with rasterio.open(writePath+"/GeoMosaic.tif", 'w', driver='GTiff',
                       width=width, height=height,
                       count=3, dtype=data.dtype, nodata=0,
                       transform=transform, crs=crs) as dst:
        dst.write(data, indexes=bands)

# Function to plot the track on a map and colour by dept
# This function will download the map data from OpenStreetMap and therefore needs internet
# input:
# pathToMeta  path to the file containing the meta data
def plotTrackWithDept(pathToMeta):
    print("####### plot track with depth #######")
    bathy_data = gpd.read_file(pathToMeta, delimiter=',', header=0)

    bathy_data['WaterDepth'] = pd.to_numeric(bathy_data['WaterDepth'])

    bathy_data = bathy_data.loc[~((bathy_data['WaterDepth'] == 0))]

    bathy_data = bathy_data[['SurveyTypeLabel','Latitude','Longitude',
                             'WaterDepth']]

    bathy_data["geometry"] = bathy_data.apply(lambda x:Point(float(x["Longitude"]),
                                                             float(x["Latitude"])), axis=1)

    geo = gpd.GeoDataFrame(bathy_data, geometry='geometry')

    geo = geo.set_crs(epsg=4326)

    data = geo.to_crs(epsg=3857)

    #fig, ax = plt.subplots(figsize=(14,6))
    fig, ax = plt.subplots()
    data.plot(ax=ax, column='WaterDepth',cmap = 'ocean_r', legend=True,
              legend_kwds={'label': "Depth (m)"})
    plt.xlabel("Longitude")
    plt.ylabel("Latitude")
    bounds = data.total_bounds
    print(bounds)
    dx=abs(bounds[2]-bounds[0])/2
    dy=abs(bounds[3]-bounds[1])/2
    toadd = max(dx,dy)
    ax.set_xlim(bounds[0]-toadd, bounds[2]+toadd)
    ax.set_ylim(bounds[1]-toadd, bounds[3]+toadd)

    print("get map from openStreetMap")
    contextily.add_basemap(ax, zoom=16, source=contextily.providers.OpenStreetMap.Mapnik)
    # plt.show()
    outName= os.path.dirname(pathToMeta)+"/TrackWithDept.png"
    plt.savefig(outName, dpi=300)

# The positions taken from GNSS are not accurate enough. Meaning there are several frames with the same position.
# This leads to problems when georeferencing it. Therefore it is necessary to stretch them apart
# This is currently done by simply looking for 2 positions that are a measurable distance apart and assigning
# the position in between linearly.
# This function will create a corrected meta data file
# input:
# filename filepath to meta data
# distance minimal distance between 2 position in meter
def fixPosition(filename,distance):
    print("fix position")
    outFile=os.path.dirname(filename)+"/slEdited.csv"
    #print("exists "+str(os.path.exists(outFile)))
    if(not os.path.exists(outFile)):
        #print("fix file "+str(filename))
        data = pd.read_csv(filename)

        Lat=data["Latitude"].copy()
        Lon=data["Longitude"].copy()
        Type=data["SurveyTypeLabel"]
        Heading=data["GNSSHeading"]

        index1=0
        index2=1
        maxIndex=len(Lat)

        tenPercent=maxIndex/10
        printCounter=0

        maxReached=False
        while(maxReached==False):
            dist=0
            point1=(Lat[index1],Lon[index1])
            while(dist<distance and index2<maxIndex):
                point2=(Lat[index2],Lon[index2])
                dist=geopy.distance.geodesic(point1, point2).m
                index2+=1
            if(index2==maxIndex):
                index2=maxIndex-1
                maxReached=True
            # print("distance "+str(dist))
            # print("index "+str(index2))

            startPoint=(Lat[index1],Lon[index1])
            dindex=index2-index1
            dlat=(Lat[index2]-Lat[index1])/dindex
            dlon=(Lon[index2]-Lon[index1])/dindex

            for i in range(0,dindex,1):
                Lat[i+index1]=startPoint[0]+i*dlat
                Lon[i+index1]=startPoint[1]+i*dlon
            index1=index2
            if(index1>printCounter):
                printCounter=printCounter+tenPercent
                print(str(int(printCounter/tenPercent)*10)+"%")

        newData=pd.DataFrame(Type)
        newData["Latitude"]=Lat
        newData["Longitude"]=Lon
        newData["GNSSHeading"]=Heading
        newData["WaterDepth"]=data["WaterDepth"]

        newData.to_csv(outFile)
        print("end of fix position")

# Function to read the meta information
# input
# path  path to meta information
def readMetaInformation(path):
    print("start read meta information")
    data = pd.read_csv(path)
    metaInfo = dt.MetaInformation()
    metaInfo.dataPrim = data[data['SurveyTypeLabel'] == 'Primary']
    metaInfo.dataDown = data[data['SurveyTypeLabel'] == 'Downscan']
    metaInfo.dataSide = data[data['SurveyTypeLabel'] == 'Sidescan']
    metaInfo.sideLat=metaInfo.dataSide.loc[:,"Latitude"].values.tolist()
    metaInfo.sideLon=metaInfo.dataSide.loc[:,"Longitude"].values.tolist()
    metaInfo.sideHeading=metaInfo.dataSide.loc[:,"GNSSHeading"].values.tolist()
    return metaInfo

# Function to sort file name in alpha numeric order
# This is important to process the files in the right order
# input
# data list with the file names
# return list with file names sorted
def sorted_alphanumeric(data):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(data, key=alphanum_key)

# This function takes a position and return a local coordinate system
# which fits
# Geocord geocoordinates
# return local geo coordinate system
def getLocalGeoCoordinateSystem(Geocord):
    utm_crs_list = query_utm_crs_info(
        datum_name="WGS 84",
        area_of_interest=AreaOfInterest(
            west_lon_degree=Geocord.west,
            south_lat_degree=Geocord.south,
            east_lon_degree=Geocord.east,
            north_lat_degree=Geocord.north,
        ),
    )
    utm_crs = CRS.from_epsg(utm_crs_list[0].code)
    return utm_crs

# convert openCV image to PIL image format
def openCvToPilImage(openCvImageIn):
    cvRGB = cv2.cvtColor(openCvImageIn, cv2.COLOR_BGR2RGB)
    pilImg = Image.fromarray(cvRGB)
    return pilImg

# convert PIL image format to openCV image format
def pilToOpenCvImage(pilImageIn):
    numpy_image=np.array(pilImageIn)
    opencv_image=cv2.cvtColor(numpy_image, cv2.COLOR_RGB2BGR)
    return opencv_image

# This function takes 2 images in and puts them in different colour canels in the final image
# This is currently used to combine the Down and the Prime scan
# src1 image 1
# scr2 image 2
def combineImages(src1,src2):

    #convert input pil to opencv, this is in RGB not in BGR but since we are working with grayscale here anyways, it doesn't matter
    src1=pilToOpenCvImage(src1)
    src2=pilToOpenCvImage(src2)

    src1=cv2.cvtColor(src1, cv2.COLOR_BGR2GRAY)
    src2=cv2.cvtColor(src2, cv2.COLOR_BGR2GRAY)

    # reshape to the shape of src1
    height, width = src1.shape[:2]
    src2=cv2.resize(src2, (width, height), interpolation = cv2.INTER_AREA)

    src2=cv2.convertScaleAbs(src2, alpha=1.5, beta=1.5)
    src1=cv2.convertScaleAbs(src1, alpha=1.5, beta=1.5)

    blank_image = np.zeros((height,width,1), np.uint8)
    blank_image.fill(100)
    dst1 = cv2.merge([blank_image,src1,src2 ])

    # average weighting
    #dst1 = cv2.addWeighted(src1, 0.5, src2, 0.5, 0.0)
    #cv2.imwrite(pathToCombined, dst1)
    return dst1

def check_if_scan_exsists(path:str, message):
    """
    Checks if the given path to scan files does exsist.
    Throws an error if not.
    Returns True if path exsists

    """
    if not os.path.exists(path):
        error_string = "####### Selected scan not found. Make sure preprocessing is enabled in config and your sonar file contains the scan. #######"
        if message == "":
            error_string = error_string
        else:
            error_string = message + error_string
        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path + "  " + error_string)
    return True


# -*- coding:utf-8 -*-
# editor: zzh
# date: 2020/10/21

import pandas as pd
import re_extract
import os
from xpinyin import Pinyin

path = r"D:\MyProject\天气灾害分类及信息提取\测试数据\第2次测试\测试文本"
classifications = re_extract.classifications
# classifications = ["冰雹灾害"]
wufiles = ["大雾迷眼车撞树 小货车看不清路撞树上[图]"
,"合宁高速发生5车相撞事故 4人死亡"
,"京港澳高速发生多起交通事故 致2人死20多人伤"
,"江西九景高速发生多起追尾事故致10死19伤"
,"沪渝高速安徽芜湖境内4车追尾致7人死亡"
,"山东平邑发生一起交通事故 4人死亡2人重伤"
,"沪渝高速安徽芜湖段四车追尾致7人死亡"
,"沈海高速连云港段发生7起交通事故致5死11伤"
,"杭浦高速20辆车“雾中”相撞 已致2人死亡"
,"京台高速安徽宿州段连环车祸致5死11伤"
,"大雾弥漫 杭浦高速20辆车雾中相撞"
,"洛阳环城高速3辆大货车连环相撞 2人受伤(图)"
,"京福高速枣庄段发生十余辆车相撞事故 致2死8伤"
,"四川成绵高速因大雾53车追尾 2死多伤"
,"浙江杭浦高速20辆车大雾中相撞 致2死8伤"
,"京藏高速吴忠段34车连撞3人轻伤 已基本恢复通行"
,"江西鄱阳湖大桥连环车祸 已造成十三人死亡"
,"京福高速德州段近20辆车连环相撞致1死8伤"
,"四川成乐高速大雾引发连环车祸 造成2死4伤(图)"
,"沪昆高速7车因大雾追尾 造成2人死亡25人受伤"
,"蓉遵高速贵州习水段发生多车连环相撞事故 涉及37辆车"
,"沪昆高速贵州境内7车连环相撞造成2死25伤"
,"湖南交通事故现场2名救援人员因二次车祸遇难"
,"山东滨莱高速6车相撞致6人死亡"
,"江苏南京宁合高速发生重大车祸 5人死亡3人重伤"
,"山东济菏高速发生追尾事故 已造成9人死亡(图)"
,"山西一高速公路因大雾发生交通事故 死伤升至44人"
,"湖北襄荆高速8车相撞造成两死六伤"
,"京港澳高速航海路十几辆车相撞 至少2人死亡"
,"大雾弥漫 呼和浩特快速路上21辆车连环追尾"
,"京福高速徐州段连续追尾事故已致6死30多伤"
,"贵州剑河12车连环相撞 致2人死亡30余人受伤"
,"山西太长高速多车相撞六人身亡 全国劳模张家胜遇难"
,"京福高速徐州段双向60余车相撞 已致6死32伤(图)"
,"山东济广高速追尾事故死亡人数上升至5人"
,"京台高速宿州段15车连环撞 致5死11伤"
,"因大雾道路湿滑 河南信阳发生交通事故致4死多伤"
,"南京大雾两车相撞造成5死3伤"
,"一小型客车在贵州毕节追尾大客车 致7人遇难"
,"除夕沪昆高速上海段11车追尾 3人遇难"
,"沪渝高速芜宣段发生四车追尾事故 导致7人死亡"
,"四川内宜高速多车连环相撞 致2死30余人受伤"
,"辽宁大雾致14车连撞 造成4死3重伤"
,"京港澳高速邢台段汽车连续追尾致3死6伤"
,"京港澳高速重大追尾事故造成3人死亡"
,"四川内宜高速多车连环相撞 2人死30余人受伤"
,"安徽境内高速路发生8车追尾致8死1伤(图)"
,"沪昆高速江西段大雾 引发连环追尾事故4死4伤(图)"
,"沪昆高速湖南境内因大雾发生9车追尾致3死8伤"
,"京台高速山东段120车连环相撞 已致7死35伤(图)"
]

def sort_(lis):
    pin = Pinyin()
    result = []
    for item in lis:
        result.append((pin.get_pinyin(item.split('_')[1]), item))
    print(result)
    result.sort()
    for i in range(len(result)):
        result[i] = result[i][1]
    return result

def tostr(data):
    if len(data) == 0:
        return ""
    else:
        return "".join([str(d)+"\n" for d in data])

def all_to_excel(filespath,savepath):
    classifications = ['低温灾害', '冰雹灾害', '台风灾害', '地质灾害', '城市内涝', '大雾灾害', '大风灾害', '干旱灾害', '暴雨洪涝', '森林火灾', '雪灾灾害', '雷电灾害']
    # classifications = ['大雾灾害']

    for classification in classifications:
        writer = pd.ExcelWriter(classification + ".xlsx")
        files = os.listdir(filespath + '\\' + classification)
        files = sort_(files)
        # files = wufiles
        data_new = pd.DataFrame()
        for file in files:
            try:
                # with open(path + '\\' + classification + '\\大雾_' + file+'.txt', 'r', encoding='utf8') as f:
                with open(path + '\\' + classification + '\\' + file, 'r', encoding='utf8') as f:
                    title = f.readline()
                    time = f.readline()
                    source = f.readline()
                    contents = f.readlines()
                    print("***" * 20)
                    print("文件名：", file.strip())
                    # print("新闻标题：", title.strip())
                    infos = re_extract.info_extract(contents)

                    data = pd.DataFrame(columns=['实际值', '输出值', '测试结果'],
                                        index=[[title, title, title, title, title, title, title, title, title, title,
                                                title, title, title, title, title, title, title, title, title, title,
                                                title],
                                               ['灾情类别', '开始日期', '开始时间', '结束日期', '结束时间', '位置', '承载体',
                                                '受灾人口', '失踪人口', '受伤人口', '转移人口', '死亡人口',
                                                '受灾面积', '受灾面积（类似）', '成灾面积', '绝收面积', '倒塌房屋', '损坏房屋', '经济损失', '积水深度',
                                                '积雪深度']])

                    print(infos)
                    k_v = {"AIMP": "受灾人口", "AMP": "失踪人口",
                           "AINP": "受伤人口", "ATP": "转移人口", "ADP": "死亡人口", "AIAC": "受灾面积", "AIAC2": "受灾面积（类似）",
                           "ASAC": "成灾面积",
                           "ATAC": "绝收面积", "AHC": "倒塌房屋", "AHC2": "损坏房屋", "AE": "经济损失", "AWD": "积水深度", "ASD": "积雪深度"}

                    for k in infos.keys():
                        data["输出值"][title][k_v[k]] = tostr(infos[k])
                    # data.to_excel(writer,sheet_name=title[:30])
                    data_new = data_new.append(data)
            except Exception as e:
                e.with_traceback()
                pass

        data_new.to_excel(writer,classification+'.xlsx')
        writer.save()

if __name__ == '__main__':
    all_to_excel(path,"测试结果")
    pass


#from django.shortcuts import render
from django.http import response
from rest_framework.views import APIView
from rest_framework.response import Response

class HelloapiView(APIView):
    """Api view de prueba"""

    def get(self, request, format=None):
        """retornar lista de caracteristicas de APIVIEW"""

        an_apiview = [
            'Usamos metodos HTTP como funciones (get, post, patch, put delete)',
            'Es similar a una vista tradicional de Django',
            'Nos da el mayor control sobre la logicade nusetra aplicacion',
            'Esta mapeado manualmente a los URLS',
        ]

        return Response({'message':'Hello', 'an_apiview': an_apiview})


import functools
import torch.nn as nn

def get_norm_layer(norm_type='instance'):
    if norm_type == 'batch':
        return functools.partial(nn.BatchNorm2d, affine=True)
    elif norm_type == 'instance':
        return functools.partial(nn.InstanceNorm2d, affine=False)
    else:
        raise NotImplementedError('Invalid normalization type: {}'.format(norm_type))

def get_param_groups(net, weight_decay, norm_suffix='weight_g', verbose=False):
    norm_params = []
    unnorm_params = []
    for n, p in net.named_parameters():
        if n.endswith(norm_suffix):
            norm_params.append(p)
        else:
            unnorm_params.append(p)

    param_groups = [{'name': 'normalized', 'params': norm_params, 'weight_decay': weight_decay},
                    {'name': 'unnormalized', 'params': unnorm_params}]

    if verbose:
        print('{} normalized parameters'.format(len(norm_params)))
        print('{} unnormalized parameters'.format(len(unnorm_params)))

    return param_groups

class WNConv2d(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, padding, bias=True):
        super(WNConv2d, self).__init__()
        self.conv = nn.utils.weight_norm(
            nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding, bias=bias))

    def forward(self, x):
        x = self.conv(x)

        return x


from collections import defaultdict
from importlib import import_module
from itertools import chain
import logging
import os
from platform import uname
import random
from select import select
import shlex
import shutil
import string
import subprocess
import sys

from builtins import range # pylint: disable=redefined-builtin
from builtins import object # pylint: disable=redefined-builtin
from future.moves.urllib.parse import parse_qs
from zc.buildout import UserError
from zc.buildout.download import Download

from dockeroo import filters
from dockeroo.filters import scm as filters_scm
from dockeroo.filters import RecipeFilter
from dockeroo.utils import ExternalProcessError
from dockeroo.utils import OptionRepository, reify
from dockeroo.utils import string_as_bool, uniq
from dockeroo.utils import mkdir

FILTERS = []

class RecipeFilterset(object):
    _filters = defaultdict(list)

    def __init__(self, recipe):
        self.recipe = recipe
        for flt in FILTERS:
            self.add(flt)

    def add(self, cls):
        self._filters[cls.filter_category].append(cls(self.recipe))

    def __call__(self, filter_category, args=None, kwargs=None, continue_on_success=False):
        args = args or []
        kwargs = kwargs or {}
        rets = []
        for fltr in self._filters[filter_category]:
            ret = fltr(*args, **kwargs)
            if ret is not None:
                if not continue_on_success:
                    return ret
                else:
                    rets.append(ret)
        return rets or None

    @classmethod
    def preload_filters(cls, base_module):
        import inspect
        from warnings import warn

        for module_name in \
            [os.path.splitext(entry)[0] \
             for entry in [entry for entry in os.listdir(os.path.dirname(base_module.__file__)) \
             if entry.endswith('.py') and not entry.startswith('__')]]:

            try:
                module = import_module("{}.{}".format(
                    base_module.__name__, module_name))
            except Exception as exc: # pylint: disable=broad-except
                warn('''Unable to import module "{}": {}'''.format(module_name, exc))
            else:
                for fltr in [entry for entry in [getattr(module, entry) for entry in dir(module)] \
                             if inspect.isclass(entry) and
                             issubclass(entry, RecipeFilter) and
                             hasattr(entry, 'filter_category') and entry.filter_category]:
                    FILTERS.append(fltr)

RecipeFilterset.preload_filters(filters)
RecipeFilterset.preload_filters(filters_scm)

class BaseRecipe(object): # pylint: disable=too-many-public-methods,too-many-instance-attributes
    loggers = [__name__, 'zc.buildout']

    def __init__(self, buildout, name, options):
        self.logger = logging.getLogger(__name__)
        self.cleanup_paths = set()
        self.name = name
        self.options = OptionRepository(options, name=self.name)
        self.buildout = buildout
        self.filterset = RecipeFilterset(self)
        self.log_handler = logging.StreamHandler(sys.stdout)
        self._save_logging = {}
        self.initialize()

    @classmethod
    def _uninstall(cls, name, options):
        self = cls({'buildout': {}}, name, options)
        if hasattr(self, 'uninstall') and callable(self.uninstall):
            return self.uninstall()

    def setup_logging(self):
        self._save_logging = {}
        for logger in [logging.getLogger(x) for x in self.loggers]:
            self.save_logger(logger)
            logging._acquireLock() # pylint: disable=protected-access
            logger.handlers = [self.log_handler]
            logging._releaseLock() # pylint: disable=protected-access
            logger.propagate = False

    def restore_logging(self):
        for logger in [logging.getLogger(x) for x in self.loggers]:
            self.restore_logger(logger)

    def save_logger(self, logger):
        logging._acquireLock() # pylint: disable=protected-access
        self._save_logging['{}_handlers'.format(
            logger.name)] = list(logger.handlers)
        logging._releaseLock() # pylint: disable=protected-access
        self._save_logging['{}_propagate'.format(
            logger.name)] = logger.propagate

    def restore_logger(self, logger):
        logging._acquireLock() # pylint: disable=protected-access
        logger.handlers = self._save_logging.pop(
            '{}_handlers'.format(logger.name))
        logging._releaseLock() # pylint: disable=protected-access
        logger.propagate = self._save_logging.pop(
            '{}_propagate'.format(logger.name))

    def set_logging(self, fmt, level, verbosity=0):
        self.log_handler.setFormatter(logging.Formatter(fmt))
        for logger in [logging.getLogger(x) for x in self.loggers]:
            logger.setLevel(level - verbosity)

    @property
    @reify
    def offline(self):
        return string_as_bool(self.buildout['buildout'].get('offline', False))

    @property
    @reify
    def newest(self):
        return string_as_bool(self.buildout['buildout'].get('newest', False))

    @property
    @reify
    def download_hashing(self):
        return string_as_bool(self.buildout['buildout'].get('hash-name', True))

    @property
    @reify
    def download_cache(self):
        return self.buildout['buildout'].get('download-cache', None)

    @property
    @reify
    def download_manager(self):
        return Download(self.buildout['buildout'],
                        hash_name=self.options.get(
                            'download-hashing',
                            default=self.buildout.get('download-hashing', True)),
                        cache=self.download_cache,
                        logger=self.logger)

    @property
    @reify
    def default_executable(self):
        return self.buildout['buildout'].get('executable', sys.executable)

    @property
    @reify
    def buildout_directory(self):
        return self.buildout['buildout'].get(
            'directory', os.path.join(os.path.dirname(sys.argv[0]), '..', 'parts'))

    @property
    @reify
    def parts_directory(self):
        return self.buildout['buildout'].get(
            'parts-directory', os.path.join(self.buildout_directory, 'parts'))

    @property
    @reify
    def default_location(self):
        return os.path.join(self.parts_directory, self.name)

    @property
    @reify
    def default_working_directory(self):
        return os.path.join(
            self.parts_directory,
            "{}.workdir{}".format(
                self.name,
                ''.join(random.choice(string.ascii_letters + string.digits) \
                    for _ in range(8))))

    @property
    @reify
    def default_log_format(self):
        return self.buildout['buildout'].get('log-format', None) or "%(name)s: %(message)s"

    @property
    @reify
    def default_log_level(self):
        level = self.buildout['buildout'].get('log-level', 'INFO')
        if level in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
            return getattr(logging, level)
        else:
            try:
                return int(level)
            except ValueError:
                raise UserError('''Invalid log level "{}"'''.format(level))

    @property
    @reify
    def default_verbosity(self):
        verbosity = self.buildout['buildout'].get('verbosity', 0)
        try:
            return int(verbosity)
        except ValueError:
            raise UserError('''Invalid verbosity "{}"'''.format(verbosity))

    def initialize_target(self, target, mandatory=False):
        if self.options.has_key('{}-target'.format(target)):
            setattr(self, '_{}_target'.format(target),
                    self.options.get('{}-target'.format(target)))
        elif self.options.has_key('target'):
            setattr(self, '_{}_target'.format(target),
                    self.options.get('target'))
        elif hasattr(self, '{}_target'.format(target)):
            setattr(self, '_{}_target'.format(target),
                    getattr(self, '{}_target'.format(target)))
        elif mandatory:
            raise UserError(
                '''You must provide a "target" or "{}-target" field.'''
                .format(target))
        else:
            self._update_target = None

    def initialize(self):
        self.set_logging(self.default_log_format,
                         self.default_log_level)

        self.initialize_target('install', mandatory=True)

        self.initialize_target('update')
        if self._update_target is not None:
            self.update = self.update_wrapper

        self.initialize_target('uninstall')
        if self._uninstall_target is not None:
            self.uninstall = self.uninstall_wrapper

    def download(self, url, params=None, force=False):
        return self.filterset('download', [url.strip()], {'params': params or {}, 'force': force})

    def extract_archive(self, src, dst, params=None):
        return self.filterset('extract.archive', [src.strip(), dst], {'params': params or {}})

    def extract_scm(self, repo_type, src, dst, params=None):
        return self.filterset('extract.scm',
                              [repo_type, src.strip(), dst],
                              {'params': params or {}})

    @property
    @reify
    def locations(self):
        return uniq([_f for _f in [self.default_location] +
                     getattr(self, 'extra_locations', []) if _f])

    @property
    @reify
    def working_directories(self):
        return uniq([self.default_working_directory])

    def install_wrapper(self):
        self.setup_logging()
        exc = False
        try:
            for location in [l for l in self.locations if os.path.exists(l)]:
                self.cleanup_paths.add(location)
                raise shutil.Error(
                    '''Directory "{}" already exists'''.format(location))
            for working_directory in self.working_directories:
                self.mkdir(working_directory)
                self.cleanup_paths.add(working_directory)
            if callable(self._install_target):
                self._install_target()
            else:
                exec(self._install_target) # pylint: disable=exec-used
        except Exception: # pylint: disable=broad-except
            self.restore_logging()
            exc = True
            raise
        finally:
            if exc and string_as_bool(self.options.get('keep-on-error', False)):
                for path in self.cleanup_paths:
                    self.logger.info('Left path "%s" as requested', path)
            else:
                for path in self.cleanup_paths:
                    self.logger.debug('Cleaning up "%s"', path)
                    self.rm(path)
        specs = self.options.get('specs', default=None)
        if specs is not None:
            self.check_specs(specs)
        self.restore_logging()
        return self.locations
    install = install_wrapper

    def update_wrapper(self):
        self.setup_logging()
        exc = False
        try:
            for working_directory in self.working_directories:
                self.mkdir(working_directory)
                self.cleanup_paths.add(working_directory)
            if callable(self._update_target):
                self._update_target() # pylint: disable=not-callable
            else:
                exec(self._update_target) # pylint: disable=exec-used
        except Exception: # pylint: disable=broad-except
            self.restore_logging()
            exc = True
            raise
        finally:
            if exc and string_as_bool(self.options.get('keep-on-error', False)):
                for path in self.cleanup_paths:
                    self.logger.info('Left path "%s" as requested', path)
            else:
                for path in self.cleanup_paths:
                    self.logger.debug('Cleaning up "%s"', path)
                    self.rm(path)
        specs = self.options.get('specs', default=None)
        if specs is not None:
            self.check_specs(specs)
        self.restore_logging()
        return self.locations

    def uninstall_wrapper(self):
        self.setup_logging()
        try:
            if callable(self._uninstall_target):
                self._uninstall_target()
            else:
                exec(self._uninstall_target) # pylint: disable=exec-used
        except Exception: # pylint: disable=broad-except
            self.restore_logging()
            raise
        self.restore_logging()

    def check_specs(self, specs):
        pass

    def pipe_command(self, command_list, *args, **kwargs):
        subprocess_list = []
        previous = None
        kwargs['stdout'] = subprocess.PIPE
        run_list = []
        for command in command_list:
            if previous is not None:
                kwargs['stdin'] = previous.stdout
            proc = subprocess.Popen(command, *args, **kwargs)
            if previous is not None:
                previous.stdout.close()
            subprocess_list.append((proc, command))
            run_list.append(' '.join(command))
            previous = proc
        self.logger.info('Running: "%s"', ' | '.join(run_list))
        subprocess_list.reverse()
        for proc, command in subprocess_list:
            proc.wait()
        for proc, command in subprocess_list:
            if proc.returncode != 0:
                raise UserError(
                    '''Failed while running command "{}"'''.format(command))

    def fail_if_path_exists(self, path): # pylint: disable=no-self-use
        if os.path.lexists(path):
            raise UserError(
                '''Path "{}" exists, cannot continue'''.format(path))

    def copy(self, origin, destination):
        if os.path.isfile(origin):
            shutil.copy(origin, destination)
        else:
            self.copy_tree(origin, destination)
        return destination

    def copy_tree(self, origin, destination, ignore=None):
        if os.path.exists(destination):
            raise shutil.Error(
                '''Destination already exists: "{}"'''.format(destination))
        self.logger.debug('Copying "%s" to "%s"', origin, destination)
        try:
            shutil.copytree(origin, destination,
                            ignore=shutil.ignore_patterns(*ignore or []))
        except (shutil.Error, OSError) as error:
            try:
                shutil.rmtree(destination, ignore_errors=True)
            except (shutil.Error, OSError) as strerror:
                self.logger.error('Error occurred when cleaning after error: "%s"', strerror)
            raise error

    @staticmethod
    def mkdir(*paths):
        mkdir(*paths)

    @staticmethod
    def rm(*paths): # pylint: disable=invalid-name
        for path in paths:
            if os.path.isdir(path):
                shutil.rmtree(path)
            elif os.path.exists(path):
                os.remove(path)

    def patch(self, patches, prefix=None, command_options=None, command_binary=None, cwd=None):
        params = {
            'command-options': shlex.split(command_options) if command_options is not None else [],
            'command-binary': command_binary.strip() if command_binary is not None else 'patch',
        }
        if cwd is not None:
            params['cwd'] = cwd.strip()
        for patch in patches:
            if '#' in patch:
                patch, params_string = patch.split('#', 1)
                params.update(parse_qs(params_string))
            if prefix is not None:
                patch = os.path.join(prefix, patch)
            self.logger.info('Applying patch: "%s"', patch)
            self.filterset('patch', [patch], {'params': params})

    def render_template(self, source, destination, **context):
        params = {
            'source': source,
            'context': context,
        }
        self.logger.info('Rendering template "%s" to "%s"', source, destination)
        self.filterset('render_template', [destination], {'params': params})

    def call(self, *args, **kwargs):
        kwargs.update(close_fds=True, stdout=subprocess.PIPE,
                      stderr=subprocess.PIPE)
        ignore_errnos = kwargs.pop('ignore_errnos', [])
        stdout_log_level = kwargs.pop(
            'stdout_log_level', self.default_log_level)
        stderr_log_level = kwargs.pop('stderr_log_level', logging.ERROR)
        popen = subprocess.Popen(args, **kwargs)
        log_level = {popen.stdout: stdout_log_level,
                     popen.stderr: stderr_log_level}

        def check_io():
            for iobuf in select([popen.stdout, popen.stderr], [], [], 1000)[0]:
                line = iobuf.readline().strip()
                if line:
                    self.logger.log(log_level[iobuf], '%s', line)
        while popen.poll() is None:
            check_io()
        check_io()
        returncode = popen.wait()
        if returncode != 0 and returncode not in ignore_errnos:
            raise subprocess.CalledProcessError(returncode, ' '.join(args))
        return returncode

    def calls(self, cmd, **kwargs):
        """Subprocesser caller which allows to pass arguments as string"""
        return self.call(shlex.split(cmd), **kwargs)

    @classmethod
    def guess_main_directory(cls, path):
        if len(os.listdir(path)) == 1:
            child = os.listdir(path)[0]
            if os.path.isdir(os.path.join(path, child)):
                return os.path.join(path, child)
        return path

class BaseSubRecipe(object):

    def __init__(self, recipe, group):
        self.recipe = recipe
        self.group = group
        self.logger = recipe.logger
        self.options = recipe.options[group]
        self.shell = self.options.get('shell', '/bin/sh')
        self.options.setdefault('location',
                                self.recipe.default_location \
                                if self.group is None or None not in self.recipe.subrecipes \
                                else self.recipe.subrecipes.get(None).location)
        self.options.setdefault('executable',
                                self.recipe.default_executable \
                                if self.group is None or None not in self.recipe.subrecipes \
                                else self.recipe.subrecipes.get(None).executable)

    @property
    def name(self):
        return self.options.get('name', None)

    @name.setter
    def name(self, value):
        self.options.set('name', value)

    @property
    @reify
    def working_directory(self):
        return self.options.get('working-directory',
                                self.recipe.default_working_directory)

    @property
    @reify
    def executable(self):
        return self.options.get('executable')

    @property
    @reify
    def environment_config(self):
        env = {}
        for line in self.options.get('environment', default='').splitlines():
            line = line.strip()
            if not line:
                continue
            if not '=' in line:
                raise UserError(
                    '''Line "{}" in environment is incorrect'''.format(line))
            key, value = line.split('=', 1)
            key = key.strip()
            value = value.strip()
            if key in env:
                raise UserError('''Key "{}" is repeated'''.format(key))
            env[key] = value
        return env

    @property
    @reify
    def environment(self):
        config = self.environment_config.copy()
        env = {}
        for key, value in os.environ.items():
            change = config.pop(key, None)
            if change is not None:
                env[key] = change % os.environ
                self.logger.info(
                    '''Environment "{}" set to "{}"'''.format(key, env[key]))
            else:
                env[key] = value
        for key, value in config.items():
            self.logger.info(
                '''Environment "{}" added with "{}"'''.format(key, value))
            env[key] = value
        return env

    @property
    @reify
    def location(self):
        return self.options.get('location')

    @property
    @reify
    def log_format(self):
        return self.options.get('log-format',
                                self.recipe.default_log_format)

    @property
    @reify
    def log_level(self):
        return self.options.get('log-level',
                                self.recipe.default_log_level)

    def initialize(self):
        self.recipe.set_logging(self.log_format,
                                self.log_level)

    @property
    @reify
    def completed(self):
        return os.path.join(self.location, '.completed')

    def mark_completed(self, files=None):
        self.recipe.mkdir(self.location)
        with open(self.completed, 'a'):
            os.utime(self.completed, None)
        return (files or []) + [self.completed]

class BaseGroupRecipe(BaseRecipe):
    subrecipe_class = NotImplemented

    def __init__(self, buildout, name, options):
        super(BaseGroupRecipe, self).__init__(buildout, name, options)
        if self.subrecipe_class is NotImplemented:
            raise ValueError("subrecipe_class has not been set")
        for group in self.options:
            self.subrecipes[group] = self.subrecipe_class(self, group) # pylint: disable=not-callable
            self.subrecipes[group].initialize()

    @property
    @reify
    def locations(self):
        return uniq([_f for _f in
                     chain(*[[x.location] + getattr(x, 'extra_locations', [])
                             for x in self.subrecipes.values()]) if _f])

    @property
    @reify
    def working_directories(self):
        return uniq([x.working_directory for x in self.subrecipes.values()])

    def initialize(self):
        super(BaseGroupRecipe, self).initialize()
        self.update = self.update_wrapper
        self.subrecipes = dict()

    def run_target(self, name, *args, **kwargs):
        for group in self.subrecipes:
            attr = getattr(self.subrecipes[group], name, None)
            if callable(attr):
                attr(*args, **kwargs)
            elif attr is not None:
                exec(attr) # pylint: disable=exec-used

    def install_target(self):
        return self.run_target('install')

    def update_target(self):
        return self.run_target('update')

    def uninstall_target(self):
        return self.run_target('uninstall')


"""
This module holds our greedy search decoder
"""
import sys

sys.path.append('..')
import torch
from torch import nn
from vocab import SOS_TOKEN

USE_CUDA = torch.cuda.is_available()
DEVICE = torch.device("cuda" if USE_CUDA else "cpu")

class GreedySearchDecoder(nn.Module):
    """Greedy searcher"""

    def __init__(self, encoder, decoder):
        super(GreedySearchDecoder, self).__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, input_seq, input_length, max_length):
        encoder_outputs, encoder_hidden = self.encoder(input_seq, input_length)
        decoder_hidden = encoder_hidden[:self.decoder.n_layers]
        decoder_input = torch.ones(1, 1, device=DEVICE,
                                   dtype=torch.long) * SOS_TOKEN
        all_tokens = torch.zeros([0], device=DEVICE, dtype=torch.long)
        all_scores = torch.zeros([0], device=DEVICE)
        for _ in range(max_length):
            decoder_output, decoder_hidden = self.decoder(
                decoder_input, decoder_hidden, encoder_outputs
            )

            decoder_scores, decoder_input = torch.max(decoder_output, dim=1)

            all_tokens = torch.cat((all_tokens, decoder_input), dim=0)
            all_scores = torch.cat((all_scores, decoder_scores), dim=0)

            decoder_input = torch.unsqueeze(decoder_input, 0)
        return all_tokens, all_scores


# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('train_u6lujuX_CVtuZ9i.csv')
testset = pd.read_csv('test_Y3wMUE5_7gLdaTN.csv')

dataset['Gender'].fillna(dataset['Gender'].mode()[0],inplace=True)
dataset['Married'].fillna(dataset['Married'].mode()[0],inplace=True)
dataset['Self_Employed'].fillna(dataset['Self_Employed'].mode()[0],inplace=True)
dataset['Loan_Status'] = dataset['Loan_Status'].map(lambda x: 1 if x=='Y' else 0)

dataset['Credit_History'].fillna(dataset['Loan_Status'], inplace = True)
#for dependents
dataset['Dependents'] = dataset['Dependents'].replace({'3+':3})
dataset['Dependents'].fillna(dataset['Dependents'].mode()[0],inplace=True)

testset['Gender'].fillna(testset['Gender'].mode()[0],inplace=True)
testset['Married'].fillna(testset['Married'].mode()[0],inplace=True)
testset['Self_Employed'].fillna(testset['Self_Employed'].mode()[0],inplace=True)
testset['Credit_History'].fillna(testset['Credit_History'].mode()[0],inplace=True)
testset['Dependents'] = testset['Dependents'].replace({'3+':3})
testset['Dependents'].fillna(testset['Dependents'].mode()[0],inplace=True)

X = dataset.iloc[:,[1,2,3,4,5,6,7,8,9,10,11]].values
Xtest = testset.iloc[:,[1,2,3,4,5,6,7,8,9,10,11]].values
y = dataset.iloc[:, -1].values

from sklearn.preprocessing import Imputer
imputer = Imputer(missing_values = 'NaN', strategy = 'most_frequent', axis = 0)
imputer = imputer.fit(X[:, 8:9])
X[:, 8:9] = imputer.transform(X[:, 8:9])
imputer1 = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
imputer1 = imputer1.fit(X[:, 7:8])
X[:, 7:8] = imputer1.transform(X[:, 7:8])

imputer2 = Imputer(missing_values = 'NaN', strategy = 'most_frequent', axis = 0)
imputer2 = imputer2.fit(Xtest[:, 8:9])
Xtest[:, 8:9] = imputer2.transform(Xtest[:, 8:9])
imputer12 = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
imputer12 = imputer12.fit(Xtest[:, 7:8])
Xtest[:, 7:8] = imputer12.transform(Xtest[:, 7:8])

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X = LabelEncoder()
X[:, 0] = labelencoder_X.fit_transform(X[:, 0])

labelencoder_X1 = LabelEncoder()
X[:, 1] = labelencoder_X1.fit_transform(X[:, 1])
#labelencoder_X111 = LabelEncoder()
#X[:, 2] = labelencoder_X111.fit_transform(X[:, 2])
labelencoder_X2 = LabelEncoder()
X[:, 3] = labelencoder_X2.fit_transform(X[:, 3])
labelencoder_X3 = LabelEncoder()
X[:, 4] = labelencoder_X3.fit_transform(X[:, 4])
#onehotencoder = OneHotEncoder(categorical_features = [3])
#X = onehotencoder.fit_transform(X).toarray()
labelencoder_X4 = LabelEncoder()
X[:, 10] = labelencoder_X4.fit_transform(X[:, 10])
onehotencoder = OneHotEncoder(categorical_features = [10])
X = onehotencoder.fit_transform(X).toarray()

X = X[:, 1:]

labelencoder_Xtest = LabelEncoder()
Xtest[:, 0] = labelencoder_Xtest.fit_transform(Xtest[:, 0])

labelencoder_Xtest1 = LabelEncoder()
Xtest[:, 1] = labelencoder_Xtest1.fit_transform(Xtest[:, 1])
#labelencoder_Xtest111 = LabelEncoder()
#Xtest[:, 2] = labelencoder_Xtest111.fit_transform(Xtest[:, 2])
labelencoder_Xtest2 = LabelEncoder()
Xtest[:, 3] = labelencoder_Xtest2.fit_transform(Xtest[:, 3])
labelencoder_Xtest3 = LabelEncoder()
Xtest[:, 4] = labelencoder_Xtest3.fit_transform(Xtest[:, 4])
#onehotencoder = OneHotEncoder(categorical_features = [3])
#Xtest = onehotencoder.fit_transform(Xtest).toarray()
labelencoder_Xtest4 = LabelEncoder()
Xtest[:, 10] = labelencoder_Xtest4.fit_transform(Xtest[:, 10])
onehotencoder = OneHotEncoder(categorical_features = [10])
Xtest = onehotencoder.fit_transform(Xtest).toarray()

Xtest = Xtest[:, 1:]
# Fitting XGBoost to the Training set
from xgboost import XGBClassifier
classifier = XGBClassifier()
classifier.fit(X, y)

y_pred = classifier.predict(Xtest)

from sklearn.model_selection import cross_val_score
accuracies = cross_val_score(estimator = classifier, X = X, y = y, cv = 10)
accuracies.mean()
accuracies.std()

y_pred=list(y_pred)
for j in range(len(y_pred)):
    if y_pred[j]==1:
        y_pred[j]='Y'
    else:
        y_pred[j]='N'


__authors__ = 'Marija Aleksejeva, Daniil Kolomeitsev, Jevgenij Mozajev,' \
              ' Nina Mustafina, Svetlana Pavlova'

# This module creates a collection of links and respective page files
# from source link using breadth-first algorithm with adjustable depth.

from bs4 import BeautifulSoup
import requests
from urllib.parse import urljoin
import os

# This function prints a set of links to file.
def links_to_file(link_list, path):
    with open(path, 'w', encoding='utf-8') as f:
        for link in link_list:
            f.write(link + '\n')

# This function prints pages to files. The link to the page is in
# the first row.
def page_to_file(file_name, dir_name, path, link, text):
    if not os.path.exists(os.path.join(path, dir_name)):
        os.makedirs(os.path.join(path, dir_name))
    with open(os.path.join(path, dir_name, str(file_name) + '.txt'),
              'w', encoding='utf-8') as f:
        new_text = (link + '\n' + text)
        f.write(new_text)

# This function creates a set of links with distance = depth from
# the source link. For each link, it initializes a save of the page
# code to a file.
def crawler(current_level, depth):
    file_counter = 0
    dir_counter = 0
    files_in_dir = 0
    link_list = set()
    while depth > 0:
        next_level = []
        for link in current_level:
            page = requests.get(link, allow_redirects=False)
            text = page.text

            page_to_file(str(file_counter), str(dir_counter),
                         'collection_pages', link, text)
            files_in_dir += 1
            file_counter += 1
            if files_in_dir == 100:
                dir_counter += 1
                files_in_dir = 0
            soup = BeautifulSoup(text, 'html.parser')
            for child_link in soup.find_all('a'):
                new_link = urljoin(link, child_link.get('href'))
                if not new_link.startswith('https://'):
                    if not new_link in link_list:
                        link_list.add(new_link)
                        next_level.append(new_link)
            if file_counter == 500:
                break
        current_level = next_level
        depth -= 1
    return link_list


import unittest
import wikitextparser as wtp

from etymmap.graph import RelationType
from etymmap.specific import TemplateHandler, LinkNormalization
from etymmap.specific_en import EtymologyTemplateHandler, configure

configure()

class TestEtymologyTemplateHandler(unittest.TestCase):
    handler: TemplateHandler = EtymologyTemplateHandler()

    def test_normalizations(self):
        def test(template, rel=None, src=None, tgt=None):
            template = wtp.parse(template).templates[0]
            n = self.handler.to_normalization(template)
            if rel:
                self.assertEqual(rel, n.relation)
            if src:
                self.assertEqual(src, n.link_source)
            if tgt:
                self.assertEqual(tgt, n.link_target)

        test(
            "{{derived|en|fr|bureaucrate}}",
            rel={"type": RelationType.DERIVATION},
            src={"language": "en"},
            tgt={"language": "fr", "term": "bureaucrate"},
        )

        test(
            "{{borrowed|en|de|Ablaut|gloss=sound gradation}}",
            rel={"type": RelationType.BORROWING},
            src={"language": "en"},
            tgt={"language": "de", "term": "Ablaut", "t": "sound gradation"},
        )

        test(
            "{{learned borrowing|en|la|aberrātiō|gloss=relief, diversion|notext=1}}",
            rel={"type": RelationType.LEARNED_BORROWING},
            src={"language": "en"},
            tgt={"language": "la", "term": "aberrātiō", "t": "relief, diversion"},
        )

        test(
            "{{desc|sq|thesar|bor=1}}",
            rel={"type": RelationType.BORROWING},
            tgt={"language": "sq", "term": "thesar"},
        )

        test(
            "{{desc|ro|tezaur|pclq=1|unc=1}}",
            rel={"type": RelationType.PARTIAL_CALQUE, "uncertain": True},
        )

        test(
            "{{back-formation|enm|waxen|t=to grow|id=to grow|nocap=1}}",
            rel={"type": RelationType.BACKFORM},
            tgt={"language": "enm", "term": "waxen", "t": "to grow", "id": "to grow"},
        )

        test(
            "{{onomatopoeic|en|title=onomatopoeic}}",
            rel={"type": RelationType.ONOM},
            tgt=LinkNormalization.NO_TARGET,
        )

        test(
            "{{unknown|la}}",
            rel={"type": RelationType.UNKNOWN},
            tgt=LinkNormalization.NO_TARGET,
        )

        test(
            "{{short for|ja|西本願寺|tr=Nishi Hongan-ji|nodot=1|sort=にし}}",
            rel={"type": RelationType.SHORTENING},
            tgt={"language": "ja", "term": "西本願寺", "tr": "Nishi Hongan-ji"},
        )

        test(
            "{{short for|vi|Thiên Can||[[celestial stem]]}}",
            rel={"type": RelationType.SHORTENING},
            tgt={
                "language": "vi",
                "term": "Thiên Can",
                "alt": "",
                "t": "celestial stem",
            },
        )

        test(
            "{{affix|en|a-|gloss1=towards|back|gloss2=back}}",
            rel={"type": RelationType.MORPHOLOGICAL},
            tgt=(
                {"language": "en", "term": "a-", "t": "towards"},
                {"language": "en", "term": "back", "t": "back"},
            ),
        )

        test(
            "{{named-after|es|nat=Genoese|occ=explorer|Christopher Columbus|wplink==|born=1451|died=1506}}",
            rel={"type": RelationType.EPONYM},
            tgt={
                "nat": "Genoese",
                "occ": "explorer",
                "name": "Christopher Columbus",
                "wplink": "=",
                "born": "1451",
                "died": "1506",
            },
        )

        test(
            "{{root|en|pie|*root1|*root2|id2=pie-root2}}",
            rel={"type": RelationType.ROOT},
            tgt=(
                {"language": "pie", "term": "*root1"},
                {"language": "pie", "term": "*root2", "id": "pie-root2"},
            ),
        )

        test(
            "{{ar-root|س|ل|م}}",
            rel={"type": RelationType.ROOT},
            tgt=({"language": "ar", "term": "س ل م"}),
        )

        test(
            "{{ja-r|羨ましい|うらやましい|t=enviable}}",
            rel={"type": RelationType.RELATED},
            tgt=(
                {"language": "ja", "term": "羨ましい", "ascii": "うらやましい", "t": "enviable"}
            ),
        )

        test(
            "{{ja-r|入手する||t=enviable|linkto=入手}}",
            rel={"type": RelationType.RELATED},
            tgt=(
                {
                    "language": "ja",
                    "term": "入手",
                    "ascii": "",
                    "alt": "入手する",
                    "t": "enviable",
                }
            ),
        )


from collections import Counter

def default(*_, **__):
    return True

def expand(sequence, mappings, iterations=10):
    for _ in range(iterations):
        i = 0
        while i < len(sequence) - 1:
            pair = "".join(sequence[i : (i + 2)])
            no_insert = mappings.get(pair, default)(sequence, i + 1)
            i += 1 + (not no_insert)
    return sequence

def get_answer(expanded):
    count = Counter(expanded).most_common()
    return count[0][1] - count[-1][1]

with open("inputs/day14.txt") as f:
    raw_input = f.read().split("\n")[:-1]

sequence = list(raw_input.pop(0))
mappings = {
    x.split(" -> ")[0]: lambda l, i, val=x.split(" -> ")[1]: l.insert(i, val)
    for x in raw_input[1:]
}

expanded = expand(sequence.copy(), mappings)
answer1 = get_answer(expanded)
print(f"Answer 1 = {answer1}")

pair_mappings = {}
raw_input.pop(0)

# Map each pair to the two pairs it forms when expanded
for i, pair in enumerate(raw_input):
    split = pair.split(" -> ")
    pair_mappings[split[0]] = [split[0][0] + split[1][0], split[1][0] + split[0][1], i]

mappings = {x.split(" -> ")[0]: x.split(" -> ")[1] for x in raw_input}
l = list(mappings.keys())

count = Counter(sequence)
count.update({k: 0 for k in set(mappings.values()) if k not in count.keys()})

pairs_count = Counter(
    ["".join(sequence[i : (i + 2)]) for i in range(len(sequence) - 1)]
)
pairs_count = {
    **pairs_count,
    **{k: 0 for k in mappings.keys() if k not in pairs_count.keys()},
}

for __ in range(40):
    temp = {k: 0 for k in pairs_count}
    for k, v in pairs_count.items():
        # Update letters count
        count[mappings[k]] += v
        # Update pairs counts
        temp[pair_mappings[k][0]] += v
        temp[pair_mappings[k][1]] += v
    # Read from, clear temp
    pairs_count = temp

answer2 = max(count.values()) - min(filter(lambda x: x > 0, count.values()))
print(f"Answer 2 = {answer2}")


from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from rest_framework import permissions, status, viewsets
from rest_framework.decorators import action, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from django_filters.rest_framework import DjangoFilterBackend

from .mixins import ListRetrieveViewSet
from .models import (
    User, Subscriber, Tag, Ingredient,
    Recipe, Amount, ShoppingCart, Favorite
)
from .filters import RecipeFilter, IngredientFilter
from .serializers import (
    TagSerializer, RecipeWriteSerializer,
    RecipeSerializerGet, FavoriteRecipeSerializer,
    IngredientSerializerGet, UserSerializer,
    LiteRecipeSerializer,
    PasswordSerializer, NewUserSerializer,
    FavoriteSerializer, ShoppingCartSerializer,
    FollowListSerializer, FollowSerializer
)
from .pagination import FoodgramPagePagination

@permission_classes([permissions.AllowAny, ])
class TagViewSet(viewsets.ReadOnlyModelViewSet):
    """
    ViewSet предназначен для взаимодействия в моделью Tag.
    Он позволяет получать данные о тэгах.
    """
    queryset = Tag.objects.all().order_by('id')
    serializer_class = TagSerializer
    pagination_class = None

@permission_classes([permissions.IsAuthenticatedOrReadOnly, ])
class RecipeViewSet(viewsets.ModelViewSet):
    """
    ViewSet предназначен для взаимодействия в моделью Recipe.
    Он позволяет получать данные о рецептах, создавать, удалять и изменять их.
    Также добавлять рецепты в раздел "Избранное" или удалять их.
    Скачивать файл со списком покупок, добалять и удалять рецепты из него.
    Доступна фильтрация по избранному, автору, списку покупок и тегам.
    """
    queryset = Recipe.objects.all()
    pagination_class = FoodgramPagePagination
    filter_backends = (DjangoFilterBackend,)
    filter_class = RecipeFilter

    def get_serializer_class(self):
        if self.request.method == 'GET':
            return RecipeSerializerGet
        return RecipeWriteSerializer

    def perform_create(self, serializer):
        serializer.save(author=self.request.user)

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context.update({'request': self.request})
        return context

    @action(detail=True, methods=['POST', 'DELETE'], url_path='favorite')
    def favorite(self, request, pk):
        current_user = self.request.user
        recipe = get_object_or_404(Recipe, pk=pk)
        recipe_in_favorite = Favorite.objects.filter(
            user=current_user, recipes=recipe
        )
        data = {'user': current_user.id, 'recipes': recipe.id}
        serializer = FavoriteSerializer(
            data=data,
            context={'request': request}
        )
        serializer.is_valid(raise_exception=True)
        if request.method == 'POST':
            Favorite.objects.create(user=current_user, recipes=recipe)
            serializer = FavoriteRecipeSerializer(
                recipe, context={'request': request}
            )
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        recipe_in_favorite.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

    @action(
        detail=True,
        methods=['POST', 'DELETE'],
        url_path='shopping_cart',
        permission_classes=[IsAuthenticated]
    )
    def shopping_cart(self, request, pk):
        recipe = get_object_or_404(Recipe, pk=pk)
        current_user = self.request.user
        recipe_in_shop = ShoppingCart.objects.filter(
            user=current_user, recipe=recipe
        )
        data = {'user': current_user.id, 'recipe': recipe.id}
        serializer = ShoppingCartSerializer(
            data=data,
            context={'request': request}
        )
        serializer.is_valid(raise_exception=True)
        if request.method == 'DELETE':
            recipe_in_shop.delete()
            return Response(status=status.HTTP_204_NO_CONTENT)
        ShoppingCart.objects.create(user=current_user, recipe=recipe)
        serializer = LiteRecipeSerializer(recipe, context={'request': request})
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    @action(
        detail=False,
        methods=['GET'],
        url_path='download_shopping_cart',
        permission_classes=[IsAuthenticated]
    )
    def download_shopping_cart(self, request):
        ingredients = Amount.objects.filter(
            recipes__shopping__user=request.user).values(
            'ingredients__name',
            'ingredients__measurement_unit',
            'amount'
        )
        list_unique_ingr = []
        shop_list = 'Список покупок \n\n'
        for i in range(len(ingredients)):
            if ingredients[i].get('ingredients__name') not in list_unique_ingr:
                list_unique_ingr.append(
                    ingredients[i].get('ingredients__name')
                )
        for unique_ingr in list_unique_ingr:
            count = 0
            ingredient = ingredients.filter(ingredients__name=unique_ingr)
            for i in range(len(ingredient)):
                count += ingredient[i].get('amount')
            shop_list += (
                f"{ingredient[i].get('ingredients__name')} "
                f"({ingredient[i].get('ingredients__measurement_unit')}) - "
                f"{count}\n")
        response = HttpResponse(shop_list, 'Content-Type: text/plain')
        response['Content-Disposition'] = 'attachment; filename="Cart.txt"'
        return response

@permission_classes([permissions.AllowAny, ])
class IngredientViewSet(ListRetrieveViewSet):
    """
    ViewSet предназначен для взаимодействия в моделью Ingredient.
    Он позволяет получать данные обо всех ингредиентах
    или о каком-то определнном.
    """
    serializer_class = IngredientSerializerGet
    queryset = Ingredient.objects.all().order_by('name')
    filter_backends = (IngredientFilter,)
    search_fields = ('^name',)

@permission_classes([permissions.AllowAny, ])
class UserViewSet(viewsets.ModelViewSet):
    """
    ViewSet предназначен для взаимодействия в моделью User.
    Он позволяет получать данные о пользователях, о текущем пользователе(me),
    создавать пользователей и менять пароль, получать данные о подписках,
    подписываться и удалять подписки.
    """
    serializer_class = UserSerializer
    queryset = User.objects.all()
    pagination_class = FoodgramPagePagination

    def get_serializer_class(self):
        if self.request.method == 'GET':
            return UserSerializer
        return NewUserSerializer

    @action(
        detail=False,
        methods=['GET'],
        url_path='me',
        permission_classes=[IsAuthenticated]
    )
    def me(self, request):
        user = request.user
        if request.method == 'GET':
            serializer = UserSerializer(user)
            return Response(serializer.data, status=status.HTTP_200_OK)

        serializer = UserSerializer(
            user,
            data=request.data,
            partial=True
        )
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return Response(serializer.data, status=status.HTTP_200_OK)

    @action(
        detail=False,
        methods=['POST'],
        url_path='set_password',
        permission_classes=[IsAuthenticated]
    )
    def change_password(self, request):
        serializer = PasswordSerializer(data=request.data)
        author = self.request.user
        if serializer.is_valid(raise_exception=True):
            if not author.check_password(
                    serializer.data.get('current_password')
            ):
                return Response(
                    {'current_password': ['Wrong password.']},
                    status=status.HTTP_400_BAD_REQUEST
                )
            author.set_password(serializer.data.get('new_password'))
            author.save()
            response = {
                'status': 'success',
                'code': status.HTTP_200_OK,
                'message': 'Password updated successfully'
            }
            return Response(response)

    @action(
        detail=False,
        methods=['GET'],
        url_path='subscriptions',
        permission_classes=(IsAuthenticated,)
    )
    def subscriptions(self, request, pk=None):
        subscriptions_list = self.paginate_queryset(
            User.objects.filter(subscribed__user=request.user)
        )
        serializer = FollowListSerializer(
            subscriptions_list, many=True, context={
                'request': request
            }
        )
        return self.get_paginated_response(serializer.data)

    @action(
        detail=True,
        methods=['POST', 'DELETE'],
        permission_classes=(IsAuthenticated,),
        url_path='subscribe'
    )
    def subscribe(self, request, pk):
        if request.method != 'POST':
            subscription = get_object_or_404(
                Subscriber,
                subscribed=get_object_or_404(User, id=pk),
                user=request.user
            )
            self.perform_destroy(subscription)
            return Response(status=status.HTTP_204_NO_CONTENT)
        serializer = FollowSerializer(
            data={
                'user': request.user.id,
                'subscribed': get_object_or_404(User, id=pk).id
            },
            context={'request': request}
        )
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return Response(serializer.data, status=status.HTTP_201_CREATED)


import os
import numpy as np
import pandas as pd
import random
import torch
import pathlib
import torch
from torch.utils.data import Dataset
from .utils import load_data
from .data_transform import standarize

def get_meta_data(config):
    data_dir = config['dataset']['input_directory']
    config_dataset_files_meta = config['dataset']['files']['meta']
    train_curated = pd.read_csv(data_dir + config_dataset_files_meta['train_curated'])
    train_noisy = pd.read_csv(data_dir + config_dataset_files_meta['train_noisy'])
    test = pd.read_csv(data_dir + config_dataset_files_meta['test'])

    train = pd.concat([train_curated, train_noisy], axis=0, sort=False, ignore_index=True)
    train['noisy_flg'] = 0
    train.loc[len(train_curated):len(train_curated) + len(train_noisy), 'noisy_flg'] = 1
    train['n_labels'] = train['labels'].apply(lambda x: len(x.split(',')))

    labels = test.columns[1:].tolist()
    n_classes = len(labels)
    y_train = np.zeros((len(train), n_classes)).astype(int)
    for i, row in enumerate(train['labels'].str.split(',')):
        for label in row:
            idx = labels.index(label)
            y_train[i, idx] = 1

    return train, test, y_train, labels

class ToTensor(object):
    """
    convert ndarrays in sample to Tensors.
    return:
        feat(torch.FloatTensor)
        label(torch.LongTensor of size batch_size x 1)
    """
    def __call__(self, data):
        data = torch.from_numpy(data).type(torch.FloatTensor)
        return data

class FAT_TrainSet_logmel(Dataset):
    def __init__(self, config, dataframe, labels, transform=None):
        self.config = config
        self.dataframe = dataframe
        self.labels = labels
        self.transform = transform

    def __len__(self):
        return len(self.dataframe)

    def __getitem__(self, idx):
        # get filename
        fe_dir = pathlib.Path(self.config['fe_dir'])
        filename = os.path.splitext(self.dataframe["fname"][idx])[0] + '.pkl'
        file_path = fe_dir / filename

        # Read and Resample the audio
        data = self._random_selection(file_path)
        label = torch.from_numpy(self.labels[idx]).float()

        if self.transform is not None:
            data = self.transform(data)

        return data, label

    def _random_selection(self, file_path):
        input_length = self.config['model']['params']['input_length']

        # Read the logmel pkl
        logmel = load_data(file_path)
        logmel = np.concatenate((logmel, logmel), axis=2)   # (3, n_mel, 2*n_times)

        # Random offset / Padding
        if logmel.shape[2] > input_length:   # logmel.shape = (3, n_mel, xx)
            max_offset = logmel.shape[2] - input_length
            offset = np.random.randint(max_offset)
            data = logmel[:, :, offset:(input_length + offset)]
        else:
            if input_length > logmel.shape[2]:
                max_offset = input_length - logmel.shape[2]
                offset = np.random.randint(max_offset)
            else:
                # input_length = logmel.shape[2]
                offset = 0
            data = np.pad(logmel, ((0, 0), (0, 0), (offset, input_length - logmel.shape[2] - offset)), "constant")

        return data

class FAT_TestSet_logmel(Dataset):
    def __init__(self, config, fnames, transform=None, tta=5):
        self.config = config
        self.fnames = fnames
        self.transform = transform
        self.tta = tta

    def __len__(self):
        return len(self.fnames) * self.tta

    def __getitem__(self, idx):
        new_idx = idx % len(self.fnames)

        fe_dir = pathlib.Path(self.config['fe_dir'])
        filename = os.path.splitext(self.fnames[new_idx])[0] + '.pkl'
        file_path = fe_dir / filename

        # Read and Resample the audio
        data = self._random_selection(file_path)
        fname = self.fnames[new_idx]

        if self.transform is not None:
            data = self.transform(data)

        return data, fname

    def _random_selection(self, file_path):
        input_length = self.config['model']['params']['input_length']

        # Read the logmel pkl
        logmel = load_data(file_path)
        logmel = np.concatenate((logmel, logmel), axis=2)   # (3, n_mel, 2*n_times)

        # Random offset / Padding
        if logmel.shape[2] > input_length:   # logmel.shape = (3, n_mel, xx)
            # Random offset
            max_offset = logmel.shape[2] - input_length
            offset = np.random.randint(max_offset)
            data = logmel[:, :, offset:(input_length + offset)]
        else:
            # Padding
            if input_length > logmel.shape[2]:
                max_offset = input_length - logmel.shape[2]
                offset = np.random.randint(max_offset)
            else:
                # input_length = logmel.shape[2]
                offset = 0
            data = np.pad(logmel, ((0, 0), (0, 0), (offset, input_length - logmel.shape[2] - offset)), "constant")

        return data