"""This scripts includes DTI registration pipeline using DTI-TK tool.
I followed the official DTI-TK tutorials.
The steps listed below are taken from here:
http://dti-tk.sourceforge.net/pmwiki/pmwiki.php?n=Documentation.Registration
"""

import os
import shutil
import logging
import subprocess
from typing import Union
from pathlib import Path
from dtip.convert import fsl_to_dtitk_multi
from dtip.utils import show_exec_time

# * Add dtitk tool to PATH
from dtip.utils import ROOT_DIR
if os.getenv('DTITK_ROOT') is None:
    # Add DTI-TK PATH as environment variable
    dtitk_maindir = f"{ROOT_DIR}/dtitk"
    os.environ["DTITK_ROOT"] = dtitk_maindir
    os.environ["PATH"] += f":{dtitk_maindir}/bin:{dtitk_maindir}/utilities:{dtitk_maindir}/scripts"

__all__ = [
    'dtitk_register_multi',
    'template_to_subject_space',
    'template_to_subject_space_multi'
]

@show_exec_time
def dtitk_register_multi(
    input_path: Union[str, Path],
    template_path: Union[None, str, Path],
    bootstrapped_template_path: Union[None, str, Path],
    output_path: Union[str, Path],
    n_diffeo_iters: int = 8,
) -> int:
    """DTI existing Template-based Image Registration using Diffusion
        Tensor Imaging ToolKit (DTI-TK)

    Args:
        input_path: folder path containing a subject's data.
        template_path: Path of the template to use for registration.
        output_path: location to save the output files.

    Returns:
        exit code 0 on successful execution.
    """

    input_path, output_path = Path(input_path), Path(output_path)
    if bootstrapped_template_path:
        bootstrapped_template_path = Path(bootstrapped_template_path)

    if bootstrapped_template_path is None:
        # * Step 1: Convert FSL format to DTI-TK format and move files
        # * to `output_path`.
        exit_code = fsl_to_dtitk_multi(input_path, output_path)
        if exit_code != 0:  # Stop here if any error
            _msg = "Error in `dtitk_register_multi` execution :(. Stopped."
            logging.error(_msg)
            raise RuntimeError(_msg)
        logging.info(
            f"Converted `{input_path}` to DTI-TK format and saved at `{output_path}`."
        )

        # * SPATIAL NORMALIZATION AND ATLAS CONSTRUCTION
        # * Step 2. Bootstrapping the initial DTI template from the input DTI volumes
        # Get subjects' DTI file paths
        subs_filepaths = [
            subject_path / "dti_dtitk.nii.gz"
            for subject_path in output_path.glob("*")
            if subject_path.is_dir()
        ]
        # Create a file with subset names
        subs_filepath = output_path / "subs.txt"
        with open(subs_filepath, "w") as subf:
            for filepath in subs_filepaths:
                subf.write(f"{filepath}\n")

        print(subs_filepath)
        # Run the `dti_template_bootstrap` command
        logging.info("Starting template bootstrapping...")
        subprocess.run(
            [
                "dti_template_bootstrap",
                template_path,
                subs_filepath,
                "EDS",
                "4",
                "4",
                "4",
                "0.0001",
            ]
        )
        logging.info("Bootstrapping complete.")
    else:

        subs_filepaths = [
            subject_path / "dti_dtitk.nii.gz"
            for subject_path in output_path.glob("*")
            if subject_path.is_dir()
        ]
        # Create a file with subset names
        subs_filepath = output_path / "subs.txt"
        with open(subs_filepath, "w") as subf:
            for filepath in subs_filepaths:
                subf.write(f"{filepath}\n")

        template_path = bootstrapped_template_path

    # * Step 3: Rigid Alignment of DTI Volumes
    # As mentioned in tutorial, rigid alignment is not required when using
    # an existing template.

    # * Step 4: Affine Alignment of DTI volumes
    logging.info("Affine aligment with template refinement")
    for i, subject_path in enumerate(subs_filepaths, start=1):
        logging.info(
            f"[{i}/{len(subs_filepaths)}] Affine alignment of `{subject_path}`..."
        )
        subprocess.run(
            [
                "dti_affine_reg",
                template_path,
                subject_path,
                "EDS",
                "4",
                "4",
                "4",
                "0.001",
            ]
        )
    logging.info("Affine alignment completed!")

    # * Step 5: Deformable alignment with template refinement
    # generate the mask image for mask argument of diffeomorphic command
    logging.debug("Extract TR image from template to create a mask...")
    ret_code = subprocess.run(
        ["TVtool", "-in", template_path, "-tr", "-out", "template_tr.nii.gz"]
    ).returncode
    if ret_code != 0:
        raise RuntimeError(
            """[Error @ `dtitk_register_multi`]
        Problem with mask generation for diffeomorphic alignment."""
        )
    logging.debug("Generating mask for diffeomorphic alignment...")
    ret_code = subprocess.run(
        [
            "BinaryThresholdImageFilter",
            "template_tr.nii.gz",
            "mask.nii.gz",
            "0.01",
            "100",
            "1",
            "0",
        ]
    ).returncode
    if ret_code != 0:
        raise RuntimeError(
            """[Error @ `dtitk_register_multi`]
        Problem with BinaryThresholdImageFilter command in subprocess."""
        )
    logging.info("Done!")
    # Run alignment subject wise
    # Get subjects' DTI file paths
    for subject_path in output_path.glob("*"):
        if subject_path.is_dir():
            logging.info(
                f"Diffeomorphic alignment of subject `{subject_path}`...")
            filepath = f"{subject_path}/dti_dtitk_aff.nii.gz"
            # Run alignment
            subprocess.run(
                [
                    "dti_diffeomorphic_reg",
                    template_path,
                    filepath,
                    "mask.nii.gz",
                    "1",
                    str(n_diffeo_iters),
                    "0.002",
                ]
            )
            logging.info("Subject alignment complete!")

    # Move extra generated files to output_path folder
    logging.info(f"Moving generated files to `{output_path}`...")
    if Path('mean_initial.nii.gz').exists():
        shutil.move('mean_initial.nii.gz', output_path/'mean_initial.nii.gz')
    if Path('mask.nii.gz').exists():
        shutil.move('mask.nii.gz', output_path/'mask.nii.gz')
    if Path('template_tr.nii.gz').exists():
        shutil.move('template_tr.nii.gz', output_path/'template_tr.nii.gz')
    if Path('bootstrapped_template.nii.gz').exists():
        shutil.move('bootstrapped_template.nii.gz',
                    output_path/'bootstrapped_template.nii.gz')
    logging.info("Registration complete!")
    return 0

def template_to_subject_space(subject_dir_path: Union[str, Path],
                              template_path: Union[str, Path],
                              transform_type: str) -> int:
    """Transform the template to the subject space.
        This function uses DTI-TK tool to perform transformations.

        This function is based on the following tutorial:
            http://dti-tk.sourceforge.net/pmwiki/pmwiki.php?n=Documentation.OptionspostReg

    Args:
        subject_dir_path: Subject folder path containing `subj.nii.gz` and
            `subj.aff` files to perform affine or diffeomorphic transformation.
            If the `transform_type` argument is `diffeo` then `subj.df.nii.gz`
            file should be present as well.
        template_path: Path to the template or atlas file to transform to
            the subject space.
        transform_type: Either `affine` for affine transformation or `diffeo`
            for diffeomorphic transformation.

    Returns:
        exit code 0 on successful execution
    """

    subject_dir_path = Path(subject_dir_path)
    if not subject_dir_path.is_dir():
        raise ValueError("`subject_dir_path` must be a folder.")
    subject_path = subject_dir_path/'dti_dtitk.nii.gz'
    aff_path = subject_dir_path/'dti_dtitk.aff'
    inv_aff_path = subject_dir_path/'dti_dtitk_inv.aff'
    if transform_type == 'affine':  # perform affine transformation
        savepath = subject_dir_path / \
            Path(template_path).stem.replace('.nii', '')
        savepath = f"{savepath}_affine_dti_space.nii.gz"
        # Compute the inverse of affine matrix
        ret_code = subprocess.run([
            'affine3Dtool', '-in', aff_path, '-invert', '-out', inv_aff_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `affine3Dtool` subprocess.")

        ret_code = subprocess.run([
            'affineScalarVolume', '-in', template_path, '-trans', inv_aff_path,
            '-target', subject_path, '-interp', '1', '-out', savepath
        ]).returncode

        if ret_code != 0:
            raise RuntimeError(
                "Something wrong with `affineScalarVolume` subprocess.")
    elif transform_type == 'diffeo':  # perform diffeomorphic transformation
        savepath = subject_dir_path / \
            Path(template_path).stem.replace('.nii', '')
        savepath = f"{savepath}_diffeo_dti_space.nii.gz"
        #* Combined displacement field from the native space to the template space
        diffeo_df_path = subject_dir_path/'dti_dtitk_aff_diffeo.df.nii.gz'
        combined_df_path = subject_dir_path/'dti_dtitk_combined.df.nii.gz'
        ret_code = subprocess.run([
            'dfRightComposeAffine',
            '-aff', aff_path,
            '-df', diffeo_df_path,
            '-out', combined_df_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `dfRightComposeAffine` subprocess.")
        #* Mapping the subject data to the template space
        subject_combined_path = subject_dir_path/'dti_dtitk_combined.nii.gz'
        ret_code = subprocess.run([
            'deformationSymTensor3DVolume',
            '-in', subject_path,
            '-trans', combined_df_path,
            '-target', template_path,
            '-out', subject_combined_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `dfRightComposeAffine` subprocess.")
        #* Combined displacement field from the template space to the native space
        ## 1. compute the inverse of the affine transformation with the
        ## command affine3Dtool
        ret_code = subprocess.run([
            'affine3Dtool', '-in', aff_path, '-invert', '-out', inv_aff_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `affine3Dtool` subprocess.")
        ## 2. compute the inverse of the deformable transformation with the
        ## command dfToInverse
        diffeo_inv_df_path = subject_dir_path/'dti_dtitk_aff_diffeo.df_inv.nii.gz'
        ret_code = subprocess.run([
            'dfToInverse', '-in', diffeo_df_path, '-out', diffeo_inv_df_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `dfToInverse` subprocess.")
        ## 3. combine the two inverted transformations with the command
        ## dfLeftComposeAffine (not dfRightComposeAffine)
        combined_inv_df_path = subject_dir_path/'dti_dtitk_combined.df_inv.nii.gz'
        ret_code = subprocess.run([
            'dfLeftComposeAffine',
            '-df', diffeo_inv_df_path,
            '-aff', inv_aff_path,
            '-out', combined_inv_df_path
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `dfLeftComposeAffine` subprocess.")
        #* Mapping the atlas space data to the subject space
        ret_code = subprocess.run([
            'deformationScalarVolume',
            '-in', template_path,
            '-trans', combined_inv_df_path,
            '-target', subject_path,
            '-interp', '1',
            '-out', savepath
        ]).returncode
        if ret_code != 0:
            raise RuntimeError("Something wrong with `deformationScalarVolume` subprocess.")
    else:
        raise NotImplementedError("Unknown `transform_type`. Only `affine` and `diffeo` is supported.")
    return 0

@show_exec_time
def template_to_subject_space_multi(subjects_dir_path: Union[str, Path],
                                    template_path: Union[str, Path],
                                    transform_type: str) -> int:
    """Transform the template to the subject space.
        This function uses DTI-TK tool to perform transformations.

        This function is based on the following tutorial:
            http://dti-tk.sourceforge.net/pmwiki/pmwiki.php?n=Documentation.OptionspostReg

    Args:
        subjects_dir_path: Parent folder path containing subject folder
            containing `subj.nii.gz` and `subj.aff` files to perform
            affine or diffeomorphic transformation. If the `transform_type`
            argument is `diffeo` then `subj.df.nii.gz` file should be present
            as well.
        template_path: Path to the template or atlas file to transform to
            the subject space.
        transform_type: Either `affine` for affine transformation or `diffeo`
            for diffeomorphic transformation.

    Returns:
        exit code 0 on successful execution
    """
    error_list = []
    template_path = Path(template_path)
    subjects_paths = [
        p for p in Path(subjects_dir_path).glob('*')
        if p.is_dir()
    ]
    total_subjects = len(subjects_paths)
    logging.debug("Starting template to subject space transform...")
    for i, subject_path in enumerate(subjects_paths, start=1):
        ret_code = template_to_subject_space(subject_dir_path=subject_path,
                                             template_path=template_path,
                                             transform_type=transform_type)
        if ret_code == 0:
            logging.info(
                f"[{i}/{total_subjects}] Transformed `{template_path.name}` to `{subject_path.name}`")
        else:
            logging.warning(
                f"Error transforming {subject_path.stem}. Skipped.")
            error_list.append(subject_path.stem)

    if error_list:
        print("=" * 10, f"{len(error_list)} Subjects with Errors", "=" * 10)
        for es in error_list:
            print(es)
        print("=" * 44)
        return len(error_list)
    else:
        print("=" * 30)
        print("All subjects completed without errors.")
        return 0


from unittest import TestCase
from solution import solution

class Test(TestCase):
    def test_solution(self):
        places = [["POOOP", "OXXOX", "OPXPX", "OOXOX", "POXXP"],
                  ["POOPX", "OXPXP", "PXXXO", "OXXXO", "OOOPP"],
                  ["PXOPX", "OXOXP", "OXPOX", "OXXOP", "PXPOX"],
                  ["OOOXX", "XOOOX", "OOOXX", "OXOOX", "OOOOO"],
                  ["PXPXP", "XPXPX", "PXPXP", "XPXPX", "PXPXP"]]
        result = [1, 0, 1, 1, 1]
        self.assertEqual(result, solution(places))


from django.conf.urls import patterns, include, url
from django.contrib import admin
from tastypie.api import Api

from VirtualResourceController.oms_resources.api import OMSInstanceResource, \
                                                        OMSHostResource

admin.autodiscover()

# setup/register API URL endpoints
v1_api = Api(api_name='v1')
v1_api.register(OMSInstanceResource())   # OMS App Deployment Instance
v1_api.register(OMSHostResource())   # OMS App Deployment Instance

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'VirtualResourceController.views.home', name='home'),
    # url(r'^VirtualResourceController/', include('VirtualResourceController.foo.urls')),
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^admin/', include(admin.site.urls)),
    (r'^api/', include(v1_api.urls)),
)


"""
Пусть на вход программы поступает массив из произвольного количества целых чисел и ещё одно целое число,
которое будем проверять на вхождение в этот массив. Задача состоит в том, чтобы вернуть индекс
первого вхождения элемента, если он входит в него, и False если не входит.
"""

def find(array, element):
    for i, a in enumerate(array):
        if a == element:
            return i
    return False

array = list(map(int, input("Введите массив чисел:").split()))
print(array)
element = int(input("Введите число:"))
print(element)

print(find(array, element))

Линейный поиск.py

Линейный поиск.py


"""
Пользователь задаёт числа a, b, c
из уравнения ax^2 + bx + c = 0,
вывести корни уравнения или
сообщить, что их нет.
"""

print("This program solves quadratic equations.")
print("You will need to input a, b, c for the equation")
print("ax^2 + bx + c = 0.")

a = input("Input a.\n")
b = input("Input b.\n")
c = input("Input c.\n")

try:
    a = float(a)
    b = float(b)
    c = float(c)
except ValueError:
    print("You should have entered three numbers!\nAbort session.")
    quit()

discriminant = b**2 - 4*a*c
if discriminant < 0:
    print("The equation does not have real solutions!")
else:
    if a!=0:
        x_1 = (-b - discriminant**0.5)/(2*a)
        x_2 = (-b + discriminant**0.5)/(2*a)
        print(f"x_1 = {x_1}")
        print(f"x_2 = {x_2}")
    else:
        if b == 0 and c == 0:
            print("The solution is any of real numbers.")
        elif b!=0:
            x_1 = -c/b
            print(f"x_1 = x_2 = {x_1}")
        elif b==0 and c!=0:
            print("The equation does not have solutions!")


from sklearn.model_selection import train_test_split as sklearn_train_test_split
from sklearn.preprocessing import LabelEncoder
from scipy.sparse import csr_matrix

import numpy as np
import pandas as pd

def get_examples(dataframe, labels_column="rating"):
    examples = dataframe[['userid', 'itemid']].values
    labels = dataframe[f'{labels_column}'].values
    return examples, labels

def train_test_split(examples, labels, test_size=0.1, verbose=0):
    if verbose:
        print("Train/Test split ")
        print(100-test_size*100, "% of training data")
        print(test_size*100, "% of testing data")

    train_examples, test_examples, train_labels, test_labels = sklearn_train_test_split(
        examples,
        labels,
        test_size=0.1,
        random_state=42,
        shuffle=True
    )

    train_users = train_examples[:, 0]
    test_users = test_examples[:, 0]

    train_items = train_examples[:, 1]
    test_items = test_examples[:, 1]

    x_train = np.array(list(zip(train_users, train_items)))
    x_test = np.array(list(zip(test_users, test_items)))

    y_train = train_labels
    y_test = test_labels

    if verbose:
        print()
        print('number of training examples : ', x_train.shape)
        print('number of training labels : ', y_train.shape)
        print('number of test examples : ', x_test.shape)
        print('number of test labels : ', y_test.shape)

    return (x_train, x_test), (y_train, y_test)

def ids_encoder(ratings):
    users = sorted(ratings['userid'].unique())
    items = sorted(ratings['itemid'].unique())

    uencoder = LabelEncoder()
    iencoder = LabelEncoder()

    uencoder.fit(users)
    iencoder.fit(items)

    ratings.userid = uencoder.transform(ratings.userid.tolist())
    ratings.itemid = iencoder.transform(ratings.itemid.tolist())

    return ratings, uencoder, iencoder

class ExplainableMatrixFactorization:

    def __init__(self, m, n, W, alpha=0.001, beta=0.01, lamb=0.1, k=10):
        self.W = W
        self.m = m
        self.n = n

        np.random.seed(64)

        self.k = k
        self.P = np.random.normal(size=(self.m,k))
        self.Q = np.random.normal(size=(self.n,k))

        self.alpha = alpha
        self.beta = beta
        self.lamb = lamb

        self.history = {
            "epochs":[],
            "loss":[],
            "val_loss":[],
        }

    def print_training_parameters(self):
        print('Training EMF')
        print(f'k={self.k} \t alpha={self.alpha} \t beta={self.beta} \t lambda={self.lamb}')

    def update_rule(self, u, i, error):
        self.P[u] = self.P[u] + \
            self.alpha*(2 * error*self.Q[i] - self.beta*self.P[u] - self.lamb*(self.P[u] - self.Q[i]) * self.W[u,i])

        self.Q[i] = self.Q[i] + \
            self.alpha*(2 * error*self.P[u] - self.beta*self.Q[i] + self.lamb*(self.P[u] - self.Q[i]) * self.W[u,i])

    def mae(self,  x_train, y_train):
        M = x_train.shape[0]
        error = 0
        for pair, r in zip(x_train, y_train):
            u, i = pair
            error += np.absolute(r - np.dot(self.P[u], self.Q[i]))
        return error/M

    def print_training_progress(self, epoch, epochs, error, val_error, steps=5):
        if epoch == 1 or epoch % steps == 0 :
                print(f"epoch {epoch}/{epochs} - loss : {round(error,3)} - val_loss : {round(val_error,3)}")

    def learning_rate_schedule(self, epoch, target_epochs = 20):
        if (epoch >= target_epochs) and (epoch % target_epochs == 0):
                factor = epoch // target_epochs
                self.alpha = self.alpha * (1 / (factor * 20))
                print("\nLearning Rate : {}\n".format(self.alpha))

    def fit(self, x_train, y_train, validation_data, epochs=10):
        self.print_training_parameters()

        x_test, y_test = validation_data

        for epoch in range(1, epochs+1):
            for pair, r in zip(x_train, y_train):
                u,i = pair
                r_hat = np.dot(self.P[u], self.Q[i])
                e = r - r_hat
                self.update_rule(u, i, error=e)

            error = self.mae(x_train, y_train)
            val_error = self.mae(x_test, y_test)
            self.update_history(epoch, error, val_error)
            self.print_training_progress(epoch, epochs, error, val_error, steps=1)

        return self.history

    def update_history(self, epoch, error, val_error):
        self.history['epochs'].append(epoch)
        self.history['loss'].append(error)
        self.history['val_loss'].append(val_error)

    def evaluate(self, x_test, y_test):
        error = self.mae(x_test, y_test)
        print(f"validation error : {round(error,3)}")

    def predict(self, userid, itemid, uencoder, iencoder):
        u = uencoder.transform([userid])[0]
        i = iencoder.transform([itemid])[0]
        r = np.dot(self.P[u], self.Q[i])
        return r

    def recommend(self, userid, uencoder, iencoder, N):
        u = uencoder.transform([userid])[0]

        predictions = np.dot(self.P[u], self.Q.T)

        top_idx = np.flip(np.argsort(predictions))[:N]

        top_items = iencoder.inverse_transform(top_idx)

        preds = predictions[top_idx]

        return top_items, preds

    def calc_mnap(self, k, df, users,uencoder, iencoder):
        if not 'itemid' in df.columns:
            errstr = "Column with items id's must be named itemid "
            raise TypeError(errstr)
        else:
            maps = []
            for user in users:
                score = 0.0
                num_hits = 0.0
                top_items, preds = self.recommend(user, uencoder, iencoder,N = 25)
                rel_items = df[df.userid == user]['itemid'].tolist()
                for i,p in enumerate(top_items[:k]):
                    if p in rel_items and p not in top_items[:i]:
                        num_hits += 1.0
                        score += num_hits / (i+1.0)
                maps.append(score / min(len(rel_items), k))
            return np.mean(maps)

    def calc_recalls(self, k, df, users, uencoder, iencoder):
        if not 'itemid' in df.columns:
            errstr = "Column with items id's must be named itemid "
            raise TypeError(errstr)
        else:
            recals = []
            for user in users:
                top_items, preds = self.recommend(user,uencoder, iencoder, N = 25)
                rel_items = df[df.userid == user]['itemid'].tolist()
                top_inter = 0
                for el in top_items[:5]:
                    if el in rel_items:
                        top_inter += 1
                recals.append(top_inter / len(top_items))
            return np.mean(recals)

    def dcg_at_k(self, r, k):
        r = r[:k]
        dcg = np.sum(r / np.log2(np.arange(2, len(r) + 2)))
        return dcg

    def ndgc_at_k(self, r, k, method = 0):
        dcg_max = self.dcg_at_k(sorted(r, reverse = True), k)
        return self.dcg_at_k(r, k) / dcg_max

    def calc_ndcg(self, predictions, k, users, ratings, movie_ids):
        ndgcs = []

        for target_users in np.unique(users):

            target_movie_id = movie_ids[target_users == users]
            target_rating = ratings[target_users == users]

            rel = target_rating[np.argsort(-predictions[target_users == users])]
            ndgc = self.ndgc_at_k(rel, k=k)
            ndgcs.append(ndgc)

        ndcg = np.mean(ndgcs)
        return ndcg


from fountains import app
from fountains.database import db, Fountain, FountainSearch
from flask import render_template
import json
from fountains.recommendation import find_closest_fountain
from fountains.recommendation import recommend_next_location

import settings

@app.route('/', methods=['GET'])
def home():
    return render_template('home.html', settings=settings)

@app.route('/map', methods=['GET'])
def map():
    return render_template('map.html', settings=settings)

@app.route('/fountains', methods=['GET'])
def fountains():
    fountains = Fountain.query.all()
    return json.dumps([
        {"latitude":fountain.latitude, "longitude":fountain.longitude}
        for fountain in fountains])

@app.route('/fountains/closest/<latitude>,<longitude>', methods=['GET'])
def closest_fountain(latitude, longitude):
    latitude = float(latitude)
    longitude = float(longitude)
    fountains = Fountain.query.all()
    coords = [(fountain.latitude, fountain.longitude)
              for fountain in fountains]
    point = find_closest_fountain((latitude, longitude), coords)

    # Store a FountainSearch for this search
    fs = FountainSearch(
            user_position=(latitude, longitude),
            fountain_position=point)
    db.session.add(fs)
    db.session.commit()

    return json.dumps({"latitude": point[0], "longitude": point[1]})

@app.route('/fountains/recommendations', methods=['GET'])
def recommended_fountains():
    searches = FountainSearch.query.all()

    coords = [(search.user_latitude, search.user_longitude,
               search.fountain_latitude, search.fountain_longitude)
               for search in searches]

    point = recommend_next_location(coords)

    return json.dumps({"latitude": point[0], "longitude": point[1]})


import Logger as lg

from Data.View.BaseDataView import BaseDataView
from Data.View.PlotFragmentationItem import PlotFragmentationItem

from Data.PeakML.Peak import Peak
from Data.PeakML.Header import Header
from Data.PeakML.PeakData import PeakData

from typing import List

class PlotFragmentationDataView(BaseDataView):
    def __init__(self):
        super().__init__(['Label','Fragments','Colour'])

    def load_plot_data_for_selected_peak(self, Peak: Peak, peakml_header: Header, colours):
        try:
            self.clear_datalist()

            for peak in Peak.peaks:
                measurement = peakml_header.get_measurement_by_id(peak.measurement_id)
                label = measurement.label
                fragments = peak.peak_data.fragments
                if fragments != []:
                    self.add_item(label,fragments,colours[f"M-{measurement.id}"])

            self.refresh_dataframe()

        except Exception as err:
            lg.log_error(f'Unable to update plot Fragmentation data: {err}')

    def add_item(self, label:str , fragments: List[str], color: str):
        self.datalist.append(PlotFragmentationItem(label,fragments,color))

    def refresh_dataframe(self):
        self.clear_dataframe()
        for item in self.datalist:
            self.dataframe = self.dataframe.append({
                                                    "UID": item.uid,
                                                    "Label": item.label,
                                                    "Fragments" : item.fragments,
                                                    #"Colour": item.colour,
                                                    "Selected": item.selected,
                                                    "Checked": item.checked,
                                                }, ignore_index=True)


import argparse

class get_test_config():
    def __init__(self):
        # Parse from command line
        self.parser = argparse.ArgumentParser(description='non-blind image deconvolution in the presence of kernel/model uncertainty')
        self.parser.add_argument('--gpu_idx', type=int, default=0, help='idx of gpu')
        self.parser.add_argument('--parallel', default=True, help='Parallel computation of the PSNR')

        # Problem Settings
        self.parser.add_argument('--dataset_name', default='Levin',choices=['Levin', 'Sun', 'Lai', 'Lai_Real'])
        self.parser.add_argument('-s', '--sigma', default = '0', choices=['0','2.55'])

        # Training Parameters
        self.parser.add_argument('--layers', type=int, default=4, help='net layers')
        self.parser.add_argument('--deep', type=int, default=17, help='one module deep')
        self.parser.add_argument('--save_img', default=False, help='save images into file')
        self.parser.add_argument('--compute_metrics', default=False, help='save images into file')
        self.parser.parse_args(namespace=self)

        # Predefined parameters
        if self.sigma == '0':
            self.taper = 'same'
            self.lmd = [0.005, 0.1, 0.1, 0.1, 0.1]
            self.test_ckp_dir = 'pretrained/noise_0pp'

        elif self.sigma == '2.55':
            self.taper = 'valid'
            self.lmd = [0.005, 0.5, 0.5, 0.5, 0.5]
            self.test_ckp_dir = 'pretrained/noise_1pp'

        # Data Preparation
        self.ker_dir = {}
        self.ker_dir['cho'] = './data/{0}_NK/BD_cho_and_lee_tog_2009/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['fergus'] = './data/{0}_NK/BD_fergus_tog_2006/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['levin'] = './data/{0}_NK/BD_levin_cvpr_2011/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['pan'] = './data/{0}_NK/BD_pan_cvpr_2016/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['sun'] = './data/{0}_NK/BD_sun_iccp_2013/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['xu'] = './data/{0}_NK/BD_Xu_eccv_2010/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['michaeli'] = './data/{0}_NK/BD_Michaeli_eccv_2014/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['xu10'] = './data/{0}_NK/BD_xu_eccv_2010/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['xu13'] = './data/{0}_NK/BD_xu_cvpr_2013/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['perrone'] = './data/{0}_NK/BD_perrone_cvpr_2014/kernel_estimates/'.format(self.dataset_name)
        self.ker_dir['lai_real'] = './data/Lai_NK/real/kernels/'
        self.test_sp_dir = './data/{0}_NK/sharp/'.format(self.dataset_name)

        # Select Blind Deconvolution Kernels by the kernels.
        if self.dataset_name == 'Sun':
            self.test_bl_dir = './data/Sun_NK/sigma_{0}_ker_levin/'.format(self.sigma)
            self.test_ker = ['cho', 'michaeli','xu']
            self.tr_ker_dir = './data/kernels/Levin09_v7.mat'
            self.bd_cut = 28
        elif self.dataset_name == 'Levin':
            self.test_bl_dir = './data/Levin_NK/sigma_{0}_ker_levin/'.format(self.sigma)
            self.test_ker = ['cho','levin','pan','sun']
            self.tr_ker_dir = './data/kernels/Levin09_v7.mat'
            self.bd_cut = 15
        elif self.dataset_name == 'Lai':
            self.test_bl_dir = './data/Lai_NK/sigma_{0}_ker_lai/'.format(self.sigma)
            self.test_ker = ['xu10','xu13','sun','perrone']
            self.bd_cut = 15
        elif self.dataset_name == 'Lai_Real':
            self.test_bl_dir = './data/Lai_NK/real/blurry/'

        if self.taper == 'valid':
            if self.dataset_name != 'Lai_Real':
                self.test_bl_dir = self.test_bl_dir[:-1] + '_valid/'
        elif self.taper == 'taper':
            self.test_bl_dir = self.test_bl_dir[:-1] + '_taper/'

        self.img_num = None

        if self.save_img:
            self.test_save_dir = './deblurred_results/noise_{0}_{1}/'.format(self.sigma,self.taper)


# coding=utf-8
# @Time    : 2021/3/4
# @Author  : ZTY

import torch
import torch.nn as nn

class GRUModel(nn.Module):
    def __init__(self, input_num, output_num):
        super(GRUModel, self).__init__()
        self.hidden_size = output_num
        self.grucell = nn.GRUCell(input_num, output_num)

    def forward(self, x, hid):
        # x : seq_lenth * batch * embed_size(=input_size)
        # hid : batch * embed_size(=hidden_size)
        output = []
        if hid is None:
            hid = torch.randn(x.shape[0], self.hidden_size)
        for i in range(x.shape[0]):
            hx = self.grucell(x[i], hid)
            output.append(hx)
        # print(output)
        output = torch.stack(output)

        return output, hx


'''
# -*- coding: utf-8 -*-
@Author  :   Vivian Li , Siara Small
@Date    :   2023/05/17
@Last Modified By :   Vivian, Siara Small
@Last Modified Time :   2023/05/17 19:50:58
'''

from typing import Dict, Union, List
import os

from .objects import SettingDict, SettingObject, PluginSuiteSetObj, EngineSetObj
from gailbot.core.utils.general import (
    is_file,
    is_directory,
    read_toml,
    get_name,
    make_dir,
    delete,
    filepaths_in_dir)
from gailbot.core.utils.logger import makelogger

logger = makelogger("setting_manager")

class ExistingSettingName(Exception):
    def __init__(self, name: str, *args: object) -> None:
        super().__init__(*args)
        self.name = name
    def __str__(self) -> str:
        return f"the setting name {self.name} already exist"

class SettingManager():
    """
    Manages all available settings
    """
    profiles : Dict[str, SettingObject] = dict()
    engine_settings  : Dict[str, EngineSetObj]   = dict()

    def __init__(self, workspace:str, load_exist: bool = True) -> None:
        """ constructing the setting manager

        Args:
            workspace (str): the path to the directory stores all the
                             setting files
            load_exist (bool, optional): if true , load existing setting in
                             workspace. Defaults to True.
        """
        self.workspace = workspace
        self.engine_set_space = os.path.join(workspace, "engine_setting")
        self.default_setting = None
        self.default_engine_setting = None

        if not is_directory(self.workspace):
            make_dir(self.workspace)

        if not is_directory(self.engine_set_space):
            make_dir(self.engine_set_space)

        if load_exist:
            engine_files = filepaths_in_dir(self.engine_set_space, ["toml"])
            for file in engine_files:
                self.load_set_from_file(file, self.add_new_engine)

            setting_files = filepaths_in_dir(self.workspace, ["toml"])
            for file in setting_files:
                self.load_set_from_file(file, self.add_new_setting)

    def load_set_from_file(self, file_path, addfun, overwrite: bool = False) ->bool:
        """ load the setting from local file

        Args:
            file_path (str): the file path
            overwrite (bool, optional): if true, the loaded
            file will overwrite existing setting with same name. Defaults to False.

        Returns:
            bool: return true if the loading is successful, false if the file
            cannot be loaded
        """
        if is_file(file_path):
            data = read_toml(file_path)
            try:
                name = get_name(file_path)
                data = read_toml(file_path)
                return addfun(name, data, overwrite)
            except Exception as e:
                logger.error(e, exc_info=e)
                return False

    #####################################################################
    #               Functions for managing engine setting               #
    #####################################################################
    def get_engine_setting_names(self) -> List[str]:
        """return a list of available engine setting name

        Returns:
            List[str]: a list of engine setting names
        """
        return list(self.engine_settings.keys())

    def add_new_engine(self, name, engine:Dict[str, str], overwrite: bool = False):
        """add a new engine setting

        Args:
            name (str): the name of the engine setting
            engine (Dict[str, str]): the data of the engine setting,
                                     one required field is the type of the
                                     engine

            overwrite (bool, optional): if True, overwrite the existing engine s
                                        etting with the same name. Defaults to False.

        Raises:
            ExistingSettingName: if the engine setting name has been taken, and overwrite is set to False

        Returns:
            bool: return true if the setting is successfully added, false otherwise
        """
        if self.is_engine_setting(name) and (not overwrite):
            raise ExistingSettingName(name)
        try:
            setting: EngineSetObj = EngineSetObj(engine, name)
            assert setting.engine_setting
            self.engine_settings[name] = setting
            self.save_engine_setting(name)
            return True
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def remove_engine_setting(self, name):
        """remove the engine setting from the disk

        Args:
            name (str): the name of the engine setting

        Returns:
            bool: return true if the engine setting is removed successfully
        """
        try:
            assert self.is_engine_setting(name)
            assert not self.engine_settings[name].is_in_use()
            del self.engine_settings[name]
            if is_file(self.get_engine_src_path(name)):
                delete(self.get_engine_src_path(name))
            return True
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def is_engine_setting_in_use(self, name)->bool:
        """check if the engine setting is in use

        Args:
            name (str): the name of the engine setting
        """
        return self.engine_settings[name].is_in_use()

    def is_engine_setting(self, name):
        """check if the given setting is engine setting

        Args:
            name (str): the name that identify the engine setting

        Returns:
            bool: true if the setting is engine setting false otherwise
        """
        return name in self.engine_settings

    def save_engine_setting(self, name:str) -> Union[bool, str]:
        """save the setting as a local file

        Args:
            name (str): the setting name

        Returns:
            Union[bool, str]: return the saved file path if the setting is
                              saved successfully, return false otherwise
        """
        try:
            out_path = self.get_engine_src_path(name)
            if is_file(out_path):
                delete(out_path)
            self.engine_settings[name].save_setting(out_path)
            return out_path
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def update_engine_setting(self, name:str, setting_data: Dict[str, str]) -> bool:
        """
        update the engine setting

        Args:
            name(str)
            setting_data(Dict[str, str])

        Returns:
            bool
        """
        if self.is_engine_setting(name):
            try:
                engine_setting = self.engine_settings[name]
                assert engine_setting.update_setting(setting_data)
                assert self.save_engine_setting(name)
                for profile in engine_setting.applied_in_profiles:
                    ## update the engine setting on the disk
                    self.save_setting(profile)
                return True
            except Exception as e:
                logger.error(e, exc_info=e)
                return False

    def get_engine_src_path(self, name:str) -> str:
        """given a engine setting name, return its path

        Args:
            name (str): the engine setting name

        Returns:
            str: a path to store the setting file

        Note:
            This is a function to form a path to the local setting file
            in a unified format, the path does not guaranteed to indicate
            an existing setting file
        """
        return os.path.join(self.engine_set_space, name + ".toml")

    def get_engine_setting_data(self, name:str) -> Union[bool, Dict[str, str]]:
        """ get the setting data of the engine setting

        Args:
            name (str): the name of the engine

        Returns:
            Union[bool, Dict[str, str]]: return the dictionary that stores the
                                         the engine data if the data engine
                                         name is a valid engine in the setting
                                         manager, else return false
        """
        if self.is_engine_setting(name):
            return self.engine_settings[name].get_setting_dict()
        else:
            return False

    def _get_profile_engine(self, profile_name: str) -> EngineSetObj:
        """return the engine used in the profile identifies by profile name

        Args:
            profile_name (str): the name of the profile to be queried

        Returns:
            EngineSetObj: the engine object
        """
        profile_obj = self.profiles[profile_name]
        engine_obj = self.engine_settings[profile_obj.engine_setting_name]
        return engine_obj

    def set_to_default_engine_setting(self, setting_name: str) -> bool:
        """set one setting to be the default setting

        Args:
            name (str): the name of the setting

        Returns:
            bool: return true if the default setting can be set,
                  false otherwise
        """
        if setting_name in self.profiles:
             self.default_engine_setting = setting_name
             return True
        else:
             return False

    def get_default_engine_setting_name(self) -> str:
        """ return the name of the default engine

        Returns:
            str: _description_
        """
        return self.default_engine_setting

    #####################################################################
    #               Functions for managing profile setting              #
    #####################################################################
    def get_setting_names(self) -> List[str]:
        """ return a list of available setting names

        Returns:
            List[str]: a list of setting names
        """
        return list(self.profiles.keys())

    def remove_setting(self, name: str) -> bool:
        """ given the setting name, remove the setting and the local
            setting file
        Args:
            name (str): the name that identify the setting

        Returns:
            bool: return true if the removal is successful, false if
                  the setting does not exist
        """
        if self.is_setting(name):
            settingObj = self.profiles.pop(name)
            self.engine_settings[settingObj.engine_setting_name].remove_applied_profile(name)
            if is_file(self.get_profile_src_path(name)):
                delete(self.get_profile_src_path(name))
            return True
        else:
            return False

    def get_setting(self, name:str) -> Union [SettingObject, bool]:
        """ given the setting name, return the corresponding setting

        Args:
            name (str): a name that identifies the setting

        Returns:
            Union [SettingObject, bool]: return the setting object if the
            setting is found, return false if the setting does not exist
        """
        if self.is_setting(name):
            return self.profiles[name]
        else:
            return False

    def add_new_setting(self,
                        name: str,
                        data: SettingDict,
                        overwrite: bool = False) -> Union[bool, str]:
        """ add a new setting

        Args:
            name (str): the setting name that identifies the setting
            setting (Dict[str, str]): a dictionary that stores the setting
            overwrite (bool, optional): if true, the given setting will overwrite
            an existing setting if a setting with the same name exist.
            Defaults to False.

        Returns:
            Union[bool, str]: True if the setting creates successfully,
                              False otherwise

        Raises:
            ExistingSettingName: raised when the setting name already exist
                                 and the overwrite option is set to false
        """
        logger.info(f"get engine {data}")
        if self.is_setting(name):
            if overwrite:
                self.remove_setting(name)
            else:
                raise ExistingSettingName(name)
        try:
            engine_set_name = data["engine_setting_name"]
            engine_obj = self.engine_settings[engine_set_name]
            plugin_obj = PluginSuiteSetObj(data["plugin_setting"])
            setting: SettingObject = SettingObject(
                engine_setting=engine_obj,
                engine_setting_name=engine_set_name,
                plugin_setting=plugin_obj,
                name=name)
            self.engine_settings[engine_set_name].add_applied_profile(name)
            assert setting and setting.engine_setting
            self.profiles[name] = setting
            self.save_setting(name)
            return True
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def is_setting(self, name: str) -> bool:
        """ tell if a setting exists in the setting manager

        Args:
            name (str): the setting name

        Returns:
            bool: return true if the given name is an existing setting, false
                  otherwise
        """
        return name in self.profiles

    def update_setting(self, name: str, setting_data: SettingDict) -> bool:
        """ update the setting

        Args:
            name (str): setting name
            setting_data (Dict[str, str]): the updated setting content

        Returns:
            bool:   return true if the setting is updated, false if the
                    setting does not exist or the new setting dictionary
                    cannot be validated
        """
        if self.is_setting(name):
            try:
                profile_setting = self.profiles[name]
                orig_engine = profile_setting.engine_setting.name
                engine_set_name = setting_data["engine_setting_name"]
                engine_obj = self.engine_settings[engine_set_name]
                plugin_obj = PluginSuiteSetObj(setting_data["plugin_setting"])
                assert profile_setting.update_setting(engine_setting=engine_obj, plugin_setting=plugin_obj)
                assert self.save_setting(name)
                return True
            except Exception as e:
                logger.error(e, exc_info=e)
        else:
            return False

    def rename_setting(self, name: str, new_name:str) ->bool:
        """ rename a setting

        Args:
            name (str): the original name of the setting
            new_name (str): the new name of the setting

        Returns:
            bool: return true if the setting can be renamed, false
                  if the setting does not exist or if the new_name
                  has been taken by other existing setting
        """
        if self.is_setting(name):
            if self.is_setting(new_name):
                logger.error(f"new name{ new_name} has been taken")
                return False

            temp = self.profiles.pop(name)
            engine_applied = self._get_profile_engine(name)
            engine_applied.remove_applied_profile(name)

            temp.name = new_name
            engine_applied.add_applied_profile(new_name)
            self.profiles[new_name] = temp
            self.save_setting(new_name)

            if is_file(self.get_profile_src_path(name)):
                delete(self.get_profile_src_path(name))
            logger.info("update_setting")
            return self.profiles[new_name] != None
        else:
            logger.error("the setting is not found")
            return False

    def save_setting(self, name:str) -> Union[bool, str]:
        """ save the setting as a local file

        Args:
            name (str): the setting name

        Returns:
            Union[bool, str]: return the saved file path if the setting
                              is saved successfully, return false otherwise

        """
        try:
            out_path = self.get_profile_src_path(name)
            if is_file(out_path):
                delete(out_path)
            self.profiles[name].save_setting(out_path)
            return out_path
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def get_setting_dict(self, setting_name:str) -> Union[bool, SettingDict]:
        """ return the setting data as a dictionary

        Args:
            setting_name (str): the name that identifies the setting

        Returns:
            Union[bool, SettingDict]: if the setting exists, return the setting
                                      data, else return false
        """
        if setting_name in self.profiles:
            return self.profiles[setting_name].get_data()
        else:
            return False

    def get_profile_src_path(self, name:str) -> str:
        """given a setting name, return its path

        Args:
            name (str): the setting name

        Returns:
            str: a path to store the setting file

        Note:
            This is a function to form a path to the local setting file
            in a unified format, the path does not guaranteed to indicate
            an existing setting file
        """
        return os.path.join(self.workspace, name + ".toml")

    def delete_all_settings(self) -> bool:
        """ delete all settings

        Returns:
            bool: True if the deletion is successful, false if not
        """
        try:
            for setting in self.get_setting_names():
                if setting != "default":
                    self.remove_setting(setting)
            return True
        except Exception as e:
            logger.error(e, exc_info=e)
            return False

    def get_all_settings_data(self) -> Dict[str, SettingDict]:
        """
        return a dictionary that stores all available setting data
        """
        setting_dict = dict()
        for key, setting_object in self.profiles.items():
            setting_dict[key] = setting_object.data

        logger.info(f"setting data {setting_dict}")
        return setting_dict

    def set_to_default_setting(self, setting_name: str) -> bool:
        """set one setting to be the default setting

        Args:
            name (str): the name of the setting

        Returns:
            bool: return true if the default setting can be set,
                  false otherwise
        """
        if setting_name in self.profiles:
             self.default_setting = setting_name
             return True
        else:
             return False

    def get_default_profile_setting_name(self) -> str:
        """get the default setting name

        Returns:
            str: the default setting
        """
        return self.default_setting

    #####################################################################
    #       function for managing plugin setting                        #
    #####################################################################

    def is_suite_in_use(self, suite_name:str) -> bool:
        """given a suite_name, check if this suite is used
           in any of the setting

        Args:
            suite_name (str): the name of the plugin suite

        Returns:
            bool: return true if the suite is used in any of the setting,
                  false otherwise
        """
        for setting_obj in self.profiles.values():
            if suite_name in setting_obj.get_plugin_setting():
                return True
        return False


#! -*- coding:utf-8 -*-

from setuptools import setup, find_packages
import glob

# get version data
with open("VERSION") as fo:
    version = fo.read()

setup(
     name='mikado-core',
     version=version,
     description='Core files for Mikado Software OSS projects',
     author='paul@mikadosoftware.com',
     packages=find_packages(exclude=('tests')),
     # Any scripts (i.e. python/bash) found here will be added to PATH
     scripts=glob.glob('bin/*')
)


import mxnet as mx
from mxnet import gluon
from mxnet import nd

class InstanceNorm(gluon.nn.Block):
  def __init__(self, channels, **kwargs):
    super(InstanceNorm, self).__init__(**kwargs)
    with self.name_scope():
      self.gamma = self.params.get('gamma', shape=(channels, ))
      self.beta = self.params.get('beta', shape=(channels, ))

  def forward(self, x):
    outputs = nd.InstanceNorm(x, gamma=self.gamma.data(), beta=self.beta.data())
    return outputs

class GLU(gluon.nn.Block):
  def __init__(self, **kwargs):
    super(GLU, self).__init__( **kwargs)

  def forward(self, x):
    a, b = nd.split(x, 2, 1)
    outputs = a * nd.sigmoid(b)
    return outputs

class PixelShuffler(gluon.nn.Block):
  def __init__(self, **kwargs):
    super(PixelShuffler, self).__init__(**kwargs)

  def forward(self, x, size=(1, 2)):
    # data.shape = 'NCHW'
    n, c, h, w = x.shape
    #print(x.shape)
    rh, rw = size
    oh, ow = h * rh, w * rw
    oc = c // (rh * rw)
    outputs = nd.reshape(data=x, shape=(n, oc, rh, rw, h, w))
    outputs = nd.transpose(data=outputs, axes=(0, 1, 4, 2, 5, 3))
    outputs = nd.reshape(data=outputs, shape=(n, oc, oh, ow))
    return outputs

class Generator(gluon.nn.Block):

  def __init__(self, **kwargs):
    super(Generator, self).__init__(**kwargs)
    with self.name_scope():
      self.Conv_in = gluon.nn.Conv2D(channels=128, kernel_size=(1, 15), strides=(1, 1), padding=(0, 7))
      self.GLU_in = GLU()

      self.Conv_d1 = gluon.nn.Conv2D(channels=256, kernel_size=(1, 5), strides=(1, 2), padding=(0, 2))
      self.IN_d1 = InstanceNorm(channels=256)
      self.GLU_d1 = GLU()
      self.Conv_d2 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 5), strides=(1, 2), padding=(0, 2)) # warning: This channel in paper is 512. I do this because GLU would half the channel.
      self.IN_d2 = InstanceNorm(channels=1024)
      self.GLU_d2 = GLU()

      self.Conv_r1_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r1_1 = InstanceNorm(channels=1024)
      self.GLU_r1_1 = GLU()
      self.Conv_r1_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r1_2 = InstanceNorm(channels=512)
      self.Conv_r2_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r2_1 = InstanceNorm(channels=1024)
      self.GLU_r2_1 = GLU()
      self.Conv_r2_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r2_2 = InstanceNorm(channels=512)
      self.Conv_r3_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r3_1 = InstanceNorm(channels=1024)
      self.GLU_r3_1 = GLU()
      self.Conv_r3_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r3_2 = InstanceNorm(channels=512)
      self.Conv_r4_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r4_1 = InstanceNorm(channels=1024)
      self.GLU_r4_1 = GLU()
      self.Conv_r4_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r4_2 = InstanceNorm(channels=512)
      self.Conv_r5_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r5_1 = InstanceNorm(channels=1024)
      self.GLU_r5_1 = GLU()
      self.Conv_r5_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r5_2 = InstanceNorm(channels=512)
      self.Conv_r6_1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r6_1 = InstanceNorm(channels=1024)
      self.GLU_r6_1 = GLU()
      self.Conv_r6_2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 3), strides=(1, 1), padding=(0, 1))
      self.IN_r6_2 = InstanceNorm(channels=512)

      self.Conv_u1 = gluon.nn.Conv2D(channels=1024, kernel_size=(1, 5), strides=(1, 1), padding=(0, 2))
      self.ps_u1 = PixelShuffler()
      self.IN_u1 = InstanceNorm(channels=512)
      self.ps_u1_g = PixelShuffler()
      self.GLU_u1 = GLU()
      self.Conv_u2 = gluon.nn.Conv2D(channels=512, kernel_size=(1, 5), strides=(1, 1), padding=(0, 2))
      self.ps_u2 = PixelShuffler()
      self.IN_u2 = InstanceNorm(channels=256)
      self.ps_u2_g = PixelShuffler()
      self.GLU_u2 = GLU()

      self.Conv_out = gluon.nn.Conv2D(channels=24, kernel_size=(1, 15), strides=(1, 1), padding=(0, 7))

  def forward(self, inputs):
    # data.shape = 'NCHW' h1-wT-c72
    # Preprocess
    inputs = self.Conv_in(inputs)
    inputs = self.GLU_in(inputs)

    # Downsample
    downsample1 = self.Conv_d1(inputs)
    downsample1 = self.IN_d1(downsample1)
    downsample1 = self.GLU_d1(downsample1)
    downsample2 = self.Conv_d2(downsample1)
    downsample2 = self.IN_d2(downsample2)
    downsample2 = self.GLU_d2(downsample2)

    # 6 residual blocks
    residual_input = downsample2
    # residual block 1
    residual_1 = self.Conv_r1_1(residual_input)
    residual_1 = self.IN_r1_1(residual_1)
    residual_1 = self.GLU_r1_1(residual_1)
    residual_1 = self.Conv_r1_2(residual_1)
    residual_1 = self.IN_r1_2(residual_1)
    residual_1 = residual_input + residual_1
    # residual block 2
    residual_2 = self.Conv_r2_1(residual_1)
    residual_2 = self.IN_r2_1(residual_2)
    residual_2 = self.GLU_r2_1(residual_2)
    residual_2 = self.Conv_r2_2(residual_2)
    residual_2 = self.IN_r2_2(residual_2)
    residual_2 = residual_1 + residual_2
    # residual block 3
    residual_3 = self.Conv_r3_1(residual_2)
    residual_3 = self.IN_r3_1(residual_3)
    residual_3 = self.GLU_r3_1(residual_3)
    residual_3 = self.Conv_r3_2(residual_3)
    residual_3 = self.IN_r3_2(residual_3)
    residual_3 = residual_2 + residual_3
    # residual block 4
    residual_4 = self.Conv_r4_1(residual_3)
    residual_4 = self.IN_r4_1(residual_4)
    residual_4 = self.GLU_r4_1(residual_4)
    residual_4 = self.Conv_r4_2(residual_4)
    residual_4 = self.IN_r4_2(residual_4)
    residual_4 = residual_3 + residual_4
    # residual block 5
    residual_5 = self.Conv_r5_1(residual_4)
    residual_5 = self.IN_r5_1(residual_5)
    residual_5 = self.GLU_r5_1(residual_5)
    residual_5 = self.Conv_r5_2(residual_5)
    residual_5 = self.IN_r5_2(residual_5)
    residual_5 = residual_4 + residual_5
    # residual block 6
    residual_6 = self.Conv_r6_1(residual_5)
    residual_6 = self.IN_r6_1(residual_6)
    residual_6 = self.GLU_r6_1(residual_6)
    residual_6 = self.Conv_r6_2(residual_6)
    residual_6 = self.IN_r6_2(residual_6)
    residual_6 = residual_5 + residual_6
    residual_out = residual_6
    # Upsample
    upsample1 = self.Conv_u1(residual_out)
    upsample1 = self.ps_u1(upsample1) # no problem
    upsample1 = self.IN_u1(upsample1)
    upsample1 = self.GLU_u1(upsample1)
    upsample2 = self.Conv_u2(upsample1)
    upsample2 = self.ps_u2(upsample2)
    upsample2 = self.IN_u2(upsample2)
    upsample2 = self.GLU_u2(upsample2)

    # Postprocess
    outputs = self.Conv_out(upsample2)

    # output.shape = (1, T, 72)
    return outputs

class Discriminator(gluon.nn.Block):

  def __init__(self, **kwargs):
    super(Discriminator, self).__init__(**kwargs)
    with self.name_scope():
      self.Conv_1 = gluon.nn.Conv2D(channels=128, kernel_size=(3, 3), strides=(1, 2), padding=(1, 1))
      self.GLU_1 = GLU()
      self.Conv_d1 = gluon.nn.Conv2D(channels=256, kernel_size=(3, 3), strides=(2, 2), padding=(1, 1))
      self.IN_d1 = InstanceNorm(channels=256)
      self.GLU_d1 = GLU()
      self.Conv_d2 = gluon.nn.Conv2D(channels=512, kernel_size=(3, 3), strides=(2, 2), padding=(1, 1))
      self.IN_d2 = InstanceNorm(channels=512)
      self.GLU_d2 = GLU()
      self.Conv_d3 = gluon.nn.Conv2D(channels=1024, kernel_size=(6, 3), strides=(1, 2), padding=(0, 1))
      self.IN_d3 = InstanceNorm(channels=1024)
      self.GLU_d3 = GLU()
      self.fc = gluon.nn.Dense(units=1, activation='sigmoid')

  def forward(self, inputs):
    # data.shape = 'NCHW' h24-wT-c1
    # pre
    inputs = self.Conv_1(inputs)
    inputs = self.GLU_1(inputs)

    # Downsample
    downsample_1 = self.Conv_d1(inputs)
    downsample_1 = self.IN_d1(downsample_1)
    downsample_1 = self.GLU_d1(downsample_1)
    downsample_2 = self.Conv_d2(downsample_1)
    downsample_2 = self.IN_d2(downsample_2)
    downsample_2 = self.GLU_d2(downsample_2)
    downsample_3 = self.Conv_d3(downsample_2)
    downsample_3 = self.IN_d3(downsample_3)
    downsample_3 = self.GLU_d3(downsample_3)

    # post
    outputs = self.fc(downsample_3)
    return outputs


"""
创建地区列表、新增列表、现有列表，至少存储3行信息
   向以上三个列表追加数据第4行数据
        在第1个位置插入第5行数据
"""

list_area=["香港","上海","新疆"]
list_increase=[15,6,0]
list_now=[393,61,49]
print(list_area)


# -*- coding: utf-8 -*-
""" Tools used by Shiba data retrieving classes"""
import re
from collections import OrderedDict

import requests
import xmltodict

from furl import furl
from lxml import etree
from lxml import objectify
from lxml.etree import XMLSyntaxError

from ftfy import fix_text

from .shibaconnection import ShibaConnection
from .shibaresponseobject import ShibaResponseObject
from .shibaexceptions import (
    ShibaParameterError,
    ShibaLoginError,
    ShibaQuotaExceededError,
    ShibaRightsError,
    ShibaConnectionError,
    ShibaUnknownServiceError,
    ShibaServiceError,
    ShibaCallingError,
)

from http.client import HTTPException

def post_request(url, data):
    """Method creating and submitting the multipart request of the wished to be imported XML file.

    :param data: raw XML as text

    :rtype: plain text from servers response
    """
    header = {
        "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE; rv:1.9.0.10) "
        "Gecko/2009042316 Firefox/3.0.10 (.NET CLR 4.0.20506)"
    }
    d = {"file": data}
    r = requests.post(url, files=d, headers=header)
    return r.text

def retrieve_obj_from_url(url, data=None):
    """Give it an URL, will send you back an object based on received XML from WebService, it removes and store
    namespace as its not really useful for common usages of this API.
    You can also give it a dict or an objectified tree to POST a file (used for import webservice)
    :param url: fully formatted URL from url_constructor
    :param data: if not None, will send a POST request to URL instead of a GET one, with multipart sending of
    this parameter content

    :rtype : ShibaResponseObject object, containing lxml.objectify class, raw XML as Unicode string and the
    namespace from the XML.
    """
    try:
        if data is not None:
            xml = post_request(url, data)
        else:
            xml = requests.get(url).text
    except requests.ConnectionError:
        raise ShibaConnectionError("HTTP error = Connection error - On URL: " + url)
    except requests.HTTPError as e:
        raise ShibaConnectionError("URL error = " + e.reason + " - On URL: " + url)
    except HTTPException:
        raise ShibaConnectionError("HTTP unknown error =" + " - On URL: " + url)

    namespace = re.search(pattern='xmlns="[^"]', string=xml)
    if namespace is not None:
        namespace = namespace.group()
    else:
        namespace = ""

    xmlepured = re.sub(pattern=' xmlns="[^"]+"', repl="", string=xml, flags=0)
    xmlepured = fix_text(xmlepured).encode("utf-8")

    try:
        obj = objectify.fromstring(xmlepured)
    except XMLSyntaxError:
        print(xmlepured)
        raise

    if _check_errors(obj) is not False:
        try:
            if "Unknown error" == obj.error.code:
                raise ShibaServiceError(
                    "Unknown error from WebService (maybe the sale isn't confirmed yet?)"
                    " : "
                    + obj.error.message
                    + " - Reason : "
                    + obj.error.details.detail
                )
        except ShibaServiceError:
            raise ShibaServiceError(
                "Unknown error from WebService (maybe the sale isn't confirmed yet?)"
                " : " + obj.error.message + " - Reason : " + obj.error.details.detail
            )
        except Exception:
            raise ShibaUnknownServiceError(
                "An unknown error from the WebService has occurred - XML dump : "
                + etree.tostring(obj)
            )
    return ShibaResponseObject(namespace, obj, xml.encode("utf-8"))

def create_xml_from_item_obj(inv):
    """Generate XML from the "inv" parameter.

    :param inv: is an object or a dict hierarchized as the XML structure described \
    in the WebServices documentation. Take a look at the xmltodict or the lxml.objectify documentation too.

    :rtype: string output from objectified element or ElementTree element tostring() methods
    """
    if type(inv) is etree._ElementTree:
        return etree.tostring(inv)
    elif type(inv) is dict or type(inv) is OrderedDict:
        try:
            return xmltodict.unparse(inv)
        except Exception:
            raise ShibaCallingError(
                "error from dictionary to xml : can't create XML from dictionary,"
                " refers to xmltodict documentation"
                "(you don't need to add a root element 'items' to your items)"
            )
    else:
        raise ShibaCallingError(
            "error : bad input parameter given, "
            "expecting dict, objectify object or ElementTree element"
        )

def inf_constructor(shibaconnection, action, **kwargs):
    """Just a selective concatenation of args given and related static data from the given action.

    :param shibaconnection: ShibaConnection class instance
    :param action: string giving the action type for the WebService
    :param kwargs: params to concatenate

    :rtype epured and updated dict
    """
    if isinstance(shibaconnection, ShibaConnection) is False:
        raise ShibaCallingError(
            "Internal parameter error : shibaconnection parameter is not "
            "a ShibaConnection instance"
        )
    if action not in shibaconnection.actionsinfo:
        raise ShibaCallingError(
            "Internal parameter error : action parameter "
            + action
            + " is unknown from the actions list"
        )
    newkwargs = {}
    for each in kwargs:
        if kwargs[each] is not None and kwargs[each] != "":
            newkwargs[each] = kwargs[each]
    newkwargs.update(shibaconnection.actionsinfo[action])
    newkwargs["action"] = action
    return newkwargs

def url_constructor(shibaconnection, params, domain=None):
    """URL constructor, formatting input and adding as many URL arguments as given.

    :param params: info dict, from inf_constructor
    :param domain: force URL domain, useful for some actions needing to go with http instead of https

    :rtype: formatted URL string
    """
    params.pop("self", None)
    category = params.pop("cat")
    ordered_params = OrderedDict(((k, params[k]) for k in sorted([k for k in params])))

    f = furl(domain or shibaconnection.domain)
    f.path.segments.append(category)
    f.path.normalize()
    f.add(ordered_params)
    return f.url

def _check_errors(obj):
    """Errors checking from the returned XML as object. Handling errors as internal exceptions raising

    :param obj: objectified element to check errors from

    :rtype: objectified element if errors have been found, False if the WebService hasn't encountered an error
    """
    if "errorresponse" in obj.tag:
        if "ServerError" == obj.error.code:
            raise ShibaParameterError(
                "Parameter error : "
                + obj.error.message
                + " - Reason : "
                + obj.error.details.detail
            )
        if "ParameterError" == obj.error.code:
            raise ShibaParameterError(
                "Parameter error : "
                + obj.error.message
                + " - Reason : "
                + obj.error.details.detail
            )
        if "InvalidUserConnection" == obj.error.code:
            raise ShibaLoginError(
                "Invalid user connection : "
                + obj.error.message
                + " - Reason : "
                + obj.error.details.detail
            )
        if "InvalidUserRights" == obj.error.code:
            if "Quota exceeded" in obj.error.message.text:
                raise ShibaQuotaExceededError(
                    "Too many requests : "
                    + obj.error.message
                    + " - Reason : "
                    + obj.error.details.detail
                )
            else:
                raise ShibaRightsError(
                    "Invalid user rights : "
                    + obj.error.message
                    + " - Reason : "
                    + obj.error.details.detail
                )
        return obj
    return False


# Webscraping https://www.payscale.com/college-salary-report/majors-that-pay-you-back/bachelors for updated data
import pandas as pd
import bs4
import requests

data_dict = {
    'rank': [],
    'major': [],
    'degree_type': [],
    'early_career_pay': [],
    'mid_career_pay': [],
    'percentage_high_meaning': [],
}

for i in range(1, 35):
    print(i)
    web_html = requests.get(f'https://www.payscale.com/college-salary-report/majors-that-pay-you-back/bachelors/page/{i}').text
    soup = bs4.BeautifulSoup(web_html, 'html.parser')

    data_names = ['rank', 'major', 'degree_type', 'early_career_pay', 'mid_career_pay', 'percentage_high_meaning']

    for data, data_name in zip(map(lambda x: x.text, soup.select('.data-table__value')), data_names*999):
        data_dict[data_name].append(data)


from osgeo import gdal
from osgeo import ogr
import numpy as np
import matplotlib.pyplot as plt
import os
import copy
import math
import argparse
from cal import calTiff
import config_this

##need to set to nodata after do atmospheric correction
def run(folder):

	#dump metadata ls8 to dictionary
	def metadata_ls8 (metadata):
		metadata = open(metadata)
		metadata_dict = dict()
		for line in metadata:
			if '    ' in line.split('=')[0]:
				try:
					D = {(line.split('=')[0]).replace('    ','').replace(' ',''):line.split('=')[1].replace(' ','').replace('\n','')}
					metadata_dict.update(D)
				except:
					'last line have no split'
		return metadata_dict

	#convert to reflectance
	def  reflectance (band,Mp,Ap,sinSE):
		band = gdal.Open(band)
		band.GetRasterBand(1)
		band = band.ReadAsArray().astype(np.float)
		band[np.where(band==0)] = np.nan
		band = (band*Mp+Ap)/sinSE
		return band
	#find F5bins as first bin that have frequency > 50 and next right to it also > 50
	def find_1st_valid_detail (band):
		bandVals = band.ravel()
		unique, counts = np.unique(bandVals, return_counts = True)

		count_next = np.append([0],counts[0:len(counts)-1])
		count_next_2 = np.append([0],count_next[0:len(count_next)-1])
		#idea is take (x1 - x) if >50 then (x2 - x1) >50 choose x1
		A = np.where(count_next-counts > 50)
		B = np.where(count_next_2-count_next > 50)
		C = np.array(B)[0] - np.array(A)[0]
		for i in C:
			if i == 1:
				break
		loc = np.array(A)[0][i]
		return unique[loc]
	def setNotadata_0(band, no):
	    band_buffer = band
	    outband = band.replace('1.TIF','2.TIF')
	    driver = gdal.GetDriverByName("GTiff")
	    band = gdal.Open(band)
	    band.GetRasterBand(1)
	    band = band.ReadAsArray().astype(np.float)
	    band[np.where(band<0)] = no
	    config_this.createTiff(band, band_buffer, outband)

	def do_ats_correct(folder, band):
		path_band = path['%s'%(band)]
		Fbin5_band =  find_1st_valid_detail(reflectance(path_band,Mp,Ap,sinSE))
		#((DN*Mp)+ Ap)/sinSE
		expression = '((%s*%f)+%f)/%f-%f' % (band, Mp, Ap, sinSE, Fbin5_band)
		out_filename = folder+'/'+os.path.basename(folder)+'_' + band + 'product1.TIF'
		calTiff (folder, expression, out_filename)
		setNotadata_0(out_filename,np.nan)

	#config section
	driver = gdal.GetDriverByName("GTiff")
	metadata = folder + '/' +os.path.basename(folder) + '_MTL.txt'
	metadata = metadata_ls8(metadata)
	SUN_ELEVATION = metadata['SUN_ELEVATION']
	SUN_ELEVATION = math.radians(float(SUN_ELEVATION))
	sinSE = math.sin(SUN_ELEVATION)

	#from landsat 8 metadata
	Mp = 0.00002
	Ap = -0.1

	Fbin5s = dict()
	path = dict()
	path['B2'] = folder + '/' +os.path.basename(folder) + '_B2.TIF'
	path['B3'] = folder + '/' +os.path.basename(folder) + '_B3.TIF'
	path['B4'] = folder + '/' +os.path.basename(folder) + '_B4.TIF'
	path['B5'] = folder + '/' +os.path.basename(folder) + '_B5.TIF'

	bands = ['B2','B3','B4','B5']

	for band in bands:
		do_ats_correct(folder,band)

def main ():
    parser = argparse.ArgumentParser()
    parser.add_argument("-f",dest = "folder",help = "insert folder need to do correction", type = str)
    args = parser.parse_args()
    run(args.folder)
   # run(arg.shp,args.date,args.province)
if __name__=='__main__':
    main()

#'LC08_L1TP_125052_20180217_20180307_01_T1'


#!/usr/bin/env python

""" Provision AWS Lex resources using python SDK
"""

# import time
# import boto3
from botocore.exceptions import ClientError

from intent_builder import IntentBuilder
# from slot_builder import SlotBuilder
from lex_helper import LexHelper
# from models.intent import Intent

class LexBotBuilder(LexHelper):

    MAX_DELETE_TRIES = 5
    RETRY_SLEEP = 5
    LOCALE = 'en-US'

    """Create/Update different elements that make up a Lex bot"""
    def __init__(self, logger, context, lex_sdk=None, intent_builder=None):
        self._logger = logger
        self._context = context
        if lex_sdk is None:
            self._lex_sdk = self._get_lex_sdk()
        else:
            self._lex_sdk = lex_sdk
        if intent_builder is None:

            self._intent_builder = IntentBuilder(self._logger, self._context, lex_sdk=self._lex_sdk)
        else:
            self._intent_builder = intent_builder

    def _replace_intent_version(self, bot_definition, intents):
        for intent in bot_definition['intents']:
            intent['intentVersion'] = intents[intent['intentName']]
        return bot_definition

    def _bot_put_properties(self, bot_name, messages, **kwargs):
        properties = {
            "name": bot_name,
            "locale": kwargs['locale'],
            "abortStatement": {
                "messages": [
                    {
                        "content": messages['abortStatement'],
                        "contentType": "PlainText"
                    }
                ]
            },
            "processBehavior": "BUILD",
            "childDirected": False,
            "clarificationPrompt": {
                "maxAttempts": 1,
                "messages": [
                    {
                        "content": messages['clarification'],
                        "contentType": "PlainText"
                    }
                ]
            },
            "description": kwargs['description'],
            "idleSessionTTLInSeconds": 3000
        }

        return properties

    def put(self, bot):
        """create bot"""
        intent_versions = self._put_intents(bot.name, bot.intents)
        self._logger.info(intent_versions)

        bot_response = self._put_bot(bot, intent_versions)
        return bot_response

    def delete(self, bot):
        """delete bot"""
        # TODO what about deleting published version(s) of the bot?
        self._delete_bot(bot.name)
        self._delete_intents(bot.name, bot.intents)

        self._logger.info('Successfully deleted bot and associated resources')

    def _put_intents(self, bot_name, intents):
        intent_versions = []
        for intent in intents:
            intent_versions.append(
                self._intent_builder.put_intent(intent)
            )

        return intent_versions

    def _delete_intents(self, bot_name, intents):
        intent_names = [intent.intent_name for intent in intents]
        # todo fix this so it just passes the intent object

        self._logger.info(intent_names)
        self._intent_builder.delete_intents(intent_names)

    def _bot_exists(self, name, versionOrAlias='$LATEST'):
        return self._get_resource(self._lex_sdk.get_bot,
                                  'get_bot',
                                  {'name': name, 'versionOrAlias': versionOrAlias})

    def _create_bot(self, bot_name, bot_properties):
        bot_exists, checksum = self._bot_exists(bot_name)
        if bot_exists:
            creation_response = self._update_lex_resource(
                self._lex_sdk.put_bot, 'put_bot', checksum, bot_properties)
            return creation_response, creation_response['checksum']

        else:
            self._logger.info(bot_properties)
            creation_response = self._create_lex_resource(
                self._lex_sdk.put_bot, 'put_bot', bot_properties)

            return creation_response, creation_response['checksum']

    def _put_bot(self, bot, intent_versions):
        """Create/Update bot"""
        self._logger.info('Put bot intent_versions %s', bot.name)
        self._logger.info(intent_versions)

        bot_properties = self._bot_put_properties(bot.name, bot.messages, **bot.attrs)
        bot_properties.update({"intents": intent_versions})

        self._logger.info("Bot properties for AWS %s", bot_properties)

        _, checksum = self._create_bot(bot.name, bot_properties)

        version_response = self._create_lex_resource(
            self._lex_sdk.create_bot_version, 'create_bot_version',
            {
                'name': bot.name,
                'checksum': checksum
            })

        self._logger.info("Created bot version %s", bot.name)
        self._logger.info(version_response)

        return version_response

    def _delete_bot(self, bot_name):
        '''Delete bot'''
        self._logger.info('deleting bot: %s', bot_name)
        while True:
            try:
                bot_exists, _ = self._bot_exists(bot_name)
                if(bot_exists):
                    self._delete_lex_resource(self._lex_sdk.delete_bot, 'delete_bot',
                                              name=bot_name)

                    self._logger.info('deleted bot: %s', bot_name)
                    break
                else:
                    break

            except ClientError:
                self._logger.warning('Lex can not call delete_bot on deleted bot %s.',
                                     bot_name)


import os
from feature_extraction.date_utils import date_features
from feature_extraction.coords_features import coord_features
from feature_extraction.other_features import raw_features, categorical_features
from feature_extraction.path_utils import project_root

from metaflow import FlowSpec, step
import pandas as pd

class UbaarFeaturesExtraction(FlowSpec):
    @step
    def start(self):

        self.data = pd.read_csv(os.path.join(project_root(), 'data', 'raw', 'ubaar-competition', 'train.csv'),
                                encoding="utf-8", index_col="ID")
        self.features = pd.DataFrame()

        self.next(self.get_day_feature)

    @step
    def get_day_feature(self):
        self.features = date_features(self.data, self.features)

        self.next(self.get_lat_long_features)

    @step
    def get_lat_long_features(self):
        self.features = coord_features(self.data, self.features)

        self.next(self.add_categorical_features)

    @step
    def add_categorical_features(self):
        self.features = categorical_features(self.data, self.features)

        self.next(self.add_raw_features)

    @step
    def add_raw_features(self):

        self.features = raw_features(self.data, self.features)

        self.next(self.remove_missing_data)

    @step
    def remove_missing_data(self):

        self.features.dropna(inplace=True)
        self.next(self.report)

    @step
    def report(self):
        print(f"Shape: {self.features.shape}")
        self.next(self.save)

    @step
    def save(self):
        self.features.to_csv(os.path.join(project_root(), 'data', 'processed', 'ubaar_features.csv'))
        self.next(self.end)

    @step
    def end(self):
        pass

if __name__ == '__main__':
    UbaarFeaturesExtraction()


import pandas as pd
from sklearn.linear_model import LinearRegression

df = pd.read_csv('heights.csv')
x = df['height']
y = df['weight']

line_fitter = LinearRegression()
data = x.values.reshape(-1, 1)
line_fitter.fit(data, y)

print(line_fitter.predict([[70]]))
print(line_fitter.coef_)
print(line_fitter.intercept_)

import matplotlib.pyplot as plt

plt.plot(x, y, 'o')
plt.plot(x, line_fitter.predict(data))
plt.show()


from jnpr.junos.exception import RpcError

def provides_facts():
    """
    Returns a dictionary keyed on the facts provided by this module. The value
    of each key is the doc string describing the fact.
    """
    return {
        "srx_cluster": "A boolean indicating if the device is part of an "
        "SRX cluster.",
        "srx_cluster_id": "A string containing the configured cluster id",
        "srx_cluster_redundancy_group": "A multi-level dictionary of "
        "information about the SRX "
        "cluster redundancy groups "
        "on the device. The first-level "
        "key is the redundancy group id. "
        "The second-level keys are: "
        "cluster_id, failover_count, "
        "node0, and node1. The node0 and "
        "node1 keys have third-level keys "
        "of priority, preempt, status, "
        "and failover_mode. The values "
        "for this fact correspond to the "
        "values of the 'show chassis "
        "cluster status' CLI command.",
    }

def get_facts(device):
    """
    Gathers facts from the <get-chassis-cluster-status/> RPC.
    """
    srx_cluster = None
    srx_cluster_id = None
    redundancy_group = None

    try:
        rsp = device.rpc.get_chassis_cluster_status(normalize=True)
        if rsp is not None:
            if rsp.tag == "error":
                srx_cluster = False
            else:
                srx_cluster = True
                srx_cluster_id = rsp.findtext("cluster-id")
                groups = rsp.findall("redundancy-group")
                if groups is not None:
                    redundancy_group = {}
                    for group in groups:
                        group_id = group.findtext("redundancy-group-id")
                        redundancy_group[group_id] = {
                            "cluster_id": group.findtext("cluster-id"),
                            "failover_count": group.findtext(
                                "redundancy-group-failover-count"
                            ),
                        }
                        # Iterate over the broken XML in <device-stats>
                        for stats in group.findall("./device-stats"):
                            for node in zip(
                                stats.findall("device-name"),
                                stats.findall("device-priority"),
                                stats.findall("redundancy-group-status"),
                                stats.findall("preempt"),
                                stats.findall("failover-mode"),
                                stats.findall("monitor-failures"),
                            ):
                                redundancy_group[group_id][node[0].text] = {
                                    "priority": node[1].text,
                                    "status": node[2].text,
                                    "preempt": node[3].text,
                                    "failover_mode": node[4].text,
                                    "monitor-failures": node[5].text,
                                }
    except RpcError:
        # Likely a device that doesn't implement the
        # <get-chassis-cluster-status/> RPC.
        # That's OK. Just ignore it and leave srx_cluster = None.
        pass
    return {
        "srx_cluster": srx_cluster,
        "srx_cluster_id": srx_cluster_id,
        "srx_cluster_redundancy_group": redundancy_group,
    }


from wolframalpha import Client
#pip install wolframalpha

def setupWolfram():
    app_id='RWAGV2-6QGHW7Y95J'
    global client
    client=Client(app_id)

def startWolfram(question):
    try:
        res=client.query(question)
        res=next(res.results).text
        print(res)
        return res
    except:
        print("Not found on Wolfram alpha")
        return None

if __name__ == "__main__":
    setupWolfram()
    startWolfram("What is 2+3/4")


# All Elements in Two Binary Search Trees

'''
Given two binary search trees root1 and root2.

Return a list containing all the integers from both trees sorted in ascending order.
'''

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right

class Solution:
    def getAllElements(self, root1: TreeNode, root2: TreeNode) -> List[int]:
        def traverse(root, res):
            if not root:
                return res
            else:
                res.append(root.val)
            # if root and not root.right and not root.left:
            #     res.append(root.val)
            #     return res
            # else:
                res = traverse(root.left, res)
                res = traverse(root.right, res)
                return res
        res1 = traverse(root1, [])
        res2 = traverse(root2, [])
        res = res1 + res2
        res.sort()
        return res


import pygame
import random
import clown

# Initialize pygame
pygame.init()

# Set display surface
WINDOW_WIDTH = 1750
WINDOW_HEIGHT = 700
display_surface = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
pygame.display.set_caption("Catch the Clown")

# Set FPS and clock
FPS = 60
clock = pygame.time.Clock()

# Set game values
PLAYER_STARTING_LIVES = 5
score = 0
player_lives = PLAYER_STARTING_LIVES

# Set colors
BLUE = (26, 130, 167)
PURPLE = (109, 61, 146)
RED = (229, 56, 73)

# Set font
font = pygame.font.Font("AlloyInk-nRLyO.ttf", 32)
end_font = pygame.font.Font("AlloyInk-nRLyO.ttf", 64)

# Set text
title_text = font.render("Catch the Clown", True, BLUE)
title_rect = title_text.get_rect()
title_rect.topleft = (50, 10)

score_text = font.render("Score: " + str(score), True, RED)
score_rect = score_text.get_rect()
score_rect.topright = ((WINDOW_WIDTH - 50, 10))

lives_text = font.render("Lives: " + str(player_lives), True, RED)
lives_rect = lives_text.get_rect()
lives_rect.topright = ((WINDOW_WIDTH - 50, 50))

game_over_text = end_font.render("Game Over", True, RED, BLUE)
game_over_rect = game_over_text.get_rect()
game_over_rect.center = (WINDOW_WIDTH // 2, WINDOW_HEIGHT // 2)

continue_text = end_font.render("Click Anywhere To Play Again", True, RED, BLUE)
continue_rect = continue_text.get_rect()
continue_rect.center = (WINDOW_WIDTH // 2, WINDOW_HEIGHT // 2 + 128)

# Set sound and music
click_sound = pygame.mixer.Sound("click_sound.wav")
click_sound.set_volume(.5)
miss_sound = pygame.mixer.Sound("miss_sound.wav")
miss_sound.set_volume(.5)
pygame.mixer.music.load("ctc_background_music.wav")

# Set images
background_image = pygame.image.load("circusresized.jpg")
background_rect = background_image.get_rect()
background_rect.topleft = (-700, -400)
# Background Attribution
# Cirque vector created by vectorpouch - www.freepik.com

# Create clown object
clown = clown.Clown()

# The main game loop
pygame.mixer.music.play(-1, 0, 0)
running = True
while running:
    # Check to see if the user wants to quit
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

        # A click is made
        if event.type == pygame.MOUSEBUTTONDOWN:
            mouse_x = event.pos[0]
            mouse_y = event.pos[1]

            # The clown was clicked
            if clown.clown_rect.collidepoint(mouse_x, mouse_y):
                click_sound.play()
                score += 1
                clown.clown_velocity += clown.CLOWN_ACCELERATION
                previous_directions = (clown.clown_dx, clown.clown_dy)
                while previous_directions == (clown.clown_dx, clown.clown_dy):
                    clown.clown_dx = random.choice([-1, 1])
                    clown.clown_dy = random.choice([-1, 1])
            # The clown was missed
            else:
                miss_sound.play()
                player_lives -= 1

    # Move clown
    clown.move()

    # Blit Background
    display_surface.blit(background_image, background_rect)

    # Update HUD
    score_text = font.render("Score: " + str(score), True, RED)
    lives_text = font.render("Lives: " + str(player_lives), True, RED)

    # Blit HUD
    display_surface.blit(title_text, title_rect)
    display_surface.blit(score_text, score_rect)
    display_surface.blit(lives_text, lives_rect)

    # Check for game over
    if player_lives == 0:
        display_surface.blit(game_over_text, game_over_rect)
        display_surface.blit(continue_text, continue_rect)
        pygame.display.update()
        is_paused = True
        pygame.mixer.music.stop()
        while is_paused:
            for event in pygame.event.get():
                # The player wants to quit
                if event.type == pygame.QUIT:
                    is_paused = False
                    running = False
                # The player wants to play again
                elif event.type == pygame.MOUSEBUTTONDOWN:
                    is_paused = False
                    score = 0
                    player_lives = 5
                    clown.clown_velocity = clown.CLOWN_STARTING_VELOCITY
                    clown.clown_dx = random.choice([-1, 1])
                    clown.clown_dy = random.choice([-1, 1])
                    clown.clown_rect.center = (WINDOW_WIDTH // 2, WINDOW_HEIGHT // 2)
                    pygame.mixer.music.play(-1, 0, 0)

    # Blit Assets
    display_surface.blit(clown.clown_image, clown.clown_rect)

    # Update display and tick clock
    pygame.display.update()
    clock.tick(FPS)

# End the game
pygame.quit()


'''
Created on 14 Jan 2015

@author: Yuan
'''
'''
Similar as Pascals Triangle.
The difference is that we don't need to return the triangle but only the last row.
So we can simplify the storage to maintain only the previous and current row.
'''
class Solution:
    # @return a list of integers
    def getRow(self, rowIndex):
        pre = [1]
        if rowIndex == 0:
            return pre
        for i in range(1, rowIndex+1):
            current = [1]
            for j in range(1, i):
                current.append(pre[j]+pre[j-1])
            current.append(1)
            pre = current
        return current


class Solution(object):
    def intersection(self, nums1, nums2):
        """
        :type nums1: List[int]
        :type nums2: List[int]
        :rtype: List[int]
        """

        nums1 = list(set(nums1))
        nums2 = list(set(nums2))

        nums1.sort()
        nums2.sort()
        result = []

        i, j = 0, 0
        while i < len(nums1) and j < len(nums2):
            if nums1[i] == nums2[j]:
                result.append(nums1[i])
                i += 1
                j += 1
            elif nums1[i] > nums2[j]:
                j += 1
            else:
                i += 1

        return result

nums1 = [1,2,3,1]
nums2 = [5,1]

obj = Solution()
print(obj.intersection(nums1, nums2))


import pathlib
import setuptools

here = pathlib.Path(__file__).parent.resolve()
readme = (here / 'README.md').read_text(encoding='utf-8')

# did not include torch and pyscf here
install_requires=['numpy', 'paramiko', 'ruamel.yaml', 'pytest-shutil', 'pydflow']

setuptools.setup(
    name="deepks2",
    version="1.2",
    setup_requires=['setuptools_scm'],
    author="Yifan Shan",
    author_email="xiaoshan@mail.ustc.edu.cn",
    description="DeePKS-flow: a 'dflow' workflow for DeePKS-kit",
    long_description=readme,
    packages=[
        "deepks2",
        "deepks2/flow",
        "deepks2/op/iter_op",
        "deepks2/step/iter_step",
        "deepks2/utils",
    ],
    long_description_content_type="text/markdown",
    # packages=setuptools.find_packages(include=['deepks2', 'deepks2.*']),
    classifiers=[
        "Programming Language :: Python :: 3.7",
    ],
    keywords='deepks2 DeePKS-flow',
    install_requires=install_requires,
    python_requires=">=3.7",
    entry_points={
        'console_scripts': [
            'deepks2=deepks2.main:main_cli',
            'dks2=deepks2.main:main_cli',
        ],
    },
)


from django import forms
from .models import Question, Comment

class QuestionForm(forms.ModelForm):
    li = [
        ('식물', '식물'),
        ('동물', '동물'),
        ('영화', '영화'),
        ('음악', '음악'),
        ('드라마', '드라마'),
    ]
    kind = forms.CharField(widget =forms.Select(choices=li))

    class Meta:
        model = Question
        fields = ('kind', 'title', 'content',)

class CommentForm(forms.ModelForm):
    choices = [(1, '추천'), (2, '비추천')]
    pick = forms.ChoiceField(choices=choices , widget=forms.RadioSelect)
    class Meta:
        model = Comment
        fields = ('pick', 'content',)


import math

def NDCG(ranked_list, ground_truth, topn):
    dcg = 0
    idcg = IDCG(ground_truth, topn)

    for i in range(topn):
        id = ranked_list[i]
        dcg += ((2 ** ground_truth[id]) - 1)/math.log(i+2, 2)

    return dcg / idcg

def IDCG(ground_truth, topn):
    t = [a for a in ground_truth]
    t.sort(reverse=True)
    idcg = 0
    for i in range(topn):
        idcg += ((2**t[i]) - 1) / math.log(i+2, 2)
    return idcg

'''
ranked_list=tensor([ 9,  2,  7,  6, 12,  4, 13,  0, 11, 10,  1,  5,  8,  3])
ground_truth=tensor([1., 0., 1., 0., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0.])
然后我们取前top—10个item进行判断，对于precision指标，首先取正样本。
所以下面的 t[:10]=[1.,1.,1.,1.,1.,1.,0.,0.,0.,0.],
ranked_list的top-10为tensor([ 9,  2,  7,  6, 12,  4, 13,  0, 11, 10]),ground_truth[9]==0,然后查看是否在t中，
如果在，说明命中了，将t中的命中的0去掉，命中数+1。不太认同这个计算方法'''
def precision(ranked_list, ground_truth, topn):
    t = [a for a in ground_truth]
    t.sort(reverse=True)
    t = t[:topn]
    hits = 0
    for i in range(topn):
        id = ranked_list[i]
        if ground_truth[id] in t:
            t.remove(ground_truth[id])
            hits += 1
    pre = hits/topn
    return pre

def add_metric(recommend_list, all_group_list, topn):

    ndcg = NDCG(recommend_list, all_group_list, topn)
    prec = precision(recommend_list, all_group_list, topn)

    return prec, ndcg