from datetime import datetime, timedelta, timezone
from unittest.mock import patch

import numpy as np
import pytest

from enforce_typing import enforce_types

from df_py.challenge import judge
from df_py.util import networkutil

def test_get_txs():
    now = datetime.now().replace(tzinfo=timezone.utc)

    six_days_ago = now - timedelta(days=6)
    one_day_ago = now - timedelta(days=1)

    with patch("df_py.util.graphutil.submit_query") as mock:
        mock.side_effect = [
            {
                "data": {
                    "nftTransferHistories": [
                        {
                            "timestamp": six_days_ago.timestamp(),
                            "nft": {"id": "0xnft1"},
                            "oldOwner": {"id": "0xfrom1"},
                        },
                        {
                            "timestamp": one_day_ago.timestamp(),
                            "nft": {"id": "0xnft2"},
                            "oldOwner": {"id": "0xfrom2"},
                        },
                    ]
                }
            },
            {"data": {"nftTransferHistories": []}},
        ]
        txs = judge._get_txs(now)

    # sort txs by nft address, to ease subsequent testing
    nft_addrs = [tx["nft"]["id"] for tx in txs]
    sorted_txs = np.argsort(nft_addrs)
    txs = [txs[i] for i in sorted_txs]

    # now test
    nft_addrs = [tx["nft"]["id"] for tx in txs]
    from_addrs = [tx["oldOwner"]["id"] for tx in txs]

    assert nft_addrs == ["0xnft1", "0xnft2"]
    assert from_addrs == ["0xfrom1", "0xfrom2"]

def test_get_txs_without_mock():
    now = datetime.now().replace(tzinfo=timezone.utc)
    txs = judge._get_txs(now)

def test_get_txs_invalid_data():
    now = datetime.now().replace(tzinfo=timezone.utc)
    with patch("df_py.util.graphutil.submit_query") as mock:
        mock.return_value = {"error": ""}
        with pytest.raises(
            Exception, match="_get_txs: An error occured, {'error': ''}"
        ):
            txs = judge._get_txs(now)

@enforce_types
def test_nft_addr_to_pred_vals():
    mumbai_chainid = 80001
    known_nft_addr = "0x471817de04faa9b616ed7644117d957439717bf9"

    networkutil.connect(mumbai_chainid)
    judge_acct = judge.get_judge_acct()
    pred_vals = judge._nft_addr_to_pred_vals(known_nft_addr, judge_acct)

    assert len(pred_vals) == 12
    assert pred_vals[0] == 1633.1790360265798

    with patch("df_py.challenge.judge.crypto.asym_decrypt") as mock:
        mock.side_effect = Exception("mocked exception")
        pred_vals = judge._nft_addr_to_pred_vals(known_nft_addr, judge_acct)

    assert pred_vals == []

    networkutil.disconnect()

@enforce_types
def test_get_cex_vals_1():
    today = datetime.today().replace(tzinfo=timezone.utc)
    deadline_dt = today - timedelta(days=2)
    deadline_dt = deadline_dt.replace(hour=12, minute=59, second=0, microsecond=0)
    cex_vals = judge._get_cex_vals(deadline_dt)
    assert len(cex_vals) == 12

@enforce_types
def test_get_cex_vals_2():
    deadline_dt = datetime(2023, 5, 3, 23, 59, tzinfo=timezone.utc)
    cex_vals = judge._get_cex_vals(deadline_dt)
    assert len(cex_vals) == 12

@enforce_types
def test_parse_deadline_str_1():
    dt = judge.parse_deadline_str("2023-05-03_23:59")
    dt_target = datetime(2023, 5, 3, 23, 59, tzinfo=timezone.utc)
    assert dt == dt_target

@enforce_types
def test_parse_deadline_str_2():
    dt = judge.parse_deadline_str()

    assert (datetime.now(timezone.utc) - dt) < timedelta(days=7)  # within 1 wk
    assert dt.weekday() == 2  # Mon is 0, Tue is 1, Wed is 2
    assert dt.hour == 23
    assert dt.minute == 59
    assert dt.second == 0
    assert dt.microsecond == 0

@enforce_types
def test_print_results():
    from_addrs = ["0xfrom1", "0xfrom2"]
    nft_addrs = ["0xnft1", "0xnft2"]
    nmses = [0.2, 1.0]
    challenge_data = (from_addrs, nft_addrs, nmses)
    judge.print_results(challenge_data)

@enforce_types
def test_get_judge_acct():
    judge_acct = judge.get_judge_acct()
    assert judge_acct.address == judge.JUDGE_ADDRESS

@enforce_types
def test_get_challenge_data():
    dt = datetime(2021, 9, 1, 12, 59, tzinfo=timezone.utc)
    judge_acct = judge.get_judge_acct()

    with patch("df_py.challenge.judge._get_cex_vals") as mock1:
        cex_vals = list(0.1 + np.arange(0.0, 12.0, 1.0))
        assert len(cex_vals) == 12
        mock1.return_value = cex_vals
        with patch("df_py.challenge.judge._get_txs") as mock2:
            tx1 = {
                "timestamp": 1683790437.215631,
                "nft": {"id": "0xnft1"},
                "oldOwner": {"id": "0xfrom1"},
            }
            tx2 = {
                "timestamp": 1684222437.215636,
                "nft": {"id": "0xnft2"},
                "oldOwner": {"id": "0xfrom2"},
            }
            # tx1, but earlier. Will be discarded by keep youngest entry logic
            tx3 = {
                "timestamp": 1683790430,
                "nft": {"id": "0xnft3"},
                "oldOwner": {"id": "0xfrom1"},
            }
            mock2.return_value = [tx1, tx2, tx3]

            with patch("df_py.challenge.judge._nft_addr_to_pred_vals") as mock3:
                # 0x123 has correct length. 0x456 doesn't, so its nmse = 1.0
                predvals_0x123 = list(0.11 + np.arange(0.0, 12.0, 1.0))
                predvals_0x456 = [0.2, 0.4]
                assert len(predvals_0x123) == len(cex_vals)
                assert len(predvals_0x456) != len(cex_vals)
                mock3.side_effect = [predvals_0x123, predvals_0x456, predvals_0x123]
                challenge_data = judge.get_challenge_data(dt, judge_acct)

    (from_addrs, nft_addrs, nmses) = challenge_data

    assert nmses == sorted(nmses), "should be sorted by lowest-nmse first"
    assert nmses[0] != 1.0
    assert len(nmses) == 1

    # second entry is invalid
    # third entry is outdated
    assert from_addrs == ["0xfrom1"]
    assert nft_addrs == ["0xnft1"]  # keep the most recent one

def test_filter_marked_indices():
    nmses = [0.5, 1.0, 0.2, 1.0, 0.8]
    from_addrs = ["addr1", "addr2", "addr3", "addr4", "addr5"]
    nft_addrs = ["nft1", "nft2", "nft3", "nft4", "nft5"]

    expected_nmses = [0.5, 0.2, 0.8]
    expected_from_addrs = ["addr1", "addr3", "addr5"]
    expected_nft_addrs = ["nft1", "nft3", "nft5"]

    (
        filtered_nmses,
        filtered_from_addrs,
        filtered_nft_addrs,
    ) = judge._filter_marked_indices(nmses, from_addrs, nft_addrs)

    assert filtered_nmses == expected_nmses
    assert filtered_from_addrs == expected_from_addrs
    assert filtered_nft_addrs == expected_nft_addrs


import matplotlib.pyplot as plt
import numpy as np

class MyVector:
    """
    the properties of each vector are:
        name as a given string
        color given as one letter(possible values: 'r', 'g', 'b', 'y' and 'm'
        type given as a positive integer greater of equal to 1
        values given as a list of numbers
    """
    def __init__(self, n, c, t, v):
        self.__name = n
        if c != "r" and c != "g" and c != "b" and c != "y" and c != "m":
            raise AttributeError("The color is not available")
        self.__color = c
        if t < 1:
            raise ValueError("The type must be greater or equal to 1")
        else:
            self.__type = t
        self.__values = v[:]

    """
    get and set the name, color, type and the list of values for a vector
    """
    def get_name(self):
        return str(self.__name)

    def get_color(self):
        return str(self.__color)

    def get_type(self):
        return self.__type

    def get_values(self):
        return self.__values[:]

    def set_name(self, n):
        self.__name = n

    def set_color(self, c):
        if c != "r" and c != "g" and c != "b" and c != "y" and c != "m":
            raise AttributeError("The color is not available")
        else:
            self.__color = c

    def set_type(self, t):
        if t > 0:
            self.__type = t
        else:
            raise ValueError("t must be greater or equal to 1")

    def set_values(self, v):
        self.__values = v[:]
    """
    the string representation of a vector
    """
    def __repr__(self):
        r = "Vector(" + str(self.__name) + ", " + str(self.__color)
        r += ", " + str(self.__type) + ", " + str(self.__values) + ")"
        return r

    """
    the functions in the 1st iteration
    """

    # Add a scalar to a vector
    def add_scalar(self,scalar):
        for i in range(len(self.__values)):
            self.__values[i] += scalar

    # Add two vectors
    def add_vectors(self,other):
        if len(self.__values) != len(other) != 0:
            raise IndexError("The vectors dont have the same length")
        else:
            for i in range(len(other)):
                self.__values[i] += other[i]

    # Subtract two vectors
    def sub_vectors(self,other):
        if len(self.__values) != len(other) != 0:
            raise IndexError("The vectors dont have the same length")
        else:
            for i in range(len(other)):
                self.__values[i] -= other[i]

    # Multiplication
    def multiplication(self,other):
        if len(self.__values) != len(other) != 0:
            raise IndexError("The vectors dont have the same length")
        else:
            p = 0
            s = 0
            for i in range(len(other)):
                p = self.__values[i] * other[i]
                s += p
        return s

    # Sum of elements in a vector
    def sum_of_elem(self):
        if len(self.__values) == 0:
            return None
        else:
            s = 0
            for i in range(len(self.__values)):
                s += self.__values[i]
            return s

    # Product of elements in a vector
    def product(self):
        if len(self.__values) == 0:
            return None
        else:
            p = 1
            for i in range(len(self.__values)):
                p *= self.__values[i]
            return p

    # Average of elements in a vector
    def average(self):
        if len(self.__values) == 0:
            return None
        else:
            s = 0
            for i in range(len(self.__values)):
                s += self.__values[i]
            avg = s / len(self.__values)
            return avg

    # Minimum of a vector
    def min_of_vect(self):
        if len(self.__values) == 0:
            return None
        else:
            mini = self.__values[0]
            for i in range(1,len(self.__values)):
                if self.__values[i] < mini:
                    mini = self.__values[i]
            return mini

    # Maximum of a vector
    def max_of_vect(self):
        if len(self.__values) == 0:
            return None
        else:
            maxi = self.__values[0]
            for i in range(1,len(self.__values)):
                if self.__values[i] > maxi:
                    maxi = self.__values[i]
            return maxi

class VectorRepository:

    """
    data is the list of vectors
    """
    def __init__(self,data):
        self.__data = data[:]
        for i in range(len(self.__data)):
            for j in range(i+1,len(self.__data)):
                if self.__data[i].get_name() == self.__data[j].get_name():
                    raise ValueError("The names cannot be the same")

    """
    the functions in the 2nd iteration
    """
    # Add a vector to the repository
    def add_to_rep(self,n,c,t,v):
        for i in range(len(self.__data)):
            if self.__data[i].get_name() == n:
                raise ValueError("The name already exists")
            else:
                vector = MyVector(n,c,t,v)
                self.__data.append(vector)

    # Get all vectors
    def get_all_vect(self):
        return self.__data[:]

    # Get a vector at a given index
    def get_vect_by_index(self,index):
        if index < 0 or index > len(self.__data)-1:
            raise IndexError("The index is not well defined")
        else:
            return self.__data[index]

    # Update a vector at a given index
    def update_vect_by_index(self,index,n,c,t,v):
        if index < 0 or index > len(self.__data) - 1:
            raise IndexError("The index is not well defined")
        else:
            self.__data[index].set_name(n)
            self.__data[index].set_color(c)
            self.__data[index].set_type(t)
            self.__data[index].set_values(v)

    # Update a vector identified by name_id
    def update_vect_by_name(self,name,n,c,t,v):
        for i in range(len(self.__data)):
            if self.__data[i].get_name() == n:
                raise AttributeError("The name is taken")
        for i in range(len(self.__data)):
            if self.__data[i].get_name() == name:
                self.__data[i].set_name(n)
                self.__data[i].set_color(c)
                self.__data[i].set_type(t)
                self.__data[i].set_values(v)

    # Delete a vector by index
    def del_vect_by_index(self,index):
        if index < 0 or index > len(self.__data) - 1:
            raise IndexError("The index is not well defined")
        else:
            del self.__data[index]

    # Delete a vector by name_id
    def del_vect_by_name(self,name):
        for i in range(len(self.__data)):
            if self.__data[i].get_name() == name:
                del self.__data[i]

    # Plot all vectors in a chart based on the type and colour of each vector
    # On the X axis
    # 1 - circle, 2 - square, 3 - triangle, others - diamond
    def PlotVectors(self):
        for i in range(len(self.__data)):
            x = []
            for j in range(len(self.__data[i].get_values())):
                x.append(j)
            t = self.__data[i].get_type()
            if t == 1:
                marker = "o"  # o - representation of a circle
            elif t == 2:
                marker = "s"  # s - representation of a square
            elif t == 3:
                marker = "v"  # v - representation of a triangle down
            else:
                marker = "D"  # D - representation of diamond in matplotlib
            plt.scatter(x,self.__data[i].get_values(),s = 100,c = self.__data[i].get_color(),marker = marker)
        plt.title("Vectors")
        plt.show()

    """
    The functions in the 3rd iteration using the functions
    in library numpy
    """

    # Add a scalar to a vector using numpy add
    def add_scalar_np(self,scalar,index):
        v = np.add(self.__data[index].get_values(),scalar)
        return v[:]

    # Add two vectors using numpy add
    def add_vectors_np(self,index,other):
        v = np.add(self.__data[index].get_values(),other)
        return v[:]

    # Subtract two vectors using numpy sub
    def sub_vectors_np(self,index,other):
        v = np.sub(self.__data[index].get_values(),other)
        return v[:]

    # Get the dot product of two vectors using numpy dot
    def multiplication_np(self,index,other):
        v = np.dot(self.__data[index].get_values(),other)
        return v

    # Get the sum of elements in the vector using numpy sum
    def sum_of_elem_np(self,index):
        v = np.sum(self.__data[index].get_values())
        return v

    # Get the product of elements in a vector using numpy prod
    def prod_of_elem_np(self,index):
        v = np.prod(self.__data[index].get_values())
        return v

    # Get the average of elements in a vector using numpy average
    def avg_of_elem_np(self,index):
        v = np.average(self.__data[index].get_values())
        return v

    # Get the minimum value in a vector using numpy min
    def min_of_vect_np(self,index):
        v = np.min(self.__data[index].get_values())
        return v

    # Get the maximum value in a vector using numpy max
    def max_of_vect_np(self,index):
        v = np.max(self.__data[index].get_values())
        return v


import json

from PatternSpider.scrapy_redis.spiders import RedisSpider
from PatternSpider.settings.spider_names import SpiderNames
from PatternSpider.tasks import TaskManage
from lxml import etree

class MarineregionsDetailSpider(RedisSpider):
    name = SpiderNames.marineregions_detail
    redis_key = "start_urls:" + name
    task_manage = TaskManage()
    custom_settings = {
        'CONCURRENT_REQUESTS': 100,
        'DOWNLOADER_MIDDLEWARES': {
            'PatternSpider.middlewares.middlewares.RandomUserAgentMiddleware': 543,
        },
        "EXTENSIONS": {
            # 'PatternSpider.extensions.RedisSpiderSmartIdleClosedExensions': 100,
        },
        "ITEM_PIPELINES": {
            # 'PatternSpider.pipelines.DownloadImagesPipeline': 1,
            'PatternSpider.pipelines.MongoPipeline': 100,
            # 'PatternSpider.pipelines.Neo4jPipeline': 500,
        }
    }

    def replace_str(self, s):
        return s.replace("\n", '').replace("\xa0", "").replace('.', '_').replace(' ', '')

    def parse_tr_table(self, tr):
        child_trs = tr.xpath('td[2]/table/tr')
        child_datas = []
        fileds = child_trs[0].xpath('td//text()')
        for child_tr in child_trs[1:]:
            child_data_dict = {}
            values = child_tr.xpath('td//text()')
            if not values:
                continue
            for filed in fileds:
                if fileds.index(filed) == len(fileds) - 1:
                    child_data_dict[self.replace_str(filed)] = self.replace_str("".join(values[fileds.index(filed):]))
                else:
                    child_data_dict[self.replace_str(filed)] = self.replace_str(values[fileds.index(filed)])
            child_datas.append(child_data_dict)
        value = child_datas if child_datas else self.replace_str("___".join(fileds))
        return value

    def parse_tr_options(self, tr):
        contents = ""
        contents += "__".join([self.replace_str(t) for t in tr.xpath('td/text()')])
        contents += "__".join([self.replace_str(t) for t in tr.xpath('td/b/text()')])
        over_options = []
        options = tr.xpath('td[2]/select/option')
        for option in options:
            over_options.append({'url': option.xpath('@value')[0], 'name': option.text})
        value = {'content': contents, 'options': over_options}
        return value

    def parse(self, response):
        task = json.loads(response.meta['task'])
        response_html = etree.HTML(response.text)
        item = {
            "id": task['raw']['id'],
            "request_url": task['url'],
            "page_datas": {},
        }
        datas = {}
        trs = response_html.xpath('//*[@id="content"]/table/tr')
        for tr in trs:
            key = tr.xpath("td[1]//text()")
            key = self.replace_str(key[0]) if key else ""
            child_trs = tr.xpath('td[2]/table/tr')
            child_option = tr.xpath('td[2]/select/option')
            if child_trs:
                value = self.parse_tr_table(tr)
            elif child_option:
                value = self.parse_tr_options(tr)
            else:
                value = tr.xpath("td[2]//text()")
                value = "___".join([self.replace_str(v) for v in value]) if value else ""
            if not key:
                continue
            datas[key] = value
        if 'Download' in datas:
            item['down_load'] = datas['Download']['options']
        item['page_datas'] = datas
        print(item)
        yield item
        # 删除任务
        self.task_manage.del_item("mirror:" + self.name, response.meta['task'])


import logging

import allure
from selenium.webdriver.common.by import By

def handle_black(func):
    logging.basicConfig(level=logging.INFO)

    def wrapper(*args, **kwargs):
        from UI_shizhan.page.base_page import BasePage
        _black_list = [
            (By.XPATH, '//*[@text="确认"]'),
            (By.XPATH, '//*[@text="下次再说"]'),
            (By.XPATH, '//*[@text="确认"]'),
        ]
        _max_num = 3
        _error_num = 0
        instance: BasePage = args[0]
        try:
            logging.info('run ' + func.__name__ + '\n args: \n' + repr(args) + '\n' + repr(kwargs))
            element = func(*args, **kwargs)
            _error_num = 0
            instance._driver.implicitly_wait(10)
            return element
        except Exception as e:
            instance.screenshot('tmp.png')  # 截图
            with open('tmp.png', 'rb') as f:
                content = f.read()
            allure.attach(content, attachment_type=allure.attachment_type.PNG)
            logging.error('element not found, handle black list')
            instance._driver.get_screenshot_as_png()
            instance._driver.implicitly_wait(1)
            if _error_num > _max_num:
                raise e
            _error_num += 1
            for ele in _black_list:
                elelist = instance.finds(*ele)
                if len(elelist) > 0:
                    elelist[0].click()
                    return wrapper(*args, **kwargs)
            raise e

    return wrapper


from flask_app import app
from flask import render_template, request, redirect
from flask_app.models.user import User

@app.route('/')
def index():
    list_users=User.get_all()
    return render_template('index.html',users=list_users)

@app.route('/user/new')
def new_user():
    return render_template('new_user.html')

@app.route('/user/create',methods=['POST'])
def create_user():
    print("************",request.form,"***************")
    User.create_user(request.form)
    return redirect('/')

@app.route('/user/show/<int:id>')
def show(id):
    data ={
        "id":id
    }
    return render_template("read_one.html",user=User.get_one(data))

@app.route('/user/edit/<int:id>')
def edit(id):
    data ={
        "id":id
    }
    return render_template("edit.html",user=User.get_one(data))

@app.route('/user/update',methods=['POST'])
def update():
    User.update(request.form)
    return redirect('/')

@app.route('/user/destroy/<int:id>')
def destroy(id):
    data ={
        'id': id
    }
    User.destroy(data)
    return redirect('/')


from flask import current_app
from flask import jsonify
from shared.settings import settings
import traceback
from shared.shared_logger import get_shared_logger
from werkzeug.exceptions import Forbidden, NotFound

logger = get_shared_logger()

@current_app.errorhandler(NotFound)
def handle_exception(e):
    payload = {'error': str(e)}
    return jsonify(payload), 404

@current_app.errorhandler(Exception)
def handle_exception(e):
    payload = {'error': str(e)}
    exc_traceback = traceback.format_exc()
    if settings.DIFFGRAM_ERROR_SEND_TRACES_IN_RESPONSE:
        payload['trace'] = exc_traceback
    logger.error(exc_traceback)
    try:
        error_code = e.code
    except:
        error_code = 500
    return jsonify(payload), error_code

@current_app.errorhandler(Forbidden)
def handle_forbidden(e):
    payload = {'error': str(e)}
    exc_traceback = traceback.format_exc()
    if settings.DIFFGRAM_ERROR_SEND_TRACES_IN_RESPONSE:
        payload['trace'] = exc_traceback
    logger.error(exc_traceback)
    return jsonify(payload), 403


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 18 2022
@author: Sylvain Brisson sylvain.brisson@ens.fr
"""

import re
import numpy as np

import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib.pyplot as plt

plt.style.use("seaborn-paper")

import os
import sys

pathdir = os.path.dirname(__file__)
pathbase = os.path.abspath(os.path.join(pathdir, os.pardir))
sys.path.append(pathbase)

from common.my_io import read_receivers
from common.barycenter_sphere import barycenter_on_sphere, compute_mean_point_stations_event

import argparse

from obspy.core.event import read_events
from obspy.geodetics.base import gps2dist_azimuth

if __name__ == "__main__":

    # setup command-line args
    parser = argparse.ArgumentParser()

    parser.add_argument('--receivers', type=str,help='receivers.dat file', required=True)
    parser.add_argument('--event', type=str,help='quakeML event file', required=True)
    parser.add_argument('--az', type=float,help='min max azimuth', nargs=2, default=[-180.,180.])

    args = parser.parse_args()

    # getting event information
    event = read_events(args.event)[0]

    origin = event.preferred_origin()

    # getting station positions
    stations = read_receivers(args.receivers)

    # computing stations azimuths
    def get_az(row):
        az = gps2dist_azimuth(origin.latitude, origin.longitude, row.lat, row.lon)[1]
        if az > 180. : return az - 360.
        return az

    stations["az"] = stations.apply(
        get_az,
        axis = 1
        )

    # filter on azimuth
    azmin,azmax = args.az
    stations = stations[(stations.az >= azmin) & (stations.az <= azmax)]

    azmin = stations["az"].min()
    azmax = stations["az"].max()

    # setting/computing limits points of profil

    lonlat1 = (0., 0.)
    lonlat2 = (100., 0.)

    # compute mean point of stations
    lat_mean_stn,lon_mean_stn = barycenter_on_sphere(stations["lat"],stations["lon"])

    fig = plt.figure()

    lat0,lon0 = compute_mean_point_stations_event(event,stations)

    proj = ccrs.Robinson(
        central_longitude = lon0,
    )

    ax = fig.add_subplot(1, 1, 1, projection=proj)

    ax.set_title(f"azrange = [{azmin:.1f},{azmax:.1f}]")

    # connect event
    def onclick(event):

        global lonlat1,lonlat2, line_gc, point1, point2

        xy_data = (event.xdata, event.ydata)
        # convert from data to cartesian coordinates

        lon,lat = ccrs.PlateCarree().transform_point(*xy_data, src_crs=proj)

        if event.button == 1:
            # left click
            lonlat1 = (lon,lat)
            print(f"lonlat1 = {lon:.1f} {lat:.1f}")
        if event.button == 3:
            # right click
            lonlat2 = (lon,lat)
            print(f"lonlat2 = {lon:.1f} {lat:.1f}")

        lon1,lat1 = lonlat1
        lon2,lat2 = lonlat2

        # update plot
        line_gc.set_data((lon1,lon2), (lat1,lat2))
        point1.set_offsets(lonlat1)
        point2.set_offsets(lonlat2)

        fig.canvas.draw()

    cid = fig.canvas.mpl_connect('button_press_event', onclick)

    ax.coastlines()
    ax.add_feature(cfeature.LAND)
    ax.gridlines(linestyle=":", color="k")

    lon1,lat1 = lonlat1
    lon2,lat2 = lonlat2

    ax.scatter(stations["lon"], stations["lat"], marker=".", color="darkgreen", s = 10, transform = ccrs.PlateCarree())

    ax.scatter([origin.longitude],[origin.latitude], transform = ccrs.PlateCarree(), color="r",ec="k")
    ax.scatter([lon_mean_stn],[lat_mean_stn], transform = ccrs.PlateCarree(), color="g",ec="k")
    point1 = ax.scatter(lon1,lat1, transform = ccrs.PlateCarree(), color="b")
    point2 = ax.scatter(lon2,lat2, transform = ccrs.PlateCarree(), color="b")

    # ax.plot((lon1,lon_mean_stn), (lat1,lat_mean_stn), "b",transform=ccrs.Geodetic())

    line_gc, = ax.plot((lon1,lon2), (lat1,lat2), "r",transform=ccrs.Geodetic())

    ax.set_global()

    # plt.ion()
    plt.show()


import math

from sqlalchemy import and_, or_
from flask import request

from flask_jwt_extended import current_user, jwt_required

from fardel.core.rest import create_api, abort, Resource
from fardel_auth_address import mod
from fardel_auth_address.models import *
from fardel.ext import db, cache

auth_address_api = create_api(mod)

def rest_resource(resource_cls):
    """ Decorator for adding resources to Api App """
    auth_address_api.add_resource(resource_cls, *resource_cls.endpoints)
    return resource_cls

@rest_resource
class UserAddressApi(Resource):
    """
    :URL: ``/api/auth/address/`` and ``/api/auth/address/<addr_id>/``
    """
    endpoints = ['/address/', '/address/<addr_id>/']

    @jwt_required()
    def get(self, addr_id=None):
        """ To get all addresses """
        uas = UserAddress.query.filter_by(user_id=current_user.id).all()
        return {"addresses": [ua.dict() for ua in uas]}

    @jwt_required()
    def post(self, addr_id=None):
        """ To create an address """
        if addr_id:
            abort(405)

        data = request.get_json()
        if (not data.get('country') or not data.get('city') or
            not data.get("phone") or not data.get("postal_code") or
            not data.get('street_address')):
            return {"message":"Submitted form is not complete"}, 422

        ua = UserAddress(
            country=data.get('country'),
            city=data.get('city'),
            phone=data.get('phone'),
            postal_code=data.get('postal_code'),
            street_address=data.get('street_address'),
            user_id=current_user.id,
        )
        db.session.add(ua)
        db.session.commit()
        return {"address": ua.dict()}

    @jwt_required()
    def patch(self, addr_id=None):
        """ To update an address. Is it required ? addr_id required """
        if not addr_id:
            abort(404)
        abort(404)

    @jwt_required()
    def delete(self, addr_id=None):
        """ To delete an address, addr_id required """
        if not addr_id:
            abort(405)

        ua = UserAddress.query.filter_by(user_id=current_user.id,
                                         id=addr_id).first_or_404()
        db.session.delete(ua)
        db.session.commit()
        return {"message":"Successfully deleted the address."}


if '__main__' == __name__:
    factors = [set() for _ in range(1000000)]

    for n in range(2, 1000000):
        if len(factors[n]) != 0:
            continue
        for m in range(n + n, 1000000, n):
            factors[m].add(n)

    ln = -1
    longest = 0
    lfactors = None

    for n, factors in enumerate(factors):
        if len(factors) > longest:
            ln = n
            longest = len(factors)
            lfactors = factors

    print(ln)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 10 01:00:07 2019

@author: thugwithyoyo
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import os

import tkinter as tk
from tkinter.filedialog import askopenfilename

def GetSessionDecodingMaximum():

    # Acquire path of workspace to load.
    root = tk.Tk()
    RestoreFilePath = askopenfilename()
    root.withdraw()

    # Open workspace.
    #exec(open('./RestoreShelvedWorkspaceScript.py').read())
    try:

        exec(open('./RestoreShelvedWorkspaceScript.py').read())

    except:

        print('Unshelving error.  Will attempt to continue...')

    # Determine parent directory and filename from complete path.
    drive, path_and_file = os.path.splitdrive(RestoreFilePath)
    path, file = os.path.split(path_and_file)

    PerfIndices = np.arange(1, Performance.shape[0])
    MaxPerfVal = Performance[0]['performance']
    MaxPerfIndex = 0

    SessionName = file[0:19]

    for i in PerfIndices:

        if Performance[i]['performance'] > MaxPerfVal:

            MaxPerfIndex = i
            MaxPerfVal = Performance[i]['performance']

    return {
            'MaxPerfIndex': np.array([MaxPerfIndex]),
            'MaxPerfVal': np.array([MaxPerfVal]),
            'MaxPerfErrorBar': np.array([ConfInts[MaxPerfIndex]['performance_SE']]),
            'ShuffledPerfMedian': np.array([EventsShuffled[MaxPerfIndex]['performance_median']]),
            'ShuffledPerfErrorBar': np.array([EventsShuffled[MaxPerfIndex]['performance_SE']]),
            'SessionName': np.array([SessionName])
            }

# Integer width to position clusters.
ClusterIncrement = 1.

# The proportion of space to use to separate bar groups.
GroupSepWidthProportion = 0.25

# Calculate barwidth
BarWidth = ClusterIncrement*(1. - GroupSepWidthProportion)/2.

# Set colorscheme.  Is this necessary?
mpl.rc('image', cmap='nipy_spectral')

SessionPerfMeans = np.empty((0,))
SessionPerfSEs = np.empty((0,))
SessionShuffledPerfMeans = np.empty((0,))
SessionShuffledPerfSEs = np.empty((0,))

SessionNames = np.array([])

#EntriesFilt = np.array([1, 3, 5, 7, 9, -1])
#EntriesFilt = EntriesFilt[::-1]

GetAnotherSession = True

while (GetAnotherSession == True):

    SessionDict = GetSessionDecodingMaximum()

#    SessionPerfMeans = np.hstack([SessionPerfMeans, np.array([MaxPerfVal])])
#    SessionPerfSEs = np.hstack([SessionPerfSEs, np.array([ConfInts[i]['performance_SE']])])
#    SessionShuffledPerfMeans = np.hstack([SessionShuffledPerfMeans, np.array([EventsShuffled[i]['performance_median']])])
#    SessionShuffledPerfSEs = np.hstack([SessionShuffledPerfSEs, np.array([EventsShuffled[i]['performance_SE']])])
#
#    SessionNames = np.hstack([SessionNames, SessionName])

    SessionPerfMeans = np.hstack([SessionPerfMeans, SessionDict['MaxPerfVal']])
    SessionPerfSEs = np.hstack([SessionPerfSEs, SessionDict['MaxPerfErrorBar']])
    SessionShuffledPerfMeans = np.hstack([SessionShuffledPerfMeans, SessionDict['ShuffledPerfMedian']])
    SessionShuffledPerfSEs = np.hstack([SessionShuffledPerfSEs, SessionDict['ShuffledPerfErrorBar']])

    SessionNames = np.hstack([SessionNames, SessionDict['SessionName']])

    root = tk.Tk()

    GetAnotherSession = tk.messagebox.askyesno(message='Include another session?')

    root.withdraw()

#
# Extract recording IDs from filenames
NiceSessionNames = SessionNames

for i in np.arange(0, SessionNames.shape[0]):

    NiceSessionNames[i] = SessionNames[i][0:10]

SortMap = np.argsort(NiceSessionNames)

# Begin plot generation
(NumSessions, ) = SessionPerfMeans.shape
fig1, ax1 = plt.subplots(nrows=1, ncols=1)

fig1.suptitle('Peak trace decoding accuracy across sessions')

xLocs = np.arange(ClusterIncrement, (NumSessions + 1)*ClusterIncrement,
                  ClusterIncrement)

# Plot bars and errorbars of OBSERVED peak decoder performance across sessions.
ax1.bar(xLocs - BarWidth/2., SessionPerfMeans[SortMap],
        color='orange', ecolor='orange', label='Observed outcomes',
        width=BarWidth)

ax1.errorbar(xLocs - BarWidth/2., SessionPerfMeans[SortMap],
             yerr=SessionPerfSEs[SortMap],
             color='black', ecolor='black', fmt=',')

# Plot bars and errorbars of SHUFFLED peak decoder performance across sessions.
ax1.bar(xLocs + BarWidth/2., SessionShuffledPerfMeans[SortMap],
        color='gray', ecolor='gray', label='Shuffled outcomes',
        width=BarWidth)

ax1.errorbar(xLocs + BarWidth/2., SessionShuffledPerfMeans[SortMap],
             yerr=SessionShuffledPerfSEs[SortMap],
             color='black', ecolor='black', fmt=',')

ax1.set_xlabel('Session')
ax1.set_ylabel('Peak Decoding Accuracy (%)')
ax1.set_ylim([0., 1.])
ax1.set_yticks([0., 0.5, 1.0])
ax1.set_yticklabels(['0', '50', '100'])

#ax1.set_xticks(range(1, NumSessions+1, 1))
ax1.set_xticks(xLocs)
#ax1.set_xticklabels(NiceSessionNames[SortMap], rotation=45, ha='center')
ax1.set_xticklabels(np.arange(1,12,1), rotation=0)
ax1.legend(loc='lower right')
ax1.spines['top'].set_visible(False)
ax1.spines['right'].set_visible(False)
ax1.set_title('PLS-DA: 400ms window, slid over 100ms increments')


"""
# TODO:
    - Make work with more than 2 players
    - Make work with uneven teams
    - Make GUI a thing (command line ofc)
"""

import json

STARTING_ELO = 1000

def expected_score(playera: str, playerb: str):
    """
    Calculates the percentage chance that Player A has to win against Player B
    :param playera: The first player
    :param playerb: The second player
    :return: The chance that the first player has to win against Player B
    :raise IndexError: Either Player A or Player B is not in ranking.json
    """
    rb = STARTING_ELO
    ra = STARTING_ELO
    init_rankings = json.load(open("ranking.json"))
    try:
        ra = init_rankings[playera]
    except KeyError:
        data = json.load(open("ranking.json"))
        data.update({playera: STARTING_ELO})

        with open("ranking.json", "w") as fout:
            json.dump(data, fout, sort_keys=True, indent=4)
        expected_score(playera, playerb)
    try:
        rb = init_rankings[playerb]
    except KeyError:
        data = json.load(open("ranking.json"))
        data.update({playerb: STARTING_ELO})

        with open("ranking.json", "w") as fout:
            json.dump(data, fout, sort_keys=True, indent=4)
        expected_score(playera, playerb)

    return 1 / (1 + 10 ** ((rb - ra) / 400))

def expected_score_as_percent(playera: str, playerb: str):
    # return expected_score(playera, playerb) * 100
    return "{}% chance {} will win".format(round((expected_score(playera, playerb) * 100), 2), playera)

def add_game(playera: str, playerb: str, winner: int):
    """
    Adds a game to games.json
    :param playera: The first player
    :param playerb: The second player
    :param winner: The player that won
    :return: The new elo of both players
    """
    data = json.load(open("games.json"))
    if type(data) is dict:
        data = [data]

    outcome = [0] * 2
    if winner == 1:
        outcome[0] = 1
    else:
        outcome[1] = 1

    data.append({
        "Player A": {
            "Name": playera,
            "Outcome": outcome[0]
        },
        "Player B": {
            "Name": playerb,
            "Outcome": outcome[1]
        }
    })

    with open("games.json", "w") as fout:
        json.dump(data, fout, sort_keys=True, indent=4)

    update_rankings(playera, playerb, winner)

def update_rankings(playera: str, playerb: str, winner: int):
    """
    Updates elo values in ranking.json
    :param playera: The first player
    :param playerb: The second player
    :param winner: The player that won
    :return: N/A
    """
    outcome = [0] * 2
    if winner == 1:
        outcome[0] = 1
    else:
        outcome[1] = 1

    init_rankings = json.load(open("ranking.json"))
    expecteda = expected_score(playera, playerb)
    expectedb = expected_score(playerb, playera)
    try:
        olda = init_rankings[playera]
    except KeyError:
        olda = STARTING_ELO
    try:
        oldb = init_rankings[playerb]
    except KeyError:
        oldb = STARTING_ELO
    newa = round((olda + 32 * (outcome.pop(0) - expecteda)), 2)
    newb = round((oldb + 32 * (outcome.pop(0) - expectedb)), 2)

    data = json.load(open("ranking.json"))
    data[playera] = newa
    data[playerb] = newb

    with open("ranking.json", "w") as fout:
        json.dump(data, fout, sort_keys=True, indent=4)

def gui():
    pass

add_game("test1", "test2", 1)


import matplotlib.pyplot as plt
import pandas as pd
import geopandas as gpd
from shapely.geometry import Point

fname = "data/calif/calif.ast"

# ASCII_data "CALIF ASCII data format"
fmt = ["station_id 1 8 char 0",
       "latitude_deg 10 11 short 0",
       "latitude_min 12 15 short 2",
       "longitude_deg 16 19 short 0",
       "longitude_min 20 23 short 2",
       "sea_level_elev_ft 24 29 long 1",
       "obs_grav 30 36 long 2",
       "accuracy_code 37 40 char 0",
       "Free_air_anom 41 46 long 2",
       "Bouguer_anom_simp 47 52 long 2",
       "terr_corr_inner 53 57 long 2",
       "terr_corr_total 58 62 long 2",
       "Hammer_zone 63 63 char 0",
       "Bouguer_anom_267 64 69 long 2",
       "isostatic_anom 70 75 long 2",
       "IGSN71_conv_factor 76 80 char 0"]

dtypes = []

f = open(fname, "r+")
data_content = f.readlines()
f.close()

allrows = []
for lineno, line in enumerate(data_content):
    row = []
    for fmtcol in fmt:
        colname, start_col, end_col, dtype, num = fmtcol.split()

        # from one-indexing to zero-indexing,=
        start_col, end_col = int(start_col)-1, int(end_col)
        num_decimal = int(num)

        if dtype == "char":
            item = line[start_col:end_col]
            if lineno == 0:
                dtypes.append((colname))

        elif num_decimal > 0:
            end_col -= num_decimal
            new_end_col = end_col + num_decimal
            stritem = line[start_col:end_col].strip()
            neg = False
            if stritem:
                if stritem[0] == "-":
                    neg = True
                    if len(stritem) == 1:
                        whole = 0
                    else:
                        whole = float(stritem[1:])
                else:
                    whole = float(stritem)
            else:
                whole = 0
            frac = float(line[end_col:new_end_col]) / (10**num_decimal)
            item = whole + frac
            if lineno == 0:
                dtypes.append((colname))
        else:
            whole = float(line[start_col:end_col])
            if line[start_col] == "-":
                whole = float(line[start_col+1:end_col])
            item = float(whole)
            if lineno == 0:
                dtypes.append((colname))

        row.append(item)
    allrows.append(row)

allrows_df = pd.DataFrame(allrows, columns=dtypes)

# 60 minutes per degree of latitude/longitude
# Converting Latitude/Longitude to decimal degrees
allrows_df["latitude"] = (allrows_df["latitude_deg"] +
                          allrows_df["latitude_min"] / 60)
allrows_df["longitude"] = -1*( allrows_df["longitude_deg"] +
                               allrows_df["longitude_min"] / 60)

df = allrows_df.drop(columns=["latitude_deg", "latitude_min",
                              "longitude_deg", "longitude_min"])

# Converting to a georeferenced data object and saving as geojson
geom = [Point(x, y) for (x, y) in df[["longitude", "latitude"]].values]
gdf = gpd.GeoDataFrame(df, geometry=geom, crs="epsg:4326")
gdf.to_file("data/geojson/calif_nev_ncei_grav.geojson", driver="GeoJSON")

# Very basic scatter plot of the data
fig, ax = plt.subplots(figsize=(12,12))
ax.scatter(df["longitude"], df["latitude"], c=df["isostatic_anom"], s=0.8)
ax.set_xlabel("Longitude")
ax.set_ylabel("Latitude")
ax.set_aspect("equal")

# Save to file
fig.savefig("figures/canv_data.png")


import numpy as np
import json
from nuscenes.nuscenes import NuScenes
from nuscenes.utils.geometry_utils import BoxVisibility
import yaml

with open("./config.yaml") as f:
    config = yaml.load(f, Loader=yaml.FullLoader)

# Settings
sensor = config["sensor"]
use_category = config["use_category"]
use_visibility = config["use_visibility"]
scene_num_train = config["scene_num_train"]
min_dist = config["min_dist"]
max_dist = config["max_dist"]

use_scene_path = config["use_scene_path"]
nuscenes_path = config["nuscenes_path"]
rawlst_paths = config["rawlst_path"]

# Compute distance between box and ego.
def box_distance(sensor_data, ann_data):
    ego_location = nusc.get("ego_pose", sensor_data["ego_pose_token"])[
        "translation"]
    box_world_location = ann_data["translation"]
    distance = np.linalg.norm(
        np.array(box_world_location) - np.array(ego_location))
    return distance

with open(use_scene_path, "r") as f:
    use_scene = json.load(f)
img_seq = []
scene_lsts = []
scene_lsts.append(use_scene[:scene_num_train])  # [0, scene_num_train) for training set
scene_lsts.append(use_scene[scene_num_train:])  # [scene_num_train, last] for validation set
seq_head = True
found_box = False
nusc = NuScenes(version='v1.0-trainval', dataroot=nuscenes_path, verbose=False)

# Iterate over selected scenes to record frame pairs
# that contain obj(s) which meet requirements.
for i, scene_lst in enumerate(scene_lsts):

    for scene in nusc.scene:

        if scene["token"] in scene_lst:

            sample = nusc.get("sample", scene["first_sample_token"])
            sensor_data = nusc.get("sample_data", sample['data'][sensor])
            curr_sensor_token = sensor_data["token"]

            # Iterate over all CAM_FRONT images in current scene (all frames)
            while curr_sensor_token:

                sensor_data = nusc.get("sample_data", curr_sensor_token)
                # Last frame of current scene.
                if not sensor_data["next"]:
                    found_box = False
                    seq_head = True
                    break
                # Ignore non-keyframe.
                if not sensor_data["is_key_frame"]:
                    curr_sensor_token = sensor_data["next"]
                    continue

                # Get boxes in CAM_FRONT of corresponding frame
                data_path, curr_boxlst, _ = nusc.get_sample_data(
                    curr_sensor_token, box_vis_level=BoxVisibility.ALL)

                for cbox in curr_boxlst:

                    if cbox.name not in use_category:
                        continue
                    annotation = nusc.get("sample_annotation", cbox.token)
                    visibility_token = annotation["visibility_token"]
                    if nusc.get("visibility", visibility_token)["level"] != use_visibility:
                        continue
                    distance = box_distance(sensor_data, annotation)
                    if distance >= min_dist and distance <= max_dist:
                        curr_instance_token = annotation["instance_token"]
                        next_sensor_token = sensor_data["next"]
                        next_sensor_data = nusc.get(
                            "sample_data", next_sensor_token)
                        while not next_sensor_data["is_key_frame"]:
                            next_sensor_token = next_sensor_data["next"]
                            if not next_sensor_token:
                                break
                            next_sensor_data = nusc.get(
                                "sample_data", next_sensor_token)

                        if not next_sensor_data:
                            break

                        next_data_path, next_boxlst, _ = nusc.get_sample_data(
                            next_sensor_data["token"], box_vis_level=BoxVisibility.ALL)

                        for nbox in next_boxlst:

                            next_annotation = nusc.get(
                                "sample_annotation", nbox.token)
                            next_instance_token = next_annotation["instance_token"]

                            # Same obj in two frames.
                            if next_instance_token == curr_instance_token:
                                next_visibility_token = next_annotation["visibility_token"]
                                if nusc.get("visibility", next_visibility_token)["level"] != use_visibility:
                                    break
                                next_sensor_data = nusc.get(
                                    "sample_data", next_sensor_token)
                                next_distance = box_distance(
                                    next_sensor_data, next_annotation)
                                if next_distance >= min_dist and next_distance <= max_dist:
                                    # If beginning of a new image sequence, record current frame also.
                                    if seq_head:
                                        img_seq.append(list())
                                        curr_lst = img_seq[-1]
                                        curr_lst.append(dict())
                                        curr_lst[-1]["imgpath"] = nusc.get_sample_data_path(
                                            curr_sensor_token)
                                        curr_lst[-1]["name"] = curr_sensor_token
                                        seq_head = False
                                    curr_lst = img_seq[-1]
                                    curr_lst.append(dict())
                                    curr_lst[-1]["imgpath"] = nusc.get_sample_data_path(
                                        next_sensor_token)
                                    curr_lst[-1]["name"] = next_sensor_token
                                    found_box = True
                                    break

                        if found_box:
                            break

                if not found_box:
                    seq_head = True

                found_box = False
                curr_sensor_token = sensor_data["next"]

    with open(rawlst_paths[i], 'w') as f:
        json.dump(img_seq, f, sort_keys=True, indent=4)


from glob import glob
import numpy as np
import math
import cv2

from utils.data_loader import DataLoader
from utils.gate_corner import chollet_gate_fix_missing
from utils.data_formater import DataFormater

def remove_empty_frames(raw_data):
    # Clean les frames de debut si pas de diff entre (n) et (n+1)
    for key, val in raw_data.items():
        data = raw_data[key] # data est une liste avec toutes les frames
        # clean empty rateThrust
        i = 0
        while i < len(data):
            if not data[i]["rateThrust"]:
                del data[i]
            else:
                i += 1
        # Clean static drone frames
        while len(data) > 1:
            frame = data[1]
            if frame and frame["droneSpeed"][0] == 0 and frame["droneSpeed"][1] == 0 and frame["droneSpeed"][2] == 0:
                del data[0]
            else:
                break
        raw_data[key] = data
    return raw_data

class GroundTruth(object):
    """docstring for GroundTruthUtils."""
    def __init__(self, path=None, json_file=None, yaml_file=None, split=0.1, nb_timesteps=6):
        self.split = split

        # Search json and yaml in path if path not None
        # else use directly json_file and yaml_file
        if path:
            path = path.rstrip('/')
            json_file = [file for file in glob("{}/*.json".format(path))]
            yaml_file = [file for file in glob("{}/*.yaml".format(path))]
            if len(json_file) != 1:
                print("No json or more than one file in the specified path.")
                exit(1)
            if len(yaml_file) != 1:
                print("No yaml or more than one file in the specified path.")
                exit(1)
            json_file = json_file[0]
            yaml_file = yaml_file[0]

        self.path = path
        self.json_file = json_file
        self.yaml_file = yaml_file

        self.data_formater = DataFormater(nb_timesteps=nb_timesteps)
        self.data_loader = DataLoader(json_file=json_file, yaml_file=yaml_file, nb_gates_nominal=2, nb_gates_ir=1)
        data = np.array(self.data_loader.data)
        self.init(data)

    def init(self, data):
        bad_runs = [
            3, 11, 25, 28, 46, 50, 51,
            54, 55, 58, 59, 76, 80,
        ]

        # Clean data and store it in a nice dictionnary
        self.data = {}
        for frame in data:
            if int(frame['run_id']) in bad_runs:
                continue
            if frame['run_id'] not in self.data:
                self.data[frame['run_id']] = []
            id = frame['run_id']
            del frame['run_id']
            self.data[id].append(frame)

        self.data = self.__cleanRuns(self.data)

        # Store informations
        self.nb_runs = len(self.data)
        self.nb_frames = 0 # TODO: Add this

        # Set the idxs for the input generator
        self.idxs = []
        for idx in self.data.keys():
            self.idxs += [(idx, n)for n in range(len(self.data[idx]))]
        self.idxs = np.array(self.idxs)

        self.__split()

    def __formatFrames(self, raw_data):
        for key, val in raw_data.items():
            data = raw_data[key] # data est une liste avec toutes les frames
            data_array = []
            for frame in data:
                # INPUT
                if not frame['eulerDroneNominal'][0]:
                    continue
                input_array = self.data_formater.format_input(frame)
                # OUTPUT
                output_array = self.data_formater.format_output(frame)
                data_array.append([input_array, output_array])
            raw_data[key] = np.array(data_array)
        return raw_data

    def __cleanRuns(self, raw_data):
        raw_data = remove_empty_frames(raw_data)
        raw_data = self.__formatFrames(raw_data)
        return raw_data

    def __split(self):

        self.nb_sample = self.idxs.shape[0]

        size_valid = int(self.split * self.nb_sample)
        size_train = self.nb_sample - size_valid

        self.idxs_train = self.idxs[size_valid:]
        self.idxs_valid = self.idxs[0:size_valid]

if __name__ == "__main__":
    import time
    import pickle
    import cv2

    PATH = "../resources/"
    json_file = PATH + "test_dumps_map1.json"
    yaml_file = PATH + "nominal_gate_locations.yaml"
    gt = GroundTruth(json_file=json_file, yaml_file=yaml_file)

    #with open('data_killian.pkl', 'r') as f:
    #    gt = pickle.load(f)

    data = []
    for k, v in gt.data.items():
        for x in v[:, 0]:
            #print('Euler 1', ["{0:0.2f}".format(i) for i in x[:3]])
            #print('Euler 2', ["{0:0.2f}".format(i) for i in x[3:6]])
            #print('drl', ["{0:0.2f}".format(i) for i in x[6:7]])
            #print('dist', ["{0:0.2f}".format(i) for i in x[7:9]])
            #print('speed', ["{0:0.2f}".format(i) for i in x[9:12]])
            #print('corners', ["{0:0.2f}".format(i) for i in x[12:20]])
            print("Next")
            w, h = 400, 400
            img = np.zeros((w, h, 3), np.uint8)
            bbox = x[12:20]
            xy = bbox.reshape(4, -1) * [w, h]
            xy = xy[:,[1, 0]]
            xy = np.round(xy)
            cv2.polylines(img, np.int32([xy]), True, (0,0,255), thickness=2)
            cv2.imshow("Gate detection", (img))
            cv2.waitKey(0)

            data += [x]
    np.array(data, dtype=np.float32)


from unittest.mock import patch

BASE_API_PATH = '/api/v1'

CONTENT_TYPE_JSON_HEADER = {'Content-Type': 'application/json', 'Accept': 'application/json'}

MOCK_ENVOY_RESPONSE = {
    "data": {
      "buzzer": 0,
      "diabled": 0,
      "distance": 8221347.672491201,
      "id": 1,
      "lat": 38.5549336,
      "lng": -121.4270026,
      "name": "MockEnvoy1",
      "serial": "4674007431",
      "speed": 0,
      "typeId": 3,
      "vin": "WVWPR7AU3JW907001",
      "volts": 1246.0
    }
  }

@patch('src.controllers.envoy.vehicles.fetch_vehicles')
def test_get_vehicles(mock_fetch_vehicles, app_client):
    mock_fetch_vehicles.return_value = [MOCK_ENVOY_RESPONSE]
    response = app_client.get(f'{BASE_API_PATH}/envoy/getStatus', headers=CONTENT_TYPE_JSON_HEADER)
    mock_fetch_vehicles.assert_called_with()
    assert response.status_code == 200
    assert response.json == [MOCK_ENVOY_RESPONSE]

@patch('src.controllers.envoy.vehicles.fetch_vehicle_by_id')
def test_get_vehicle_by_id(mock_fetch_vehicles, app_client):
    mock_fetch_vehicles.return_value = MOCK_ENVOY_RESPONSE
    response = app_client.get(f'{BASE_API_PATH}/envoy/getStatus/1', headers=CONTENT_TYPE_JSON_HEADER)
    mock_fetch_vehicles.assert_called_with(vehicle_id=1)
    assert response.status_code == 200
    assert response.json == MOCK_ENVOY_RESPONSE


import os, sys
import gettext

from twisted.web.woven import page, widgets
from twisted.web import microdom

t = gettext.translation(
    'remusserver', '/usr/local/share/locale', fallback=True)
_ = t.gettext

document = microdom.parseString("<xml />", caseInsensitive=0, preserveCase=0)

class Stylesheet(widgets.Widget):
    tagName = 'link'

    def initialize(self, link=""):
        self.link = link

    def setLink(self, link):
        self.link = link

    def setUp(self, request, node, data):
        href = self.link or data
        self['rel'] = "stylesheet"
        self['type'] = "text/css"
        self['href'] = href

class Menu(widgets.Widget):
    tagName = 'div'

    def setUp(self, request, node, data):
        for menuitem in self.model:
            link = document.createElement("a")
            link.attributes.update({
                'href':  menuitem.orig["link"],
                'title': menuitem.orig["tooltip"]})
            link.appendChild(document.createTextNode(menuitem.orig["text"]))

            span = document.createElement("span")
            span.attributes.update({
                'class':  "menuitem"})
            span.appendChild(link)
            self.appendChild(span)
            self["class"] = "topmenu"

def remus_page_hierarchy(cls, l=None):
    "Returns a list of inherited classes up to RemusPage"
    if l == None:
        l = []
    if issubclass(cls, RemusPage):
        l.append(cls)
        for base in cls.__bases__:
            remus_page_hierarchy(base, l)
    return l

def get_template(cls):
    if cls.template:
        template = cls.template
    elif cls.templateFile:
        if not cls.templateDirectory:
            mod = sys.modules[cls.__module__]
            if hasattr(mod, '__file__'):
                cls.templateDirectory = os.path.split(mod.__file__)[0]

        template = open(os.path.join(
            cls.templateDirectory, cls.templateFile)).read()
    return template

def merge_templates(cls, template):
    base_template = get_template(cls)
    return base_template.replace("<child/>", template)

class RemusPage(page.Page):

    title = "Remus page"

    templateFile = "RemusPage.html"

    templateDirectory = "/usr/local/libdata/remus"

    menu = (
        {
            'text': _('Home'),
            'link': '/',
            'tooltip': _('To Remus homepage')
        },
        {
            'text': _('Browse'),
            'link': '/music/',
            'tooltip': _('Browse the music database')
        },
        {
            'text': _('Playlists'),
            'link': '/playlists/',
            'tooltip': _('Manage playlists')
        },
        {
            'text': _('Server statistics'),
            'link': '/stats/',
            'tooltip': _('Various server statistics')
        }
        )

    def __init__(self, *args, **kwargs):
        page.Page.__init__(self, *args, **kwargs)

        hier = remus_page_hierarchy(self.__class__)

        template = get_template(self)

        for cls in hier[1:]:
            template = merge_templates(cls, template)
        self.template = template

    def wvfactory_Stylesheet(self, request, node, model):
        return Stylesheet(model)

    def wmfactory_base_url(self, request):
        return request.prePathURL()

    def wmfactory_title(self, request):
        return self.title

    def wmfactory_stylesheet(self, request):
        return "/styles/remus.css"

    def wvfactory_Menu(self, request, node, model):
        return Menu(model)

    def wmfactory_menu(self, request):
        return self.menu


#typefunction
a=10
b=2.3
c='sagar'
d=3+2j
f=True

print(type(a))                      #<class 'int'>
print(type(b))                      #<class 'float'>
print(type(c))                      #<class 'str'>
print(type(d))                      #<class 'complex'>
print(type(f))                      #<class 'bool'>

#typeconversion

num=20
d=str(num)                          #inttostrconverted
print(d)                            #inttostr
print(type(d))
'''
num='sagar'
d=int(num)                          #strtoint not converted
print(d)                            #invaild strtoint
print(type(d))

num='20'
d=int(num)                          #strtoint not converted
print(d)                            #it wont convert from strtoint
print(type(d))
'''
num = 1.2
print(type(num))                #<class 'float'>
d=int(num)                      #1
print(d)                        #<class 'int'>
print(type(d))

#comments 1single#  2multiple"""  """ or '''  '''

a=10                           #a is int var
b=20
c=a+b                          #add
print(c)                       #30

#control statements 1)conditional statements:

1 # a==b
2 # a!=b
3 # a<=b
4 # a.=b

a=1
b=3
if a>b:
    print("a is smaller than b")           #a<b a is smaller than b
elif a>b:
    print("a is smaller than b")           #a<b a is smaller than b
else:
    print("b is smaller than a")           #b is smaller than a

a=1
b=22
if a<b: print('this is short hand if')          #this is shor hand if

a=1
b=22
print ('this is short hand if') if a<b else print('this is short hand if else')         #this is short hand

if True:
    print('this is if')           #this is if
if False:
    print('this is if')
else:
    print('this is else')         #this is else

#logical ops

a=200
b=33
c=500
if a>b and  c>b:
    print('and')                #and

else:
    print('else')

a=200
b=33
c=500
if a>b and  c<b:
    print('and')                    #else

else:
    print('else')

a=200
b=33
c=500
if a>b or  c<b:
    print('or')                    #or

else:
    print('else')

a=200
b=33
c=500                       #ntng
if b<a:
    pass


import time
import os.path
import numpy as np
import pandas as pd
import pickle
import scipy
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import f1_score, confusion_matrix
#from sklearn.neighbors import KNeighborsClassifier
import hyperopt
from hyperopt import tpe
from hpsklearn import HyperoptEstimator, k_neighbors_classifier, svc, lightgbm_classification, gaussian_nb, any_preprocessing
import lightgbm
from datetime import timedelta, date
from tenacity import retry, stop_after_attempt

import sys, errno

def tsf_targets_import(file_path):

    """
    This function imports wideband frequency response .csv files exported from
    Echoview and performs some basic housekeeping.

    PARAMETERS:
        file_path: Path to wideband frequency response .csv file from Echoview.

    RETURNS:
        df: Pandas dataframe.
    """

    def get_datetime(df, avg_flag):
        """
        Convert Echoview timestamps in imported .csv to datetime.
        """
        if avg_flag:
            df['Ping_microseconds_start'] = df.Ping_millisecond_start * 1000
            df['Ping_microseconds_start'] = [timedelta(microseconds = i) for i in df['Ping_microseconds_start']]
            df['Datetime'] = df['Ping_date_start_Ping_time_start'] + df['Ping_microseconds_start']
            df.drop(columns=['Ping_date_start_Ping_time_start',
                             'Ping_millisecond_start',
                             'Ping_microseconds_start'],
                    inplace=True)

        else:
            df['Ping_microseconds'] = df.Ping_milliseconds * 1000
            df['Ping_microseconds'] = [timedelta(microseconds = i) for i in df['Ping_microseconds']]
            df['Datetime'] = df['Ping_date_Ping_time'] + df['Ping_microseconds']
            df.drop(columns=['Ping_date_Ping_time',
                             'Ping_milliseconds',
                             'Ping_microseconds'],
                    inplace=True)

        return df

    if "trackavg.csv" in file_path:
        parse_cols = ['Ping_date_start', 'Ping_time_start']
        ind_col = None
        avg_flag = True
        transpose = True
    elif "target" in file_path:
        parse_cols = ['Ping_date', 'Ping_time']
        ind_col = None
        avg_flag = False
        transpose = False
    else:
        parse_cols = ['Ping_date', 'Ping_time']
        ind_col = 'Target_index'
        avg_flag = False
        transpose = True

    if transpose:
        transposed_path = file_path[:-4] + '_transposed.csv'
        if os.path.isfile(transposed_path) == True:
            df = pd.read_csv(transposed_path,
                             index_col=ind_col,
                             skiprows=1,
                             skipfooter=2,
                             engine='python',
                             parse_dates=[parse_cols])
        else:
            df = pd.read_csv(file_path, low_memory=False).T
            file_path = transposed_path
            df.to_csv(path_or_buf=file_path)
            print('A transposed TS(f) file was created at: \n' + str(file_path))
            df = pd.read_csv(file_path,
                             index_col=ind_col,
                             skiprows=1,
                             skipfooter=2,
                             engine='python',
                             parse_dates=[parse_cols])
    else:
        df = pd.read_csv(file_path,
                 index_col=ind_col,
                 skiprows=0,
                 skipfooter=2,
                 engine='python',
                 parse_dates=[parse_cols])

    df = get_datetime(df, avg_flag)

    return df

@retry(stop=stop_after_attempt(10))
def nested_cv(X, y, model, n_splits, n_folds, unique_id, le, path):

    """
    This function performs nested cross-validation with Bayesian hyperparameter
    optimisation. It uses stratified k-fold cross-validation in both the inner
    and outer loops. After each outer loop, it outputs the results to a .pkl
    file. As there is an element of randomness to the optimisation procedure,
    sometimes all trials will fail. If you re-run the script, it will import
    the incomplete .pkl file and try again.

    Note that this is a modified version that uses F1 score as the evaluation
    metric. It also calculates class-specific F1 scores and confusion matrices,
    which are added to the output dataframe.

    PARAMETERS:
        X: data minus labels
        y: labels
        model: HyperoptEstimator object
        n_splits: # of splits to use in outer K-fold cross-validation
        n_folds: # of folds to use in inner K-fold cross-validation
        unique_id: Unique name string for file output path
    Written by Chelsey McGowan-Yallop
    """
    cv = StratifiedKFold(n_splits=n_splits,
                         shuffle=True,
                         random_state=42) # Outer CV

    i_start = 0
    i_list = []
    results_df = None
    cv_path = path + unique_id + '_NestedCV.pkl'

    if os.path.isfile(cv_path) == True: # If CV is incomplete, resume
        results_df = pd.read_pickle(cv_path)
        i_start = results_df.Outer_fold.max() + 1
        print('Resuming cross-validation from fold ' + str(i_start + 1))

    # Generate indices to split data by StratifiedKFold
    # Append indices for each fold to list
    for tr_i, te_i in cv.split(X,y):
        i_list.append([tr_i, te_i])

    # For each fold...
    for i in range(i_start, len(i_list)):
        results_list = []
        print('Beginning fold ' + str(i+1) + ' of ' + str(len(i_list)))

        # Split data into training and test tests
        X_train = X[i_list[i][0]]
        y_train = y[i_list[i][0]]
        X_test = X[i_list[i][1]]
        y_test = y[i_list[i][1]]

        start = time.time()

        # Fit the HyperoptEstimator to training data (optimise model)
        model.fit(X_train,
                  y_train,
                  n_folds=n_folds, # Inner stratified k-fold CV
                  cv_shuffle=True)

        end = time.time()
        duration = end - start

        # Use optimised model to predict labels for test data
        y_pred = model.predict(X_test)
        score = f1_score(y_test, y_pred, average='weighted') # Evaluate

        # Everything below: formats and/or calculates results for output file
        sorted_labels = np.sort(np.unique(y_test))
        unweighted_score = f1_score(y_test, y_pred,
                                    average=None,
                                    labels=sorted_labels)
        c_matrix = confusion_matrix(y_test, y_pred,
                                    labels=sorted_labels)

        for trial in range(len(model.trials.trials)):
                if model.trials.trials[trial].get('result').get('status') == 'ok':
                    trial_loss = model.trials.trials[trial].get('result').get('loss')
                    trial_duration = model.trials.trials[trial].get('result').get('duration')
                else:
                    trial_loss = np.nan
                    trial_duration = np.nan

                results_list.append([i,
                                     score,
                                     unweighted_score,
                                     le.inverse_transform(sorted_labels),
                                     c_matrix,
                                     duration,
                                     trial,
                                     trial_loss,
                                     trial_duration])

        append_df = pd.DataFrame(results_list,
                                 columns=['Outer_fold',
                                          'Outer_score',
                                          'Outer_unweighted_scores',
                                          'Outer_unweighted_score_labels',
                                          'Outer_confusion_matrix',
                                          'Outer_training_duration',
                                          'Trial',
                                          'Trial_loss',
                                          'Trial_duration'])
        if i == i_start:
            if results_df is not None:
                final_df = pd.concat([results_df,
                                      append_df],
                                      ignore_index=True)
            else:
                final_df = append_df
            final_df.to_pickle(cv_path)

        else:
            results_df = pd.read_pickle(cv_path)
            final_df = pd.concat([results_df,
                                  append_df],
                                  ignore_index=True)
            final_df.to_pickle(cv_path)
    return

def f1_loss(y_true, y_pred):
    """
    Custom loss function for HyperOpt.
    Uses F1 score instead of accuracy score, as the latter is inappropriate
    for multi-class classification.
    """
    return 1.0 - f1_score(y_true, y_pred, average='weighted')

def main_classify(df, clf, unique_id,path, preprocessing=[],  ex_preprocessing=[], timeout=300, n_jobs=-1, max_evals=50, n_splits = 10,
n_folds = 10):
    """
    Function to run nested cross validation then apply fit to whole dataset
    Uses F1 score instead of accuracy score, as the latter is inappropriate
    for multi-class classification.
    """

    # -- WRANGLE DATA ---------------------------------------------------------
    df_train = df.loc[:,:].copy()

    le = LabelEncoder() # Maps labels -> int (e.g. Atlantic cod -> 0, Polar cod -> 1)
    df_train['Species_le'] = le.fit_transform(df_train.Species)
    X = df_train.iloc[:,:-2].to_numpy() # Features, TS(f) only
    y = df_train.loc[:,'Species_le'].to_numpy() # Labels

    # -- NESTED CROSS-VALIDATION ----------------------------------------------

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)
    model

    nested_cv(X, y, model, n_splits, n_folds, unique_id, le, path)

    # -- RETRAIN MODEL --------------------------------------------------------

    print('Retraining model on full dataset')

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)

    model.fit(X, y, n_folds=n_folds, cv_shuffle=True)

    with open(path + unique_id + '_BestParams.pkl', 'wb') as handle:
        pickle.dump(model.best_model(), handle)
    return

def print_F1_mean(classifier):
    cv_df = classifier['cv_df']

    print('Mean class-weighted F1 score for : '
          + str(round(cv_df.Outer_score.mean(), 2))
          + ' ± '
          + str(round(cv_df.Outer_score.std(), 2)))

    mean_class_scores = np.mean(np.vstack(cv_df.Outer_unweighted_scores.values), axis=0)
    std_class_scores = np.std(np.vstack(cv_df.Outer_unweighted_scores.values), axis=0)

    for i, species in enumerate(cv_df.Outer_unweighted_score_labels.values[0]):
        print('Mean F1 score for ' + species + ': '
             + str(round(mean_class_scores[i], 2))
             + ' ± '
             + str(round(std_class_scores[i], 2)))

def read_results(unique_id ,classifypath):
    ' read the classifier results and predictions from the selected classifier'

    main_path = f'{classifypath}/{unique_id}'
    cv_path = '_NestedCV.pkl'
    best_params = '_BestParams.pkl'
    pred_path = '_Predictions.pkl'

    # Load dataframes
    cv_df = pd.read_pickle(main_path + cv_path) # Nested CV results
    best_params = pd.read_pickle(main_path + best_params)
    #pred_df = pd.read_pickle(main_path+pred_path)

    return {'name':unique_id, 'cv_df':cv_df, 'best_params':best_params}#,  'pred_df':pred_df,}

def main_classify_test(df, df_new, clf, unique_id,path, preprocessing=[],  ex_preprocessing=[], timeout=300, n_jobs=-1, max_evals=50, n_splits = 10, n_folds = 10):
    """
    Function to run nested cross validation then apply fit to whole dataset
    Uses F1 score instead of accuracy score, as the latter is inappropriate
    for multi-class classification.
    """

    # -- WRANGLE DATA ---------------------------------------------------------
    df_train = df.loc[:,:].copy()

    le = LabelEncoder() # Maps labels -> int (e.g. Atlantic cod -> 0, Polar cod -> 1)
    df_train['Species_le'] = le.fit_transform(df_train.Species)
    X = df_train.iloc[:,:-2].to_numpy() # Features, TS(f) only
    y = df_train.loc[:,'Species_le'].to_numpy() # Labels

# -- WRANGLE TEST DATA ---------------------------------------------------------
    measured_X = df_new

    # -- NESTED CROSS-VALIDATION ----------------------------------------------

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)
    model

    nested_cv(X, y, model, n_splits, n_folds, unique_id, le, path)

    # -- RETRAIN MODEL --------------------------------------------------------

    print('Retraining model on full dataset')

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)

    model.fit(X, y, n_folds=n_folds, cv_shuffle=True)

    # -- PREDICT CLASSES FOR NEW DATA -----------------------------------------

    print('Classifying new data')

    y_pred = model.predict(measured_X) # Predict classes for measured TS(f)
    y_pred = le.inverse_transform(y_pred) # Transform labels back to species

    # -- OUTPUT RESULTS -------------------------------------------------------

    measured_X['Prediction'] = y_pred
    measured_X.to_pickle(path + unique_id + '_Predictions.pkl')

    with open(path + unique_id + '_BestParams.pkl', 'wb') as handle:
        pickle.dump(model.best_model(), handle)
    return

def main_classify_test_old(df, df_new, clf, unique_id,path, preprocessing=[],  ex_preprocessing=[], timeout=300, n_jobs=-1, max_evals=50, n_splits = 10, n_folds = 10):
    """
    Function to run nested cross validation then apply fit to whole dataset
    Uses F1 score instead of accuracy score, as the latter is inappropriate
    for multi-class classification.
    """

    # -- WRANGLE DATA ---------------------------------------------------------
    df_train = df.loc[:,:].copy()

    le = LabelEncoder() # Maps labels -> int (e.g. Atlantic cod -> 0, Polar cod -> 1)
    df_train['Species_le'] = le.fit_transform(df_train.Species)
    X = df_train.iloc[:,:-2].to_numpy() # Features, TS(f) only
    y = df_train.loc[:,'Species_le'].to_numpy() # Labels

# -- WRANGLE TEST DATA ---------------------------------------------------------
    measured_X = df_new.copy()

    # -- NESTED CROSS-VALIDATION ----------------------------------------------

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)
    model

    nested_cv(X, y, model, n_splits, n_folds, unique_id, le, path)

    # -- RETRAIN MODEL --------------------------------------------------------

    print('Retraining model on full dataset')

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)

    model.fit(X, y, n_folds=n_folds, cv_shuffle=True)

    # -- PREDICT CLASSES FOR NEW DATA -----------------------------------------

    print('Classifying new data')

    y_pred = model.predict(measured_X) # Predict classes for measured TS(f)
    y_pred = le.inverse_transform(y_pred) # Transform labels back to species

    # -- OUTPUT RESULTS -------------------------------------------------------

    measured_X['Prediction'] = y_pred
    measured_X.to_pickle(path + unique_id + '_Predictions.pkl')

    with open(path + unique_id + '_BestParams.pkl', 'wb') as handle:
        pickle.dump(model.best_model(), handle)
    return

def main_classify_all(df, df_SED, df_track, df_trackavg, clf, unique_id,path, preprocessing=[],  ex_preprocessing=[], timeout=300, n_jobs=-1, max_evals=50, n_splits = 10, n_folds = 10):
    """
    Function to run nested cross validation then apply fit to whole dataset
    Uses F1 score instead of accuracy score, as the latter is inappropriate
    for multi-class classification.
    """

    # -- WRANGLE DATA ---------------------------------------------------------
    df_train = df.loc[:,:].copy()

    le = LabelEncoder() # Maps labels -> int (e.g. Atlantic cod -> 0, Polar cod -> 1)
    df_train['Species_le'] = le.fit_transform(df_train.Species)
    X = df_train.iloc[:,:-2].to_numpy() # Features, TS(f) only
    y = df_train.loc[:,'Species_le'].to_numpy() # Labels

# -- WRANGLE TEST DATA ---------------------------------------------------------
    measured_XSED = df_SED.copy()
    measured_Xtrack = df_track.copy()
    measured_Xtrackavg = df_trackavg.copy()

    # -- NESTED CROSS-VALIDATION ----------------------------------------------

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)

    nested_cv(X, y, model, n_splits, n_folds, unique_id, le, path)

    # -- RETRAIN MODEL --------------------------------------------------------

    print('Retraining model on full dataset')

    model = HyperoptEstimator(classifier = clf,
                              preprocessing = preprocessing,
                              ex_preprocs = ex_preprocessing,
                              algo = tpe.suggest,
                              trial_timeout = timeout,
                              loss_fn = f1_loss,
                              max_evals = max_evals,
                              n_jobs = n_jobs)

    model.fit(X, y, n_folds=n_folds, cv_shuffle=True)

    # -- PREDICT CLASSES FOR NEW DATA -----------------------------------------

    print('Classifying new data')

    y_predSED = model.predict(measured_XSED) # Predict classes for measured TS(f)
    y_predSED = le.inverse_transform(y_predSED) # Transform labels back to species

    y_predtrack = model.predict(measured_Xtrack) # Predict classes for measured TS(f)
    y_predtrack = le.inverse_transform(y_predtrack) # Transform labels back to species

    y_predtrackavg = model.predict(measured_Xtrackavg) # Predict classes for measured TS(f)
    y_predtrackavg = le.inverse_transform(y_predtrackavg) # Transform labels back to species

    # -- OUTPUT RESULTS -------------------------------------------------------

    measured_XSED['Prediction'] = y_predSED
    measured_Xtrack['Prediction'] = y_predtrack
    measured_Xtrackavg['Prediction'] = y_predtrackavg
    measured_XSED.to_pickle(path + unique_id + '_Predictions_SED.pkl')
    measured_Xtrack.to_pickle(path + unique_id + '_Predictions_track.pkl')
    measured_Xtrackavg.to_pickle(path + unique_id + '_Predictions_trackavg.pkl')

    with open(path + unique_id + '_BestParams.pkl', 'wb') as handle:
        pickle.dump(model.best_model(), handle)
    return

def read_results_all(unique_id ,classifypath):
    ' read the classifier results and predictions from the selected classifier'

    main_path = f'{classifypath}/{unique_id}'
    cv_path = '_NestedCV.pkl'
    best_params = '_BestParams.pkl'
    predSED_path = '_Predictions_SED.pkl'
    predtrack_path = '_Predictions_track.pkl'
    predtrackavg_path = '_Predictions_trackavg.pkl'

    # Load dataframes
    cv_df = pd.read_pickle(main_path + cv_path) # Nested CV results
    best_params = pd.read_pickle(main_path + best_params)
    predSED_df = pd.read_pickle(main_path+predSED_path)
    predtrack_df = pd.read_pickle(main_path+predtrack_path)
    predtrackavg_df = pd.read_pickle(main_path+predtrackavg_path)

    return {'name':unique_id, 'cv_df':cv_df, 'best_params':best_params,  'predSED_df':predSED_df, 'predtrack_df':predtrack_df, 'predtrackavg_df':predtrackavg_df,}


import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
'''
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['mape'])
history = model.fit(X_train, y_train, epochs=10, batch_size=16)           # default: epochs=1, batch_size=32
print(hisotry.history.keys())                   # dict_keys(['loss', 'mape', 'val_loss', 'val_mape'])
'''
# X_train: numpy array: (num_samples, num_features)
# y_train: numpy array: (num_samples, num_classes)  (categorical_crossentropy) (label is a vector)
# y_train: numpy array: (num_samples, )  (sparse_categorical_crossentropy)  (label is a single integer)
# verbose: 0 (silent), 1 (all), 2 (one line for one epoch)

''' Example '''
from tensorflow.keras.preprocessing import image
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# 1. load data
fashion_mnist_data = tf.keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist_data.load_data()
print(train_images.shape)
print(train_labels[0])

train_images = train_images / 255
test_images = test_images / 255

labels = [
    'T-shirt/top',
    'Trouser',
    'Pullover',
    'Dress',
    'Coat',
    'Sandal',
    'Shirt',
    'Sneaker',
    'Bag',
    'Ankle boot'
]
i=0
img=train_images[i,:,:]
plt.imshow(img)
print("img: \n", img)

# 2. Build model
model = Sequential([
    Conv2D(16, (3,3), activation='relu' ,input_shape=(28, 28, 1)),
    MaxPooling2D((3, 3)),
    Flatten(),
    Dense(10, activation='softmax')
])
model.summary()

# 3. Compile and train
opt = tf.keras.optimizers.Adam(learning_rate=0.005)
acc = tf.keras.metrics.SparseCategoricalAccuracy()
mae = tf.keras.metrics.MeanAbsoluteError()
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=[acc, mae])

history = model.fit(train_images[...,np.newaxis], train_labels, epochs=8, batch_size=256)       # need a channel_dim
# X_train: numpy array: (num_samples, num_features)
# y_train: numpy array: (num_samples, num_classes)  (categorical_crossentropy) (label is a vector)
# y_train: numpy array: (num_samples, )  (sparse_categorical_crossentropy)  (label is a single integer)
# verbose: 0 (silent), 1 (all), 2 (one line for one epoch)

# 4. Plot
df = pd.DataFrame(history.history)
print(df.head())

loss_plot = df.plot(y='loss', title="Loss vs. Epoch", legend=False)
loss_plot = df.plot(y='mean_absolute_error', title="Mae vs. Epoch", legend=False)
loss_plot.set(xlabel='Epochs', ylabel='Loss')
plt.show()


#Filename:getZhihu.py
import re,os
import urllib2
from bs4 import BeautifulSoup
import sys
import time

reload(sys)
sys.setdefaultencoding("utf-8")

def getHtml(url):
    header={'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1','Referer' : '******'}
    request=urllib2.Request(url,None,header)
    response=urllib2.urlopen(request)
    text=response.read()
    return text

def mkDir():
    date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
    os.mkdir(str(date))

def saveText(text):
    date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
    dir_name="/home/wang/Documents/py/Zhihu/"+date
    soup=BeautifulSoup(text)
#    i=1
#    for i in soup.h2:
#        i=i+1
    if soup.h2.get_text()=='':
        filename=dir_name+"/ad.txt"
        fp=file(filename,'w')
        content=soup.find('div',"content")
        content=content.get_text()
        fp.write(content)
        fp.close()

#    elif i > 1:
#        filename=dir_name+"/kiding.txt"
#        contents=soup.findAll('div',"content")+soup.findAll("div","question")
#        contents=contents.get_text()
#        fp=file(filename,'w')
#        fp.write(contents)
#        fp.close()

    else:
        filename=dir_name+"/"+soup.h2.get_text()+".txt"
        fp=file(filename,'w')
        content=soup.find('div',"content")
        content=content.get_text()
        fp.write(content)
        fp.close()

#   print content #test

def getUrl(url):
    html=getHtml(url)

#   print html
    soup=BeautifulSoup(html)
    urls_page=soup.find('div',"post-body")
#   print urls_page

    urls=re.findall('"((http)://.*?)"',str(urls_page))
    return urls

def main():
    mkDir()
    page="http://zhihudaily.ahorn.me"
    urls=getUrl(page)
    for url in urls:
        text=getHtml(url[0])
        saveText(text)

if __name__=="__main__":
    main()

#url="http://daily.zhihu.com/story/4671450"
#url="http://zhihudaily.ahorn.me"
#text=getHtml(url)
#saveText(text)
#getUrl(url)


largest = None
smallest = None
while True:
	try:
		num = input("Enter a number: ")
		if num == "done" : break
		fnum = int(num)
		if largest < fnum :
			largest = fnum
		if smallest == None or smallest > fnum:
			smallest = fnum
	except:
		print("Invalid input")

print("Maximum is", largest)
print("Minimum is", smallest)


import asyncio
import random

async def send_show(device, command, semaphore):
    print("^^^ Waiting")
    async with semaphore:
        print(f">>> Connect {device}")
        await asyncio.sleep(random.random() * 3)
        print(f"<<< Done    {device}")

async def main():
    s = asyncio.Semaphore(10)
    coroutines = [send_show(i, "sh clock", s) for i in range(50)]
    await asyncio.gather(*coroutines)

if __name__ == "__main__":
    asyncio.run(main())


from rest_framework import serializers
from account.models import User, Plan, ConfirmationCode, SerialNumber, Opt

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = (
            'id', 'full_name',
            'phone',
        )

class ConfirmationCodeSerializer(serializers.ModelSerializer):
    serial_number = serializers.SlugRelatedField(slug_field='number', read_only=True)

    class Meta:
        model = ConfirmationCode
        fields = (
            'id', 'serial_number', 'code',
            'expiry', 'activation_date',
        )

class SerialNumberSerializer(serializers.ModelSerializer):
    user = serializers.SlugRelatedField(slug_field='full_name', read_only=True)
    # code = serializers.SerializerMethodField()
    code = ConfirmationCodeSerializer(many=False)

    class Meta:
        model = SerialNumber
        fields = (
            'id', 'number', 'user',
            'created_at', 'days_charge', 'code'
        )

    # def create(self, validated_data):
    #     """
    #     Overriding the default create method of the Model serializer.
    #     :param validated_data: data containing all the details of student
    #     :return: returns a successfully created student record
    #     """
    #     user_data = validated_data.pop('user')
    #     user = UserSerializer.create(UserSerializer(), validated_data=user_data)
    #     student, created = UnivStudent.objects.update_or_create(user=user,
    #                         subject_major=validated_data.pop('subject_major'))
    #     return student

class PlanSerializer(serializers.ModelSerializer):
    class Meta:
        model = Plan
        fields = (
            'id', 'title', 'price', 'days',
        )


import pandas as pd
import sys
import pickle

def main():
    df = pd.read_csv(sys.argv[1])
    phecodes = pd.read_csv(sys.argv[2], dtype=str)
    clf =  pickle.load(open(sys.argv[3], 'rb'))
    phe_list = list(phecodes.PHECODE.unique())
    #predict phe_list of df using clf
    predicted_probs = clf.predict_proba(df[phe_list])
    df['control_prob'] = predicted_probs[:, 0]
    df['case_prob'] = predicted_probs[:, 1]
    #Save predictions to file

if __name__=='__main__':
    main()


import logging
from fastapi import FastAPI
import azure.functions as func
from routers import applicant
from fastapi.middleware.cors import CORSMiddleware

app = FastAPI(title="Resume Parser API on Azure Functions")
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"]
)
app.include_router(applicant.router)

def main(req: func.HttpRequest, context:func.Context) -> func.HttpResponse:
    logging.info('Python HTTP trigger function processed a request.')
    return func.AsgiMiddleware(app).handle(req, context)


from day7_data import data

# part one

bags = dict()
for line in data.split('\n'):

    bag, content = line.split(' bags contain ')

    if content == "no other bags.":
        content = []

    else:
        content = [(c.strip().split(" ")[:-1]) for c in content.split(",")]
        content = [[" ".join(c[1:3]), c[0]] for c in content]

    bags[bag] = content

new_shiny_set = {'shiny gold'}
shiny_bags = set()

while new_shiny_set:

    old_shiny_set = new_shiny_set
    new_shiny_set = set()

    for bag, content in bags.items():
        for c in content:
            if c[0] in old_shiny_set:
                new_shiny_set.add(bag)

    shiny_bags = shiny_bags.union(new_shiny_set)

    for k in new_shiny_set:
        del bags[k]

print(len(shiny_bags - {'shiny gold'}))

# part two

def get_bag_count(bag):
    n = 0
    for b, k in bags[bag]:
        b_in = get_bag_count(b)
        n += int(k) * (1 + b_in)
    return n

print(get_bag_count("shiny gold"))


from PIL import Image
import numpy as np
from pathlib import Path
import os, webbrowser

img_path = Path("./test_set/data/")
save_path = Path("./test_set/eval/")

for i in range(20):
    print("Processing pic " + str(i))
    x_img = Image.open(img_path / 'x' / (str(i) + '_x.png'))
    y_img = Image.open(img_path / 'y' / (str(i) + '_y.png'))

    y_np = np.array(y_img.convert('RGBA'))

    y_np[:,:, 3] = (255 * (y_np[:, :, :3] != 255).any(axis=2)).astype(np.uint8)/4

    #y_np = np.where(y_np == [255, 255, 255], [0,0,0,255], y_np)

    y_img = Image.fromarray(y_np, mode='RGBA')
    x_img.convert('RGBA')
    #y_img = Image.blend(x_img, y_img, 0.25)

    comb_img = Image.alpha_composite(x_img, y_img)
    path = (save_path / (str(i) + "_comb.png"))
    comb_img.save(path, format='PNG')


import datetime
import json
from multiprocessing import Pool

import pandas as pd
import requests
from tqdm import tqdm

def write_log(message):
    with open('/tmp/chrome_extensions.log', 'a') as f:
        f.write(f'{datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")} -- {message}\n')

class ChromeExtension(object):
    def __init__(self, ext_id):
        self.ext_id = ext_id
        self.report = None

    @staticmethod
    def get_headers():
        with open('apikey.txt', 'r') as f:
            return {'API-Key': f.read().strip()}

    def submit_for_scan(self):
        api_url = 'https://api.crxcavator.io/v1/submit'
        try:
            response = requests.post(
                api_url,
                headers=self.get_headers(),
                json={'extension_id': f'{self.ext_id}'}
            )
            if response.status_code == 200:
                # write_log(f'Successfully submitted extension {self.ext_id} for scanning.')
                return True
            else:
                write_log(f'ERROR: Status code [{response.status_code}] while submitting extension: {self.ext_id}'
                          f'\r\n\t\tResponse:{response.text}')
                return False
        except requests.exceptions.ConnectionError as e:
            write_log(f'Connection error received when submitting {self.ext_id} to be scanned, trying again.')
            return self.submit_for_scan()

    def crxcavator_lookup(self):
        # Lookup the risk score in crxcavator as it has everything that I planned to do and more.
        api_url = f'https://api.crxcavator.io/v1/report/{self.ext_id}'
        try:
            response = requests.get(
                api_url,
                headers=self.get_headers()
            )
            if response.status_code == 200 and response.json():
                self.report = response.json()[-1]
            else:
                self.report = {}
                self.submit_for_scan()
        except requests.exceptions.ConnectionError as e:
            write_log(f'Connection error received when querying {self.ext_id}, trying again.')
            self.crxcavator_lookup()

    def get_risk_info(self, submit=False):
        if self.report is None:
            self.crxcavator_lookup()
        if self.report == {}:
            write_log(f'No data for extension {self.ext_id}')
            if submit is not False:
                self.submit_for_scan()
            return {'extension_id': self.ext_id, 'name': 'No data available.'}
        risk = self.report['data']['risk'] if 'risk' in self.report['data'].keys() else None
        report_data = {
            'extension_id': self.ext_id,
            'version': self.report['version'],
            'total_risk': risk['total'] if (risk and ('total' in risk.keys())) else None,
            'URLs': self.report['data']['extcalls'] if 'extcalls' in self.report['data'].keys() else None,
            'dangerous_fns': list(self.report['data']['dangerousfunctions'].keys()) if 'dangerousfunctions' in
                                                                                self.report['data'].keys() else None,
            'entrypoints': self.report['data']['entrypoints'] if 'entrypoints' in self.report['data'].keys() else None,
            'csp_risk': risk['csp']['total'] if 'csp' in risk.keys() else None,
            'permissions_risk': risk['permissions']['total'] if 'permissions' in risk.keys() else None,
            'webstore_risk': risk['webstore']['total'] if 'webstore' in risk.keys() else None,
            'manifest': self.report['data']['manifest'] if 'manifest' in self.report['data'].keys() else None
        }
        webstore = self.report['data']['webstore']
        if webstore:
            report_data.update({
                'name': webstore['name'],
                'short_description': webstore['short_description'],
                'version': webstore['version'],  # available in manifest and base object
                'last_updated': webstore['last_updated'],
                'offered_by': webstore['offered_by'],  # Looks worse than mine
                'rating': webstore['rating'],
                'rating_users': webstore['rating_users'],
                'users': webstore['users'],
                'size': webstore['size'],
                'type': webstore['type'],
                'permission_warnings': webstore['permission_warnings'],
                'email': webstore['email'],
                'address': webstore['address'],
                'privacy_policy': webstore['privacy_policy']
            })
        else:
            report_data.update({
                'name': None,
                'short_description': None,
                'version': None,  # available in manifest and base object
                'last_updated': None,
                'offered_by': None,  # Looks worse than mine
                'rating': None,
                'rating_users': None,
                'users': None,
                'size': None,
                'type': None,
                'permission_warnings': None,
                'email': None,
                'address': None,
                'privacy_policy': None
            })
        return report_data

class ExtensionList(object):
    def __init__(self, ext_ids, pool_size=20):
        self.extensions = [ChromeExtension(ext_id) for ext_id in ext_ids]
        self.pool_size = pool_size

    def run_async(self, func, data, progress=True, task=None):
        with Pool(self.pool_size) as p:
            if progress:
                return_data = list(tqdm(p.imap(func, data), total=len(data), desc=task))
            else:
                return_data = list(p.imap(func, data))
        return return_data

    def get_extension_data(self):
        print(f'#### Getting CRXcavator data for {len(self.extensions)} extensions ####')
        self.extensions = self.run_async(self._lookup, self.extensions, progress=True, task='API Queries')

    @staticmethod
    def _lookup(extension) -> ChromeExtension:
        extension.crxcavator_lookup()
        return extension

    def get_data_as_df(self):
        return pd.DataFrame(self.run_async(self._export_data, self.extensions, progress=True, task='Getting CRX data'))

    @staticmethod
    def _export_data(extension) -> ChromeExtension:
        return extension.get_risk_info()

    def submit_for_scans(self):
        results = self.run_async(self._scan_crx, self.extensions, progress=True, task='Submitting CRX for scanning.')
        print(f'Submission Successes: {len([x for x in results if x is True])}')
        print(f'Submission Failures: {len([x for x in results if x is False])}')

    @staticmethod
    def _scan_crx(extension) -> ChromeExtension:
        return extension.submit_for_scan()

    def to_csv(self, csv_path):
        ext_df = self.get_data_as_df()
        columns = [
            'extension_id', 'name', 'short_description', 'version', 'last_updated', 'total_risk',
            'offered_by', 'rating', 'rating_users', 'users', 'size', 'type', 'permission_warnings', 'email',
            'address', 'privacy_policy', 'entrypoints', 'csp_risk', 'permissions_risk',
            'webstore_risk'
        ]
        ext_df.to_csv(csv_path, columns=columns, index=False, encoding='utf-8')