from inspect import isgenerator
import json
from uuid import UUID

from fluiddb.cache.cache import BaseCache, CacheResult
from fluiddb.cache.factory import CachingAPIFactory
from fluiddb.data.user import User, Role
from fluiddb.model.user import getUser, UserAPI

def cachingGetUser(username):
    """Get a L{User} for a given username.

    The L{User}s will be fetched from the cache, otherwise it will be fetched
    directly from the database. Cache misses will be added to the cache. See
    L{fluiddb.model.user.getUser}.

    @param username: The username of the user.
    @return: A L{User} instance corresponding to the given username.
    """
    cache = UserCache()
    cachedResult = cache.get(username)
    if cachedResult.results is not None:
        return cachedResult.results
    else:
        user = getUser(username)
        if user is not None:
            cache.save(user)
        return user

class CachingUserAPI(object):
    """The public API to secure L{User}-related functionality.

    @param user: The L{User} to perform operations on behalf of.
    """

    def __init__(self):
        self._api = UserAPI(factory=CachingAPIFactory())

    def create(self, values, createPrivateNamespace=None):
        """See L{UserAPI.create}."""
        return self._api.create(values, createPrivateNamespace)

    def delete(self, usernames):
        """See L{UserAPI.delete}."""
        if isgenerator(usernames):
            usernames = list(usernames)
        cache = UserCache()
        for username in usernames:
            cache.clear(username)
        return self._api.delete(usernames)

    def get(self, usernames):
        """See L{UserAPI.get}."""
        return self._api.get(usernames)

    def set(self, values):
        """See L{UserAPI.set}."""
        cache = UserCache()
        for username, password, fullname, email, role in values:
            cache.clear(username)
        return self._api.set(values)

class UserCache(BaseCache):
    """Provides caching functions for the L{getUser} function."""

    keyPrefix = u'user:'

    def get(self, username):
        """Get a L{User} object from the cache.

        @param username: The username of the L{User} as C{unicode}.
        @return: A L{CacheResult} object with the L{User} in the L{results}
            field if it's found or the username in the C{uncachedValues}
            field if it's not found.
        """
        result = self.getValues([username])
        if result is None or result == [None]:
            return CacheResult(None, username)

        userdict = json.loads(result[0])

        user = User(username=userdict['username'],
                    passwordHash=str(userdict['passwordHash']),
                    fullname=userdict['fullname'],
                    email=userdict['email'],
                    role=Role.fromID(userdict['role']))
        user.id = userdict['id']
        user.objectID = UUID(userdict['objectID'])

        return CacheResult(user, None)

    def save(self, user):
        """Store a L{User} in the cache.

        @param user: A L{User} object to save in the cache.
        """
        userdict = {'id': user.id,
                    'objectID': str(user.objectID),
                    'username': user.username,
                    'passwordHash': user.passwordHash,
                    'fullname': user.fullname,
                    'email': user.email,
                    'role': user.role.id}

        self.setValues({user.username: json.dumps(userdict)})

    def clear(self, username):
        """Delete a L{User} from the cache.

        @param username: The username of the L{User} as C{unicode}.
        """
        self.deleteValues([username])


import requests
import json
import os
import time,datetime

def updateJsonFile( path, data ):
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

# Actual temperature
waterData = {}
url = 'https://data.bs.ch/api/v2/catalog/datasets/100046/records?order_by=startzeitpunkt%20DESC&limit=2&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

if d['records'][0]['record']['fields']['rus_w_o_s3_te'] is not None:
    waterData['actualValue'] = d['records'][0]['record']['fields']['rus_w_o_s3_te']
    waterData['lastUpdate'] = d['records'][0]['record']['fields']['startzeitpunkt']
else:
    print('newest temp is null we take next older one')
    if d['records'][1]['record']['fields']['rus_w_o_s3_te'] is not None:
        waterData['actualValue'] = d['records'][1]['record']['fields']['rus_w_o_s3_te']
        waterData['lastUpdate'] = d['records'][1]['record']['fields']['startzeitpunkt']
    else:
        print('no valid temp')
        waterData['actualValue'] = 0
        waterData['lastUpdate'] = None

# Weekly data
waterData['chart'] = {}
waterData['chart']['week'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100046/records?select=avg(rus_w_o_s3_te)%20as%20temp&where=startzeitpunkt%3E%3Dnow(days%3D-7)&group_by=range(startzeitpunkt%2C%2012%20hour)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    if  e['record']['fields']['temp'] is not None:
        waterData['chart']['week'].append(e['record']['fields']['temp'])
        oldT = e['record']['fields']['temp']
    else:
        waterData['chart']['week'].append(oldT | 0)

# Monthly data
waterData['chart']['month'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100046/records?select=avg(rus_w_o_s3_te)%20as%20temp&where=startzeitpunkt%3E%3Dnow(days%3D-30)&group_by=range(startzeitpunkt,2days)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    if e['record']['fields']['temp'] is not None:
        waterData['chart']['month'].append(e['record']['fields']['temp'])
        oldT = e['record']['fields']['temp']
    else:
        waterData['chart']['month'].append(oldT | 0)

updateJsonFile( 'data/data/waterData.json', waterData)

# Air temperature
airData = {}
url = 'https://data.bs.ch/api/v2/catalog/datasets/100009/records?select=meta_airtemp%20as%20temp&where=name_original="0340A010"&limit=1&pretty=false&timezone=UTC&order_by=record_timestamp%20DESC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

try:
    d['records'][0]['record']['fields']['temp']
except NameError:
    print('air temp not defined')
    try:
        d['records'][1]['record']['fields']['rus_w_o_s3_te']
    except NameError:
        print('last air temp not defined')
        airData['actualValue'] = 0
        airData['lastUpdate'] = None
    else:
        airData['actualValue'] = d['records'][1]['record']['fields']['temp']
        airData['lastUpdate'] = d['records'][1]['record']['timestamp']
else:
    airData['actualValue'] = d['records'][0]['record']['fields']['temp']
    airData['lastUpdate'] = d['records'][0]['record']['timestamp']

# Weekly data
airData['chart'] = {}
airData['chart']['week'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100009/records?select=avg(meta_airtemp)%20as%20temp&where=name_original="0340A010"%20and%20record_timestamp%3E%3Dnow(days%3D-7)&group_by=range(record_timestamp%2C%206%20hour)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    try:
        e['record']['fields']['temp']
    except NameError:
        airData['chart']['week'].append(oldT | 0)
    else:
        airData['chart']['week'].append(e['record']['fields']['temp'])
        oldT = e['record']['fields']['temp']

# Monthly data
airData['chart']['month'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100009/records?select=avg(meta_airtemp)%20as%20temp&where=name_original="0340A010"%20and%20record_timestamp%3E%3Dnow(days%3D-30)&group_by=range(record_timestamp,2days)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    try:
        e['record']['fields']['temp']
    except NameError:
        airData['chart']['month'].append(oldT | 0)
    else:
        airData['chart']['month'].append(e['record']['fields']['temp'])
        oldT = e['record']['fields']['temp']

updateJsonFile( 'data/data/airData.json', airData)

# Water level
levelData = {}
url = 'https://data.bs.ch/api/records/1.0/analyze?dataset=100089&y.pegel.func=AVG&y.pegel.expr=pegel&precision=year&x=timestamp&sort=-x&exclude.pegel=0'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)
lastAvg = d[1]['pegel']

url = 'https://data.bs.ch/api/v2/catalog/datasets/100089/records?select=pegel&limit=1&pretty=false&timezone=UTC&order_by=record_timestamp%20DESC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)
try:
    d['records'][0]['record']['fields']['pegel']
except NameError:
    print('level not defined')
    levelData['actualValue'] = 0
    levelData['lastUpdate'] = None
else:
    levelData['actualValue'] = d['records'][0]['record']['fields']['pegel'] - lastAvg
    levelData['lastUpdate'] = d['records'][0]['record']['timestamp']

# Weekly data
levelData['chart'] = {}
levelData['chart']['week'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100089/records?select=avg(pegel)%20as%20pegel&where=record_timestamp%3E%3Dnow(days%3D-7)&group_by=range(record_timestamp%2C%206%20hour)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    try:
        e['record']['fields']['pegel']
    except NameError:
        levelData['chart']['week'].append(oldT | 0)
    else:
        levelData['chart']['week'].append(e['record']['fields']['pegel'])
        oldT = e['record']['fields']['pegel']

# Monthly data
levelData['chart']['month'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100089/records?select=avg(pegel)%20as%20pegel&where=record_timestamp%3E%3Dnow(days%3D-30)&group_by=range(record_timestamp,2days)%20as%20time&limit=900&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldT = ''
for e in d['records']:
    try:
        e['record']['fields']['pegel']
    except NameError:
        levelData['chart']['month'].append(oldT | 0)
    else:
        levelData['chart']['month'].append(e['record']['fields']['pegel'])
        oldT = e['record']['fields']['pegel']

updateJsonFile( 'data/data/levelData.json', levelData)

# Quality data
qualityData = {}

qualityData['quality'] = 2

# Global radiation
# Weekly data
globalRadiationData = {}
globalRadiationData['measure'] = 'globalRadiation'
globalRadiationData['chart'] = {}
globalRadiationData['chart']['week'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100254/records?select=gre000d0%20as%20globalRadiation&limit=7&pretty=false&timezone=UTC&order_by=date%20DESC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldD = ''
for e in d['records']:
    try:
        e['record']['fields']['globalRadiation']
    except NameError:
        globalRadiationData['chart']['week'].append(oldD | 0)
    else:
        globalRadiationData['chart']['week'].append(e['record']['fields']['globalRadiation'])
        oldD = e['record']['fields']['globalRadiation']

qualityData['lastUpdate'] = d['records'][0]['record']['timestamp']

globalRadiationLast72h = globalRadiationData['chart']['week'][0] + globalRadiationData['chart']['week'][1] + globalRadiationData['chart']['week'][2]
globalRadiationLast48h = globalRadiationData['chart']['week'][0] + globalRadiationData['chart']['week'][1]

# Monthly data
globalRadiationData['chart']['month'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100254/records?select=avg(gre000d0)%20as%20globalRadiation&order_by=date%20DESC&group_by=range(date,2days)%20as%20date&limit=15&pretty=false&timezone=UTC'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldD = ''
for e in d['records']:
    try:
        e['record']['fields']['globalRadiation']
    except NameError:
        globalRadiationData['chart']['month'].append(oldD | 0)
    else:
        globalRadiationData['chart']['month'].append(e['record']['fields']['globalRadiation'])
        oldD = e['record']['fields']['globalRadiation']

qualityData['data'] = []
qualityData['data'].append(globalRadiationData)

# Rain
# Weekly data
rainData = {}
rainData['measure'] = 'rain'
rainData['chart'] = {}
rainData['chart']['week'] = []
url = "https://data.bs.ch/api/v2/catalog/datasets/100009/records?select=max(meta_rain24h_sum)%20as%20rain&where=name_original=\"034001AF\"&limit=7&pretty=false&timezone=UTC&order_by=date%20DESC&group_by=date_format(dates_max_date, 'yyyyMMdd') as date"
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldD = ''
for e in d['records']:
    try:
        e['record']['fields']['rain']
    except NameError:
        rainData['chart']['week'].append(oldD | 0)
    else:
        rainData['chart']['week'].append(e['record']['fields']['rain'])
        oldD = e['record']['fields']['rain']

maxRainLast72h = rainData['chart']['week'][0]
maxRainLast48h = maxRainLast72h
if rainData['chart']['week'][1] > maxRainLast72h:
    maxRainLast72h = rainData['chart']['week'][1]
if rainData['chart']['week'][2] > maxRainLast72h:
    maxRainLast72h = rainData['chart']['week'][2]
if rainData['chart']['week'][1] > maxRainLast48h:
    maxRainLast48h = rainData['chart']['week'][1]
if rainData['chart']['week'][2] > maxRainLast48h:
    maxRainLast48h = rainData['chart']['week'][2]

# Monthly data
rainData['chart']['month'] = []
url = 'https://data.bs.ch/api/v2/catalog/datasets/100009/records?select=avg(meta_rain24h_sum)%20as%20rain&where=name_original=\"034001AF\"&limit=15&pretty=false&timezone=UTC&order_by=date%20DESC&group_by=range(dates_max_date,2days) as date'
try:
    resp = requests.get(url=url)
    d = resp.json()
except requests.exceptions.RequestException as e:
    raise SystemExit(e)

oldD = ''
for e in d['records']:
    try:
        e['record']['fields']['rain']
    except NameError:
        rainData['chart']['month'].append(oldD | 0)
    else:
        rainData['chart']['month'].append(e['record']['fields']['rain'])
        oldD = e['record']['fields']['rain']

qualityData['data'].append(rainData)

# Quality value
dailyRainTreshold = 2
dailyRadiationTreshold = 170

print(f'dailyRainTreshold: {dailyRainTreshold}')
print(f'dailyRadiationTreshold: {dailyRadiationTreshold}')
print(f'maxRainLast72h: {maxRainLast72h}')
print(f'maxRainLast48h: {maxRainLast48h}')
print(f'globalRadiationLast72h: {globalRadiationLast72h}')
print(f'globalRadiationLast48h: {globalRadiationLast48h}')

if maxRainLast72h < dailyRainTreshold and globalRadiationLast72h > ( dailyRadiationTreshold * 3 ):
    qualityData['quality'] = 3
elif maxRainLast48h < dailyRainTreshold and globalRadiationLast48h > ( dailyRadiationTreshold * 2 ):
    qualityData['quality'] = 2
else:
    qualityData['quality'] = 1

updateJsonFile( 'data/data/qualityData.json', qualityData)


# 将数据进行整理，分为训练数据90% 和测试数据10%

import pandas as pd
import os
dir = '/Users/icdi/Desktop/3'
train_text = []
train_label = []
test_text = []
test_label = []
name_to_index = {}
prop = 0.9
frame = pd.read_excel(dir+'/index_name.xls')

for key in frame.index:
    values = frame.loc[key].values
    name_to_index[values[1]] = values[0]

for file in os.listdir(dir):
    if(file.endswith('.txt')):
        name = file[:file.index('_')]
        name_content = open(dir + '/' + file).read().split('\n')[:-1]

        trainprop = int(prop*len(name_content))
        train_text.extend(name_content[:trainprop])
        test_text.extend(name_content[trainprop:])

        label = [str(name_to_index[name]) for i in range(len(name_content))]
        train_label.extend(label[:trainprop])
        test_label.extend(label[trainprop:])

# 将文本进行保存
train_text_file = open('train_text.txt','w')
train_text_file.write('\n'.join(train_text))

test_text_file = open('test_text.txt','w')
test_text_file.write('\n'.join(test_text))

train_label_file = open('train_label.txt','w')
train_label_file.write('\n'.join(train_label))

test_label_file = open('test_label.txt','w')
test_label_file.write('\n'.join(test_label))


import os
import sys
import argparse

class MyWC():
    def count_lines(self, data):
        print(data.count('\n'), end=' ')

    def count_words(self, data):
        print(len(data.split(None)), end=' ')

    def count_bytes(self, data):
        print(len(data.encode('utf-8')), end=' ')

    def counter(self, data, args):
        if args.l:
            self.count_lines(data)
        if args.w:
            self.count_words(data)
        if args.c:
            self.count_bytes(data)
        if args.l == args.w == args.c == False:
            self.count_lines(data)
            self.count_words(data)
            self.count_bytes(data)

    def file_open(self, path):
        with open(path) as f:
            data = f.read()
        return data

    def args_parser(self, data_input):
        parser = argparse.ArgumentParser(prog='wc.py')
        parser.add_argument('file', nargs='?', help='filename', default=sys.stdin)
        parser.add_argument('-l', action='store_true', help='print the new line counts')
        parser.add_argument('-w', action='store_true', help='print the word counts')
        parser.add_argument('-c', action='store_true', help='print the byte count')
        return parser.parse_args()

if __name__ == '__main__':
    data_input = sys.stdin
    wc = MyWC()
    args = wc.args_parser(data_input)
    filename = args.file

    if data_input.isatty():
        try:
            if os.path.isfile(filename):
                data = wc.file_open(filename)
                wc.counter(data, args)
                print('\t{}'.format(filename))
        except TypeError:
            data = sys.stdin.read()
            wc.counter(data, args)
            print('')
    else:
        data = filename.read()
        wc.counter(data, args)
        print('')


@profile
def foo():
	sum_count = 0
	for i in range(100000):
		for j in range(1000):
			sum_count += 1

if __name__ == '__main__':
	# from timer import Timer
	# with Timer() as t:
	# 	foo()
	# print t.secs
	foo()


from flask import Flask, request
from flask_restful import Resource, Api, reqparse
from flask_jwt import JWT, jwt_required
from security import authenticate, identity

app = Flask(__name__)
app.secret_key = "evgen"
api = Api(app)

jwt = JWT(app, authenticate, identity)

items = [
    {
        "name":"MyItem",
        "price":150.99
    }
]

class Items(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('price',
            type=float,
            required=True,
            help="This field cannot be left blank!"
        )

    @jwt_required()
    def get(self, name):
        item = next(filter(lambda x : x['name'] == name, items), None)
        if item:
            return item, 200
        return {"message":"Item not found"}, 404

    def post(self, name):
        if next(filter(lambda x : x['name'] == name, items), None):
            return {"message" : "This item is already  here"}, 400
        data = Item.parser.parse_args()
        item = {"name":data['name'], "price":data['price']}
        items.append(item)
        return item, 201

    def delete(self, name):
        global items
        items = list(filter(lambda x: x['name'] != name, items), None)
        return {"message":"Item deleted"}, 200

    def put(self, name):
        data = Item.parser.parse_args()
        #data = request.get_json()
        item = next(filter(lambda x : x['name'] == name, items), None)
        if item is None:
            item = {'name': name, 'price' :data['price']}
            items.append(item)
            return item, 201
        else:
            item.update(data)
        return item

class ItemList(Resource):
    @jwt_required()
    def get(self):
        return {"items" : items} , 200

api.add_resource(Items, '/item/<string:name>')
api.add_resource(ItemList, '/items')
app.run(port = 8080, debug = True)


import math
import traci
import json

class SumoObject(object):

    def __init__(self, SumoID):
        self.ID = str(SumoID)  # VehicleID
        try:

            self.height = traci.vehicle.getHeight(self.ID)

            self.ObjType = traci.vehicle.getTypeID(self.ID)
            self.Route = traci.vehicle.getRouteID(self.ID)
            self.Edge = traci.vehicle.getRoadID(self.ID)
            self.Length = traci.vehicle.getLength(self.ID)
            self.Width = traci.vehicle.getWidth(self.ID)

            self.__CalculateSizeClass() #Vehicle size category
            if traci.vehicle.getSignals(self.ID) & 8 == 8: #Bitmask - 8 for brake light
                self.StBrakePedal = True
            else:
                self.StBrakePedal = False

            tmp_pos = traci.vehicle.getPosition(self.ID)  # position: x,y
            self.PosX_FrontBumper = tmp_pos[0] # X position (front bumper, meters)
            self.PosY_FrontBumper = tmp_pos[1] # Y position (front bumper, meters)
            self.Velocity = traci.vehicle.getSpeed(self.ID)
            self.Heading = traci.vehicle.getAngle(self.ID)
            self.Color = traci.vehicle.getColor(self.ID)

            self.__CalculateCenter() #self.PosX_Center, self.PosY_Center (center, meters)

            self.vehicleClass = traci.vehicle.getVehicleClass(self.ID)

        except:
            print("Error creating container for SUMO vehicle: ", self.ID)

    def UpdateVehicle(self):

        if traci.vehicle.getSignals(self.ID) & 8 == 8: #Bitmask - 8 for brake light
            self.StBrakePedal = True
        else:
            self.StBrakePedal = False

        tmp_pos = traci.vehicle.getPosition(self.ID)  # position: x,y
        self.PosX_FrontBumper = tmp_pos[0]  # X position (front bumper, meters)
        self.PosY_FrontBumper = tmp_pos[1]  # Y position (front bumper, meters)
        self.Velocity = traci.vehicle.getSpeed(self.ID)
        self.Heading = traci.vehicle.getAngle(self.ID)
        self.Edge = traci.vehicle.getRoadID(self.ID)

        self.__CalculateCenter()  # self.PosX_Center, self.PosY_Center (center, meters)

    def ReinsertVehicle(self):

        LaneIndex = 1  # dummy
        KeepRouteMode = 1  # KeepRoute: 2 = Free move.

        try:
            traci.vehicle.add(self.ID, self.Route)  # Try to put it back
        except:
            pass #If already there, do nothing
        try:
            traci.vehicle.moveToXY(self.ID, self.Edge, LaneIndex, self.PosX_FrontBumper, self.PosY_FrontBumper, self.Heading, KeepRouteMode)
            traci.vehicle.setSpeed(self.ID, self.Velocity)
        except:
            print("Error reinserting SUMO vehicle: ", self.ID)

    #Transforms SUMO X-Y coordinate system to GPS
    def __TransformGPS(self, PosX, PosY):
        Lon, Lat = traci.simulation.convertGeo(PosX, PosY)
        return Lon, Lat

    # Transforms front bumper position to center
    def __CalculateCenter(self):
        self.PosX_Center = self.PosX_FrontBumper - (math.sin(math.radians(self.Heading)) * (self.Length / 2))
        self.PosY_Center = self.PosY_FrontBumper - (math.cos(math.radians(self.Heading)) * (self.Length / 2))

    # Defines vehicle category based on its size
    def __CalculateSizeClass(self):

        if self.Length < 1:
            self.SizeClass = 1 #pedestrian (workaround)
        elif  self.Length < 4:
            self.SizeClass = 11 #Small car
        elif self.Length < 5:
            self.SizeClass = 12 #Medium car
        else:
            self.SizeClass = 13 #Large car

    def updateFromUnity(self, data):
        self.PosX_Center = data["position"][0]
        self.PosY_Center = data["position"][1]
        self.Heading = data["heading"]

        self.calculateBumper()

    def calculateBumper(self):
        self.PosX_FrontBumper = self.PosX_Center + (math.sin(math.radians(self.Heading)) * (self.Length / 2))
        self.PosY_FrontBumper = self.PosY_Center + (math.cos(math.radians(self.Heading)) * (self.Length / 2))

    def jsonDeleteData(self):
        data = {}

        data["vehicleId"] = self.ID

        return json.dumps(data)

    def jsonInitData(self):
        data = {}

        data["vehicleId"] = self.ID
        data["sizeClass"] = self.SizeClass
        data["vehicleClass"] = self.vehicleClass
        data["colorHex"] = '#%02x%02x%02x' % self.Color[0:3]
        data["length"] = self.Length
        data["width"] = self.Width
        data["height"] = self.height

        return json.dumps(data)

    def jsonUpdateData(self):
        data = {}

        data["vehicleId"] = self.ID
        data["speed"] = self.Velocity
        data["heading"] = self.Heading
        data["position"] = [self.PosX_Center, self.PosY_Center]
        data["brake"] = self.StBrakePedal
        data["currentEdge"] = self.Edge

        return json.dumps(data)


import modules.pumpingsystem as ps
import numpy as np

schedule = np.array([[70, 70, 70],
                     [75, 75, 75]])

pump_system = ps.PumpSystem('D1')
pump_system.add_level(ps.PumpingLevel('0', 1000000, 0, 15, 0, schedule, 0, 20))

pump_system.perform_simulation('1-factor', save=True)


# Swap and maximize
# Given an array, consider array as a circular array and the task is to
# find max sum of abs diff between consecutive ele with rearrangements

def maxSum(arr, n):

    arr.sort()

    min_ele = 0
    max_ele = n-1

    ans = 0

    while min_ele < max_ele:

        ans += abs(arr[min_ele] - arr[max_ele])

        if min_ele <= (n - 1) - max_ele:
            min_ele += 1
        else:
            max_ele -= 1

    ans += (arr[max_ele] - arr[0])

    return ans

print(maxSum([4, 2, 1, 8], 4))


def solution(n, words):
    answer = []
    wrong = False

    stack = []
    for i in words:
        if len(stack) < 1:
            stack.append(i)
            continue
        if i in stack or stack[len(stack)-1][len(stack[len(stack)-1])-1] != i[0]:
            stack.append(i)
            wrong = True
            break
        stack.append(i)
    if not wrong:
        return [0, 0]
    answer.append(len(stack)%n)
    if answer == [0]:
        answer[0] = n
    if len(stack)%n == 0:
        answer.append(len(stack)//n)
    else:
        answer.append(len(stack)//n+1)
    return answer

print(solution(3, ["tank", "kick", "know", "wheel", "land", "dream", "mother", "robot", "tank"]))
print(solution(5, ["hello", "observe", "effect", "take", "either", "recognize", "encourage", "ensure", "establish", "hang", "gather", "refer", "reference", "estimate", "executive"]))
print(solution(2, ["hello", "one", "even", "never", "now", "world", "draw"]))
print(solution(2, ["hello", "two"]))


from typing import Dict, NamedTuple
from enum import Enum

import sip
from PyQt5.QtWidgets import QDialogButtonBox, QScrollArea

from ..widget import SEToolsWidget

class AnalysisSection(Enum):

    """Groupings of analysis tabs"""

    Analysis = 1
    Components = 2
    General = 3
    Labeling = 4
    Other = 5
    Rules = 6

TAB_REGISTRY: Dict[str, type] = {}

class TabRegistry(sip.wrappertype):

    """
    Analysis tab registry metaclass.  This registers tabs to be used both for
    populating the content of the "choose analysis" dialog and also for
    saving tab/workspace info.
    """

    def __new__(cls, *args, **kwargs):
        classdef = super().__new__(cls, *args, **kwargs)

        clsname = args[0]
        attributedict = args[2]
        if clsname != "AnalysisTab":
            assert "section" in attributedict, "Class {} is missing the section value, " \
                "this is an setools bug".format(clsname)

            assert "tab_title" in attributedict, "Class {} is missing the tab_title value, " \
                "this is an setools bug".format(clsname)

            assert "mlsonly" in attributedict, "Class {} is missing the mlsonly value, " \
                "this is an setools bug".format(clsname)

            # ensure there is no duplication of class name or title
            for existing_tabname, existing_class in TAB_REGISTRY.items():
                if existing_tabname == clsname:
                    raise TypeError("Analysis tab {} conflicts with registered tab {}, "
                                    "this is an setools bug".format(clsname, existing_tabname))

                if existing_class.tab_title == attributedict["tab_title"]:
                    raise TypeError("Analysis tab {}'s title \"{}\" conflicts with registered tab "
                                    "{}, this is an setools bug.".
                                    format(clsname, attributedict["tab_title"], existing_tabname))

            TAB_REGISTRY[clsname] = classdef

        return classdef

# pylint: disable=invalid-metaclass
class AnalysisTab(SEToolsWidget, QScrollArea, metaclass=TabRegistry):

    """Base class for Apol analysis tabs."""

    # A QButtonBox which has an Apply button
    # for running the analysis.
    buttonBox = None

    # The set of tab fields that are in error
    errors = None

    # Normal and error palettes to use
    orig_palette = None
    error_palette = None

    #
    # Tab error state
    #
    def set_criteria_error(self, field, error):
        """Set the specified widget to an error state."""
        field.setToolTip("Error: {0}".format(error))
        field.setPalette(self.error_palette)
        self.errors.add(field)
        self._check_query()

    def clear_criteria_error(self, field, tooltip):
        """Clear the specified widget's error state."""
        field.setToolTip(tooltip)
        field.setPalette(self.orig_palette)
        self.errors.discard(field)
        self._check_query()

    def _check_query(self):
        button = self.buttonBox.button(QDialogButtonBox.StandardButton.Apply)
        enabled = not self.errors
        button.setEnabled(enabled)
        button.setToolTip("Run the analysis." if enabled else "There are errors in the tab.")

    #
    # Save/Load tab
    #
    def save(self):
        raise NotImplementedError

    def load(self, settings):
        raise NotImplementedError

    #
    # Results runner
    #
    def run(self, button):
        raise NotImplementedError

    def update_complete(self, count):
        raise NotImplementedError


import argparse
import logging
import sys
from pathlib import Path

from maple_test.utils import ALL, BASE_DIR, OS_SEP
from maple_test.utils import complete_path, read_config, get_config_value, is_relative

TEST_CONFIG = {}
LOGGER = None
LOG_CONFIG = {}

def parse_args():
    parser = argparse.ArgumentParser()
    test_framework_parser = parser.add_argument_group("Test FrameWork arguments")
    test_framework_parser.add_argument(
        "--cfg",
        default=Path(BASE_DIR / "maple_test.cfg"),
        type=complete_path,
        help="Test framework configuration file",
    )
    test_framework_parser.add_argument(
        "-j",
        type=bigger_than_one_integer,
        metavar="<mum>",
        dest="processes",
        default=1,
        help="Run <num> cases in parallel",
    )
    test_framework_parser.add_argument(
        "--retry",
        metavar="<num>",
        default=0,
        type=bigger_than_one_integer,
        help="Re-run unsuccessful test cases",
    )
    test_framework_parser.add_argument(
        "--output",
        metavar="<file>",
        type=complete_path,
        help="Store test result at <file>",
    )
    test_framework_parser.add_argument(
        "--xml_output",
        metavar="<file>",
        type=complete_path,
        help="Store test result as xunit xml format at <file>",
    )
    test_framework_parser.add_argument(
        "--json_output",
        metavar="<file>",
        type=complete_path,
        help="Store test result as josn format at <file>",
    )
    test_framework_parser.add_argument(
        "--debug", action="store_true", help="keep test temp file"
    )
    test_framework_parser.add_argument(
        "--fail_exit",
        action="store_true",
        help="Execute test framework with a non-zero exit code if any tests fail",
    )
    test_framework_parser.add_argument(
        "-p",
        type=str,
        dest="print_type",
        action="append",
        default=[],
        choices=ALL[:],
        help="Print test cases with specified results, "
        "-pPASS -pFAIL, to print all test case that failed or passed， "
        "UNRESOLVED test case results are not displayed by default.",
    )
    test_framework_parser.add_argument(
        "--progress",
        choices=["silent", "normal", "no_flush_progress"],
        default="normal",
        help="set progress type, silent: Don't show progress, "
        "normal: one line progress bar, update per second,"
        "no_flush_progress: print test progress per 10 seconds",
    )

    test_suite_parser = parser.add_argument_group("Test Suite arguments")
    test_suite_parser.add_argument(
        "test_paths", nargs="*", type=complete_path, help="Test suite path",
    )
    test_suite_parser.add_argument(
        "--test_cfg",
        metavar="<TEST_CFG_FILE>",
        type=complete_path,
        help="test suite config file, "
        "needed when run a single case or with --test_list",
    )
    test_suite_parser.add_argument(
        "--test_list",
        metavar="<TEST_LIST_FILE>",
        type=complete_path,
        default=None,
        help="testlist path for filter test cases",
    )
    test_suite_parser.add_argument(
        "-c",
        "--config_set",
        action="append",
        dest="user_config_set",
        default=[],
        metavar="config set path",
        help="Run a test set with the specified config set path",
        type=complete_path,
    )
    test_suite_parser.add_argument(
        "-C",
        "--config",
        dest="user_config",
        metavar="key=value",
        help="Add 'key' = 'val' to the user defined configs",
        type=str,
        action=StoreDictKeyPair,
        default={},
    )
    test_suite_parser.add_argument(
        "-E",
        "--env",
        dest="user_env",
        metavar="key=value",
        help="Add 'key' = 'val' to the user defined environment variable",
        action=StoreDictKeyPair,
        default={},
    )

    running_parser = parser.add_argument_group("Running arguments")
    running_parser.add_argument(
        "--temp_dir",
        metavar="<TEMP_DIR_PATH>",
        type=complete_path,
        default=None,
        help="Location for test execute. ",
    )
    running_parser.add_argument(
        "--timeout", type=float, dest="timeout", default=600, help="test case timeout"
    )

    log_parser = parser.add_argument_group("Log arguments")
    log_parser.add_argument(
        "--log_dir",
        metavar="<LOG_DIR_FILE_PATH>",
        type=complete_path,
        default=None,
        help="Where to store test log",
    )
    log_parser.add_argument(
        "--log_level",
        "-l",
        type=get_level_name,
        default=None,
        help="set log level from: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET",
    )
    log_parser.add_argument(
        "--verbose", action="store_true", help="enable verbose output"
    )
    log_parser.add_argument(
        "--fail-verbose",
        dest="fail_verbose",
        action="store_true",
        help="enable not pass testcase verbose output",
    )

    args = parser.parse_args()

    test_framework_config = {
        "cfg": args.cfg,
        "processes": args.processes,
        "retry": args.retry,
        "output": args.output,
        "xml_output": args.xml_output,
        "json_output": args.json_output,
        "debug": args.debug,
        "fail_exit": args.fail_exit,
        "print_type": args.print_type,
        "progress": args.progress,
        "fail_verbose": args.fail_verbose,
    }

    test_suite_config = {
        "test_paths": args.test_paths or None,
        "test_cfg": args.test_cfg,
        "cli_running_config": {
            "test_list": args.test_list,
            "user_config_set": args.user_config_set,
            "user_config": args.user_config,
            "user_env": args.user_env,
        },
    }

    running_config = {
        "temp_dir": args.temp_dir,
        "timeout": args.timeout,
    }

    log_config = {
        "dir": args.log_dir,
        "level": args.log_level,
        "verbose": args.verbose,
    }

    return test_framework_config, test_suite_config, running_config, log_config

def parser_maple_test_config_file(maple_test_cfg_file):
    raw_config = read_config(maple_test_cfg_file)
    test_paths = get_config_value(raw_config, "test-home", "dir")
    if test_paths:
        test_paths = test_paths.replace("\n", "").split(":")
    else:
        test_paths = []
    test_suite_config = {
        "test_paths": [complete_path(BASE_DIR / path) for path in test_paths if path],
    }
    log_config = {
        "dir": complete_path(
            BASE_DIR / Path(get_config_value(raw_config, "logging", "name"))
        ),
        "level": get_level_name(get_config_value(raw_config, "logging", "level")),
    }

    running_config = {
        "temp_dir": complete_path(
            BASE_DIR / Path(get_config_value(raw_config, "running", "temp_dir"))
        ),
    }

    return test_suite_config, running_config, log_config

def init_config():
    global TEST_CONFIG
    (
        TEST_CONFIG,
        cli_test_suite_config,
        cli_running_config,
        cli_log_config,
    ) = parse_args()

    (
        file_test_suite_config,
        file_running_config,
        file_log_config,
    ) = parser_maple_test_config_file(TEST_CONFIG.get("cfg"))

    cli_test_paths = cli_test_suite_config.get("test_paths") or []
    file_test_paths = file_test_suite_config.get("test_paths") or []

    for path1 in cli_test_paths:
        for path2 in file_test_paths:
            if is_relative(path1, path2):
                user_test_cfg = cli_test_suite_config.get("test_cfg")
                if not user_test_cfg:
                    cli_test_suite_config["test_cfg"] = complete_path(
                        path2 / "test.cfg"
                    )

    test_suite_config = merge_config(cli_test_suite_config, file_test_suite_config)
    running_config = merge_config(cli_running_config, file_running_config)
    log_config = merge_config(cli_log_config, file_log_config)

    if log_config.get("dir") is None:
        log_dir = complete_path(Path("./maple_test_log"))
        print(
            "No log dir find in cfg file and cli args, will create lod gir at: {}".format(
                log_dir
            )
        )
        log_config["dir"] = log_dir

    global LOGGER
    maple_test_log_config = log_config.copy()
    maple_test_log_config["verbose"] = True
    maple_test_log_config["level"] = min(20, maple_test_log_config.get("level"))
    stream_fmt = logging.Formatter("%(message)s")
    log_dir = maple_test_log_config.get("dir")
    if not log_dir.exists():
        log_dir.mkdir(parents=True, exist_ok=True)
    LOGGER = construct_logger(
        maple_test_log_config,
        "Maple_Test",
        stream_fmt=stream_fmt,
        stream_level=min(20, maple_test_log_config.get("level")),
    )
    LOGGER.info("Test log saved to {}".format(log_dir))
    running_config["log_config"] = log_config.copy()

    return test_suite_config, running_config, log_config

def merge_config(cli_config, maple_test_config):
    config = {}
    config.update(maple_test_config)
    for key, value in cli_config.items():
        if value is not None:
            config[key] = value
    return config

def get_val(key):
    global TEST_CONFIG
    return TEST_CONFIG.get(key)

class StoreDictKeyPair(argparse.Action):
    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        self._nargs = nargs
        super().__init__(option_strings, dest, nargs=nargs, **kwargs)

    def __call__(self, parser, namespace, value, option_string=None):
        my_dict = getattr(namespace, self.dest)
        k = value.strip().split("=")[0].strip()
        v = "=".join(value.strip().split("=")[1:]).strip()
        my_dict[k] = v
        setattr(namespace, self.dest, my_dict)

def construct_logger(
    log_config,
    name,
    stream_fmt=None,
    file_fmt=None,
    stream_level=None,
    file_level=logging.DEBUG,
):
    name = str(name)
    log_dir = str(log_config["dir"])
    level = log_config["level"]
    verbose = log_config["verbose"]
    formatter = logging.Formatter(
        "%(asctime)s %(levelname)s %(name)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
    )
    if stream_fmt is None:
        stream_fmt = formatter
    if file_fmt is None:
        file_fmt = formatter
    logger = logging.getLogger(name)
    if logger.handlers:
        return logger
    logger.setLevel(logging.DEBUG)

    handler1 = logging.StreamHandler()
    handler1.setFormatter(stream_fmt)
    if not verbose:
        level = logging.WARNING
    else:
        level = logging.DEBUG
    if stream_level:
        level = stream_level
    handler1.setLevel(level)
    logger.addHandler(handler1)

    log_path = "{}{}{}.log".format(log_dir, OS_SEP, name)
    handler2 = logging.FileHandler(filename=log_path, mode="w")
    handler2.setLevel(file_level)
    handler2.setFormatter(file_fmt)
    logger.addHandler(handler2)

    logger.debug("Log file at: {}".format(log_path))
    return logger

def get_level_name(name: str):
    name = name.upper()
    return logging.getLevelName(name)

def bigger_than_one_integer(num):
    inum = int(num)
    if inum < 1:
        raise argparse.ArgumentTypeError("{} is not bigger than one".format(num))
    return inum


import sys
sys.stdin = open('11404.txt', 'r')

T = int(input())
for tc in range(1, T + 1):
    H, W = map(int, input().split()) # H : 행, 가로 W : 열, 세로

    print(f'#{tc}')
    for i in range(1, H * W + 1):
        if i % W == 1 and i != 1:
            print(end='\n')
        print(i, end=" ")
    print()

제어문_반복문_연습/11404_숫자사각형1.py


import matplotlib.pyplot as plt
import numpy as np
from numpy.testing import assert_allclose

from compart.marks import Ellipse

def test_points_on_circumference():
    radius = 2
    n_points = 100
    circ = Ellipse(center=np.zeros(2), width=radius, height=radius)

    points = circ.gen_points(n=n_points, fuzz=0)

    assert points.shape == (n_points, 2, 2)
    assert_allclose(np.sum(points ** 2, axis=2), radius ** 2)

def test_plot_works():
    circ = Ellipse(center=np.zeros(2), width=1, height=1)
    fig, ax = plt.subplots()
    circ.plot(ax=ax, color="orange")


import pandas as pd
import matplotlib.pyplot as plt

date_time = ["2021-01-01", "2021-01-02", "2021-01-03"]
date_time = pd.to_datetime(date_time)
data = [1, 2, 3]

DF = pd.DataFrame()
DF['value'] = data
DF = DF.set_index(date_time)
plt.plot(DF)
plt.gcf().autofmt_xdate()
plt.show()


import re, os, sys
from .s3 import upload_to_s3
from util import randomStr

parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, parentdir)
from config import *

combine_video_name = "combine-video-" + randomStr(5) + ".mp4"

# 生成无声视频
def video_slience(video_path):
    video_slience_name = re.sub(r"(.*).mp4", r"\1-silence.mp4", video_path)
    cmd = "%s -i %s -vcodec copy -an %s -y" % (
        ffmpeg_path,
        video_path,
        video_slience_name,
    )
    os.system(cmd)
    return video_slience_name

# 生成新视频
def combine(video_path):
    video_slience_name = video_slience(video_path)
    cmd = "%s -i %s -i output.wav -c:v copy -c:a aac -strict experimental %s -y" % (
        ffmpeg_path,
        video_slience_name,
        combine_video_name,
    )
    os.system(cmd)
    # video_url = upload_to_s3(combine_video_name)
    # os.remove(combine_video_name)
    return ""

if __name__ == "__main__":
    print(combine("demo.mp4"))


"""Test the entire low-resolution correction process at low-resolution.

Compare mass
fractions corrected by IsoCor to theoretical isotopologues distributions
(with algebraic equations provided in the code for manual check).
"""

import numpy as np
import pytest
import isocor as hrcor

# Tests the entire correction process at low resolution
# (here pXY represents natural abundance of isotope X of element Y)
@pytest.mark.parametrize("data", [{"formula": "C2H2",
                                   "tracer": "13C",
                                   "v_expected": "[1., 0., 0.]",
                                   "v_measured": "[p1H * p1H, p1H * p2H * 2, p2H * p2H]",
                                   "correct_NA_tracer": False,
                                   "tracer_purity": [0.0, 1.0]},
                                  {"formula": "C2H2P10",
                                   "tracer": "13C",
                                   "v_expected": "[0.7, 0.2, 0.1]",
                                   "v_measured": "[0.7*p1H * p1H, 0.7*p1H * p2H * 2+0.2*p1H * p1H, 0.7*p2H * p2H+0.2*p1H * p2H * 2+0.1*p1H * p1H]",
                                   "correct_NA_tracer": False,
                                   "tracer_purity": [0.0, 1.0]},
                                  {"formula": "OH2",
                                   "tracer": "18O",
                                   "v_expected": "[0.6, 0.4]",
                                   "v_measured": "[0.6*p1H**2, 0.6*p2H**2+0.4*p1H**2]",
                                   "correct_NA_tracer": False,
                                   "tracer_purity": [0.0, 0.0, 1.0]},
                                  {"formula": "OH2",
                                   "tracer": "17O",
                                   "v_expected": "[0.6, 0.4]",
                                   "v_measured": "[0.6*p1H**2, 0.6*p1H*p2H*2+0.4*p1H**2]",
                                   "correct_NA_tracer": False,
                                   "tracer_purity": [0.0, 1.0, 0.0]},
                                  {"formula": "C2H2P10",
                                   "tracer": "13C",
                                   "v_expected": "[0.7, 0.2, 0.1]",
                                   "v_measured": "[0.7*p1H*p1H*p12C**2, 0.7*p1H*p2H*2*p12C**2 + 0.7*p1H*p1H*p12C*p13C*2 + 0.2*p1H*p1H*p12C, 0.7*p2H*p2H*p12C**2 + 0.7*p1H*p2H*2*p12C*p13C*2 + 0.7*p1H*p1H*p13C**2 +0.2*p1H*p1H*p13C + 0.2*p1H*p2H*2*p12C + 0.1*p1H*p1H]",
                                   "correct_NA_tracer": True,
                                   "tracer_purity": [0.0, 1.0]},
                                  {"formula": "C2H2P10",
                                   "tracer": "13C",
                                   "v_expected": "[0.7, 0.2, 0.1]",
                                   "v_measured": "[0.7*p1H**2 + 0.2*p1H**2*pur12C + 0.1*p1H**2*pur12C**2, 0.7*p1H*p2H*2 + 0.2*p1H**2*pur13C + 0.2*p1H*p2H*2*pur12C + 0.1*p1H**2*pur12C*pur13C*2 + 0.1*p1H*p2H*2*pur12C**2, 0.7*p2H**2 + 0.2*p1H*p2H*2*pur13C + 0.2*p2H**2*pur12C + 0.1*p1H**2*pur13C**2 + 0.1*p1H*p2H*2*pur12C*pur13C*2 + 0.1*p2H**2*pur12C**2]",
                                   "correct_NA_tracer": False,
                                   "tracer_purity": [0.3, 0.7]}])
def test_low_res_correction(data, data_iso, usr_tolerance):
    """Correction of non-tracer elements depending on resolution and formula.

    Measured values should be corrected for the incorporation of natural isotopes of
    non-tracers elements if the resolution was not high enough to distinguish them.
    The resolution depends on m/z ratio and thus the formula of the metabolite.
    The higher the mass, the higher the resolution should be used to distinguish all
    isotopomers and avoid to need correction.
    """
    formula = data["formula"]
    # Observed abundance of each isotopomers given the natural abundance
    # of the H isotopes
    p1H = data_iso["H"]["abundance"][0]
    p2H = data_iso["H"]["abundance"][1]
    p12C = data_iso["C"]["abundance"][0]
    p13C = data_iso["C"]["abundance"][1]
    pur12C = data["tracer_purity"][0]
    pur13C = data["tracer_purity"][1]
    v_measured = eval(data["v_measured"])
    # Expected correction
    v_expected = eval(data["v_expected"])
    # Perform correction
    metabolite = hrcor.LowResMetaboliteCorrector(formula, data["tracer"],
                                                 data_isotopes=data_iso,
                                                 correct_NA_tracer=data["correct_NA_tracer"],
                                                 derivative_formula=None,
                                                 tracer_purity=data["tracer_purity"])
    _, v_corrected, _, _ = metabolite.correct(v_measured)
    # Compare corrected vs. expected data
    np.testing.assert_allclose(v_corrected, v_expected, rtol=1e-7)


import pandas as pd
# from dashboard.backend import dataset_interactions

import dash_html_components as html
from dash.dependencies import Input, Output, State
from sherlock.sherlockinterface import Sherlock
# from dashboard.utils import components_layout_helper
# from dashboard.pages import report_layout
import io
import available_settings
import base64
from dashboard.utils import callbackshelpers

variables_storage = {}

def input_content_callbacks(app):
    @app.callback(
        Output("output-data-upload", "children"),
        [Input("upload-data", "contents")],
        [State("upload-data", "filename")]
    )
    def upload_data_file(uploaded_file_contents, uploaded_filename):
        """Uploaded data file"""
        if uploaded_file_contents is not None:
            df = callbackshelpers.parse_spreadsheet_contents(uploaded_file_contents, uploaded_filename)
            if df is not None:
                variables_storage['data'] = df
                variables_storage['sherlock'] = Sherlock()
                variables_storage['sherlock'].load_data(df)

                return [''.join(['File ', uploaded_filename, ' was uploaded'])]
        return ['File was not uploaded']

    @app.callback(
        [Output("load-data-spinner", "children"),
         Output('loaded-data-description', 'children'),
         Output('loaded-variables-description', 'children')],
        [Input("load-data-button", "n_clicks")]
    )
    def build_sherlock(n):
        """Uploaded data file"""
        if n:
            df = variables_storage['data']
            sh_ai = Sherlock()
            variables_storage['sherlock'] = sh_ai
            sh_ai.load_data(df)
            report = sh_ai.visualize_variables_descriptions(None)
            data_report = callbackshelpers.data_description_visual(report['data'])
            variables_report = callbackshelpers.variables_description_visual(report['variables'])

            return ['Built Sherlock...'], data_report, variables_report

        return ['File was not uploaded'], None, None

def variables_inspection_callbacks(app):
    @app.callback(
        [Output("variables-distributions-results-container", "children"),
         Output("variables-distribution-spinner", "children")],
        [Input('variables-distribution-button', 'n_clicks')],
        [State('n-var-per-page', 'value')]
    )
    def run_variables_visualization(n, variables_per_page):
        # available_companies_list = glob.glob("/data/*.csv")
        if n:
            print('graphs to show')
            settings = {'num_var_per_graph': int(variables_per_page)}

            sh_ai = variables_storage['sherlock']
            figs = sh_ai.visualize_variables_distributions(settings)

            imgs = []
            for fig in figs:
                out_url = callbackshelpers.fig_to_uri(fig, bbox_inches='tight')
                imgs.append(html.Img(src=out_url))

            return imgs, 'Done'

        return [], None

    @app.callback(
        [Output("variables-network-results-container", "children"),
         Output("variables-network-spinner", "children")],
        [Input('variables-network-button', 'n_clicks')],
        [State('network-threshold-input', 'value'), State('network-pval-input', 'value')
            , State('correlated-threshold-text', 'value'), State('nodes-sizes-text', 'value')]
    )
    def run_variables_network_visualization(n, network_threshold, p_val, correlated_threshold, nodes_sizes):
        # available_companies_list = glob.glob("/data/*.csv")
        if n:
            sh_ai = variables_storage['sherlock']
            settings = {'network_connections_threshold_variables_relations': float(network_threshold),
                        'p_value_correlations': float(p_val), 'correlated_threshold': float(correlated_threshold),
                        'node_size': int(nodes_sizes), 'show_report': False, 'fig_size': (12, 10),
                        'max_nodes_to_draw_graph': 90}

            fig = sh_ai.visualize_variables_network(settings)
            img = None

            if fig is not None:
                out_url = callbackshelpers.fig_to_uri(fig)
                img = html.Img(src=out_url)

            return img, 'Done'
            # new_tags = input_tags.split(';')

        return [], None

    @app.callback(
        [Output("variables-response-results-container", "children"),
         Output("variables-response-spinner", "children")],
        [Input('variables-response-button', 'n_clicks')],
        [State('use-correction', 'value'), State('p-var-res-relation', 'value')]
    )
    def run_variables_response_visualization(n, use_correction, p_value):
        # available_companies_list = glob.glob("/data/*.csv")
        if n:
            sh_ai = variables_storage['sherlock']
            settings = {'p_variables_response_relation': float(p_value), 'use_correction': use_correction == 'True'}

            figs = sh_ai.visualize_variables_response_relationship(settings)

            imgs = []
            for fig in figs:
                out_url = callbackshelpers.fig_to_uri(fig, bbox_inches='tight')
                imgs.append(html.Img(src=out_url))

            return imgs, 'Done'
            # new_tags = input_tags.split(';')

        return [], None

    @app.callback(
        [Output("all-data-model-container", "children"),
         Output("all-data-model-spinner", "children")],
        [Input('all-data-model-button', 'n_clicks')],
    )
    def run_model_potential(n):
        # available_companies_list = glob.glob("/data/*.csv")
        if n:
            sh_ai = variables_storage['sherlock']
            settings = None
            fig = sh_ai.visualize_all_data_model(settings)

            out_url = callbackshelpers.fig_to_uri(fig, bbox_inches='tight')
            img = html.Img(src=out_url)

            return img, 'Done'
            # new_tags = input_tags.split(';')

        return [], None

def insights_callbacks(app):
    @app.callback(
        [Output("comparison-against-shuffle-container", "children"),
         Output("compare-against-shuffle-spinner", "children")],
        [Input('compare-against-shuffle-button', 'n_clicks')],
        [State('number-runs', 'value'), State('bagging-perc', 'value'),
         State('select-model', 'value'), State('select-scaler', 'value'), State('select-random', 'value'),
         State('train-perc', 'value')]
    )
    def compare_against_shuffle(n, n_runs, bagging_perc, model, scaler, random_selector, train_perc):
        if n:
            scaler_object = available_settings.available_models_preprocessing[scaler]
            model_object = available_settings.available_models[model]
            settings = {'scaler': scaler_object, 'train_perc': train_perc, 'runs': n_runs,
                        'selection_type': random_selector, 'model': model_object,
                        'bootstrap_aggregate_percentage': bagging_perc, 'fig_size': (12, 10)}

            sh_ai = variables_storage['sherlock']
            fig = sh_ai.visualize_compare_against_shuffle(settings)
            out_url = callbackshelpers.fig_to_uri(fig, bbox_inches='tight')
            img = html.Img(src=out_url)

            return img, [f'Done']

            # new_tags = input_tags.split(';')

        return None, [f'Not compared']

    @app.callback(
        [Output("insights-container", "children"),
         Output("insights-spinner", "children")],
        [Input('insights-button', 'n_clicks')],
        [State('number-runs-insights', 'value'), State('bagging-perc-insights', 'value'),
         State('select-model-insights', 'value'), State('select-scaler-insights', 'value'),
         State('select-random-insights', 'value'), State('train-perc-insights', 'value')]
    )
    def insight(n, n_runs, bagging_perc, model, scaler, random_selector, train_perc):
        if n:
            scaler_object = available_settings.available_models_preprocessing[scaler]
            model_object = available_settings.available_models[model]
            settings = {'scaler': scaler_object, 'train_perc': train_perc, 'runs': n_runs,
                        'selection_type': random_selector, 'model': model_object,
                        'bootstrap_aggregate_percentage': bagging_perc, 'fig_size': (12, 10)}

            sh_ai = variables_storage['sherlock']
            figs = sh_ai.visualize_insights(settings)
            imgs = []
            for fig in figs:
                out_url = callbackshelpers.fig_to_uri(fig, bbox_inches='tight')
                imgs.append(html.Img(src=out_url))

            # out_url = fig_to_uri(fig, bbox_inches='tight')
            # img = html.Img(src=out_url)

            return imgs, [f'Done']

            # new_tags = input_tags.split(';')

        return None, [f'Not compared']


import torch
import torch.nn as nn
import os
from itertools import zip_longest
import numpy as np

class Learner():
    def __init__(self, net, results_dir='./tmp/results/'):
        self.net = net  # Neural network model
        self.loss = []  # List to store different loss functions

        # Initialize the appropriate loss function for each output
        for i in range(self.net.num_outputs):
            if self.net.regression[i]:
                self.loss.append(nn.MSELoss())
            else:
                self.loss.append(nn.BCEWithLogitsLoss())
        # Use Adam optimizer for training
        self.optimizer = torch.optim.Adam(self.net.parameters(), lr=0.01)

        self.T = 0
        self.observed_tasks = set()  # Set to store observed tasks
        self.results_dir = results_dir  # Directory to store results
        self.init_trainloaders = None  # Store initial training data loaders

    def train(self, *args, **kwargs):
        # Placeholder for the main training loop
        raise NotImplementedError('Training loop is algorithm specific')

    def init_train(self, trainloader, task_id, num_epochs, save_freq=1, testloaders=None):
        # This function initializes training for a given task
        if self.init_trainloaders is None:
            self.init_trainloaders = {}
        self.init_trainloaders[task_id] = trainloader

        eval_bool = testloaders is not None  # Boolean indicating if there are test loaders

        if len(self.init_trainloaders) == self.net.num_init_tasks:
            for i in range(num_epochs):
                # Iterates over batches from different tasks in parallel
                for XY_all in zip_longest(*self.init_trainloaders.values()):
                    for task, XY in zip(self.init_trainloaders.keys(), XY_all):
                        if XY is not None:
                            X, Y = XY
                            X = X.to(self.net.device, non_blocking=True)
                            Y = Y.to(self.net.device, non_blocking=True)
                            self.gradient_step(X, Y, task)
                # Evaluate and save data periodically
                if i % save_freq == 0 or i == num_epochs - 1:
                    if eval_bool:
                        self.evaluate(testloaders)
                    self.save_data(i + 1, task_id, save_eval=eval_bool)
            for task, loader in self.init_trainloaders.items():
                self.update_multitask_cost(loader, task)

    def evaluate(self, testloaders):
        # This function evaluates the model using test data
        was_training = self.net.training
        prev_reduction = []

        # Store current loss reduction method and set to sum for evaluation
        for i in range(self.net.num_outputs):
            prev_reduction.append(self.loss[i].reduction)
            self.loss[i].reduction = 'sum'

        self.net.eval()  # Set the model to evaluation mode

        with torch.no_grad():  # No gradient computation during evaluation
            self.test_loss = {}
            self.test_acc = {}
            for task, loader in testloaders.items():
                # Variables to store cumulative loss and accuracy
                l = [0.] * self.net.num_outputs
                acc = [0.] * self.net.regression.count(False)
                n = len(loader.dataset)  # Number of samples
                cnt_not_regression = 0
                for X, Y in loader:
                    X = X.to(self.net.device, non_blocking=True)
                    Y = Y.to(self.net.device, non_blocking=True)
                    Y_hat = self.net(X, task).unsqueeze(1)  # Model predictions

                    # Calculate and accumulate loss and accuracy for each output
                    for i in range(self.net.num_outputs):
                        l[i] += self.loss[i](Y_hat[:, :, i], Y[:, :, i])
                        if not self.net.regression[i]:
                            acc[cnt_not_regression] += ((Y_hat[:, :, i] > 0.5) == (Y[:, :, i] == 1)).sum().item()
                            cnt_not_regression += 1

                # Store computed test loss and accuracy for each task
                cnt_not_regression = 0
                for i in range(self.net.num_outputs):
                    if self.net.regression[i]:
                        self.test_loss.setdefault(task, []).append(np.sqrt(l[i] / n))
                    else:
                        self.test_loss.setdefault(task, []).append(l[i] / n)
                        self.test_acc.setdefault(task, []).append(acc[cnt_not_regression] / n)
                        cnt_not_regression += 1

        # Restore the previous loss reduction method
        for i in range(self.net.num_outputs):
            self.loss[i].reduction = prev_reduction[i]
        if was_training:
            self.net.train()  # Return model to training mode

    def gradient_step(self, X, Y, task_id):
        # This function performs a single optimization step using the provided batch
        Y_hat = self.net(X, task_id=task_id).unsqueeze(1)
        total_loss = 0.

        # Calculate total loss across all outputs
        for i in range(self.net.num_outputs):
            total_loss += self.loss[i](Y_hat[:, :, i], Y[:, :, i])
        mean_loss = total_loss / self.net.num_outputs

        # Backpropagate the error and update model parameters
        self.optimizer.zero_grad()
        mean_loss.backward()
        self.optimizer.step()

    def save_data(self, epoch, task_id, save_eval=False):
        # This function saves model checkpoints and evaluation logs
        task_results_dir = os.path.join(self.results_dir, 'task_{}'.format(task_id))
        os.makedirs(task_results_dir, exist_ok=True)

        # Save model checkpoints periodically
        if epoch == 0 or epoch % 10 == 0:
            path = os.path.join(task_results_dir, 'checkpoint.pt'.format(epoch))
            torch.save({
                'epoch': epoch,
                'model_state_dict': self.net.state_dict(),
                'optimizer_state_dict': self.optimizer.state_dict(),
                'observed_tasks': self.observed_tasks,
                }, path)

        # Log evaluation results if required
        if save_eval:
            log_open_mode = 'a' if epoch > 0 else 'w'
            with open(os.path.join(task_results_dir, 'log.txt'), log_open_mode) as f:
                line = 'epochs: {}, training task: {}\n'.format(epoch, task_id)
                f.write(line)
                print(line, end='')
                for task in self.test_loss:
                    line = '\ttask: {}'.format(task)
                    for i, loss in enumerate(self.test_loss[task]):
                        if type(loss) == tuple:
                            line += '\tloss{}: ({:.4f}, {:.4f})'.format(i+1, loss[0], loss[1])
                        else:
                            line += '\tloss{}: {:.4f}'.format(i+1, loss)
                    line += '\n'
                    f.write(line)
                    print(line, end='')

    def update_multitask_cost(self, loader, task_id):
        # Placeholder for updating multitask cost
        raise NotImplementedError('Update update_multitask is algorithm specific')

class CompositionalLearner(Learner):
    """Define a compositional learning approach."""

    def train(self, trainloader, task_id, component_update_freq=100, num_epochs=100, save_freq=1, testloaders=None):

        # Check if the current task_id has been observed before
        if task_id not in self.observed_tasks:
            self.observed_tasks.add(task_id)
            self.T += 1

        # Check if testloaders are provided and evaluate the network.
        eval_bool = testloaders is not None
        if eval_bool:
            self.evaluate(testloaders)
        self.save_data(0, task_id, save_eval=eval_bool)

         # If number of observed tasks is within the limit of initial tasks, use the initial training method.
        if self.T <= self.net.num_init_tasks:
            self.net.freeze_structure()
            self.init_train(trainloader, task_id, num_epochs, save_freq, testloaders)
        else:
            # Else, freeze modules and adapt structure.
            self.net.freeze_modules()
            self.net.freeze_structure()     # freeze structure for all tasks
            self.net.freeze_structure(freeze=False, task_id=task_id)    # except current one
            for i in range(num_epochs):
                if (i + 1) % component_update_freq == 0:
                    self.update_modules(trainloader, task_id)   # replace one structure epoch with one module epoch
                else:
                    for X, Y in trainloader:
                        X = X.to(self.net.device, non_blocking=True)
                        Y = Y.to(self.net.device, non_blocking=True)
                        self.update_structure(X, Y, task_id)
                if i % save_freq == 0 or i == num_epochs - 1:
                    if eval_bool:
                        self.evaluate(testloaders)
                    self.save_data(i + 1, task_id, save_eval=eval_bool)
            self.update_multitask_cost(trainloader, task_id)

    # Update the network structure using gradients.
    def update_structure(self, X, Y, task_id):
        self.gradient_step(X, Y, task_id)    # assume shared parameters are frozen and just take a gradient step on the structure

    def update_modules(self, *args, **kwargs):
        raise NotImplementedError('Update modules is algorithm specific')

class JointLearner(Learner):
    pass

class NoComponentsLearner(Learner):
    pass

class CompositionalDynamicLearner(CompositionalLearner):
    """This class extends the functionality of CompositionalLearner by allowing dynamic changes to the network components."""

    def train(self, trainloader, task_id, valloader, component_update_freq=100, num_epochs=100, save_freq=1, testloaders=None):

        # Initial training steps are similar to CompositionalLearner.
        if task_id not in self.observed_tasks:
            self.observed_tasks.add(task_id)
            self.T += 1
        eval_bool = testloaders is not None
        if eval_bool:
            self.evaluate(testloaders)
        self.save_data(0, task_id, save_eval=eval_bool)
        if self.T <= self.net.num_init_tasks:
            self.net.freeze_structure()
            self.init_train(trainloader, task_id, num_epochs, save_freq, testloaders)
        else:
            # Add dynamic handling of network components.
            self.net.freeze_modules()
            self.net.freeze_structure()     # freeze structure for all tasks
            self.net.add_tmp_module(task_id)   # freeze original modules and structure
            self.optimizer.add_param_group({'params': self.net.components[-1].parameters()})
            if hasattr(self, 'preconditioner'):
                self.preconditioner.add_module(self.net.components[-1])

            self.net.freeze_structure(freeze=False, task_id=task_id)    # unfreeze (new) structure for current task

            for i in range(num_epochs):
                if (i + 1) % component_update_freq == 0:
                    self.update_modules(trainloader, task_id)
                else:
                    for X, Y in trainloader:
                        X = X.to(self.net.device, non_blocking=True)
                        Y = Y.to(self.net.device, non_blocking=True)
                        self.update_structure(X, Y, task_id)
                        self.net.hide_tmp_module()
                        self.update_structure(X, Y, task_id)
                        self.net.recover_hidden_module()
                if i % save_freq == 0 or i == num_epochs - 1:
                    if eval_bool:
                        self.evaluate(testloaders)
                    self.save_data(i + 1, task_id, save_eval=eval_bool)

            # Decide whether to keep the temporary module or not
            self.conditionally_add_module(valloader, task_id)
            self.save_data(num_epochs + 1, task_id, save_eval=False, final_save=True)
            self.update_multitask_cost(trainloader, task_id)

    # Method to evaluate whether to keep a temporary module or not
    def conditionally_add_module(self, valloader, task_id):
        test_loss = self.test_loss
        test_acc = self.test_acc
        self.evaluate({task_id: valloader})
        W_losses, WO_losses = [], []
        for output in self.test_loss[task_id]:
            W_losses.append(output[0])
            WO_losses.append(output[1])
        update_loss = np.mean(W_losses)
        no_update_loss = np.mean(WO_losses)
        print('W/update: {}, WO/update: {}'.format(update_loss, no_update_loss))
        if (no_update_loss - update_loss) / update_loss > .05:
            print('Keeping new module. Total: {}\n'.format(self.net.num_components))
        else:
            self.net.remove_tmp_module()
            print('Not keeping new module. Total: {}\n'.format(self.net.num_components))

        self.test_loss = test_loss
        self.test_acc = test_acc

    # Evaluate network performance on multiple tasks
    def evaluate(self, testloaders, eval_no_update=True):
        was_training = self.net.training
        prev_reduction = []
        for i in range(self.net.num_outputs):
            prev_reduction.append(self.loss[i].reduction)
            self.loss[i].reduction = 'sum'     # make sure the loss is summed over instances
        self.net.eval()
        with torch.no_grad():
            self.test_loss = {}
            self.test_acc = {}
            for task, loader in testloaders.items():
                l = [0.] * self.net.num_outputs
                acc = [0.] * self.net.regression.count(False)
                n = len(loader.dataset)
                cnt_not_regression = 0
                for X, Y in loader:
                    X = X.to(self.net.device, non_blocking=True)
                    Y = Y.to(self.net.device, non_blocking=True)
                    Y_hat = self.net(X, task).unsqueeze(1)
                    for i in range(self.net.num_outputs):
                        l[i] += self.loss[i](Y_hat[:, :, i], Y[:, :, i])
                        if not self.net.regression[i]:
                            acc[cnt_not_regression] += ((Y_hat[:, :, i] > 0.5) == (Y[:, :, i] == 1)).sum().item()
                            cnt_not_regression += 1
                cnt_not_regression = 0
                if eval_no_update and task == self.T - 1 and self.T > self.net.num_init_tasks:
                    self.net.hide_tmp_module()
                    l1 = [0.] * self.net.num_outputs
                    acc1 = [0.] * self.net.regression.count(False)
                    for X, Y in loader:
                        X = X.to(self.net.device, non_blocking=True)
                        Y = Y.to(self.net.device, non_blocking=True)
                        Y_hat = self.net(X, task).unsqueeze(1)
                        for i in range(self.net.num_outputs):
                            l1[i] += self.loss[i](Y_hat[:, :, i], Y[:, :, i])
                            if not self.net.regression[i]:
                                acc1[cnt_not_regression] += ((Y_hat[:, :, i] > 0.5) == (Y[:, :, i] == 1)).sum().item()
                                cnt_not_regression += 1
                    cnt_not_regression = 0
                    for i in range(self.net.num_outputs):
                        if self.net.regression[i]:
                            if task not in self.test_loss.keys():
                                self.test_loss[task] = [(np.sqrt(l[i] / n), np.sqrt(l1[i] / n))]
                            else:
                                self.test_loss[task].append((np.sqrt(l[i] / n), np.sqrt(l1[i] / n)))
                        else:
                            if task not in self.test_loss.keys():
                                self.test_loss[task] = [(l[i] / n, l1[i] / n)]
                            else:
                                self.test_loss[task].append((l[i] / n, l1[i] / n))
                            if task not in self.test_acc.keys():
                                self.test_acc[task] = [(acc[cnt_not_regression] / n, acc1[cnt_not_regression] / n)]
                            else:
                                self.test_acc[task] = [(acc[cnt_not_regression] / n, acc1[cnt_not_regression] / n)]
                            cnt_not_regression += 1
                    self.net.recover_hidden_module()
                else:
                    for i in range(self.net.num_outputs):
                        if self.net.regression[i]:
                            if task not in self.test_loss.keys():
                                self.test_loss[task] = [np.sqrt(l[i] / n)]
                            else:
                                self.test_loss[task].append(np.sqrt(l[i] / n))
                        else:
                            if task not in self.test_loss.keys():
                                self.test_loss[task] = [l[i] / n]
                            else:
                                self.test_loss[task].append(l[i] / n)
                            if task not in self.test_acc.keys():
                                self.test_acc[task] = [acc[cnt_not_regression] / n]
                            else:
                                self.test_acc[task] = [acc[cnt_not_regression] / n]
                            cnt_not_regression += 1

        for i in range(self.net.num_outputs):
            self.loss[i].reduction = prev_reduction[i]
        if was_training:
            self.net.train()

    # Save training data
    def save_data(self, epoch, task_id, save_eval=False, final_save=False):
        super().save_data(epoch, task_id, save_eval)
        if final_save:
            task_results_dir = os.path.join(self.results_dir, 'task_{}'.format(task_id))
            with open(os.path.join(task_results_dir, 'num_components.txt'), 'w') as f:
                line = 'final components: {}'.format(self.net.num_components)
                f.write(line)


import os

from qgis.PyQt import uic
from qgis.PyQt import QtWidgets
from qgis.core import QgsProject
from qgis.core import *

# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
FORM_CLASS, _ = uic.loadUiType(os.path.join(
    os.path.dirname(__file__), 'loadlayer_dialog_base.ui'))

class PluginLoadDialog(QtWidgets.QDialog, FORM_CLASS):
    def __init__(self, parent=None):
        """Constructor."""
        super(PluginLoadDialog, self).__init__(parent)
        # Set up the user interface from Designer through FORM_CLASS.
        # After self.setupUi() you can access any designer object by doing
        # self.<objectname>, and you can use autoconnect slots - see
        # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html
        # #widgets-and-dialogs-with-auto-connect
        self.setupUi(self)
        self.pushButton.clicked.connect(self.updateLabelText) # call function
        self.toolButton.clicked.connect(self.getOpenFilename) # function open file
        self.loadComboBoxLayers() #call function

    def updateLabelText(self):
        text = self.lineEdit.text()
        self.label.setText(text)

    def loadComboBoxLayers(self):
        self.comboBox.clear()
        layers = [layer.name() for layer in QgsProject.instance().mapLayers().values()]
        self.comboBox.addItems(layers)

    def getOpenFilename(self):
        fn = QtWidgets.QFileDialog.getOpenFileName(self, 'Open file')[0]
        self.lineEdit_2.setText(fn)


# based on
# https://stackoverflow.com/questions/27863832/calling-python-2-script-from-python-3
# https://github.com/pytest-dev/execnet/issues/26
#
# This is used to call python functions with another python version
#
# Use in the following way for example
#   bridge = Bridge(version="2.7", module="my_module", path="../path/to/source")
#   my_function = bridge.connect("my_function")
#   result = my_function(call,with,parameters,as,usual)
#
# Construction a bridge at first and then calling the .connect(str) function is done this
# way to be able to connect more then one function from a given module easily.
# keyworded argumetnts are not supported yet

import execnet
import sys
import os

class BridgeConnection:
    def __init__(self, version, module, function_name, path=None):
        self.version = version
        self.module = module
        self.function_name = function_name
        self.path = path

    def __call__(self, *args):
        gw = execnet.makegateway("popen//python=python%s" % self.version)

        script = """
                        import sys

                        outchan = channel.gateway.newchannel()
                        sys.stdout = outchan.makefile("w")
                        channel.send(outchan)
                        errchan = channel.gateway.newchannel()
                        sys.stderr = errchan.makefile("w")
                        channel.send(errchan)
                        """
        if self.path is not None:
            script += """
                        sys.path.append('%s')
                        """ % (os.path.abspath(self.path))
        script += """
                        from %s import %s as the_function

                        result = the_function(*channel.receive())

                        channel.send(result)

                        sys.stdout = sys.__stdout__
                        sys.stderr = sys.__stderr__
                        try:
                            sys.stdout.flush()
                            sys.stdout.close()
                        except:
                            pass
                        try:
                            sys.stderr.flush()
                            sys.stderr.close()
                        except:
                            pass

                        outchan.close()
                        errchan.close()
                        """ % (self.module, self.function_name)

        channel = gw.remote_exec(script)
        outchan = channel.receive()
        errchan = channel.receive()
        outchan.setcallback(lambda data: sys.stdout.write(str(data)))
        errchan.setcallback(lambda data: sys.stderr.write(str(data)))
        channel.send(args)
        result = channel.receive()
        # wait for proper shutdown
        # https://bitbucket.org/hpk42/execnet/issues/26/redirecting-stdout-stderr-in-the-remote
        outchan.waitclose()
        errchan.waitclose()
        return result

class Bridge:
    def __init__(self, version, module, path=None):
        self.version = version
        self.module = module
        self.path = path

    def __getattr__(self, item):
        return BridgeConnection(self.version, self.module, item, self.path)


import json
import time

def grapfdata():
    # define a function to process the original json data to make it simplified in order to improve reaction speed.

    # load the json file
    try:
        bd_json = open('static/BikesData.json')
    except IOError:
        print("file dublin.json not found")
    else:
        print('successfully open file dublin.json ')
    finally:
        gdata = json.load(bd_json)
        bd_json.close()
    stations = {}

    # deploy the empty dictionary 'stations', and use the station name as keys.
    for i in gdata:
        if i["name"] not in stations:
            stations[i["name"]] = {}

    for m in stations:
        # ab: available bikes, abs: available bike stands
        # ab_time_usage: available bikes at different times, abs_time_usage: available bike stands at different times.
        weekday_data = {
            "Mon": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Tue": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Wed": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Thu": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Fri": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Sat": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
            "Sun": {'ab': 0, 'abs': 0, 'ab_time_usage': {}, 'abs_time_usage': {}},
        }
        count = {}  # used to count how many values are added, will be used for average value calculation.
        count_h = {}
        # print(count_h)
        for j in weekday_data.keys():
            count[j] = 0
            count_h[j] = {}
            for i in range(0, 24):
                weekday_data[j]['ab_time_usage'][str(i)] = 0
                weekday_data[j]['abs_time_usage'][str(i)] = 0
        for i in count_h:
            for j in range(0, 24):
                count_h[i][str(j)] = 0

        for i in gdata:
            if i['name'] == m:
                for j in weekday_data.keys():
                    if j in i['DayOfWeek']:
                        weekday_data[j]['ab'] += i['available_bikes'] #sum the available bikes to get average data
                        weekday_data[j]['abs'] += i['available_bike_stands']
                        count[j] += 1

                        t = str((time.localtime(i['last_update']/1000)).tm_hour)
                        weekday_data[j]['ab_time_usage'][t] += i['available_bikes']
                        weekday_data[j]['abs_time_usage'][t] += i['available_bike_stands']
                        count_h[j][t] += 1
        for k in weekday_data.keys():
            weekday_data[k]['ab'] = round(weekday_data[k]['ab']/count[k])
            weekday_data[k]['abs'] = round(weekday_data[k]['abs']/count[k])
            for hour in weekday_data[k]['ab_time_usage']:
                weekday_data[k]['ab_time_usage'][hour] = round(weekday_data[k]['ab_time_usage'][hour]/count_h[k][hour])
            for hour in weekday_data[k]['abs_time_usage']:
                weekday_data[k]['abs_time_usage'][hour] = round(weekday_data[k]['abs_time_usage'][hour]/count_h[k][hour])
        stations[m] = weekday_data

    f = open("./static/processedBikeData.json", 'w')
    json.dump(stations, f)
    f.close()

grapfdata()


import torch

from ...interfaces import SamplerInterface
from ..denoisers import MaskedCompVisDenoiser, MaskedCompVisVDenoiser
from ..utils import latent_to_images, to_d, linear_multistep_coeff, get_ancestral_step

class KSampler(SamplerInterface):
	def __init__(self, schedule, **schedule_args):
		self.schedule = schedule
		self.schedule_args = schedule_args

	def use_model(self, model):
		self.model = model
		self.denoiser = (
			MaskedCompVisVDenoiser(model) if model.is_v_model
			else MaskedCompVisDenoiser(model)
		)

	def sample(self, ctx, noise, cond, uncond, steps, init_latent=None, mask=None, image_conditioning=None):
		sigmas = self.denoiser.get_sigmas(ctx.params.steps)

		if init_latent is not None:
			offset = ctx.params.steps - steps - 1
			x = noise * sigmas[offset]
			x = x + init_latent
			sigmas = sigmas[offset:]
		else:
			x = noise * sigmas[0]

		if mask is not None:
			mask = torch.nn.functional.interpolate(mask, size=noise.shape[-2:])

		def denoise(x, sigma):
			denoised = self.denoiser(
				x,
				sigma=sigma,
				cond=cond,
				uncond=uncond,
				cond_scale=ctx.params.cfg_scale,
				init_latent=init_latent,
				mask=mask,
				image_conditioning=image_conditioning
			)

			if ctx.wants_intermediate():
				ctx.put_intermediate(latent_to_images(denoised, model=self.model))

			return denoised

		scheduler = self.schedule(sigmas, denoise=denoise, **self.schedule_args)

		steps = len(sigmas) - 1

		for i in ctx.make_sampling_iter(range(steps)):
			x = scheduler.step(x, i)

		return x

class common_gamma:
	def __init__(self, sigmas, denoise, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.):
		self.denoise = denoise
		self.sigmas = sigmas
		self.s_churn = s_churn
		self.s_tmin = s_tmin
		self.s_tmax = s_tmax
		self.s_noise = s_noise

	def apply_gamma(self, x, i):
		gamma = min(self.s_churn / (len(self.sigmas) - 1), 2 ** 0.5 - 1) if self.s_tmin <= self.sigmas[i] <= self.s_tmax else 0.
		eps = torch.randn_like(x) * self.s_noise
		sigma_hat = self.sigmas[i] * (gamma + 1)

		if gamma > 0:
			x = x + eps * (sigma_hat ** 2 - self.sigmas[i] ** 2) ** 0.5

		return x, sigma_hat

class schedule_euler(common_gamma):
	def step(self, x, i):
		x, sigma_hat = self.apply_gamma(x, i)
		denoised = self.denoise(x, sigma=sigma_hat * x.new_ones([x.shape[0]]))

		d = to_d(x, sigma_hat, denoised)
		dt = self.sigmas[i + 1] - sigma_hat
		x = x + d * dt

		return x

class schedule_heun(common_gamma):
	def step(self, x, i):
		x, sigma_hat = self.apply_gamma(x, i)
		denoised = self.denoise(x, sigma=sigma_hat * x.new_ones([x.shape[0]]))

		d = to_d(x, sigma_hat, denoised)
		dt = self.sigmas[i + 1] - sigma_hat

		if self.sigmas[i + 1] == 0:
			x = x + d * dt
		else:
			x_2 = x + d * dt
			denoised_2 = self.denoise(x_2, sigma=self.sigmas[i + 1] * x.new_ones([x.shape[0]]))
			d_2 = to_d(x_2, self.sigmas[i + 1], denoised_2)
			d_prime = (d + d_2) / 2
			x = x + d_prime * dt

		return x

class schedule_dpm_2(common_gamma):
	def step(self, x, i):
		x, sigma_hat = self.apply_gamma(x, i)
		denoised = self.denoise(x, sigma=sigma_hat * x.new_ones([x.shape[0]]))

		d = to_d(x, sigma_hat, denoised)
		dt = self.sigmas[i + 1] - sigma_hat

		if self.sigmas[i + 1] == 0:
			dt = self.sigmas[i + 1] - sigma_hat
			x = x + d * dt
		else:
			sigma_mid = sigma_hat.log().lerp(self.sigmas[i + 1].log(), 0.5).exp()
			dt_1 = sigma_mid - sigma_hat
			dt_2 = self.sigmas[i + 1] - sigma_hat
			x_2 = x + d * dt_1
			denoised_2 = self.denoise(x_2, sigma=sigma_mid * x.new_ones([x.shape[0]]))
			d_2 = to_d(x_2, sigma_mid, denoised_2)
			x = x + d_2 * dt_2

		return x

class schedule_euler_ancestral:
	def __init__(self, sigmas, denoise, eta=1.):
		self.denoise = denoise
		self.sigmas = sigmas
		self.eta = eta

	def step(self, x, i):
		denoised = self.denoise(x, sigma=self.sigmas[i] * x.new_ones([x.shape[0]]))
		sigma_down, sigma_up = get_ancestral_step(self.sigmas[i], self.sigmas[i + 1], eta=self.eta)

		d = to_d(x, self.sigmas[i], denoised)
		dt = sigma_down - self.sigmas[i]

		x = x + d * dt
		x = x + torch.randn_like(x) * sigma_up

		return x

class schedule_dpm_2_ancestral:
	def __init__(self, sigmas, denoise, eta=1.):
		self.denoise = denoise
		self.sigmas = sigmas
		self.eta = eta

	def step(self, x, i):
		denoised = self.denoise(x, sigma=self.sigmas[i] * x.new_ones([x.shape[0]]))
		sigma_down, sigma_up = get_ancestral_step(self.sigmas[i], self.sigmas[i + 1], eta=self.eta)

		d = to_d(x, self.sigmas[i], denoised)

		if sigma_down == 0:
			dt = sigma_down - self.sigmas[i]
			x = x + d * dt
		else:
			sigma_mid = self.sigmas[i].log().lerp(sigma_down.log(), 0.5).exp()
			dt_1 = sigma_mid - self.sigmas[i]
			dt_2 = sigma_down - self.sigmas[i]
			x_2 = x + d * dt_1
			denoised_2 = self.denoise(x_2, sigma=sigma_mid * x.new_ones([x.shape[0]]))
			d_2 = to_d(x_2, sigma_mid, denoised_2)
			x = x + d_2 * dt_2
			x = x + torch.randn_like(x) * sigma_up

		return x

class schedule_lms:
	def __init__(self, sigmas, denoise, order=4):
		self.denoise = denoise
		self.sigmas = sigmas
		self.order = order
		self.ds = []

	def step(self, x, i):
		cur_order = min(i + 1, self.order)
		denoised = self.denoise(x, sigma=self.sigmas[i] * x.new_ones([x.shape[0]]))

		d = to_d(x, self.sigmas[i], denoised)
		self.ds.append(d)

		if len(self.ds) > self.order:
			self.ds.pop(0)

		coeffs = [
			linear_multistep_coeff(cur_order, self.sigmas.cpu(), i, j)
			for j in range(cur_order)
		]

		return x + sum(coeff * d for coeff, d in zip(coeffs, reversed(self.ds)))

class schedule_dpmpp_2m:
	def __init__(self, sigmas, denoise):
		self.denoise = denoise
		self.sigmas = sigmas
		self.old_denoised = None

	def sigma_fn(self, t):
		return t.neg().exp()

	def t_fn(self, i):
		return self.sigmas[i].log().neg()

	def step(self, x, i):
		denoised = self.denoise(x, sigma=self.sigmas[i] * x.new_ones([x.shape[0]]))

		t, t_next = self.t_fn(i), self.t_fn(i + 1)
		h = t_next - t

		if self.old_denoised is None or self.sigmas[i + 1] == 0:
			x = (self.sigma_fn(t_next) / self.sigma_fn(t)) * x - (-h).expm1() * denoised
		else:
			h_last = t - self.t_fn(i - 1)
			r = h_last / h
			denoised_d = (1 + 1 / (2 * r)) * denoised - (1 / (2 * r)) * self.old_denoised
			x = (self.sigma_fn(t_next) / self.sigma_fn(t)) * x - (-h).expm1() * denoised_d

		self.old_denoised = denoised

		return x


import xml.etree.ElementTree as ET

file_name = "./map.osm"

tree = ET.parse(file_name)
root = tree.getroot()
file_name = file_name.replace('.xml', '.txt')
f=open(file_name, "w")
for node in root.iter('node'):
    ident = node.attrib['id']
    lat = node.attrib['lat']
    lon = node.attrib['lon']
    f.write("{} {} {}\n".format(ident,lat,lon))
f.close()