import networkx as nx
import matplotlib.pyplot as plt
import csv
import sys
import random

reload(sys)
sys.setdefaultencoding('utf8')

raw_top_nodes = {}

G = nx.DiGraph()
pos = {}

# f2 = open('../../Output/citation_data_mining/case_study_M/final_ifCS=0_k=10_alpha=0.9_N=100_M=10_ifSub=1_ifDelta=1','rb')
f2 = open('../../Output/venue/case_study_M/final_ifCS=0_k=20_alpha=0.9_N=20_M=10_ifSub=1_ifDelta=1','rb')
line = f2.readline()
line = f2.readline()
while line :
	row = line.split(' ')
	line = f2.readline()
	raw_top_nodes[int(row[0])] = 'top_list'
f2.close()

f1 = open('../../emb/venue/venue_indexNetworkMin10.txt','rb')
line = f1.readline()
while line:
	row = line.split(' ')
	line = f1.readline()
	pos[int(row[0])] = (float(row[1]), float(row[2]))
f1.close()

f1 = open('../../Dataset/venue/indexNetwork.txt','rb')
line = f1.readline()
while line:
	row = line.split(' ')
	if pos.has_key(int(row[0])) and pos.has_key(int(row[1])) and int(row[2]) >= 16:
		G.add_edge(int(row[0]), int(row[1]))
	line = f1.readline()
f1.close()

labels = {}
f1 = open('../../Dataset/venue/indexToName.txt','rb')
line = f1.readline()
while line:
	row = line.split(' ')
	labels[int(row[0])] = row[1].split('\t')[1]
	line = f1.readline()
f1.close()

plt.figure(figsize = (10,10), dpi = 500)
nx.draw(G, pos, node_color = 'grey', edge_color = 'grey', arrows=False, node_size = 1, alpha = 0.3)

G2 = nx.DiGraph()

top_labels = {}
for i in raw_top_nodes:
	G2.add_node(i)
	top_labels[i] = labels[i]

print(len(G2.nodes()))
nx.draw(G2, pos, labels=top_labels, node_color = 'red', node_size = 3, alpha = 1, font_size=10)

# plt.savefig('backbone_citation-DM')
plt.savefig('backbone_venue')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jan  6 17:09:03 2019

@author: myidispg
"""

import cv2
import os
import numpy as np
from imutils import face_utils
import dlib
from landmarks_calculation import left, right

from emotion_key import emotions_list

# For command line arguments
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-i', '-image', required=True, help='The path of the image with a face')
args = vars(parser.parse_args())

casc_directory = 'face-cascades/'
def find_face(gray):
    # Create the haar cascade
    faceCascade1 = cv2.CascadeClassifier(os.path.join(casc_directory, 'haarcascade_frontalface_alt.xml'))
    faceCascade2 = cv2.CascadeClassifier(os.path.join(casc_directory, 'haarcascade_frontalface_alt2.xml'))
    faceCascade3 = cv2.CascadeClassifier(os.path.join(casc_directory, 'haarcascade_frontalface_alt_tree.xml'))
    faceCascade4 = cv2.CascadeClassifier(os.path.join(casc_directory, 'haarcascade_frontalface_default.xml'))
    # Detect faces in the image using 4 different classifiers
    faces1 = faceCascade1.detectMultiScale(
                gray,
                scaleFactor=1.2,
                minNeighbors=5
                )
    faces2 = faceCascade2.detectMultiScale(
                gray,
                scaleFactor=1.2,
                minNeighbors=5
                )
    faces3 = faceCascade3.detectMultiScale(
                gray,
                scaleFactor=1.2 ,
                minNeighbors=5
                )
    faces4 = faceCascade4.detectMultiScale(
                gray,
                scaleFactor=1.2,
                minNeighbors=5
                )

    # Go over the detected faces of all the classifiers and select one.
    if len(faces1) == 1:
        faces = faces1
    elif len(faces2) == 1:
        faces = faces2
    elif len(faces3) == 1:
        faces = faces3
    elif len(faces4) == 1:
        faces = faces4
    else:
        print('No faces found')
        return (False, 0)

    return (True, faces)

# Open the image from path specified by cmd line

image = cv2.imread(args['i'], 0)
#image = cv2.imread('/home/myidispg/My Files/Machine-Learning-Projects/Emotion-Dataset/cohn-kanade-images/S061/001/S061_001_00000012.png', 0)
#image = cv2.imread('mummy/2.jpeg', 0)

p = "shape_predictor_68_face_landmarks.dat"
face_detector = dlib.get_frontal_face_detector()
landmark_predictor = dlib.shape_predictor(p)

# Get a cropped image with dimensions similar to cohn kanade images.
from crop_face_area import CropImage
crop_image = CropImage(image)
image = crop_image.get_cropped_image()

cv2.imshow('image', image)
cv2.waitKey()
cv2.destroyAllWindows()

rects = face_detector(image)

if len(rects) == 0:
    rects = find_face(image)
    if not rects[0]:
        import sys
        sys.exit()
    else:
        rects = dlib.rectangle(left = rects[1][0][0], top=rects[1][0][1], right=rects[1][0][2], bottom=rects[1][0][3])
        print("rects- {}".format(rects))
        print("left- {}, right- {}, top- {}, bottom- {}".format(rects.left(), rects.right(), rects.top(), rects.bottom()))
        shape = landmark_predictor(image, rects)
        # Convert to numpy array
        shape = face_utils.shape_to_np(shape)
else:
    print("rects- {}".format(rects))
    print("left- {}, right- {}, top- {}, bottom- {}".format(rects[0].left(), rects[0].right(), rects[0].top(), rects[0].bottom()))
    rects = dlib.rectangle(left = rects[0].left(), top=rects[0].top(), right=rects[0].right(), bottom= rects[0].bottom())
    # Find landmark
    shape = landmark_predictor(image, rects)
    # Convert to numpy array
    shape = face_utils.shape_to_np(shape)

# Required landmark calculations for a single image
x_coords = []
y_coords = []
for i in range(len(left)):
    x_coords.append(shape[left[i]][0] - shape[right[i]][0])
    y_coords.append(shape[left[i]][1] - shape[right[i]][1])

x_coords = np.asarray(x_coords)
y_coords = np.asarray(y_coords)
distance_between = np.sqrt(np.square(x_coords) + np.square(y_coords))

from sklearn.externals import joblib

classifier = joblib.load('logistic-regression-emotion-recognition.pkl')

distance_between = distance_between.reshape(1, -1)

predict = classifier.predict(distance_between)

print("The person's emotion seems to be- {}".format(emotions_list[predict[0]]))


""" Module to perform unit tests against Subscriber class for methods that
execute and can be tested independently of the publish/subscribe network """
import unittest
import sys
from src.unit_tests import *
from src.lib.zookeeper_client import ZookeeperClient
connected = False
class TestZookeeperClient(unittest.TestCase):
    broker = None
    def __init__(self, *args, **kwargs):
        try:
            global connected
            zk = ZookeeperClient(zookeeper_hosts=['127.0.0.1:2181'],use_logger=True)
            connected = True
        except Exception as e:
            print(e)
            print("You need to start the ZooKeeper service (port 2181) before running these tests")
            sys.exit(1)
        super(TestZookeeperClient, self).__init__(*args, **kwargs)
        self.zookeeper_client = ZookeeperClient(zookeeper_hosts=['127.0.0.1:2181'], verbose=True, use_logger=True)
        self.zookeeper_client.connect_zk()
        self.zookeeper_client.start_session()

    def tearDown(self):
        try:
            self.zookeeper_client.delete_znode(znode_name='/test_znode')
        except:
            pass

    def test_create_znode(self):
        global connected
        if not connected:
            self.skipTest(not connected, "Not connected to ZooKeeper. You need to start ZK service.")
        assert(self.zookeeper_client.create_znode(znode_name='/test_znode'))

    def test_delete_znode(self):
        global connected
        if not connected:
            self.skipTest(not connected, "Not connected to ZooKeeper. You need to start ZK service.")
        assert(self.zookeeper_client.create_znode(znode_name='/test_znode'))
        assert(self.zookeeper_client.delete_znode(znode_name='/test_znode'))

    def test_get_znode_value(self):
        global connected
        if not connected:
            self.skipTest(not connected, "Not connected to ZooKeeper. You need to start ZK service.")
        self.zookeeper_client.create_znode(znode_name='/test_znode', znode_value="Test Value")
        assert(self.zookeeper_client.get_znode_value(znode_name='/test_znode') == "Test Value")

    def test_modify_znode_value(self):
        global connected
        if not connected:
            self.skipTest(not connected, "Not connected to ZooKeeper. You need to start ZK service.")
        # Create a znode then set its value
        self.zookeeper_client.create_znode(znode_name='/test_znode')
        assert(self.zookeeper_client.modify_znode_value(
            znode_name='/test_znode',
            znode_value="this is a new value") == "this is a new value")


#25.Escribe un algoritmo o el respectivo diagrama de flujo que lea un número y si este es mayor o igual a 10 devuelva el triple de este, de lo contrario la cuarta parte de este.
print("______________________________________________________")

num = float(input("Por favor ingrese un número: "))

if num >= 10: #Si el número es mayor o igual a 10, la variable ingresada se multiplicara por tres de los contrario la variable se divide en 4, terminando imprimiendo el resultado.
    num *= 3
    print("El triple del número ingresado es: ","{0:g}".format(num))
else:
    num /= 4
    print("La cuarta parte del número ingresado es: ","{0:g}".format(num))


import geopandas as gpd
import pandas as pd
import matplotlib.pyplot as plt

# Read in the UK postal code boundary data
uk_postcodes = gpd.read_file('https://github.com/missinglinkai/uk-postcodes/raw/main/postcodes.geojson')

# Download the UK population data from the ONS website
url = 'https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationestimates/datasets/populationestimatesforukenglandandwalesscotlandandnorthernireland/mid2020april2021localauthoritydistrictcodes/ukmidyearestimates20202021ladcodes.xls'
pop_data = pd.read_excel(url, sheet_name='MYE2 - Persons', header=4)

# Select the relevant columns and rename them for clarity
pop_data = pop_data[['Code', 'Name', 'All ages']]
pop_data = pop_data.rename(columns={'Code': 'ladcd', 'Name': 'ladnm', 'All ages': 'population'})

# Merge the population data with the postal code data using the local authority district code (ladcd)
uk_pop = uk_postcodes.merge(pop_data, on='ladcd')

# Plot the chloropleth map
uk_pop.plot(column='population', cmap='OrRd', legend=True)
plt.title('UK Population by Postal Code')
plt.show()


from player import Player
from deck import Deck

class Game:
    def __init__(self, playerA = Player('player1'), playerB = Player('player2')):
        self.playerA = playerA
        self.playerB = playerB
        self.deck = Deck.createDeck()
        self.turn = 'A'

    def isWinningStateReached(self):
        if self.playerA.getPoints() == 21:
            return True, self.playerA.getUsername()

        if self.playerB.getPoints() == 21:
            return True, self.playerA.getUsername()

        if self.playerA.getPoints() > 21:
            return True, self.playerB.getUsername()

        if self.playerB.getPoints() > 21:
            return True, self.playerA.getUsername()

        if self.playerA.currentChoice == self.playerB.currentChoice == 's':
            if self.playerA.getPoints() == self.playerB.getPoints():
                return True, 'no one. DRAW'

            return ((True, self.playerA.getUsername()) if self.playerA.getPoints() > self.playerB.getPoints()
                        else (True, self.playerB.getUsername))

        return False, None

    def startGame(self):
        isGameEnd, winner = False, None
        while not isGameEnd:
            if self.turn == 'A':
                self.playerA.currentChoice = input("Player A choice = ")
                if self.playerA.currentChoice == 's': pass
                elif self.playerA.currentChoice == 'd':
                    card = self.playerA.updatePoints(self.deck.drawCard())
                    print(f"{card.cardSymbol} of {card.suite} was drawn.")
                else:
                    print('Invalid choice.')
                    continue
            elif self.turn == 'B':
                self.playerB.currentChoice = input("Player B choice = ")
                if self.playerB.currentChoice == 's': pass
                elif self.playerB.currentChoice == 'd':
                    card = self.playerB.updatePoints(self.deck.drawCard())
                    print(f"{card.cardSymbol} of {card.suite} was drawn.")
                else:
                    print('Invalid choice.')
                    continue

            self.turn = 'B' if self.turn == 'A' else 'A'
            print(f"Score A={self.playerA.getPoints()} B={self.playerB.getPoints()}\n")
            isGameEnd, winner = self.isWinningStateReached()

        print(f"Winner is {winner}")

g = Game()
g.startGame()


import socket
import requests
import time
import sys
import threading

def listenThread():
	global s
	while True:
		data = s.recv(4096).decode()
		if not data:
			break
		print(data)

port = 81
IP_addr = "172.20.10.9"

if len(sys.argv) == 2:
    IP_addr = sys.argv[1]
elif len(sys.argv) == 3:
    IP_addr = sys.argv[1]
    port = int(sys.argv[2])

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((IP_addr, port))

listen_thread = threading.Thread(target=listenThread, daemon=True)
listen_thread.start()
time.sleep(1)
r = requests.post(url = "http://172.20.10.9/", data = "/capture")
print(r.text)


from re import findall

def preprocessing(input):
    all_bags = {}
    input = input.replace("bags","").replace("bag","").replace(".","")
    for rule in input.splitlines():
        color, bags = rule.split("  contain ")[:2]
        if 'no other' in rule: bags = []
        else:
            bags = [b.strip().split(" ",1) for b in bags.split(",")]
            bags = [[int(n), b] for n, b in bags]
        all_bags[color] = bags
    return all_bags

def solver(rules):
    candidates = ["shiny gold"]
    answer = set()
    while candidates:
        candidate = candidates.pop()
        for color in rules.keys():
            if candidate in (c for n,c in rules.get(color)):
                candidates.append(color)
                answer.add(color)
    yield len(answer)
    yield count_bags("shiny gold", rules) - 1

def count_bags(color, bags):
    return 1 + sum([n * count_bags(c, bags) for n,c in bags[color]])


if __name__ == '__main__':
    n = int(input())
    student_marks = {}
    for _ in range(n):
        name, *line = input().split()
        scores = list(map(float, line))
        student_marks[name] = scores
    query_name = input()
    final = sum(student_marks[query_name])/3.00
    format_final = "{:.2f}".format(final)
    print(format_final)


# 基数排序又称桶子法，是桶排序的扩展，用于排序数字或者字母类型的数据。
# 1. 将所有待排序整数（注意，必须是非负整数）统一为位数相同的整数，位数较少的前面补零。
#   一般用10进制，也可以用16进制甚至2进制。所以前提是能够找到最大值，得到最长的位数，设 k 进制下最长为位数为 d 。
# 2. 从最低位开始，依次进行一次稳定排序。这样从最低位一直到最高位排序完成以后，整个序列就变成了一个有序序列。

# 举个例子，有一个整数序列，0, 123, 45, 386, 106，下面是排序过程：
#       第一次排序，个位，000 123 045 386 106，无任何变化
#       第二次排序，十位，000 106 123 045 386
#       第三次排序，百位，000 045 106 123 386
#       最终结果，0, 45, 106, 123, 386, 排序完成。

# 首先，基数排序和计数排序都可以看作是桶排序。
# 计数排序本质上是一种特殊的桶排序，当桶的个数取最大( maxV-minV+1 )的时候，就变成了计数排序。

# 基数排序也是一种桶排序。桶排序是按值区间划分桶，基数排序是按数位来划分；
# 基数排序可以看做是多轮桶排序，每个数位上都进行一轮桶排序。

# 当用最大值作为基数时，基数排序就退化成了计数排序。
# 当使用2进制时，k=2最小，位数d最大，时间复杂度O(nd)会变大，空间复杂度O(n+k)会变小。
# 当用最大值作为基数时，k=maxV最大，d=1 最小，此时时间复杂度O(nd)变小，
# 但是空间复杂度O(n+k)会急剧增大，此时基数排序退化成了计数排序。

# 基数排序在进制k和位数d之间折中，在空间复杂度和时间复杂度之间进行折中。

def radix_sort(s):
    """基数排序"""
    i = 0  # 记录当前正在排那一位，最低位为1
    max_num = max(s)  # 最大值
    j = len(str(max_num))  # 记录最大值的位数
    while i < j:
        bucket_list = [[] for _ in range(10)]  # 初始化桶数组
        for x in s:
            bucket_list[int(x / (10 ** i)) % 10].append(x)  # 找到位置放入桶数组
        print(bucket_list)
        s.clear()
        for x in bucket_list:  # 放回原序列
            for y in x:
                s.append(y)
        i += 1

if __name__ == '__main__':
    a = [334, 5, 67, 345, 7, 345345, 99, 4, 23, 78, 45, 1, 3453, 23424]
    radix_sort(a)
    print(a)


import functools

import torchbearer
from torchbearer.callbacks import Callback
from torchbearer.bases import get_metric

import torch

import warnings

class TorchScheduler(Callback):
    def __init__(self, scheduler_builder, monitor=None, step_on_batch=False):
        self._scheduler_builder = scheduler_builder
        self._monitor = monitor
        self._scheduler = None
        self._step_on_batch = step_on_batch

        from distutils.version import LooseVersion
        version = torch.__version__ if str(torch.__version__) is torch.__version__ else "0.4.0"
        self._newstyle = LooseVersion(version) > LooseVersion("1.1.0")

    def _step(self, state, current=None):
        if state[torchbearer.MODEL].training is False:
            return

        if self._newstyle or self._step_on_batch:
            if current is None:
                self._scheduler.step()
            else:
                self._scheduler.step(current)
        else:
            if current is None:
                self._scheduler.step(epoch=state[torchbearer.EPOCH])
            else:
                self._scheduler.step(current, epoch=state[torchbearer.EPOCH])

    def on_start(self, state):
        try:
            self._scheduler = self._scheduler_builder(state[torchbearer.OPTIMIZER],
                                                      last_epoch=state[torchbearer.EPOCH] - 1)
        except TypeError:
            self._scheduler = self._scheduler_builder(state[torchbearer.OPTIMIZER])

        if state[torchbearer.EPOCH] > 0 and self._step_on_batch:
            warnings.warn('Resuming schedulers with the `step_on_batch` option is not currently supported and may cause'
                          ' unexpected behaviour.')

    def on_sample(self, state):
        if not self._newstyle and self._step_on_batch and self._monitor is None:
            self._step(state)

    def on_step_training(self, state):
        if self._step_on_batch:
            if self._monitor is not None:
                current = get_metric('Scheduler', state, self._monitor)
                if current is None:
                    return
                self._step(state, current)
            elif self._newstyle:
                self._step(state)

    def on_start_training(self, state):
        if not self._newstyle and not self._step_on_batch and self._monitor is None:
            self._step(state)

    def on_end_epoch(self, state):
        if not self._step_on_batch:
            if self._monitor is not None:
                current = get_metric('Scheduler', state, self._monitor)
                if current is None:
                    return
                self._step(state, current)
            else:
                self._step(state)

class LambdaLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import LambdaLR

        # Example Trial which performs the two learning rate lambdas from the PyTorch docs
        >>> lambda1 = lambda epoch: epoch // 30
        >>> lambda2 = lambda epoch: 0.95 ** epoch
        >>> scheduler = LambdaLR(lr_lambda=[lambda1, lambda2])
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).run(1)

    Args:
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch LambdaLR <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.LambdaLR>`_
    """
    def __init__(self, lr_lambda, step_on_batch=False):
        super(LambdaLR, self).__init__(functools.partial(torch.optim.lr_scheduler.LambdaLR, lr_lambda=lr_lambda),
                                       step_on_batch=step_on_batch)

class StepLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import StepLR

        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 60
        >>> # lr = 0.0005   if 60 <= epoch < 90
        >>> scheduler = StepLR(step_size=30, gamma=0.1)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).run(1)

    Args:
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch StepLR <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.StepLR>`_
    """
    def __init__(self, step_size, gamma=0.1, step_on_batch=False):
        super(StepLR, self).__init__(functools.partial(torch.optim.lr_scheduler.StepLR,
                                                       step_size=step_size, gamma=gamma),
                                     step_on_batch=step_on_batch)

class MultiStepLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import MultiStepLR

        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 80
        >>> # lr = 0.0005   if epoch >= 80
        >>> scheduler = MultiStepLR(milestones=[30,80], gamma=0.1)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).run(1)

    Args:
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch MultiStepLR <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.MultiStepLR>`_
    """
    def __init__(self, milestones, gamma=0.1, step_on_batch=False):
        super(MultiStepLR, self).__init__(functools.partial(torch.optim.lr_scheduler.MultiStepLR,
                                                            milestones=milestones, gamma=gamma),
                                          step_on_batch=step_on_batch)

class ExponentialLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import ExponentialLR

        >>> # Example scheduler which multiplies the learning rate by 0.1 every epoch
        >>> scheduler = ExponentialLR(gamma=0.1)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).run(1)

    Args:
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch ExponentialLR <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.ExponentialLR>`_
    """
    def __init__(self, gamma, step_on_batch=False):
        super(ExponentialLR, self).__init__(functools.partial(torch.optim.lr_scheduler.ExponentialLR, gamma=gamma),
                                            step_on_batch=step_on_batch)

class CosineAnnealingLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import CosineAnnealingLR

        >>> # Example scheduler which uses cosine learning rate annealing - see PyTorch docs
        >>> scheduler = MultiStepLR(milestones=[30,80], gamma=0.1)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).run(1)

    Args:
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch CosineAnnealingLR <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.CosineAnnealingLR>`_
    """
    def __init__(self, T_max, eta_min=0, step_on_batch=False):
        super(CosineAnnealingLR, self).__init__(functools.partial(torch.optim.lr_scheduler.CosineAnnealingLR,
                                                                  T_max=T_max, eta_min=eta_min),
                                                step_on_batch=step_on_batch)

class ReduceLROnPlateau(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import ReduceLROnPlateau

        >>> # Example scheduler which divides the learning rate by 10 on plateaus of 5 epochs without significant
        >>> # validation loss decrease, in order to stop overshooting the local minima. new_lr = lr * factor
        >>> scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).for_val_steps(10).run(1)

    Args:
        monitor (str): The name of the quantity in metrics to monitor. (Default value = 'val_loss')
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch ReduceLROnPlateau <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.ReduceLROnPlateau>`_
    """
    def __init__(self,  monitor='val_loss', mode='min', factor=0.1, patience=10, verbose=False, threshold=1e-4,
                 threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-8, step_on_batch=False):
        super(ReduceLROnPlateau, self).__init__(functools.partial(torch.optim.lr_scheduler.ReduceLROnPlateau,
                                                mode=mode, factor=factor, patience=patience, verbose=verbose,
                                                threshold=threshold, threshold_mode=threshold_mode,
                                                cooldown=cooldown, min_lr=min_lr, eps=eps), monitor=monitor,
                                                step_on_batch=step_on_batch)

class CyclicLR(TorchScheduler):
    """
    Example: ::

        >>> from torchbearer import Trial
        >>> from torchbearer.callbacks import CyclicLR

        >>> # Example scheduler which cycles the learning rate between 0.01 and 0.1
        >>> scheduler = CyclicLR(0.01, 0.1)
        >>> trial = Trial(None, callbacks=[scheduler], metrics=['loss'], verbose=2).for_steps(10).for_val_steps(10).run(1)

    Args:
        monitor (str): The name of the quantity in metrics to monitor. (Default value = 'val_loss')
        step_on_batch (bool): If True, step will be called on each training iteration rather than on each epoch

    See:
        `PyTorch ReduceLROnPlateau <http://pytorch.org/docs/master/optim.html#torch.optim.lr_scheduler.ReduceLROnPlateau>`_
    """
    def __init__(self,  base_lr, max_lr, monitor='val_loss', step_size_up=2000, step_size_down=None, mode='triangular',
                 gamma=1., scale_fn=None, scale_mode='cycle', cycle_momentum=True, base_momentum=0.8, max_momentum=0.9,
                 step_on_batch=False):
        from distutils.version import LooseVersion
        version = torch.__version__ if str(torch.__version__) is torch.__version__ else "0.4.0"
        if LooseVersion(version) > LooseVersion("1.0.0"):  # CyclicLR is implemented
            super(CyclicLR, self).__init__(functools.partial(torch.optim.lr_scheduler.CyclicLR,
                                           base_lr=base_lr, max_lr=max_lr, step_size_up=step_size_up,
                                           step_size_down=step_size_down, mode=mode, gamma=gamma,
                                           scale_fn=scale_fn, scale_mode=scale_mode,
                                           cycle_momentum=cycle_momentum, base_momentum=base_momentum,
                                           max_momentum=max_momentum),
                                           monitor=monitor, step_on_batch=step_on_batch)
        else:
            raise NotImplementedError('CyclicLR scheduler was not implemented in PyTorch versions less than 1.1.0. '
                                      'Update PyTorch or use the CyclicLR callback from an older Torchbearer version.')


"""Test fg + titan + gridpp + obsmon."""
import json
import os
import shutil

import numpy as np
import pytest

from pysurfex.cli import (
    cli_merge_qc_data,
    cli_oi2soda,
    first_guess_for_oi,
    gridpp,
    qc2obsmon,
    titan,
)

an_time = "2020022006"

def hm_env():
    env = {
        "NPATCH": "2",
        "ARCHIVE_ROOT_FG": "/firstguess",
        "LSPBDC": "no",
        "COMPILE_DABYFA": "no",
        "ANASURF": "CANARI_OI_MAIN",
        "TSPAN": "5400",
        "BDDIR": "/archive/ECMWF/@YYYY@/@MM@/@DD@/@HH@",
        "MAIL_ON_ABORT": "",
        "COMPILE_ENKF": "no",
        "ARCH": "linuxgfortran",
        "FESTAT": "no",
        "SLAFDIFF": "0",
        "NLEV": "60",
        "LETKF_LAG": "no",
        "NNCV": "1,1,1,1",
        "XCH_COEFF1": "15.0",
        "NNCO": "1,1,0,0,1",
        "STREAM": "",
        "HH_LIST": "00-21:3",
        "SPPT": "no",
        "ARCHIVE_ROOT": "/archive",
        "LSMIXBC": "yes",
        "ARSTRATEGY": "climate:fg:verif:odb_stuff:               [an|fc]_fa:pp_gr:fldver",
        "TAU_SPP": "21600.",
        "DISPLAY": ":1",
        "XCLIP_RATIO_SDT": "5.0",
        "ECFSGROUP": "hirald",
        "EPERT_MODE": "after",
        "ENS_BD_CLUSTER": "no",
        "OBDIR": "/archive/observations/@YYYY@/@MM@/@DD@/@HH@",
        "FCINT_FG": "03",
        "SFXSELTIMES": "0-540:15",
        "AD_TEST": "yes",
        "TSTEP4D": "120,120",
        "VLEV": "65",
        "PERTSURF": "none",
        "SCALE_PERT": "yes",
        "OBSMONITOR": "obstat",
        "LOCAL_DEFINITION_TEMPLATES": "LOCAL_DEFINITION_TEMPLATES",
        "LUNBC": "yes",
        "PERTDIA_BDINT": "6",
        "GRID_TYPE": "CUSTOM",
        "NOUTERLOOP": "2",
        "XZ0SN": "0.003",
        "MSG_PATH": "/CLOUDS/",
        "INT_SINI_FILE": "//SURFXINI.fa",
        "ALARO_VERSION": "0",
        "CNMEXP": "HARM",
        "ANAATMO": "3DVAR",
        "SWRITUPTIMES": "0-540:60",
        "CSNOW": "D95",
        "SURFEX_OFFLINE_BINARIES": "no",
        "JB_REF_DOMAIN": "METCOOP25B",
        "FREQ_RESET_TEMP": "3",
        "TOPO_SOURCE": "gmted2010",
        "TAU_SDT": "28800",
        "CLIMDIR": "/climate/METCOOP25D",
        "EXP": "HM2PYSURFEX",
        "IO_SERVER": "yes",
        "ARCHIVE_ENSMBR": "",
        "BUFRTAB_DIR": "",
        "RSMIN_CONIFEROUS_FACTOR": "1.44",
        "BDSTRATEGY": "simulate_metcoop",
        "LSPG_SDT": ".FALSE.",
        "DFI": "none",
        "ENSINIPERT": "bnd",
        "EXTRARCH": "/archive/extract",
        "SFXWFTIMES": "0-540:60",
        "VERT_DISC": "vfd",
        "XCD_COEFF1": "10.0",
        "XCD_COEFF2": "5.0",
        "XGRASS_H_DNM": "3.0",
        "LDB_VERSION": "3.0",
        "DESKTOP_SESSION": "ubuntu",
        "PWRITUPTIMES": "0-09:15",
        "USE_MSG": "yes",
        "DOMAIN": "METCOOP25S",
        "CREATE_CLIMATE": "yes",
        "HGT_QS": "yes",
        "FP_PRECISION": "double",
        "TAUS": "5400",
        "MODIFY_LAKES": "T",
        "LGRADSP": "no",
        "ECFSLOC": "ec",
        "RSMIN_C3_FACTOR": "1.5",
        "CH_RES_SPEC": "yes",
        "ILRES": "2,2",
        "XRIMAX": "0.0",
        "SPP": "no",
        "MAKEODB2": "no",
        "GRIB_API": "GRIB_API",
        "INT_BDFILE": "//ELSCFHARMALBC@NNN@",
        "BDINT": "1",
        "RSMIN_DECIDUOUS_FACTOR": "1.13",
        "BINDIR": "/bin",
        "CV_HIGHVEG": "1.0E-5",
        "USE_REARCHIVE_EPS_EC_OPER": "no",
        "XDG_SESSION_TYPE": "x11",
        "RSMIN_C4_FACTOR": "1.13",
        "CROUGH": "NONE",
        "SLAFLAG": "0",
        "MULTITASK": "yes",
        "SPGADTMIN_SDT": "0.15",
        "MASS_FLUX_SCHEME": "edmfm",
        "HOST_SURFEX": "no",
        "FULLFAFTIMES": "0-540:15",
        "TFLAG_FG": "h",
        "CISBA": "3-L",
        "PERTATMO": "none",
        "XCGMAX": "2.0E-5",
        "COMPCENTRE": "MET",
        "SURFEX_LAKES": "FLAKE",
        "FLDEXTR": "yes",
        "REARCHIVE_EPS_EC_OPER": "no",
        "NBDMAX": "4",
        "SST_SOURCES": "IFS NEMO",
        "CAERO": "tegen",
        "CFORCING_FILETYPE": "NETCDF",
        "GRIB_API_LIB": "GRIB_API_LIB",
        "FLDEXTR_TASKS": "4",
        "BUILD": "yes",
        "ECOCLIMAP_VERSION": "SG",
        "SURFEX_SEA_ICE": "sice",
        "XLCOR_SPP": "1000000.",
        "ENSCTL": "",
        "PFFULLWFTIMES": "-1",
        "LL_LIST": "03",
        "HOST_MODEL": "ifs",
        "XSCALE_H_TREE": "0.658",
        "XCSMAX": "2.0E-4",
        "SPGADTMIN_SPP": "0.15",
        "AUXLIBS": "AUXLIBS",
        "TEND_DIAG": "no",
        "ODB_VERSION": "CY33R1.007",
        "WRK": "/",
        "EREF": "35000.",
        "OBSEXTR": "yes",
        "HWRITUPTIMES": "0-540:15",
        "CONT_ON_FAILURE": "0",
        "TSTEPTRAJ": "600",
        "XZ0HSN": "0.0003",
        "SFXSWFTIMES": "-1",
        "BDLIB": "ECMWF",
        "QT_IM_MODULE": "ibus",
        "RUNNING_MODE": "research",
        "SIMULATION_TYPE": "nwp",
        "CONVERTFA": "yes",
        "XLCOR_SDT": "2000000",
        "TL_TEST": "yes",
        "RCR_POSTP": "no",
        "FLDVER": "no",
        "ARCHIVE_ECMWF": "yes",
        "JB_INTERPOL": "yes",
        "SDEV_SDT": "0.20",
        "TESTBED_CASES": "1",
        "LISBA_CANOPY": ".FALSE.",
        "SLAFK": "1.0",
        "CV_LOWVEG": "2.0E-5",
        "SOIL_TEXTURE_VERSION": "SOILGRID",
        "MEPS_VERSION": "test",
        "Q_IN_SP": "no",
        "SHLVL": "1",
        "MAKEGRIB_LISTENERS": "1",
        "GRIB_API_INCLUDE": "GRIB_API_INCLUDE",
        "ECFS_EPS_EC_BD_PATH": "ECFS_EPS_EC_BD_PATH",
        "ODB_DIR": "/disk1/odb",
        "SURFEX_LSELECT": "yes",
        "PHYSICS": "arome",
        "ARCHIVE_FORMAT": "GRIB2",
        "VERITIMES": "00-540:60",
        "HARATU": "yes",
        "BDCLIM": "/ECMWF/climate",
        "SINGLEOBS": "no",
        "IO_SERVER_BD": "yes",
        "SPGADTMAX_SDT": "3.0",
        "FORCE2": "no",
        "FORCE1": "no",
        "CHKEVO": "no",
        "SDEV_SPP": "0.2",
        "XALLEN_TERM": "2.5",
        "DYNAMICS": "nh",
        "ECMWF_LOCAL_TABLE_PATH": "ECMWF_LOCAL_TABLE_PATH",
        "TESTBED_LIST": "TESTBED_LIST",
        "OBSMON_SYNC": "no",
        "LETKF_3DSCREEN": "yes",
        "TFLAG": "min",
        "FLDVER_HOURS": "06 12 18 24 30 36 42 48 54 60 66",
        "ANASURF_MODE": "before",
        "AUGMENT_CV": "NO",
        "ENSMSEL": "",
        "HYBRID": "no",
        "FREQ_RESET_GUST": "1",
        "NCNAMES": "nwp",
        "STATNW": "yes",
        "ENSBDMBR": "",
        "POSTP": "inline",
        "ECOCLIMAP_PARAM_BINDIR": "/climate",
        "SPGADTMAX_SPP": "3.0",
        "SPGQ_SDT": "0.5",
        "ANASURF_OI_COEFF": "POLYNOMES_ISBA_MF6",
        "VFLDEXP": "HM2PYSURFEX",
        "AI_ACCUMULATION_HOURS": "720",
        "TASK_LIMIT": "-1",
        "EZONE": "1",
        "GSIZE": "10000.0",
        "LAT0": "59.5",
        "LATC": "60.0",
        "LON0": "9.0",
        "LONC": "10.0",
        "NDGUXG": "19",
        "NDLUXG": "9",
        "NLAT": "20",
        "NLON": "10",
        "NMSMAX": "-1",
        "NNOEXTZX": "0",
        "NNOEXTZY": "0",
        "NSMAX": "-1",
        "SINLAT0": "-1",
        "TSTEP": "90",
        "FCINT": "3",
        "TEFRCL": "3600",
        "NXGSTPERIOD": "3600",
        "MODEL": "MASTERODB",
        "ENSMSELX": "-1",
        "ENSMBR": "-1",
        "LL": "03",
        "METER_UNIT": "hhmm",
    }
    for key, value in env.items():
        os.environ[key] = value

def create_titan_settings(qc_fname, first_guess_file, blacklist_fname, json_obs_file):
    qc_settings = {
        "t2m": {
            "do_test": True,
            "plausibility": {"minval": 200, "maxval": 350},
            "firstguess": {
                "fg_file": first_guess_file,
                "fg_var": "air_temperature_2m",
                "negdiff": 2,
                "posdiff": 3,
                "do_test": False,
            },
            # Not working yet
            "buddy": {"do_test": False},
            "climatology": {"do_test": False, "minval": 270, "maxval": 275},
            "sct": {},
            "redundancy": {},
            "blacklist": {},
            "domain": {},
            "nometa": {},
            "fraction": {
                "fraction_file": first_guess_file,
                "fraction_var": "land_area_fraction",
            },
            "sets": {
                "label": {
                    "filepattern": json_obs_file,
                    "filetype": "json",
                    "varname": "airTemperatureAt2M",
                    "tests": {"firstguess": {"do_test": True}},
                    "sigmao": 0.5,
                }
            },
        },
        "rh2m": {
            "do_test": True,
            "plausibility": {"minval": 0, "maxval": 1},
            "firstguess": {
                "fg_file": first_guess_file,
                "fg_var": "relative_humidity_2m",
                "negdiff": 0.2,
                "posdiff": 0.2,
                "do_test": False,
            },
            # Not working yet
            "buddy": {"do_test": False},
            "climatology": {"do_test": False, "minval": 0, "maxval": 1},
            "sct": {},
            "redundancy": {},
            "blacklist": {},
            "domain": {},
            "nometa": {},
            "fraction": {
                "fraction_file": first_guess_file,
                "fraction_var": "land_area_fraction",
            },
            "sets": {
                "label": {
                    "filepattern": json_obs_file,
                    "filetype": "json",
                    "varname": "relativeHumidityAt2M",
                    "tests": {"firstguess": {"do_test": True}},
                    "sigmao": 0.5,
                }
            },
        },
        "sd": {
            "do_test": True,
            "plausibility": {"minval": 0, "maxval": 50},
            "firstguess": {
                "fg_file": first_guess_file,
                "fg_var": "surface_snow_thickness",
                "negdiff": 0.4,
                "posdiff": 0.4,
                "do_test": True,
            },
            # Not working yet
            "buddy": {"do_test": False},
            "climatology": {"do_test": False, "minval": 0, "maxval": 1},
            "sct": {},
            "redundancy": {},
            "blacklist": {},
            "domain": {},
            "nometa": {},
            "fraction": {
                "fraction_file": first_guess_file,
                "fraction_var": "land_area_fraction",
            },
            "sets": {
                "label": {
                    "filepattern": json_obs_file,
                    "filetype": "json",
                    "varname": "totalSnowDepth",
                    "tests": {"firstguess": {"do_test": True}},
                    "sigmao": 0.5,
                }
            },
        },
    }
    with open(qc_fname, mode="w", encoding="utf-8") as file_handler:
        json.dump(qc_settings, file_handler)
    with open(blacklist_fname, mode="w", encoding="utf-8") as file_handler:
        json.dump({}, file_handler)

def create_obs_data(var, obs_fname):
    if var == "t2m":
        name = "airTemperatureAt2M"
        val = 273
    elif var == "rh2m":
        name = "relativeHumidityAt2M"
        val = 85
    elif var == "sd":
        name = "totalSnowDepth"
        val = 0.25
    else:
        raise NotImplementedError
    qc_data = {
        "0": {
            "varname": name,
            "obstime": "202002200600",
            "lon": 6.9933000000000005,
            "lat": 62.191,
            "stid": "1111",
            "elev": 900.0,
            "value": val,
            "flag": 0.0,
            "epsilon": 0.8,
            "laf": 1.0,
            "provider": "bufr",
            "fg_dep": np.nan,
            "an_dep": np.nan,
            "passed_tests": [
                "domain",
                "blacklist",
                "nometa",
                "plausibility",
                "redundancy",
                "firstguess",
                "fraction",
                "sct",
            ],
        },
        "1": {
            "varname": name,
            "obstime": "202002200600",
            "lon": 7.8173,
            "lat": 59.767500000000005,
            "stid": "NA",
            "elev": 1340.0,
            "value": val,
            "flag": 199.0,
            "epsilon": 1.2,
            "laf": 1.0,
            "provider": "bufr",
            "fg_dep": np.nan,
            "an_dep": np.nan,
            "passed_tests": [],
        },
        "2": {
            "varname": name,
            "obstime": "202002200600",
            "lon": 9.99,
            "lat": 60.191,
            "stid": "NA",
            "elev": 900.0,
            "value": val,
            "flag": 0.0,
            "epsilon": 0.9,
            "laf": 1.0,
            "provider": "bufr",
            "fg_dep": np.nan,
            "an_dep": np.nan,
            "passed_tests": [
                "domain",
                "blacklist",
                "nometa",
                "plausibility",
                "redundancy",
                "firstguess",
                "fraction",
                "sct",
            ],
        },
    }
    json.dump(qc_data, open(obs_fname, mode="w", encoding="utf-8"))

@pytest.fixture(params=["t2m", "rh2m", "sd"])
def _qc_gridpp_obsmon(
    tmp_path_factory, request, conf_proj_domain_file, firstguess4gridpp, hm
):
    harmonie = []
    if hm == "harmonie":
        hm_env()
        harmonie = ["--harmonie"]
    var = request.param
    translation = {
        "t2m": {
            "elevGradient": "-0.0065",
            "nc_name": "air_temperature_2m",
            "hor": "30000.0",
            "vert": "300.0",
        },
        "rh2m": {
            "elevGradient": "0.0",
            "nc_name": "relative_humidity_2m",
            "hor": "40000.0",
            "vert": "400.0",
        },
        "sd": {
            "elevGradient": "0.0",
            "nc_name": "surface_snow_thickness",
            "hor": "60000.0",
            "vert": "500.0",
        },
    }

    first_guess_file = firstguess4gridpp
    # Create observations
    obs_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/obs_{var}.json"
    create_obs_data(var, obs_fname)

    # Titan
    qc_settings_fname = (
        f"{tmp_path_factory.getbasetemp().as_posix()}/qc_settings_{var}.json"
    )
    qc_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/qc_{var}.json"
    blacklist_fname = f"{tmp_path_factory.getbasetemp().as_posix()}/blacklist_{var}.json"
    create_titan_settings(qc_settings_fname, first_guess_file, blacklist_fname, obs_fname)

    with pytest.raises(SystemExit):
        titan(argv=["fail"])

    argv = [
        "-i",
        qc_settings_fname,
        "-v",
        var,
        "-dtg",
        an_time,
        "--blacklist",
        blacklist_fname,
        "--domain",
        conf_proj_domain_file,
        "-o",
        qc_fname,
        "--debug",
        "domain",
        "blacklist",
        "nometa",
        "plausibility",
        "redundancy",
        "firstguess",
        "fraction",
        "buddy",
        "climatology",
        "sct",
    ]
    argv += harmonie
    titan(argv=argv)

    qc_titan_obs = json.load(open(qc_fname, "r"))
    assert qc_titan_obs["0"]["epsilon"] == 0.5
    shutil.copy(qc_fname, f"{qc_fname}-1")
    shutil.copy(qc_fname, f"{qc_fname}-2")
    argv = [
        "-t",
        an_time,
        "-i",
        f"{qc_fname}-1",
        f"{qc_fname}-2",
        "-o",
        f"{qc_fname}-merged",
    ]
    cli_merge_qc_data(argv=argv)

    # gridpp
    with pytest.raises(SystemExit):
        gridpp(argv=["fail"])

    analysis_file = f"{tmp_path_factory.getbasetemp().as_posix()}/an_{var}.nc"
    argv = [
        "-i",
        first_guess_file,
        "-o",
        analysis_file,
        "-obs",
        obs_fname,
        "-hor",
        translation[var]["hor"],
        "-vert",
        translation[var]["vert"],
        "-v",
        translation[var]["nc_name"],
        "--elevGradient",
        translation[var]["elevGradient"],
    ]
    print(qc_fname)
    gridpp(argv=argv)

    output = f"{tmp_path_factory.getbasetemp().as_posix()}/OBSERVATIONS_200330H06.DAT"
    with pytest.raises(SystemExit):
        cli_oi2soda(argv=["fail"])

    argv = [
        "--t2m_file",
        first_guess_file,
        "--t2m_var",
        "air_temperature_2m",
        "--rh2m_file",
        first_guess_file,
        "--rh2m_var",
        "relative_humidity_2m",
        "--sd_file",
        first_guess_file,
        "--sd_var",
        "surface_snow_thickness",
        "--debug",
        "-o",
        output,
        "2020033006",
    ]
    cli_oi2soda(argv=argv)

    # Obsmon
    db_file = f"{tmp_path_factory.getbasetemp().as_posix()}/ecma.db"
    obsmon_test(var, obs_fname, first_guess_file, analysis_file, db_file)

def obsmon_test(var, qc_fname, first_guess_file, analysis_file, db_file):

    translation = {
        "t2m": "air_temperature_2m",
        "rh2m": "relative_humidity_2m",
        "sd": "surface_snow_thickness",
    }
    nc_name = translation[var]

    with pytest.raises(SystemExit):
        qc2obsmon(argv=["fail"])

    argv = [
        an_time,
        var,
        qc_fname,
        "--fg_file",
        first_guess_file,
        "--an_file",
        analysis_file,
        "--file_var",
        nc_name,
        "-o",
        db_file,
    ]
    qc2obsmon(argv=argv)

@pytest.mark.usefixtures("_qc_gridpp_obsmon")
@pytest.mark.parametrize("hm", ["no-harmonie", "harmonie"])
def test_qc_gridpp_obsmon():
    _qc_gridpp_obsmon

@pytest.mark.parametrize("hm", ["no-harmonie", "harmonie"])
def test_first_guess(tmp_path_factory, conf_proj_2x3_file, data_thredds_nc_file, hm):
    output = f"{tmp_path_factory.getbasetemp().as_posix()}/FirstGuess4gridpp_output.nc"
    harmonie = []
    if hm == "harmonie":
        hm_env()
        harmonie = ["--harmonie"]

    with pytest.raises(SystemExit):
        first_guess_for_oi(argv=["fail"])

    argv = [
        "-c",
        "pysurfex/cfg/first_guess.yml",
        "-i",
        data_thredds_nc_file,
        "-if",
        "netcdf",
        "-dtg",
        an_time,
        "-d",
        conf_proj_2x3_file,
        "--laf_converter",
        "none",
        "--sd_converter",
        "sweclim",
        "--debug",
        "-o",
        output,
        "air_temperature_2m",
        "relative_humidity_2m",
        "surface_snow_thickness",
    ]
    argv += harmonie
    first_guess_for_oi(argv=argv)


import os,sys
sys.path.append('..')
import logging
import math

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertModel
from torch.autograd import Variable

from transformers import BertConfig

from misc.config import Config

class MLMHead(nn.Module):
    def __init__(self, bert_config):
        super(MLMHead, self).__init__()
        self.transform = nn.Linear(bert_config.hidden_size, bert_config.hidden_size)
        self.act = nn.GELU()
        self.LayerNorm = nn.LayerNorm(bert_config.hidden_size, eps=bert_config.layer_norm_eps)
        self.decoder = nn.Linear(bert_config.hidden_size, bert_config.vocab_size - 1) # classification layer for MLM

    def forward(self, seq_feat):
        pred = self.decoder(self.LayerNorm(self.act(self.transform(seq_feat)))) # [B, L, 512] --> [B, L, vocab-1]
        return pred

def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

class JointTransformer(nn.Module):
    """Container module with an encoder, a recurrent or transformer module, and a decoder."""

    def __init__(self, bert_config, nlayers, output_channels, dropout=0.5):
        super(JointTransformer, self).__init__()
        try:
            from torch.nn import TransformerEncoder, TransformerEncoderLayer
        except:
            raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower.')
        self.model_type = 'Transformer'

        encoder_layers = TransformerEncoderLayer(bert_config.hidden_size #512
                                                 , bert_config.num_attention_heads #8
                                                 , bert_config.intermediate_size #2048
                                                 , dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.mlm =  MLMHead(bert_config)
        self.avgpool = nn.AdaptiveAvgPool2d((2,2))
        self.sent_fc = nn.Linear(output_channels, output_channels) # projection layer for sent
        self.word_fc = nn.Linear(output_channels, output_channels) # projection layer for word

        self.global_image_fc = nn.Linear(2048, output_channels) # projection layer for images features after joint transformer
        self.region = conv1x1(512, output_channels) # output of tranformer is 512 channels hence input to conv1x1 is 512

    def forward(self, image_features, text_features, mask, task = 'itm'):
        h,w = image_features.shape[2], image_features.shape[3]
        batch_size, EmbSize = image_features.shape[0], image_features.shape[1]
#         print(text_features.is_contiguous(),',',image_features.is_contiguous())
#         print('image features shape:',(batch_size, EmbSize,h,w))
        image_features =  image_features.view(batch_size, EmbSize, -1).contiguous().permute(2,0,1).contiguous() # from [B, EmbSize, 16x16] to [Seq_Len_Img, B, EmbSize]
#         print(text_features.is_contiguous(),',',image_features.is_contiguous())
#         print('image features shape for transformer input:',image_features.shape)
        image_features_seq_len = image_features.shape[0]
        text_features = text_features.permute(2,0,1).contiguous() # from [B, EmbDim, Seqlen] to [Seq_Len_Txt, B, EmbSize]
#         print(text_features.is_contiguous(),',',image_features.is_contiguous())
#         print('text features shape for transformer input:',text_features.shape)
        combined_features = torch.cat([image_features,text_features], dim = 0)
#         print(text_features.is_contiguous(),',',image_features.is_contiguous(),combined_features.is_contiguous())
#         print('combined features shape for transformer input:',combined_features.shape)
        transformed_feats = self.transformer_encoder(combined_features)  # [Seq_Len_Img + Seq_Len_Txt, batchsize, feats_size]
#         print('transformed features shape for transformer input:',transformed_feats.shape)
        text_feats = transformed_feats[image_features_seq_len:,:,:].permute(1,0,2).contiguous()
#         print(text_feats.is_contiguous())
#         print('post transformed text feats shape:',text_feats.shape)
        if task == 'mlm':
            return self.forward_mlm(text_feats)
        elif task == 'itm':
            return self.forward_itm(transformed_feats[:image_features_seq_len,:,:].permute(1,2,0).view(batch_size, EmbSize, h, w)
                                    ,transformed_feats[image_features_seq_len:,:,:].permute(1,0,2), mask)
        else:
            raise ValueError('invalid task for text encoder')

    def forward_mlm(self, seq_feat):

        pred = self.mlm(seq_feat) # [B, L, vocab-1]

        return pred.transpose(1,2) # need transpose pred for loss_fn

    def forward_itm(self, vis_feat, seq_feat, mask):
        #-----------------------------------------
        region_feat = self.region(vis_feat) # [B, C, H, W]
        #-----------------------------------------
        x = self.avgpool(vis_feat)
        z = torch.flatten(x, 1) # batch_size, 2048
        global_feat = self.global_image_fc(z) # [B, C]
        #-----------------------------------------
        sent_feat = self.sent_fc(seq_feat[:,0]) # [B, 0th [CLS] token]
        #-----------------------------------------
        word_feat = self.word_fc(seq_feat) # [B, L, C]

        return region_feat, global_feat, word_feat.transpose(1,2), sent_feat

class ImageText_Classification(nn.Module):
    def __init__(self, num_class=14, img_encoder_path=None, txt_encoder_path=None, pretrained=False, cfg=None, bert_config=None):
        super(ImageText_Classification, self).__init__()
        # batchsize, timesteps, 1
        self.img_encoder = ImageEncoder(output_channels=cfg.hidden_dim)
        self.txt_encoder = TextEncoder(bert_config=bert_config, output_channels=cfg.hidden_dim)

        if cfg.pretrained_text_encoder_path != '':
            print('Initiate text encoder from MLM pretrained parameters from:', cfg.pretrained_text_encoder_path)
            state_dict = torch.load(cfg.pretrained_text_encoder_path, map_location='cpu')
            self.txt_encoder.load_state_dict(state_dict['model'], strict=False)
#         print(misskey)
#         print(unknowkey)

        if pretrained:
            print('Load image encoder from:', img_encoder_path)
            state_dict = torch.load(img_encoder_path, map_location='cpu')
            if 'model' in state_dict.keys():
                self.img_encoder.load_state_dict(state_dict['model'])
            else:
                self.img_encoder.load_state_dict(state_dict)

            print('Load text encoder from:', txt_encoder_path)
            state_dict = torch.load(txt_encoder_path, map_location='cpu')
            if 'model' in state_dict.keys():
                self.txt_encoder.load_state_dict(state_dict['model'])
            else:
                self.txt_encoder.load_state_dict(state_dict)

#         self.soft_attention = SoftAttention(in_groups=1, m_heads=16, in_channels=512)
        ## image feature input ##
        self.gap = nn.AdaptiveAvgPool2d(1) # gap = GlobalAveragePooling, pooling region features into one feature
        self.proj_region = nn.Linear(cfg.hidden_dim, 256) # projection pooled region feature, 512 --> 256
        self.proj_global = nn.Linear(cfg.hidden_dim, 256) # projection global image feature, 512 --> 256
        ## text feature input ##
        self.proj_sent = nn.Linear(cfg.hidden_dim, 256) # projection global sentence feature, 512 --> 256
        self.proj_word = nn.Linear(cfg.hidden_dim, 256) # projection masked averaged words/phrases feature, 512 --> 256

        self.relu = nn.ReLU(inplace=True) # activation function
        self.fc1 = nn.Linear(1024, 512) # FC1 after concat image + text projected features, (4*256)
        self.fc2 = nn.Linear(512, 256) # FC2
        self.do = nn.Dropout(0.5) # dropout layer
        self.output = nn.Linear(256, num_class) # 14 classes including no findings

    def forward(self, x, t, mask):
        ## image input ##
        f_r, f_g, attn_maps_l4, learnable_scalar_l4, attn_maps_l5, learnable_scalar_l5 = self.img_encoder(x) # N, 512, 16, 16; N, 512
#         f_r,attn_maps = self.soft_attention(f_r)
        g_p = torch.squeeze(self.gap(f_r)) # N, 512
        f_r = self.relu(self.proj_region(self.do(g_p))) # N, 256, projection region feature
        f_g = self.relu(self.proj_global(self.do(f_g))) # N, 256, projection global feature
        ## text input ##
        f_w, f_b, f_t, f_s = self.txt_encoder(t, mask) # N, 160, 512; N, 159, 512; N, 158, 512; N, 512
        f_w_avg = (f_w * mask.unsqueeze(-1)).sum(1) / mask.sum(1).unsqueeze(-1) # [N, 512] masked average pooling from word_feat
        f_b_avg = (f_b * mask[:,:-1].unsqueeze(-1)).sum(1) / mask[:,:-1].sum(1).unsqueeze(-1) # [N, 512] masked average pooling from bigram_feat
        f_t_avg = (f_t * mask[:,:-2].unsqueeze(-1)).sum(1) / mask[:,:-2].sum(1).unsqueeze(-1) # [N, 512] masked average pooling from trigram_feat
        f_wp = (f_w_avg + f_b_avg + f_t_avg) / 3.0 # merge 3 level word/phrase features into one feature [N, 512]
        f_wp = self.relu(self.proj_word(self.do(f_wp))) # N, 256, projection word feature
        f_s = self.relu(self.proj_sent(self.do(f_s))) # N, 256, projection sent feature
        ## classification head ##
        net = torch.cat((f_g, f_r, f_s, f_wp), dim=-1) #  N, 512 + 512 (1024 --> 512 --> 256 --> 14)
        net = self.relu(self.fc1(self.do(net)))  # N, 512
        net = self.relu(self.fc2(self.do(net)))  # N, 256
        out = self.output(self.do(net)) # N, 14
        return out, attn_maps_l4, learnable_scalar_l4, attn_maps_l5, learnable_scalar_l5

### add new code here ###
class TextEncoderMLM(nn.Module):
    def __init__(self, bert_config, output_channels=512):
        super(TextEncoderMLM, self).__init__()

        print('Bert encoder with MaskedLMhead.')

        self.bert = BertModel(bert_config, add_pooling_layer=False)
#         self.sent = nn.Linear(bert_config.hidden_size, output_channels)
        self.transform = nn.Linear(bert_config.hidden_size, bert_config.hidden_size)
        self.act = nn.GELU()
        self.LayerNorm = nn.LayerNorm(bert_config.hidden_size, eps=bert_config.layer_norm_eps)
        self.mlm = nn.Linear(bert_config.hidden_size, bert_config.vocab_size - 1) ## classification layer for MLM

    def forward(self, x, mask):
        word_feat = self.bert(input_ids=x, attention_mask=mask)[0]
        return word_feat # word_feat.transpose(1,2), sent_feat

class Text_Classification(nn.Module):
    def __init__(self, num_class=14, txt_encoder_path=None, pretrained=False, cfg=None, bert_config=None):
        super(Text_Classification, self).__init__()
        # batchsize, timesteps, 1
        self.txt_encoder = TextEncoderMLM(bert_config=bert_config, output_channels=cfg.hidden_dim)

        if cfg.pretrained_text_encoder_path != '':
            print('Initiate text encoder from MLM pretrained parameters from:', cfg.pretrained_text_encoder_path)
            state_dict = torch.load(cfg.pretrained_text_encoder_path, map_location='cpu')
            self.txt_encoder.load_state_dict(state_dict['model'], strict=False)

        ## text feature input ##
#         self.proj_sent = nn.Linear(cfg.hidden_dim, 256) # projection global sentence feature, 512 --> 256
        self.proj_word = nn.Linear(cfg.hidden_dim, 512) # projection masked averaged words/phrases feature, 512 --> 256

        self.relu = nn.ReLU(inplace=True) # activation function
#         self.fc1 = nn.Linear(1024, 512) # FC1 after concat image + text projected features, (4*256)
        self.fc1 = nn.Linear(512, 256) # FC2
        self.fc2 = nn.Linear(256, 128) # FC2
        self.do = nn.Dropout(0.5) # dropout layer
        self.output = nn.Linear(128, num_class) # 14 classes including no findings

    def forward(self, t, mask):
        ## text input ##
        f_w = self.txt_encoder(t, mask) # N, 160, 512; N, 159, 512; N, 158, 512; N, 512
        f_w_avg = (f_w * mask.unsqueeze(-1)).sum(1) / mask.sum(1).unsqueeze(-1) # [N, 512] masked average pooling from word_feat
#         f_b_avg = (f_b * mask[:,:-1].unsqueeze(-1)).sum(1) / mask[:,:-1].sum(1).unsqueeze(-1) # [N, 512] masked average pooling from bigram_feat
#         f_t_avg = (f_t * mask[:,:-2].unsqueeze(-1)).sum(1) / mask[:,:-2].sum(1).unsqueeze(-1) # [N, 512] masked average pooling from trigram_feat
#         f_wp = (f_w_avg + f_b_avg + f_t_avg) / 3.0 # merge 3 level word/phrase features into one feature [N, 512]
        f_wp = self.relu(self.proj_word(self.do(f_w_avg))) # N, 256, projection word feature
#         f_s = self.relu(self.proj_sent(self.do(f_s))) # N, 256, projection sent feature
        ## classification head ##
#         net = torch.cat((f_s, f_wp), dim=-1) #  N, 512 (512 --> 256 --> 14)
#         net = self.relu(self.fc1(self.do(net)))  # N, 512
        net = self.relu(self.fc1(self.do(f_wp)))  # N, 256
        net = self.relu(self.fc2(self.do(net)))  # N, 128
        out = self.output(self.do(net)) # N, 14
        return out


# @Time : 2022-10-02 12:15
# @Author : Phalange
# @File : 2160. 拆分数位后四位数字的最小和.py
# @Software: PyCharm
# C'est la vie,enjoy it! :D

class Solution:
    def minimumSum(self, num: int) -> int:
        num = "".join([c for c in str(num)])
        num = "".join(sorted(num))
        num = num.replace("0","")
        num = [int(c) for c in num]
        if len(num) <=2:
            return sum(num)
        else:
            if len(num) == 3:
                return num[0] * 10 + num[1] + num[2]
            else:
                return num[0] * 10 + num[1]*10 + num[2] + num[3]

print(Solution().minimumSum(3000))

字符串/2160. 拆分数位后四位数字的最小和.py

2160. 拆分数位后四位数字的最小和.py


"""
Given an array of n numbers, calculate the maximum subarray sum, i.e., the
largest possible sum of a sequence of consecutive values in the array. The
problem is interesting when there may be negative values in the array.

For example, in the array: [-1, 2, 4, -3, 5, 2, -5, 2]
the following subarray produces the maximum sum 10: [2, 4, -3, 5, 2]

We assume that an empty subarray is allowed, so the maximum subarray sum is
always at least 0.
"""
from util import timeit

@timeit
def algorithm_1(array: list) -> int:
    """
    Algorithm 1 - Brute Force

    A straightforward way to solve the problem is to go through all possible
    subarrays, calculate the sum of values in each subarray and maintain the
    maximum sum.

    The variables [i] and [j] fix the first and last index of the subarray, and
    the sum of values is calculated to the variable [cur]. The variable [best]
    contains the maximum sum found during the search.

    The time complexity of the algorithm is O(n^3), because it consists of
    three nested loops that go through the input.
    """
    best = 0
    size = len(array)
    for i in range(0, size):
        for j in range(i, size):
            curr = 0  # sum of current subarray
            for k in range(i, j):
                curr += array[k]
            best = max(best, curr)

    return best

@timeit
def algorithm_2(array: list) -> int:
    """
    Algorithm 2 - Brute Force Optimized

    It is easy to make Algorithm 1 more efficient by removing one loop from it.
    This is possible by calculating the sum at the same time when the right end
    of the subarray moves.

    The time complexity is O(n^2).
    """
    best = 0
    size = len(array)
    for i in range(0, size):
        curr = 0
        for j in range(i, size):
            curr += array[j]
            best = max(best, curr)

    return best

@timeit
def algorithm_3(arr: list) -> int:
    """
    Algorithm 3 - Kadane’s Algorithm

    The idea is to calculate, for each array position, the maximum sum of a
    subarray that ends at that position. After this, the answer for the problem
    is the maximum of those sums.

    Consider the subproblem of finding the maximum-sum subarray that ends at
    position k. There are two possibilities:
    . The subarray only contains the element at position k
    . The subarray consists of a subarray that ends at position k−1, followed
      by the element at position k
    In the latter case, since we want to find a subarray with maximum sum, the
    subarray that ends at position k −1 should also have the maximum sum. Thus,
    we can solve the problem efficiently by calculating the maximum subarray
    sum for each ending position from left to right.

    The algorithm only contains one loop that goes through the input, so the
    time complexity is O(n). This is also the best possible time complexity,
    because any algorithm for the problem has to examine all array elements at
    least once.
    """
    best, curr = 0, 0
    for elem in arr:
        curr = max(elem, curr + elem)
        best = max(best, curr)

    return best

arr = [-1, 2, 4, -3, 5, 2, -5, 2]
algorithm_1(arr)
algorithm_2(arr)
algorithm_3(arr)
''' terminal
run algorithm_1([-1, 2, 4, -3, 5, 2, -5, 2])
got '10' in 0.0001549000 secs.

run algorithm_2([-1, 2, 4, -3, 5, 2, -5, 2])
got '10' in 0.0001001000 secs.

run algorithm_3([-1, 2, 4, -3, 5, 2, -5, 2])
got '10' in 0.0000287000 secs.
'''


from collections import deque
from logging import getLogger
from scipy.signal import savgol_filter
import numpy as np

_logger = getLogger(__name__)

initialized = False

def initialize(params):
    global initialized, buffer, device, step_length, edge_type, refinement, dark_event, fel_on_event, use_dark, calib, use_filter, filter_window

    device = params["device"]
    step_length = params["step_length"]
    edge_type = params["edge_type"]
    refinement = params["refinement"]
    dark_event = params["dark_event"]
    fel_on_event = params["fel_on_event"]
    buffer = deque(maxlen=params["buffer_length"])
    use_dark = params["use_dark"]
    calib = params["calib"]
    filter_window = params["filter_window"]
    use_filter = params['filter']
    initialized = True

def _interpolate_row(y_known, x_known, x_interp):
    y_interp = np.interp(x_interp, x_known, y_known)
    return y_interp

def find_edge(data, step_length=50, edge_type="falling", refinement=1):
    # refine data
    data_length = data.shape[1]
    refined_data = np.apply_along_axis(
        _interpolate_row,
        axis=1,
        arr=data,
        x_known=np.arange(data_length),
        x_interp=np.arange(0, data_length - 1, refinement),
    )

    # prepare a step function and refine it
    step_waveform = np.ones(shape=(step_length,))
    if edge_type == "rising":
        step_waveform[: int(step_length / 2)] = -1
    elif edge_type == "falling":
        step_waveform[int(step_length / 2) :] = -1

    step_waveform = np.interp(
        x=np.arange(0, step_length - 1, refinement), xp=np.arange(step_length), fp=step_waveform
    )

    # find edges
    xcorr = np.apply_along_axis(np.correlate, 1, refined_data, v=step_waveform, mode="valid")
    edge_position = np.argmax(xcorr, axis=1).astype(float) * refinement
    xcorr_amplitude = np.amax(xcorr, axis=1)

    # correct edge_position for step_length
    edge_position += np.floor(step_length / 2)

    return {"edge_pos": edge_position, "xcorr": xcorr, "xcorr_ampl": xcorr_amplitude, "signal":data}

def process(data, pulse_id, timestamp, params):
    if not initialized:
        initialize(params)

    # Read stream inputs
    prof_sig = data[params["prof_sig"]]
    if use_filter:
        prof_sig  = savgol_filter(prof_sig,filter_window,3)
    events = data[params["events"]]

    if prof_sig.ndim == 1:
        prof_sig = prof_sig[np.newaxis, :]

    if events[dark_event] and use_dark:
        buffer.append(prof_sig)
        edge_results = {"edge_pos": np.nan, "xcorr": np.nan, "xcorr_ampl": np.nan, "signal":np.nan}
    else:
        if events[fel_on_event] and buffer:
            prof_sig = prof_sig / np.mean(buffer, axis=0)
            edge_results = find_edge(prof_sig, step_length, edge_type, refinement)
        elif events[fel_on_event] and not use_dark:
            edge_results = find_edge(prof_sig, step_length, edge_type, refinement)
        else:
            edge_results = {"edge_pos": np.nan, "xcorr": np.nan, "xcorr_ampl": np.nan, "signal":np.nan}

    # calib edge
    edge_results["arrival_time"] = edge_results["edge_pos"] * calib

    # Set bs outputs
    output = {}
    for key, value in edge_results.items():
        output[f"{device}:{key}"] = value

    return output


#!/usr/bin/env python
# Move videos from downloaded TV series to structured directories by season

import os
import re
import shutil
from pathlib import Path

import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('-I', '--info', action='store_true')
parser.add_argument('-i', '--input')
parser.add_argument('-o', '--output')
parser.add_argument('-s', '--simulate', action='store_true')
args = parser.parse_args()

extensions = ('mkv', 'avi', 'mp4', 'srt', 'ass')

if args.input is not None and args.output is not None:
    if args.input.__len__() and args.output.__len__():
        input_dir = str(args.input.rstrip('/'))
        output_dir = str(args.output.rstrip('/'))
        #
        # if args.verbose:
        #     print('Checking if output directory exists')
        # if not os.path.exists(output_dir):
        #     if args.verbose:
        #         print('Creating directory: ' + input_dir)
        #     os.makedirs(output_dir)

        if os.path.exists(input_dir):
            pathlist = Path(input_dir).glob('**/*.*')
            count = {}
            total = 0
            try:
                for index, path in enumerate(pathlist):
                    extension = os.path.splitext(str(path))[1][1:]
                    # if args.verbose:
                    #     print('Checking file extension ' + extension)
                    if extension.lower() in extensions:
                        filename = os.path.basename(str(path))
                        # m = re.search('(.+)(?:s([0-9]{2,})e([0-9]{2,})|([0-9]+)([0-9]{2,})|([0-9]+)x([0-9]{2,})).*/i', filename)
                        # m = re.search('(.+).*s([0-9]+)e([0-9])+.*', filename)
                        m = re.search('(.+)[sS]([0-9]{2,})[eE]([0-9]{2,}).*', filename)

                        if m is not None:
                            if args.verbose:
                                print('\nProcessing ' + filename)

                            final_filename = re.sub(
                                r'[^\x20\x30-\x39\x41-\x5A\x61-\x7A]+', ' ', str(m.group(1))
                            ).strip().lower()
                            # print('Title: ' + final_filename
                            #       + ', season: ' + str(m.group(2))
                            #       + ', episode: ' + str(m.group(3)))

                            season_number = 0
                            episode_number = 0

                            if m.group(2) is not None:
                                season_number = m.group(2)
                                episode_number = str(int(m.group(3)))
                            # if m.group(4) is not None:
                            #     season_number = m.group(4)
                            #     episode_number = str(int(m.group(5)))
                            # if m.group(6) is not None:
                            #     season_number = m.group(6)
                            #     episode_number = str(int(m.group(7)))

                            season_number = str(int(season_number))
                            episode_number = str(int(episode_number))
                            target_dir = os.path.join(str(output_dir), final_filename)
                            target_dir = os.path.join(str(target_dir), 'season ' + season_number)

                            if args.verbose:
                                print('Season: ' + season_number + ', episode: ' + episode_number)

                            if not os.path.exists(target_dir):
                                if args.verbose:
                                    print('Creating directory: ' + target_dir)
                                if not args.simulate:
                                    os.makedirs(target_dir)
                            else:
                                if args.verbose:
                                    print('It\'s ok, directory "' + target_dir + '" already exists')

                            if args.verbose:
                                print('Moving ' + filename)
                            if not args.simulate:
                                shutil.move(str(path), str(target_dir))

                            total = total + 1

                            if args.verbose:
                                print('Saved ' + filename)
            except KeyboardInterrupt:
                print('\n\nOh, no, a KeyboardInterrupt :(\nLet\'s see how was going on...')
                pass
            print('Total: ' + str(total) + ' processed files')
        else:
            print('Invalid input directory, it cannot be opened')
    else:
        print('Invalid arguments, input and output must be set')
else:
    print('Invalid arguments, input and output must be set')


'''
Hashtag split based on word weighing algorithm.
'''

from preprocessing_interface import PreprocessingInterface
from wordsegment import load, segment

class HashtagSplit(PreprocessingInterface):

    def run(self):
        super().run();

        load()

        output = open(self.output, 'w+')
        with open(self.input, mode='r') as input:
            for line in input:
                for word in line.split():
                    if word[0] == '#':
                        split = segment(word[1:])

                        for split_word in split:
                            output.write(split_word + ' ')

                    else:
                        output.write(word + ' ')

                output.write('\n')

        output.close()


import torch
import numpy as np
from torch import nn
from torch.nn import functional as F
from torch.autograd import Variable
from pytorch_pra.softmax import model

model = model.LogisticRegression()
checkpoint = torch.load('./softmax_model.pth')
model.load_state_dict(checkpoint['net'])

model.eval()
print("hello world")
with open('./test.txt','r') as f:
    data_list = f.readlines()
    #data_list = [i.split('\n')[0] for i in data_list]
    data_list = [i.split() for i in data_list]
    data = [(float(i[0]),float(i[1]),float(i[2])) for i in data_list]
    label = [int(i[3])-1 for i in data_list]

data_max = np.max(data,0)
data_min = np.min(data,0)
data = (data-data_min)/data_max

correct = 0
for i in range(len(label)):
    d = torch.Tensor(np.array(data[i]).reshape(1,-1))
    la = torch.Tensor(label[i])
    out = model(d)
    out = torch.argmax(out,1)
    if(out.data[0]==label[i]):
        correct += 1
print(correct/len(label))


import random
from math import pi, sqrt

import cairo

from lib import colors, palettes

class Drops:
    def __init__(self, filename="output.png", img_width=2000, img_height=2000, count=5, palette=random.choice(palettes.PALETTES)):
        self.filename = filename
        self.palette = palette
        self.width = img_width
        self.height = img_height
        self.count = count
        self.ims = cairo.ImageSurface(
            cairo.FORMAT_ARGB32, self.width, self.height)
        self.ctx = cairo.Context(self.ims)
        self.make_solid_background()

    def make_solid_background(self):
        self.ctx.rectangle(0, 0, self.width, self.height)
        self.ctx.set_source_rgb(
            *palettes.hex_to_tuple(self.palette['background']))
        self.ctx.fill()

    def write_to_file(self):
        self.ims.write_to_png(self.filename)

    def make_drop(self, x, y, width, height):
        """Make a drop inside the given bounding box."""
        self.ctx.save()
        self.ctx.translate(x, y)

        [color, inner_color] = [ palettes.hex_to_tuple(c) for c in random.sample(self.palette['colors'], 2) ]
        radius = width // 2
        if random.choice([True, False]):
            # Pointing up
            cx = radius
            cy = height - radius
            point = (cx, 0)
        else:
            # Pointing down
            cx = radius
            cy = radius
            point = (cx, height)

        (tang1, tang2) = self.tangential_points(cx, cy, radius, *point)
        self.ctx.move_to(*point)
        self.ctx.line_to(*tang1)
        self.ctx.line_to(*tang2)
        self.ctx.set_source_rgb(*color)
        self.ctx.fill()

        self.ctx.arc(cx, cy, radius, 0, 2 * pi)
        self.ctx.set_source_rgb(*color)
        self.ctx.fill()

        if random.choice([True, False]):
            inner_radius = radius // 2
            self.ctx.arc(cx, cy, inner_radius, 0, 2 * pi)

        self.ctx.restore()

    def tangential_points(self, cx, cy, radius, px, py):
        """Returns the two tangential points from points (px, py) to circle (cx, cy, radius)"""
        dx, dy = px - cx, py - cy
        dxr, dyr = -dy, dx
        d = sqrt(dx**2 + dy**2)
        if d < radius:
            raise ValueError(
                f"({px}, {py}) is inside ({cx}, {cy}, {radius}). No tangential possible.")

        rho = radius / d
        ad = rho**2
        bd = rho*sqrt(1-rho**2)
        tx1 = cx + ad*dx + bd*dxr
        ty1 = cy + ad*dy + bd*dyr
        tx2 = cx + ad*dx - bd*dxr
        ty2 = cy + ad*dy - bd*dyr
        return ((tx1, ty1), (tx2, ty2))

    def dream(self):
        x_interval = self.width // (self.count * 2)
        y_interval = self.height // self.count
        margin = 5
        for y in range(0, self.height, y_interval):
            for x in range(0, self.width, x_interval):
                self.make_drop(x + margin, y + margin, x_interval - 2 * margin, y_interval - 2 * margin)
        self.write_to_file()

    @staticmethod
    def make_random(filename="output.png", p=random.choice(palettes.PALETTES), img_width=2000, img_height=2000):
        c = random.randint(3, 10)
        drops = Drops(filename=filename, palette=p, count=c,
                      img_height=img_height, img_width=img_width)
        drops.dream()

if __name__ == "__main__":
    for idx in range(4):
        Drops.make_random(filename="output-{}.png".format(idx),
                          p=random.choice(palettes.PALETTES))


# -*- coding: utf-8 -*-
import tensorflow as tf
import numpy as np

'''
周莫烦的第一个TensorFlow demo 1初始化data 2 初始化TensorFlow 结构 2.1 随机生成weight  bias 2.2 lossfunction的选取
2.3 optimizer选取 2.4 train目的的选取 2.5 正式初始化tensorflow
3 tf使用session 来控制整个流程
'''

# generate some data
x_data =np.random.rand(100).astype(np.float32)
y_data = x_data*0.1+0.3

# create tensorlow structure
# initial weight and biases
Weights = tf.Variable(tf.random_uniform([1],-1.0,1.0))
biases = tf.Variable(tf.zeros([1]))

y = Weights*x_data + biases
loss= tf.reduce_mean(tf.square(y-y_data))
## 0.5 means learning rate,choosing a optimizer. then train it
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
## initial tf
init = tf.initialize_all_variables()
##tensorflow structure end

sess = tf.Session()
sess.run(init)

for step in range(201):
    sess.run(train)
    if step %20 ==0:
        print(step,sess.run(Weights),sess.run(biases))


from view.todo_pyqt import Window, QApplication
from view.alpha import run_alpha
import sys

if __name__ == "__main__":
    if input("console run[empty for `no`]"):
        app = QApplication(sys.argv)
        screen = Window()
        screen.show()
        sys.exit(app.exec_())
    else:
        run_alpha()


from django.db.models import Min, Max
from django.dispatch import receiver
from django.db.models.signals import post_delete, post_save

from shop.models import SalesBid, TransProduct, PurchaseBid, ProductImage, StoreProduct

@receiver(post_delete, sender=ProductImage)
def remove_file_from_s3(sender, instance, using, **kwargs):
    instance.image.delete(save=False)

@receiver(post_delete, sender=SalesBid)
def modify_product_price_salesbid_deleted(sender, instance, using, **kwargs):
    product = instance.product
    product.purchase_price = SalesBid.objects.filter(product=product).aggregate(price=Min('price'))['price']
    product.save()
    if product.info.transproduct_set.count()>1:
        modelproduct = TransProduct.objects.get(info=product.info, size='ALL')
        modelproduct.purchase_price = SalesBid.objects.filter(product__info=product.info).exclude(product__size='ALL').aggregate(
        price=Min('price'))['price']
        modelproduct.save()

@receiver(post_delete, sender=PurchaseBid)
def modify_product_price_purchasebid_deleted(sender, instance, using, **kwargs):
    product = instance.product
    product.sales_price = PurchaseBid.objects.filter(product=product).aggregate(price=Max('price'))['price']
    product.save()
    if product.info.transproduct_set.count() > 1:
        modelproduct = TransProduct.objects.get(info=product.info, size='ALL')
        modelproduct.sales_price = \
            PurchaseBid.objects.filter(product__info=product.info).exclude(product__size='ALL').aggregate(
                price=Max('price'))['price']
        modelproduct.save()


#!/usr/bin/env python3
"""Prompt Vector for user behavior when idling."""
import random
from time import sleep

import rospy
from rospy import Publisher, Subscriber
from std_msgs.msg import String, Float32, Bool
from anki_vector_ros.msg import RobotStatus

class Prompts:
    """Controller for prompting user behavior."""

    def __init__(self):
        self.anim_pub = Publisher("/anim/play", String, queue_size=1)
        self.speech_pub = Publisher("/behavior/say_text", String, queue_size=1)
        self.lift_pub = Publisher("/behavior/lift_height", Float32, queue_size=1)
        self.light_flash_pub = Publisher("/cube/flash_lights", Bool, queue_size=1)
        self.head_angle_pub = Publisher("/behavior/head_angle", Float32, queue_size=1)
        self.idle_pub = Publisher("/labdemo/idle", Bool, queue_size=1)
        self.prompt_pub = Publisher("/labdemo/prompt", Bool, queue_size=1)

        self.idling = False
        self.prompting = True
        self.prompt_queued = False
        self.prompts = [self.prompt_fistbump, self.prompt_pet, self.prompt_cube]
        self.last_prompt = -1

        Subscriber("/labdemo/idle", Bool, self.idle_callback)
        Subscriber("/labdemo/prompt", Bool, self.prompting_callback)

    def idle_callback(self, msg):
        self.idling = msg.data

        if msg.data and not self.prompt_queued:
            # Let Vector idle for some period
            self.prompt_queued = True
            sleep(random.randint(3, 7))

            # Robot is no longer idling, no need to prompt
            if not self.idling:
                self.prompt_queued = False
                return

            # Robot is still idling, prompt user
            self.idle_pub.publish(False)
            self.prompt_pub.publish(True)

            # Prevent same prompts in a row
            prompt_i = random.randint(0, len(self.prompts) - 1)
            while prompt_i == self.last_prompt:
                prompt_i = random.randint(0, len(self.prompts) - 1)

            self.last_prompt = prompt_i
            prompt = self.prompts[prompt_i]
            prompt()

            sleep(15.0)  # Wait for user response
            self.prompt_queued = False
            # User hasn't evoked a response from Vector yet, return to idle
            if self.prompting:
                self.prompt_pub.publish(True)
                self.idle_pub.publish(True)

    def prompting_callback(self, msg):
        self.prompting = msg.data

    def prompt_fistbump(self):
        self.lift_pub.publish(0.0)
        self.speech_pub.publish("Give me a fist bump!")
        sleep(1.0)
        self.lift_pub.publish(0.6)

    def prompt_pet(self):
        self.speech_pub.publish("Do you want to pet my back?")

    def prompt_cube(self):
        self.anim_pub.publish("anim_lookinplaceforfaces_keepalive_short")
        self.speech_pub.publish("Can you show me the cube?")
        sleep(1.0)

        # Set head to easily see the cube
        self.head_angle_pub.publish(0.2)
        self.light_flash_pub.publish(True)

if __name__ == "__main__":
    rospy.init_node("vector_prompts")
    rospy.wait_for_message("/status", RobotStatus)
    Prompts()
    rospy.spin()


from __future__ import print_function

import argparse
import sys

import configparser
from typing import Any, Dict, List, Optional

from . import (confirm_licenses_package, get_access_token, retrieve_licenses_package,
               save_license_package_to_json)
from ..exceptions import AuthError, BiloServerError, LicenseSaveError, ScriptError

def parse_args(args=None):  # type: (Optional[List[str]]) -> argparse.Namespace
    parser = argparse.ArgumentParser(description="Retrieve license package")
    parser.add_argument(
        "--pickup-number", required=True, help="Pickup Number given when ordering license"
    )
    parser.add_argument(
        "--school", required=True, help="School abbreviation for which the license should be retrieved"
    )
    parser.add_argument(
        "--config-file", required=False, help="A path to a file which contains all config options for this command."
    )

    return parser.parse_args(args)

def retrieve_licenses(config, pickup_number):
    """
    Retrieve, save and confirm the retrieval of the license package

    During the workflow, the retrieval and confirmation
    endpoints can return the following status codes:
    Retrieval: 200 / 208
    Confirmation: 200 / 409
    The following scenarios are considered to be viable:
        - 200 -> 200 - The license package was retrieved, saved and confirmed for the first time
        - 208 -> 200 - The license package was already retrieved,
                       but has failed to be saved last time
        - 208 -> 409 - The license package was already retrieved
                       and was successfully confirmed

    The scenario where the license was newly acquired (200),
    but the confirmation has already happened (409) is considered
    to be erroneous and will raise an exception
    """
    if config is None:
        config = get_config_from_file('/etc/bildungslogin/config.ini')
    client_id = config["client_id"]
    client_secret = config["client_secret"]
    scope = config["scope"]
    auth_server = config["auth_server"]
    resource_server = config["resource_server"]

    try:
        access_token = get_access_token(client_id, client_secret, scope, auth_server)
    except Exception as exc:
        raise AuthError(
            "Unable to get access: {}".format(exc.message)
        )
    retrieve_response_code, license_response = \
        retrieve_licenses_package(access_token, resource_server, pickup_number)
    try:
        license_path = save_license_package_to_json(license_response, pickup_number)
    except Exception as exc:
        raise LicenseSaveError(
            "Unable to save license as a json: {}".format(exc.message)
        )

    confirm_response_code = confirm_licenses_package(access_token, resource_server, pickup_number)
    if (retrieve_response_code, confirm_response_code) == (200, 409):
        raise BiloServerError("Server error: New license was already confirmed")
    return license_path, license_response['licenses']

def get_config(args):  # type: (argparse.Namespace) -> Dict[str, Any]
    config = get_config_from_file(args.config_file)

    has_missing_args = False
    for required_field in ["client_id", "client_secret", "scope"]:
        if not config.get(required_field):
            has_missing_args = True
            print(
                "'%s' is missing. Add it via --config-file or --%s"
                % (required_field, required_field.replace("_", "-"))
            )
    if has_missing_args:
        sys.exit(1)
    return config

def get_config_from_file(filename):  # type: (str) -> Dict[str, Any]
    config = {}
    cp = configparser.ConfigParser()
    try:
        with open(filename, "r") as fd:
            cp.read_file(fd)
    except EnvironmentError as exc:
        raise ScriptError("Failed to load config from {!r}: {!s}".format(filename, exc))
    else:
        if cp.has_option("Auth", "ClientId"):
            config["client_id"] = cp["Auth"]["ClientId"]
        if cp.has_option("Auth", "ClientSecret"):
            config["client_secret"] = cp["Auth"]["ClientSecret"]
        if cp.has_option("Auth", "Scope"):
            config["scope"] = cp["Auth"]["Scope"]
        if cp.has_option("APIEndpoint", "AuthServer"):
            config["auth_server"] = cp["APIEndpoint"]["AuthServer"]
        if cp.has_option("APIEndpoint", "ResourceServer"):
            config["resource_server"] = cp["APIEndpoint"]["ResourceServer"]
    return config

def main():  # type: () -> None
    try:
        args = parse_args()
        retrieve_licenses(get_config(args), args.pickup_number)
    except ScriptError as err:
        print("Script Error: %s" % (err,), file=sys.stderr)


from controller import Robot

from rcj_soccer_robot import RCJSoccerRobot, TIME_STEP
import math
import statistics

TIME_STEP = 64
from intercepts import interceptCalculator
from CoordinateRecalculator import coor_recalc, robot_pos_recalc
from GoToFunc import goTo
from lidar_processor import Get_Lidar_Range
from MovementCalculator import fit_parabola, get_tangent_point, passes_boundary, scores_own_goal

INTERCEPT_CONST = 0.03

class MyScoringRobot(RCJSoccerRobot):
    def getIntercepts(self, robot_pos, Team):
        #get our robots
        r1 = robot_pos

        #prep dictionary

        #get all the intercepts, with a lota samples
        intercept = self.intercept_c.calculateOptimumIntercept(r1, Team, sample_count=200)

        #dict with times

        #dict with our robots intercepts

        return intercept
    def be_attacker(self, myi, robot_pos, team, ball_pos, target):
        stuff = 0
        point = 0
        if (myi['x'] < robot_pos['x']) if team else (myi['x'] > robot_pos['x']):
            x = fit_parabola(myi, robot_pos ,target)
            x['a'] = 1*x['a']
            if not passes_boundary(x):
                point = get_tangent_point(robot_pos, x, team)
                ball_angle, robot_angle = self.get_angles(point, robot_pos)

                return goTo(point['x'], point['y'], robot_pos, robot_angle, team,should_soften=False, ball_pos=ball_pos)

            else:
                ball_angle, robot_angle = self.get_angles(myi, robot_pos)

                return goTo(myi['x'], myi['y'], robot_pos, robot_angle, team, should_soften=False, ball_pos=ball_pos)

        else:
            ball_angle, robot_angle = self.get_angles(myi, robot_pos)

            return goTo(myi['x'], myi['y'], robot_pos, robot_angle, team, should_soften=False, ball_pos=ball_pos)
    def run(self):
        median_filter = [0.5, 0.5, 0.5, 0.5, 0.5,0.5, 0,5]
        iterator = 0

        prev_ball_x = 0
        self.intercept_c = interceptCalculator(3)
        Team = (self.team != "B")
        myi = None
        cur_gap = None
        ball_pos = None
        while self.robot.step(TIME_STEP) != -1:
            # Get the position of our robot

            robot_pos = self.get_posfrom_devices()

            #print(math.degrees(robot_pos['orientation'])-90)
            #print(robot_pos)
            # Get lidar ranges in an array indexed 0..359, index 0 = front, 90-1 = right, 180-1 = back, 270-1 = left
            lidar_ranges = self.lidar.getRangeImage()
            # just for experiments: print sensor values
            #print(robot_pos, 'front range:', lidar_ranges[0])
            gap_y = Get_Lidar_Range(robot_pos, Team,lidar_ranges)

            if gap_y != -1:
                cur_gap = {'x': 0.95 if not Team else 0.05, 'y': gap_y}
                median_filter[iterator] = gap_y
                iterator+=1
                if iterator>7:
                    iterator = 0
            # for i in range(len(lidar_ranges)):
            #     print(i, lidar_ranges[i])

            if self.is_new_data():

                data = self.get_new_data()
                #due to extensive openAI gym testing we know that desync DOES occur
                while self.is_new_data():
                    data = self.get_new_data()

                # Get the position of the ball
                ball_pos = data['ball']
                robot_pos = robot_pos_recalc(robot_pos, Team)
                ball_pos = coor_recalc(data['ball']['x'], data['ball']['y'], Team)
                # YOUR CODE HERE

                self.intercept_c.pushPoint(ball_pos)
                target = {'x':1 if not Team else 0, 'y': statistics.median(median_filter)}
                print(target)

                myi = self.getIntercepts(robot_pos, Team)

                out = self.be_attacker(myi, robot_pos, Team, ball_pos, target)
                if abs(prev_ball_x - ball_pos['x']) > 0.1:
                    Team = not Team
                    cur_gap=None
                    print("SWITCHERO")

                self.left_motor.setVelocity(out[1])
                self.right_motor.setVelocity(out[0])
                prev_ball_x = ball_pos['x']
                # At first you might want to consider some experiments with still robot, then turning faster and faster
            elif myi != None and ball_pos != None and target != None:
                robot_pos = robot_pos_recalc(robot_pos, Team)
                out = self.be_attacker(myi, robot_pos, Team, ball_pos, target)
                self.left_motor.setVelocity(out[1])
                self.right_motor.setVelocity(out[0])

my_scoring_robot = MyScoringRobot(Robot())
my_scoring_robot.run()