# Builtins
import unittest
import pathlib
import shutil

# Submodule imports
from harvest.trader import BackTester
from harvest.algo import BaseAlgo
from harvest.api.dummy import DummyStreamer
from harvest.utils import *

class TestTester(unittest.TestCase):
    def tear_up_down(func):
        def wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            finally:
                path = pathlib.Path("data")
                shutil.rmtree(path)

        return wrapper

    @tear_up_down
    def test_start_do_nothing(self):
        """Do a quick run-through of the BackTester
        to ensure it can run without crashing.
        """
        s = DummyStreamer()
        t = BackTester(s)
        t.set_symbol("A")
        t.set_algo(BaseAlgo())
        t.start("1MIN", ["5MIN"], period="1DAY")
        self.assertTrue(True)

    @tear_up_down
    def test_check_aggregation(self):
        """ """
        t = BackTester(DummyStreamer())
        t.set_symbol("A")
        t.set_algo(BaseAlgo())
        t.start("1MIN", ["1DAY"], period="1DAY")

        minutes = list(t.storage.load("A", Interval.MIN_1)["A"]["close"])[-200:]
        days_agg = list(t.storage.load("A", int(Interval.DAY_1) - 16)["A"]["close"])[
            -200:
        ]

        self.assertListEqual(minutes, days_agg)

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-

# A very simple setup script to create a single executable
#
# hello.py is a very simple 'Hello, world' type script which also displays the
# environment in which the script runs
#
# Run the build process :
#
#    'python setup.py build'
# or
#    'python setup.py bdist_msi'
#
# If everything works well you should find a subdirectory in the build
# subdirectory that contains the files needed to run the script without Python

import sys
from cx_Freeze import setup, Executable
from ngchecker import __version__

options = {
    'build_exe': {
        'compressed': True,
        'include_files': [
            'readme.md',
            'ngchecker.conf',
            'check.list',
            'modules/',
            'log/',
        ],
        'excludes' : ['Tkinter'],
        'path': sys.path + ['modules']
    }
}

executables = [
    Executable('ngchecker.py')
]

setup(name    = 'ngchecker',
      version = __version__,
      auther  = "Junest LX",
      auther_emai = "support@junest.com",
      options = options,
      description ='FlexiNG Smart Checker',
      executables =executables
      )


# test_print_contour.py by CoccaGuo at 2021/09/28 18:13

import cv2
import numpy as np
from DataProcessor import *
from ContourDetecter import *
from utils import Color
from ColorBlender import *
from RingFactoryMaker import getRingFactory

def result_plot(image_path: str):
    def print_color(ring, color):
        if ring.colored == False:
            ring.colored = True
            pts = rf.contour_list[ring.id]
            # singleLayerBlender(image, ring, rf.contour_list[ring.id], color, Color.BG_COLOR, k=3)
            multiLayerBlender(image, ring, rf.contour_list[ring.id], color, Color.BG_COLOR, k=4)
            cv2.fillConvexPoly(image, pts, color.value)

    image = cv2.imread(image_path)
    rf = getRingFactory(image)

    for ring in rf.ring_list:

        if ring.sides == 7:
            print_color(ring, Color.SLATEBLUE)

            for nnRing in rf.findRingsByID(rf.nearestNeighbor(ring.id)):
                if nnRing.sides == 6:
                    print_color(nnRing, Color.IVORY)

        if ring.sides == 5:
            print_color(ring, Color.WHEAT)

            for nnRing in rf.findRingsByID(rf.nearestNeighbor(ring.id)):
                if nnRing.sides == 6:
                    print_color(nnRing, Color.IVORY)

        if ring.sides == 6:
            flag = True
            for nnRing in rf.findRingsByID(rf.nearestNeighbor(ring.id)):
                if nnRing.sides != 6:
                    flag = False
            if flag:
                print_color(ring, Color.YELLOW)

        # cv2.putText(image, str(ring.id), ring.position, cv2.FONT_HERSHEY_SIMPLEX, 0.4, Color.RED.value)

    cv2.imshow("pic", image)
    cv2.waitKey()
    cv2.destroyAllWindows()
    cv2.imwrite('readme.assert/final_1.png', image)

if __name__ == '__main__':
    image_path = 'data/raw2.png'
    result_plot(image_path)


from tealight.art import (fill_polygon, color, line, spot, circle, box, image, text, background, polygon)
import random
from math import cos, sin, pi
import math

from tealight.art import screen_width, screen_height, test_polygon
from tealight.utils import github_load, sleep
foo = github_load("c-ryan747", "art", "carClass")
foo2 = github_load("c-ryan747", "art", "mapClass")

##########################################################
##########################################################
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
## This is the wrong one go to racetrack.py
##########################################################
##########################################################

# Car stats
acc = 0.3
max_speed = 10
turn_speed = 0.07
friction = 0.05

# Bounce stats
side_bounce = -2

sleep(100)

player_won = 0

lap1 = 1
lap2 = 1
time = 0

car_x = 200
car_y = 160
car_v = 0
car_a = 0
direction = 0
spin = 0

car_x2 = 200
car_y2 = 240
car_v2 = 0
car_a2 = 0
direction2 = 0
spin2 = 0

player_1 = foo.car(car_x,car_y,direction)
player_2 = foo.car(car_x2,car_y2,direction2)
track = foo2.track()

##########################################################
### Frame event
##########################################################

def handle_frame():
  global time, friction, car_x, car_y, car_v, car_a, direction, lap1, car_x2, car_y2, car_v2, car_a2, direction2, lap2, player_won

  # refreshes screen
  color("rgba(255,255,255,0.1)")

  box(0,screen_height/10,screen_width,screen_height/10*9)

  # Does time
  if lap1 < 4:
    time = time + 0.04
  color( "rgba(150,150,150,0.7)")
  box(0,0,screen_width,screen_height/10)
  color("blue")
  #text(40,20,str(math.ceil(time)))
  text(180,10,"Polygon Racers!")
  text(20,36,"arrow keys lap: " + str(math.floor(lap1))[0]+"/4")
  text(280,36,"wasd keys lap: " + str(math.floor(lap2))[0]+"/4")
  # Does laps for player 1
  if track.test_in_top_detector(player_1.points[0],player_1.points[1]) == True:
    if ((lap1-0.5) % 1) == 0:
      lap1 = lap1 + 0.5
      if lap1 == 4.0 and lap2 < 4:
        player_won = 1
  if track.test_in_bottom_detector(player_1.points[0],player_1.points[1]) == True:
    if (lap1 % 1) == 0:
      lap1 = lap1 + 0.5

  # Does laps for player 2
  if track.test_in_top_detector(player_2.points[0],player_2.points[1]) == True:
    if ((lap2-0.5) % 1) == 0:
      lap2 = lap2 + 0.5
      if lap2 == 4.0 and lap1 < 4:
        player_won = 2
  if track.test_in_bottom_detector(player_2.points[0],player_2.points[1]) == True:
    if (lap2 % 1) == 0:
      lap2 = lap2 + 0.5

  ################################
  # Collisions with the polygons
  ################################

  if not track.test_point(player_1.points[0],player_1.points[1]):
    #if track.test_point(player_1.points[2],player_1.points[3]) and track.test_point(player_1.points[4],player_1.points[5]):
    car_v = car_v*-0.6
    car_x = car_x + -5 * cos(direction)
    car_y = car_y - -5 * sin(direction)

  if not track.test_point(player_1.points[2],player_1.points[3]):
    if car_v > 0:
      car_v = car_v*-0.6
      if track.test_point(player_1.points[4],player_1.points[5]):
        car_x = car_x - side_bounce * sin(direction)
        car_y = car_y + side_bounce * cos(direction)
      car_x = car_x - 4 * sin(direction+(pi/2))
      car_y = car_y + 4 * cos(direction+(pi/2))
    else:
      #if track.test_point(player_1.points[0],player_1.points[1]):

      car_v = 5
      #if track.test_point(player_1.points[0],player_1.points[1]):
      #car_x = car_x - 4 * sin(direction+(pi/2))
      #car_y = car_y + 4 * cos(direction+(pi/2))
      #print "should be working"

  elif not track.test_point(player_1.points[4],player_1.points[5]):
    if car_v > 0:
      car_v = car_v*-0.6
      if track.test_point(player_1.points[2],player_1.points[3]):
        car_x = car_x - side_bounce * sin(direction)
        car_y = car_y + side_bounce * cos(direction)
      car_x = car_x - 4 * sin(direction-(pi/2))
      car_y = car_y + 4 * cos(direction-(pi/2))
    else:
      car_v = 5
      #if track.test_point(player_1.points[0],player_1.points[1]):

      #if track.test_point(player_1.points[0],player_1.points[1]):
      #car_x = car_x - 4 * sin(direction-(pi/2))
      #car_y = car_y + 4 * cos(direction-(pi/2))
      #print "should be working2"

  #############################
  # Colisions for second car

  if not track.test_point(player_2.points[0],player_2.points[1]):
    car_v2 = car_v2*-0.6
    car_x2 = car_x2 + -5 * cos(direction2)
    car_y2 = car_y2 - -5 * sin(direction2)

  if not track.test_point(player_2.points[2],player_2.points[3]):
    if car_v2 > 0:
      car_v2 = car_v2*-0.6
      if track.test_point(player_2.points[2],player_2.points[3]):
        car_x2 = car_x2 - side_bounce * sin(direction2)
        car_y2 = car_y2 + side_bounce * cos(direction2)
    else:
      car_v2 = 2
      #car_x = car_x - 10 * sin(direction)
      #car_y = car_y + 10 * cos(direction)
      #print "should be working"

  if not track.test_point(player_2.points[4],player_2.points[5]):
    if car_v2 > 0:
      car_v2 = car_v2*-0.6
      if track.test_point(player_2.points[2],player_2.points[3]):
        car_x2 = car_x2 - side_bounce * sin(direction2)
        car_y2 = car_y2 + side_bounce * cos(direction2)
    else:
      car_v2 = 2
      #car_x = car_x - 10 * sin(direction)
      #car_y = car_y + 10 * cos(direction)
      #print "should be working2"

  ##########
  # Physics

  #car1
  if car_v > 0:
    car_v -= friction
  else:
    car_v += friction

  car_v = min(car_v + car_a,10)

  car_x = car_x + car_v * cos(direction)
  car_y = car_y - car_v * sin(direction)

  direction = direction + spin*min(1,car_v/5)

  #car2
  if car_v2 > 0:
    car_v2 -= friction
  else:
    car_v2 += friction

  car_v2 = min(car_v2 + car_a2,10)

  car_x2 = car_x2 + car_v2 * cos(direction2)
  car_y2 = car_y2 - car_v2 * sin(direction2)

  direction2 = direction2 + spin2*min(1,car_v2/5)

  ################
  # Drawing bits
  ################
  color("black")
  track.draw_polygons()
  player_1.move(car_x,car_y,direction)
  player_2.move(car_x2,car_y2,direction2)

  if player_won == 1:
    image(screen_width/2 - 320,screen_height/2 - 200,"http://i.imgur.com/OQ5AJok.png")
  if player_won == 2:
    image(screen_width/2 - 320,screen_height/2 - 200,"http://i.imgur.com/Bzd2sgV.png")

##########################################################
### Keypressing events
##########################################################

def handle_keydown(key):
  global spin, turn_speed, car_a, acc, spin2, car_a2

  if key == "left":
    spin = turn_speed
  elif key == "right":
    spin = -turn_speed
  elif key == "up":
    car_a = acc
  elif key == "down":
    car_a = -acc/1.5

  if key == "a":
    spin2 = turn_speed
  elif key == "d":
    spin2 = -turn_speed
  elif key == "w":
    car_a2 = acc
  elif key == "s":
    car_a2 = -acc/1.5

def handle_keyup(key):
  global spin, car_a,spin2, car_a2

  if key == "left" or key == "right":
    spin = 0
  elif key == "up" or key == "down":
    car_a = 0

  if key == "a" or key == "d":
    spin2 = 0
  elif key == "s" or key == "w":
    car_a2 = 0

def draw_static_things():
  blue_top = [(0,0),
              (0,screen_height/10),
              (screen_width,screen_height/10),
              (screen_width,0)]
  color("blue")
  fill_polygon(blue_top)

draw_static_things()


import torch
from PIL import Image
from torchvision import transforms
from torch.autograd import Variable
from torch.nn import functional
from model import *
import numpy
from sklearn.externals import joblib
from sklearn import svm
import os

rgb = RGB_model()
rgb.load_state_dict(torch.load(r'C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\rgb.pkl'))
rgb.eval().cuda()
hsv = HSV_model()
hsv.load_state_dict(torch.load(r'C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\hsv.pkl'))
hsv.eval().cuda()
ycbcr = YCbCr_model()
ycbcr.load_state_dict(torch.load(r'C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\ycbcr.pkl'))
ycbcr.eval().cuda()
patch = patch_model()
patch.load_state_dict(torch.load(r'C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\patch.pkl'))
patch.eval().cuda()
lbp = LBP_model()
lbp.load_state_dict(torch.load(r'C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\lbp.pkl'))
lbp.eval().cuda()

path = r'E:/database/nuaa/data/test/0/'
lbp_path = r'E:/database/nuaa/LBP/test/0/'
# path = r'E:/database/CBSR-Antispoofing/croped/test/0/'
# lbp_path = r'E:/database/CBSR-Antispoofing/LBP/test/0/'
t = transforms.Compose([
    transforms.Resize((256, 256)),
    transforms.ToTensor()])
tr = transforms.Compose([
    transforms.Resize((256, 256)),
    transforms.RandomCrop(96),
    transforms.Resize((256, 256)),
    transforms.ToTensor()])

num_true = 0
for test_img in os.listdir(path):
    pil_img = Image.open(path + test_img).convert('RGB')
    lbp_img = Image.open(lbp_path + test_img).convert('RGB')
    img = t(pil_img)
    img.unsqueeze_(dim=0)
    img = Variable(img).cuda()
    imge = tr(pil_img)
    imge.unsqueeze_(dim=0)
    imge = Variable(imge).cuda()
    limg = t(lbp_img)
    limg.unsqueeze_(dim=0)
    limg = Variable(limg).cuda()

    rgb_out = rgb(img)
    rgb_out = functional.softmax(rgb_out, dim=1)
    rgb_out = rgb_out.cpu()
    rgb_out = rgb_out.detach().numpy()
    #rgb_result = int(numpy.argmax(rgb_out.cpu().detach().numpy()))
    hsv_out = hsv(img)
    hsv_out = functional.softmax(hsv_out, dim=1)
    hsv_out = hsv_out.cpu()
    hsv_out = hsv_out.detach().numpy()
    #hsv_result = int(numpy.argmax(hsv_out.cpu().detach().numpy()))
    ycbcr_out = ycbcr(img)
    ycbcr_out = functional.softmax(ycbcr_out, dim=1)
    ycbcr_out = ycbcr_out.cpu()
    ycbcr_out = ycbcr_out.detach().numpy()
    #ycbcr_result = int(numpy.argmax(ycbcr_out.cpu().detach().numpy()))
    patch_out = patch(imge)
    patch_out = functional.softmax(patch_out, dim=1)
    patch_out = patch_out.cpu()
    patch_out = patch_out.detach().numpy()
    #patch_result = int(numpy.argmax(patch_out.cpu().detach().numpy()))
    lbp_out = lbp(limg)
    lbp_out = functional.softmax(lbp_out, dim=1)
    lbp_out = lbp_out.cpu()
    lbp_out = lbp_out.detach().numpy()

    #out = out.detach().numpy()
    #ind = int(numpy.argmax(out))
    # ind = out[0, 0]+out[0, 1]
    #print(rgb_out, hsv_out, ycbcr_out, patch_out)
    #print(rgb_result, hsv_result, ycbcr_result, patch_result)

    clf = joblib.load(r"C:\Users\Neticle\Desktop\bishedaima\Fusing Multiple plus\casia_model\svm.m")
    test_X = [[rgb_out[0, 0], hsv_out[0, 0], ycbcr_out[0, 0], lbp_out[0, 0], patch_out[0, 0]]]
    #print(clf.predict(test_X)[0])
    if clf.predict(test_X)[0] == 0:
        num_true +=1
    print(num_true)


# /*
#  * CIS2750 F2017
#  * Assignment 3
#  * Jackson Zavarella 0929350
#  * This file is a python wrapper for the c functions in CalendarParser.h
#  * No code was used from previous classes/ sources
#  */

from ctypes import *

class CalendarDriver(object):
    """docstring for CalendarDriver."""
    def __init__(self, sofile):
        self.sofile = sofile
        self.calLib = CDLL(sofile)

        self.calLib.openCalendarPython.restype = c_void_p
        self.calLib.openCalendarPython.argtypes = [c_char_p]
        self.calLib.deleteCalendar.argtypes = [c_void_p]
        self.calLib.writeCalendar.argtypes = [c_char_p, c_void_p]
        self.calLib.writeCalendar.restype = c_uint

        # Set return and argument types
        self.calLib.printError.restype = c_char_p
        self.calLib.validateCalendarPython.restype = c_uint;
        self.calLib.validateCalendarPython.argtypes = [c_char_p]
        self.calLib.getCalendarComponentsPython.restype = c_char_p
        self.calLib.getCalendarComponentsPython.argtypes = [c_void_p]
        self.calLib.getComponentPropsPython.restype = c_char_p
        self.calLib.getComponentPropsPython.argtypes = [c_void_p, c_int]
        self.calLib.getComponentAlarmsPython.restype = c_char_p
        self.calLib.getComponentAlarmsPython.argtypes = [c_void_p, c_int]
        self.calLib.validateVersionPython.restype = c_int
        self.calLib.validateVersionPython.argtypes = [c_char_p]
        self.calLib.matchTEXTField.restype = c_int
        self.calLib.matchTEXTField.argtypes = [c_char_p]
        self.calLib.matchDATEField.restype = c_int
        self.calLib.matchDATEField.argtypes = [c_char_p]

        self.calLib.getComponentPropertiesDatabasePython.argtypes = [c_void_p, c_int]
        self.calLib.getComponentPropertiesDatabasePython.restype = c_char_p

        self.calLib.createBasicCalendarPython.restype = c_void_p
        self.calLib.createBasicCalendarPython.argtypes = [c_float, c_char_p]

        self.calLib.addEventPython.argtypes = [c_void_p, c_char_p, c_char_p, c_char_p]
        self.calLib.safelyFreeString.argtypes = [c_void_p]

    def addEvent(self, calPointer, uid, date, start):
        self.calLib.addEventPython(calPointer, self.encode(uid), self.encode(date), self.encode(start))

    def createBasicCalendar(self, version, prodId):
        return self.calLib.createBasicCalendarPython(c_float(version), self.encode(prodId))

    def writeCalendar(self, fileName, calPointer):
        response = self.calLib.writeCalendar(self.encode(fileName), calPointer)
        return self.printError(response)

    def printError(self, error):
        charp = self.calLib.printError(error)
        humanReadable = charp.decode()
        # self.calLib.safelyFreeString(charp)
        return humanReadable

    def openCalendar(self, fileName):
        return self.calLib.openCalendarPython(self.encode(fileName))

    def validateCalendar(self, fileName):
        response = self.calLib.validateCalendarPython(self.encode(fileName))
        return self.printError(response)

    def getCalendarComponents(self, calPointer):
        response = self.calLib.getCalendarComponentsPython(calPointer)
        components = response.decode().split("\"\\")
        parsedComponents = []
        for component in components:
            if component == '':
                continue
            parsedComponents.append(component.split("\\\"")) # Add this component to the list
        return parsedComponents

    def deleteCalendar(self, calPointer):
        self.calLib.deleteCalendar(calPointer)

    def getComponentAlarms(self, calPointer, compNum):
        response = self.calLib.getComponentAlarmsPython(calPointer, c_int(compNum))
        if response is None:
            return None
        return "Component Alarms:\n" + response.decode()

    def getComponentProperties(self, calPointer, compNum):
        response = self.calLib.getComponentPropsPython(calPointer, c_int(compNum))
        if response is None:
            return None
        return "Component Properties:\n" + response.decode()

    def getComponentPropertiesDatabase(self, calPointer, compNum):
        response = self.calLib.getComponentPropertiesDatabasePython(calPointer, c_int(compNum))
        if response is None:
            return None
        return response.decode()

    def encode(self, string):
        return string.encode('utf-8')

    def validateVersion(self, version):
        response = self.calLib.validateVersionPython(self.encode(version))
        if response == 1:
            return True
        return False

    def matchDATEField(self, date):
        response = self.calLib.matchDATEField(self.encode(date))
        if response == 1:
            return True
        return False

    def matchTEXTField(self, text):
        response = self.calLib.matchTEXTField(self.encode(text))
        if response == 1:
            return True
        return False


import os
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions
import numpy as np
import functools
import tfmpl
import mnist_data

tf.logging.set_verbosity(tf.logging.INFO)

tf.app.flags.DEFINE_enum("algo", "nis_vae_proposal",
                         ["nis_vae_proposal", "nis_gaussian_proposal",
                           "vae_gaussian_prior", "vae_nis_prior"],
                         "Algorithm to run.")
tf.app.flags.DEFINE_integer("latent_dim", 50,
                            "Dimension of the latent space of the VAE.")
tf.app.flags.DEFINE_integer("K", 128,
                            "Number of samples for NIS model.")
tf.app.flags.DEFINE_float("scale_min", 1e-5,
                             "Minimum scale for various distributions.")
tf.app.flags.DEFINE_float("learning_rate", 3e-4,
                           "The learning rate to use for ADAM or SGD.")
tf.app.flags.DEFINE_integer("batch_size", 4,
                             "The number of examples per batch.")
tf.app.flags.DEFINE_string("logdir", "/tmp/nis",
                            "Directory for summaries and checkpoints.")
tf.app.flags.DEFINE_integer("max_steps", int(1e6),
                            "The number of steps to run training for.")
tf.app.flags.DEFINE_integer("summarize_every", int(1e3),
                            "The number of steps between each evaluation.")
FLAGS = tf.app.flags.FLAGS

def mlp(inputs,
        layer_sizes,
        hidden_activation=tf.math.tanh,
        final_activation=tf.math.log_sigmoid,
        name=None):
  """Creates a simple multi-layer perceptron."""
  with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
    for i, s in enumerate(layer_sizes[:-1]):
      inputs = tf.layers.dense(inputs,
                               units=s,
                               activation=hidden_activation,
                               kernel_initializer=tf.initializers.glorot_uniform,
                               name="layer_%d" % (i+1))
    output = tf.layers.dense(inputs,
                             units=layer_sizes[-1],
                             activation=final_activation,
                             kernel_initializer=tf.initializers.glorot_uniform,
                             name="layer_%d" % (len(layer_sizes)+1))
  return output

def conditional_normal(
        inputs,
        data_dim,
        hidden_sizes,
        hidden_activation=tf.math.tanh,
        scale_min=1e-5,
        name=None):
    raw_params = mlp(inputs,
                     hidden_sizes + [2*data_dim],
                     hidden_activation=hidden_activation,
                     final_activation=None,
                     name=name)
    loc, raw_scale = tf.split(raw_params, 2, axis=-1)
    scale = tf.math.maximum(scale_min, tf.math.softplus(raw_scale))
    return tfd.MultivariateNormalDiag(loc=loc, scale_diag=scale)

def conditional_bernoulli(
        inputs,
        data_dim,
        hidden_sizes,
        hidden_activation=tf.math.tanh,
        name=None):
    bern_logits = mlp(inputs,
                      hidden_sizes + [data_dim],
                      hidden_activation=hidden_activation,
                      final_activation=None,
                      name=name)
    return tfd.Bernoulli(logits=bern_logits)

def vae(data,
        latent_dim,
        q_fn,
        prior_fn,
        generative_fn):
  batch_size = tf.shape(data)[0]
  data_dim = data.get_shape().as_list()[1]

  # Construct approximate posterior and sample z.
  q = q_fn(data)
  z = q.sample()
  log_q_z = q.log_prob(z)

  # compute the prior prob of z
  log_p_z = prior_fn(z)

  # Compute the model logprob of the data
  p_x_given_z = generative_fn(z)
  log_p_x_given_z = p_x_given_z.log_prob(data)

  elbo = log_p_z + log_p_x_given_z - log_q_z
  return elbo

def make_vae_with_nis_prior(
        data,
        latent_dim,
        K,
        nis_hidden_sizes,
        q_hidden_sizes,
        p_x_hidden_sizes,
        scale_min=1e-5):

  q_fn = functools.partial(
          conditional_normal,
          data_dim=latent_dim,
          hidden_sizes=q_hidden_sizes,
          scale_min=scale_min,
          name="q")

  prior_fn = functools.partial(
          make_nis_lower_bound,
          K=K,
          hidden_layer_sizes=nis_hidden_sizes)

  generative_fn = functools.partial(
          conditional_normal,
          data_dim=data.get_shape().as_list()[1],
          hidden_sizes=p_x_hidden_sizes,
          scale_min=scale_min,
          name="vae_p_x")

  return vae(data, latent_dim, q_fn, prior_fn, generative_fn)

def make_vae_with_gaussian_prior(
        data,
        latent_dim,
        q_hidden_sizes,
        p_x_hidden_sizes,
        scale_min=1e-5):

  dtype = data.dtype
  q_fn = functools.partial(
          conditional_normal,
          data_dim=latent_dim,
          hidden_sizes=q_hidden_sizes,
          scale_min=scale_min,
          name="q")

  prior_fn = tfd.MultivariateNormalDiag(loc=tf.zeros([latent_dim], dtype=dtype),
                                        scale_diag=tf.ones([latent_dim], dtype=dtype)).log_prob

  generative_fn = functools.partial(
          conditional_normal,
          data_dim=data.get_shape().as_list()[1],
          hidden_sizes=p_x_hidden_sizes,
          scale_min=scale_min,
          name="vae_p_x")

  return vae(data, latent_dim, q_fn, prior_fn, generative_fn)

def make_nis_with_gaussian_proposal(target_samples, K, hidden_layer_sizes):
  """Constructs an NIS distribution for the given target samples and parameters.

  Args:
    target_samples: [batch_size, data_size]
  """
  batch_size = tf.shape(target_samples)[0]
  data_size = tf.shape(target_samples)[1]
  dtype = target_samples.dtype
  proposal = tfd.MultivariateNormalDiag(loc=tf.zeros([data_size], dtype=dtype),
                                        scale_diag=tf.ones([data_size], dtype=dtype))
  proposal_samples = proposal.sample([batch_size, K])  #[batch_size, K, data_size]

  mlp_fn = functools.partial(
             mlp,
             layer_sizes=hidden_layer_sizes + [1],
             final_activation=None,
             name="nis_mlp")

  log_energy_target = tf.reshape(mlp_fn(target_samples), [batch_size])
  log_energy_proposal = tf.reshape(mlp_fn(proposal_samples), [batch_size, K])

  proposal_lse = tf.reduce_logsumexp(log_energy_proposal, axis=1) # [batch_size]

  #[batch_size]
  denom = tf.reduce_logsumexp(tf.stack([log_energy_target, proposal_lse], axis=-1), axis=1)
  denom -= tf.log(tf.to_float(K+1))

  lower_bound = proposal.log_prob(target_samples) +  log_energy_target - denom
  return lower_bound

def make_nis_with_vae_proposal(
        data,
        K,
        vae_latent_dim,
        nis_hidden_sizes,
        q_hidden_sizes,
        p_x_hidden_sizes,
        scale_min=1e-5,
        lr=1e-4):
  batch_size = data.get_shape().as_list()[0]
  data_size = data.get_shape().as_list()[1]
  dtype = data.dtype
  # Sample z_1:K from the VAE
  vae_prior = tfd.MultivariateNormalDiag(loc=tf.zeros([vae_latent_dim], dtype=dtype),
                                         scale_diag=tf.ones([vae_latent_dim], dtype=dtype))
  z = vae_prior.sample([batch_size, K])  #[batch_size, K, vae_latent_dim]
  # Use zs to sample xs
  p_x_given_z_fn = functools.partial(conditional_normal,
          data_dim=data_size,
          hidden_sizes=p_x_hidden_sizes,
          scale_min=scale_min,
          name="p_x_given_z")
  p_x_given_z = p_x_given_z_fn(z)
  x = p_x_given_z.sample() #[batch_size, K, data_size]

  # compute lower bound on log prob of data
  q = conditional_normal(
          data, vae_latent_dim, q_hidden_sizes, scale_min=scale_min, name="q")
  z_q = q.sample() # [batch_size, vae_latent_dim]
  p_x_given_z_q = p_x_given_z_fn(z_q)
  log_p_data_lb = vae_prior.log_prob(z_q) + p_x_given_z_q.log_prob(data) - q.log_prob(z_q)

  mlp_fn = functools.partial(
             mlp,
             layer_sizes=nis_hidden_sizes+ [1],
             final_activation=None,
             name="nis_mlp")

  log_energy_target = tf.reshape(mlp_fn(data), [batch_size])
  log_energy_proposal = tf.reshape(mlp_fn(x), [batch_size, K])

  proposal_lse = tf.reduce_logsumexp(log_energy_proposal, axis=1) # [batch_size]

  #[batch_size]
  denom = tf.reduce_logsumexp(tf.stack([log_energy_target, proposal_lse], axis=-1), axis=1)
  denom -= tf.log(tf.to_float(K+1))

  lower_bound = log_p_data_lb + log_energy_target - denom
  return lower_bound

def make_log_hooks(global_step, elbo):
  hooks = []
  def summ_formatter(d):
    return ("Step {step}, elbo: {elbo:.5f}".format(**d))
  elbo_hook = tf.train.LoggingTensorHook(
      {"step": global_step, "elbo": elbo},
      every_n_iter=FLAGS.summarize_every,
      formatter=summ_formatter)
  hooks.append(elbo_hook)
  if len(tf.get_collection("infrequent_summaries")) > 0:
    infrequent_summary_hook = tf.train.SummarySaverHook(
        save_steps=10000,
        output_dir=FLAGS.logdir,
        summary_op=tf.summary.merge_all(key="infrequent_summaries")
    )
    hooks.append(infrequent_summary_hook)
  return hooks

def main(unused_argv):
  FLAGS.logdir = os.path.join(FLAGS.logdir, FLAGS.algo)
  g = tf.Graph()
  with g.as_default():

    data_batch, _, _ = mnist_data.get_mnist(
            batch_size=FLAGS.batch_size,
            split="train")

    if FLAGS.algo == "nis_vae_proposal":
      print("Running NIS with VAE proposal")
      elbo = make_nis_with_vae_proposal(
              data_batch,
              K=FLAGS.K,
              vae_latent_dim=FLAGS.latent_dim,
              nis_hidden_sizes=[200, 100],
              q_hidden_sizes=[300, 300],
              p_x_hidden_sizes=[300, 300],
              scale_min=FLAGS.scale_min)
    elif FLAGS.algo == "nis_gaussian_proposal":
      print("Running NIS with Gaussian proposal")
      elbo = make_nis_with_gaussian_proposal(
              data_batch,
              K=FLAGS.K,
              hidden_layer_sizes=[200,100])
    elif FLAGS.algo == "vae_nis_prior":
      print("Running VAE with NIS prior")
      elbo = make_vae_with_nis_prior(
              data_batch,
              latent_dim=FLAGS.latent_dim,
              K=FLAGS.K,
              nis_hidden_sizes=[200, 100],
              q_hidden_sizes=[300, 300],
              p_x_hidden_sizes=[300, 300],
              scale_min=FLAGS.scale_min)
    elif FLAGS.algo == "vae_gaussian_prior":
      print("Running VAE with gaussian prior")
      elbo = make_vae_with_gaussian_prior(
              data_batch,
              latent_dim=FLAGS.latent_dim,
              q_hidden_sizes=[300, 300],
              p_x_hidden_sizes=[300, 300],
              scale_min=1e-5)

    # Finish constructing the graph
    elbo_avg = tf.reduce_mean(elbo)
    tf.summary.scalar("elbo", elbo_avg)
    global_step = tf.train.get_or_create_global_step()
    opt = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
    grads = opt.compute_gradients(-elbo_avg)
    train_op = opt.apply_gradients(grads, global_step=global_step)

    log_hooks = make_log_hooks(global_step, elbo_avg)

    with tf.train.MonitoredTrainingSession(
        master="",
        is_chief=True,
        hooks=log_hooks,
        checkpoint_dir=FLAGS.logdir,
        save_checkpoint_secs=120,
        save_summaries_steps=FLAGS.summarize_every,
        log_step_count_steps=FLAGS.summarize_every) as sess:
      cur_step = -1
      while cur_step <= FLAGS.max_steps and not sess.should_stop():
        _, cur_step = sess.run([train_op, global_step])

if __name__ == "__main__":
  tf.app.run(main)


"""
Code for comparing clustering redshifts to photo-z
"""
from __future__ import print_function, division
from .comparator import Comparator

import numpy as np
from scipy import stats

class ClusterComparator(Comparator):
    def __init__(self, z_bins, z, pz, cov, z_pz, pz_pz, z_min, z_max, weight_fit=1):
        """Reference cluster redshifts are over z_bins and are pz with
        covariance cov normalized over range z_min to z_max

        Parameters
        ----------
        z_bins : array (Ntomo, Nbins,)
            Bins used for clustering redshifts

        z_pz : array (Ntomo, Npz,)
            clustering redshifts

        pz : array (Ntomo, Nbins-1)
            Clustering Redshift Estimates

        cov : array (Ntomo, Nbins-1, Nbins-1)
            Covariance matrix of clustering redshift estimates

        z_pz : array (Ntomo, Npz,)
            redshift of Pz estimate

        pz_pz : array (Ntomo, Npz)
            Pz estimates

        z_min : (Ntomo,)
            Lower normalization ranges

        z_max : (Ntomo,)
            Upper normalization ranges

        Notes
        -----
        Priors are all truncorm (mu, sigma, +-cutoff) :

            b : (0, 0.05, 0.4)
            k : (0, 0.1, 1)
            gamma : (0, 0.5, 3)
        """
        self.z_bins = z_bins
        # self.z = [0.5 * (z_bins_i[1:] + z_bins_i[:-1])
        #           for z_bins_i in self.z_bins]
        self.z = z
        # should probably check z_bins vs z here
        self.pz = pz
        self.cov = cov
        self.z_min = z_min
        self.z_max = z_max

        self.z_pz = z_pz
        self.pz_pz = pz_pz

        self.labels = np.array([['$b_{{{0}}}$'.format(i + 1),
                                 '$k_{{{0}}}$'.format(i + 1)]
                                for i in range(len(self.pz))]).flatten().tolist()
        self.labels +=  ['$\gamma$']
        self.prior = [stats.truncnorm(a=(-0.4 - 0) / 0.05, b=(0.4 - 0) / 0.05,
                                      loc=0, scale=0.05),
                      stats.truncnorm(a=(-5 - 0) / 1., b=(5 - 0) / 1.,
                                      loc=0, scale=1.)] * len(self.pz) + \
                     [stats.uniform(loc=-8, scale=16)]
                     # [stats.truncnorm(a=(-3 - 0) / 0.5, b=(3 - 0) / 0.5,
                     #                  loc=0, scale=0.5)]

        self.weight_fit = weight_fit

    def distribution_clustering(self, z, pz, cov, p, z_min, z_max, **kwargs):
        """Function that modifies the clustering redshift distribution

        Parameters
        ----------
        z : redshift centers
        pz : redshift distribution
        p : array
            Parameter values

        Returns
        -------
        z_prime     :   redshift centers
        pz_prime     :   redshift distribution
        cov_prime   :   shifted covariance matrix

        """
        log_amplitude = p[1:-1:2]
        gamma = p[-1]

        z_prime = []
        pz_prime = []
        cov_prime = []
        for z_i, pz_i, cov_i, z_min_i, z_max_i, log_amplitude_i in zip(
                z, pz, cov, z_min, z_max, log_amplitude):
            z0 = 0.5 * (z_min_i + z_max_i)
            amplitude = np.exp(log_amplitude_i)
            mult = ((1. + z_i) / (1 + z0)) ** gamma * amplitude
            z_prime.append(z_i)
            pz_prime.append(pz_i * mult)
            cov_prime.append(cov_i * mult[None] * mult[:, None])

        return z_prime, pz_prime, cov_prime

    def distribution_photoz(self, z_bins, z, pz, p, **kwargs):
        """Function that returns the shifted photo-z distribution

        Parameters
        ----------
        z : redshift centers
        pz : redshift distribution
        p : array
            Parameter values

        Returns
        -------
        z_prime : redshift centers
        pz_prime : redshift distribution

        """
        bias = p[:-1:2]

        z_prime = []
        pz_prime = []

        for z_i, pz_i, z_bins_i, bias_i in zip(
                z, pz, z_bins, bias):
            z_prime_i, pz_prime_i = self.rebin(z_i - bias_i, pz_i, z_bins_i)
            z_prime.append(z_prime_i)
            pz_prime.append(pz_prime_i)

        return z_prime, pz_prime

    def distribution(self, p, z_bins, z, pz, cov, z_pz, pz_pz, z_min, z_max, **kwargs):
        """Given parameter p values, return modified clustering and photo-z

        Parameters
        ----------
        p : array
            Parameter values

        Returns
        -------
        z_pz_p  :   photoz redshift centers
        pz_pz_p  :   photoz redshift distribution
        z_p     :   clustering redshift centers
        pz_p     :   clustering redshift distribution
        cov_p   :   clustering covariance matrix

        """
        z_p, pz_p, cov_p = self.distribution_clustering(z, pz, cov, p, z_min, z_max)
        # get the photoz modifications
        z_pz_p, pz_pz_p = self.distribution_photoz(z_bins, z_pz, pz_pz, p)

        return z_pz_p, pz_pz_p, z_p, pz_p, cov_p

    def lnlike(self, p):
        """Function that evaluates the goodness of fit of parameters

        Parameters
        ----------
        p : array
            Parameter values

        Returns
        -------
        chi squares
        """

        z_pz_p, pz_pz_p, z_p, pz_p, cov_p = self.distribution(p,
            self.z_bins, self.z, self.pz, self.cov, self.z_pz, self.pz_pz,
            self.z_min, self.z_max)

        # get the modifications to the clustering redshifts
        chi2s = [self._chi2(pz_pz_p[i], pz_p[i], cov_p[i])
                 for i in range(len(pz_p))]
        return -0.5 * sum(chi2s) * self.weight_fit


"""change int to bigint for downloads in recent table

Revision ID: c81b3715b9e5
Revises: 9116cea0e0d7
Create Date: 2018-04-05 00:56:02.276823

"""
# flake8: noqa
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'c81b3715b9e5'
down_revision = '9116cea0e0d7'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('recent', 'downloads',
               existing_type=sa.INTEGER(),
               type_=sa.BigInteger(),
               existing_nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('recent', 'downloads',
               existing_type=sa.BigInteger(),
               type_=sa.INTEGER(),
               existing_nullable=False)
    # ### end Alembic commands ###


r"""
Interface for extracting data and generating images from Jmol readable files.

JmolData is a no GUI version of Jmol useful for extracting data from files Jmol
reads and for generating image files.

AUTHORS:

- Jonathan Gutow (2012-06-14): complete doctest coverage
- Jonathan Gutow (2012-03-21): initial version
"""

#*******************************************************************************
#       Copyright (C) 2012 Jonathan Gutow (gutow@uwosh.edu)
#
#  Distributed under the terms of the GNU General Public License (GPL)
#  as published by the Free Software Foundation; either version 2 of
#  the License, or (at your option) any later version.
#                  http://www.gnu.org/licenses/
#*******************************************************************************
from __future__ import print_function

from sage.structure.sage_object import SageObject

from sage.env import SAGE_LOCAL
from sage.misc.temporary_file import tmp_filename

import subprocess
import os

class JmolData(SageObject):
    r"""
    .. todo::

       Create an animated image file (GIF) if spin is on and put data
       extracted from a file into a variable/string/structure to return
    """
    def __init__(self):
        """
        EXAMPLES:

        Create a JmolData object::

            sage: from sage.interfaces.jmoldata import JmolData
            sage: JData = JmolData()
        """
        pass

    def is_jvm_available(self):
        """
        Returns True if the Java Virtual Machine is available and False if not.

        EXAMPLES:

        Check that it returns a boolean::

            sage: from sage.interfaces.jmoldata import JmolData
            sage: JData = JmolData()
            sage: type(JData.is_jvm_available())
            <type 'bool'>
        """
        try:
            version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
        except (subprocess.CalledProcessError, OSError):
            return False

        import re
        java_version = re.search("version.*[1][.][78]", version)
        return java_version is not None

    def export_image(self,
        targetfile,
        datafile, #name (path) of data file Jmol can read or script file telling it what to read or load
        datafile_cmd='script', #"script" or "load"
        image_type ='PNG', #PNG, JPG, GIF
        figsize=5,
        **kwds):
        r"""
        This executes JmolData.jar to make an image file.

        INPUT:

        - targetfile -- the full path to the file where the image
          should be written.

        - datafile -- full path to the data file Jmol can read or
          text of a script telling Jmol what to read or load.
          If it is a script and the platform is cygwin, the filenames in
          the script should be in native windows format.

        - datafile_cmd -- (default ``'script'``)  ``'load'`` or ``'script'``
          should be ``"load"`` for a data file.

        - image_type -- (default ``"PNG"``) ``'PNG'`` ``'JPG'`` or ``'GIF'``

        - figsize -- number (default 5) equal to (pixels/side)/100

        OUTPUT:

        Image file, .png, .gif or .jpg (default .png)

        .. note::

            Examples will generate an error message if a functional Java Virtual Machine (JVM)
            is not installed on the machine the Sage instance is running on.

        .. warning::

            Programmers using this module should check that the JVM is
            available before making calls to avoid the user getting
            error messages.  Check for the JVM using the function
            :meth:`is_jvm_available`, which returns True if a JVM is available.

        EXAMPLES:

        Use Jmol to load a pdb file containing some DNA from a web data
        base and make an image of the DNA. If you execute this in the
        notebook, the image will appear in the output cell::

            sage: from sage.interfaces.jmoldata import JmolData
            sage: JData = JmolData()
            sage: script = "load =1lcd;display DNA;moveto 0.0 { -473 -713 -518 59.94} 100.0 0.0 0.0 {21.17 26.72 27.295} 27.544636 {0.0 0.0 0.0} -25.287832 64.8414 0.0;"
            sage: testfile = tmp_filename(ext="DNA.png")
            sage: JData.export_image(targetfile=testfile,datafile=script,image_type="PNG")  # optional -- java internet
            sage: print(os.path.exists(testfile)) # optional -- java internet
            True

        Use Jmol to save an image of a 3-D object created in Sage.
        This method is used internally by plot3d to generate static images.
        This example doesn't have correct scaling::

            sage: from sage.interfaces.jmoldata import JmolData
            sage: JData = JmolData()
            sage: D=dodecahedron()
            sage: from sage.misc.misc import SAGE_TMP
            sage: archive_name=os.path.join(SAGE_TMP, "archive.jmol.zip")
            sage: D.export_jmol(archive_name)  #not scaled properly...need some more steps.
            sage: archive_native = archive_name
            sage: import sys
            sage: if sys.platform == 'cygwin':
            ....:     from subprocess import check_output, STDOUT
            ....:     archive_native = check_output(['cygpath', '-w', archive_native],
            ....:                                   stderr=STDOUT).rstrip()
            sage: script = 'set defaultdirectory "{0}"\n script SCRIPT\n'.format(archive_native)
            sage: testfile = os.path.join(SAGE_TMP, "testimage.png")
            sage: JData.export_image(targetfile=testfile, datafile=script, image_type="PNG") # optional -- java
            sage: print(os.path.exists(testfile)) # optional -- java
            True
        """
        # Set up paths, file names and scripts
        jmolpath = os.path.join(SAGE_LOCAL, "share", "jmol", "JmolData.jar")
        target_native = targetfile
        import sys
        if sys.platform == 'cygwin':
            jmolpath      = subprocess.check_output(['cygpath', '-w', jmolpath],
                                                    stderr=subprocess.STDOUT).rstrip()
            target_native = subprocess.check_output(['cygpath', '-w', target_native],
                                                    stderr=subprocess.STDOUT).rstrip()
            if (datafile_cmd != 'script'):
                datafile  = subprocess.check_output(['cygpath', '-w', datafile],
                                                    stderr=subprocess.STDOUT).rstrip()
        launchscript = ""
        if (datafile_cmd!='script'):
            launchscript = "load "
        launchscript = launchscript + datafile
        imagescript = "write " + image_type + " " + target_native + "\n"

        sizeStr = "%sx%s" %(figsize*100,figsize*100)
        # Scratch file for Jmol errors
        scratchout = tmp_filename(ext=".txt")
        with open(scratchout, 'w') as jout:
            # Now call the java application and write the file.
            env = dict(os.environ)
            env['LC_ALL'] = 'C'
            env['LANG'] = 'C'
            subprocess.call(["java", "-Xmx512m", "-Djava.awt.headless=true",
                "-jar", jmolpath, "-iox", "-g", sizeStr,
                "-J", launchscript, "-j", imagescript],
                stdout=jout, stderr=jout, env=env)
        if not os.path.isfile(targetfile):
            raise RuntimeError("Jmol failed to create file %s, see %s for details"%(repr(targetfile), repr(scratchout)))
        os.unlink(scratchout)


"""This module Creates a awesome Engendro bot."""
import error_handler
import global_messages
import commands
import logging
import os
import sys
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters

class Engendro(object):
    """Awesome random behaviour Telegram bot."""
    def __init__(self, token):
        """
        Args:
            config (obj): Configuration object parameter
        """

        self.__updater = Updater(token=token)

        dispatcher = self.__updater.dispatcher
        dispatcher.add_handler(CommandHandler('uptime', commands.uptime))
        dispatcher.add_handler(CommandHandler('roll', commands.roll))

        dispatcher.add_handler(
            MessageHandler(Filters.text, global_messages.handle))

        dispatcher.add_error_handler(error_handler.handle)

    def start(self):
        """Starts Engendro bot."""
        self.__updater.start_polling(clean=True)
        self.__updater.idle()

if __name__ == '__main__':
    # Config logger
    logging.basicConfig(
        level=logging.WARNING,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    if 'DEBUG' in os.environ and os.environ['DEBUG'] == '1':
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG)

    # Start bot
    if len(sys.argv) > 1:
        engendro = Engendro(sys.argv[1])
    elif 'BOT_TOKEN' in os.environ:
        engendro = Engendro(os.environ['BOT_TOKEN'])
    else:
        raise Exception("BOT_TOKEN not defined")
    engendro.start()


#!/usr/bin/env python3
import re
import sys
import os
from traceback import print_exception
from openpyxl import load_workbook
from pandas import DataFrame
import numpy as np
import pickle

# Parses all experiment files (xls or xlsx format), looks for 'freezes' worksheet,
# assembles into csv output.

# --------- func defs -----------------

def parse_file(exp_file):
	try:
		wb = load_workbook(exp_file,read_only=True, data_only=True)
	except:
		return(None)
	if "freezes" in wb.sheetnames:
		ws = wb['freezes']
		return(ws)
	else:
		return(None)

# -------- end func defs ------------

if __name__ == '__main__':
# gets all experiment files
	path = "~/Desktop/experiments"
	files = os.listdir(os.path.expanduser(path))
	files = [os.path.abspath(os.path.join(os.path.expanduser(path),x)) for x
						in files if x[-4:] == ".xls" or
						x[-5:] == ".xlsx"]
#print("\n".join(files))

	dat = map(parse_file,files)
	res = DataFrame()

	for i,n in enumerate(dat):
		print(f'processing {i+1}/{len(files)} {files[i]}')
		try:
			if n is None:
				continue
			# reads sheet by DataFrame
			df = DataFrame([[c.value for c in r] for r in n.iter_rows()])
			df.columns = df.iloc()[0,]
			df.drop(index=0,axis=0,inplace=True)
			temp = np.array(df.columns)
#		print('cols',temp)
#		pickle.dump(temp,open('/Users/sku/Desktop/temp.pkl','wb'))
			mask = [x is None for x in temp]
			drops = df.columns[mask]
#		print('drops',drops)
			df.drop(labels=drops,axis=1,inplace=True)

			# appends to res
			res = res.append(df,ignore_index=True, sort=True)
		except:
			print('iter',i)
			print(df)
			print('----------')
			print(res)
			raise

	res.dropna(how='all',inplace=True)
#pickle.dump(res[['date','sampleID','type','location','notes']],
#	open('/Users/sku/Desktop/results.pkl','wb'))
#res = res[['date','sampleID','type','location','notes']]
	res.to_excel(os.path.expanduser('~/Desktop/freezes.xlsx'),index=False,
		freeze_panes=(1,0))
#print(res.to_csv(None))

#	for m in n.iter_rows():
#		if m is None:
#			continue
#		ret = [str(x.value) for x in m]
#		if all([x == "None" for x in ret]):
#			next
#		else:
#			temp = ','.join(ret)
#			print(temp)


# Initial node class

class Node:
	# simply initialize the class, with itself and the dataval
	def __init__(self, dataval=None):
		self.dataval = dataval
		# this itself stores a node class vs a pointer to it in C
		self.nextval = None

class head:
	#head basically
	def __init__(self):
		self.headval = None

list1 = head()
list1.headval = Node("Mon")
e2 = Node("Tue")
e3 = Node("Wed")
# Link first Node to second node
list1.headval.nextval = e2

# Link second Node to third node
e2.nextval = e3


try:
    import torch
    from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel
except ImportError:
    # No installation required if not using this function
    pass

from nlpaug.model.lang_models import LanguageModels

class Gpt2(LanguageModels):
    SUBWORD_PREFIX = 'Ġ'

    def __init__(self, model_path='gpt2', device='cuda'):
        super().__init__()
        self.model_path = model_path
        self.device = device

        self.tokenizer = GPT2Tokenizer.from_pretrained(model_path)
        self.model = GPT2LMHeadModel.from_pretrained(model_path)
        self.model.to(device)
        self.model.eval()

    def id2token(self, _id):
        return self.tokenizer.decode(_id, clean_up_tokenization_spaces=True).strip()

    def predict(self, text, target_word=None, top_n=5):
        # Convert feature
        input_idxes = self.tokenizer.encode(text)
        input_idxes = torch.tensor(input_idxes, device=self.device).unsqueeze(0).repeat(1, 1)

        # Prediction
        with torch.no_grad():
            outputs = self.model(input_idxes)
        target_token_logits = outputs[0][0][-1]  # GPT2 should only predict last token

        # Generate candidates
        candidate_ids, candidate_probas = self.prob_multinomial(target_token_logits, top_n=top_n + 20)
        results = self.get_candidiates(candidate_ids, candidate_probas, target_word, top_n)

        return results


import pandas as pd
import os
import glob
import numpy as np
from keras.models import load_model
from keras.layers import Input, Dense
from keras.models import Model
import keras.optimizers
import keras.callbacks
from keras import regularizers
from sklearn.model_selection import StratifiedShuffleSplit
import Saver
import Reader
import importlib
importlib.reload(Saver)
importlib.reload(Reader)

gpu = input("GPU number: ")

os.environ["CUDA_VISIBLE_DEVICES"] = gpu

name = input("Output dir: ")
epochs = int(input("Epochs: "))
periods = int(input("Period: "))
all_cls = int(input("All class? (0-1): "))
resume = int(input("Resume? (0-1): "))

spec_dir = "../Surveys/css-sloan-x-match/CSDR1/specs/"
labels_path = "../Surveys/css-sloan-x-match/CSDR1/keys-cls.csv"
spec_dim = 3795
model_dir = "./models/"+name+"/"

auto = None
init_epoch = 0

if resume==1:
    files = glob.glob(model_dir+"*.hdf5")
    path = max(files, key = os.path.getctime)
    auto = load_model(path)
    init_epoch = int(path.split(".")[-2])

else:
    units = input("Encoding dim: ")
    units = [int(u) for u in units.split(",")]
    reg = int(input("Reg to encoding? (0-1): "))
    in_spec = Input(shape=(spec_dim,))
    layer = in_spec
    if len(units)>1:
        layer = Dense(units=units[-2], activation="relu")(in_spec)
    if reg==1:
        reg_type = int(input("Reg type (1-2): "))
        w = float(input("Reg weight: "))
        if reg_type ==1:
            layer = Dense(units=units[-1], activation="relu", activity_regularizer=regularizers.l1(w))(layer)
        else:
            layer = Dense(units=units[-1], activation="relu", activity_regularizer=regularizers.l2(w))(layer)
    else:
        layer = Dense(units=units[-1], activation="relu")(layer)
    if len(units)>1:
        layer = Dense(units=units[-2], activation="sigmoid")(layer)
    out_spec = Dense(units=spec_dim, activation="sigmoid")(layer)

    auto = Model(in_spec, out_spec)
    auto.compile(optimizer="Adam", loss='mean_squared_error')

auto.summary()

db = Reader.DB()
if all_cls==1:
    db.build_db(spec_dir, down_sample=True, normed=True, labeled=True, labels_path=labels_path, down_cls=True)
else:
    db.build_db(spec_dir, down_sample=True, normed=True, labeled=True, labels_path=labels_path, one_class=5)

sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=0)
train, test = None, None
X_train, X_test = None, None

for train_index, test_index in sss.split(np.zeros(len(db.labels)), db.labels):
    train, test = train_index, test_index
    X_train, X_test = db.flux[train_index], db.flux[test_index]

if not os.path.exists(model_dir):
    os.mkdir(model_dir)
file_path = model_dir+name+".{epoch:02d}.hdf5"
model_checkpoint =  keras.callbacks.ModelCheckpoint(file_path, period=periods)

his = auto.fit(X_train, X_train,
                epochs = init_epoch+epochs,
                shuffle = True,
                batch_size=len(X_train),
                validation_data=(X_test, X_test),
                initial_epoch = init_epoch,
                callbacks=[model_checkpoint],
               )
his_files = glob.glob(model_dir+"*.npy")

if resume and his_files:
    path = max(his_files , key = os.path.getctime)
    his_last = np.load(path).item()
    loss = his_last["loss"] + his.history["loss"]
    val_loss = his_last["val_loss"] + his.history["val_loss"]
    his_all = {"loss":loss, "val_loss":val_loss}
    np.save(model_dir+name+".npy", his_all)

else:
    np.save(model_dir+name+".npy", his.history)


"""
Stream helper for liquidpy
"""
import io
from .defaults import LIQUID_DEBUG_SOURCE_CONTEXT

def words_to_matrix(words):
	"""
	Convert words to matrix for searching.
	For example:
	```
	['{%', '{%-', '{{'] => [
		{'{': 0},     3 shares, 0 endings
		{'%': 1, '{': 1},
		{'-': 1}
	]
	```
	@params:
		words: The words to be converted
	@returns:
		The converted matrix
	"""
	matrix = [{} for _ in range(max(len(word) for word in words))]
	for word in words:
		for i, char in enumerate(word):
			matrix[i].setdefault(char, 0)
			if i == len(word) - 1:
				matrix[i][char] += 1
	return matrix

class LiquidStream:
	"""
	The stream helper for liquidpy
	"""
	def __init__(self, stream):
		"""
		Initialize the stream
		@params:
			stream (Stream): A python stream
		"""
		self.stream = stream
		self.cursor = stream.tell()

	@staticmethod
	def from_file(path):
		"""
		Get stream of a file
		@params:
			path (str): The path of the file
		@returns:
			LiquidStream
		"""
		return LiquidStream(io.open(path, mode = 'r', encoding = 'utf-8'))

	@staticmethod
	def from_string(string):
		"""
		Get stream of a string
		@params:
			string (str): The string
		@returns:
			LiquidStream
		"""
		return LiquidStream(io.StringIO(string))

	@staticmethod
	def from_stream(stream):
		"""
		Get stream of a stream
		@params:
			stream (Stream): A stream
		@returns:
			LiquidStream
		"""
		return LiquidStream(stream)

	def __del__(self):
		"""
		Close the stream when GC
		"""
		self.close()

	def close(self):
		"""
		Close the stream
		"""
		if self.stream and not self.stream.closed:
			self.stream.close()

	def next(self):
		"""
		Read next character from the stream
		@returns:
			str: the next character
		"""
		ret = self.stream.read(1)
		self.cursor += len(ret.encode('utf-8'))
		return ret

	def back(self):
		"""
		Put cursor 1-character back
		"""
		self.cursor -= 1
		self.stream.seek(self.cursor)

	def rewind(self):
		"""
		Rewind the stream
		"""
		self.stream.seek(0)
		self.cursor = 0

	def eos(self):
		"""
		Tell if the stream is ended
		@returns:
			`True` if it is else `False`
		"""
		nchar = self.next()
		if not nchar:
			return True
		self.cursor -= 1
		self.stream.seek(self.cursor)
		return False

	def dump(self):
		"""
		Dump the rest of the stream
		@returns:
			str: The rest of the stream
		"""
		return self.stream.read()

	def readline(self):
		"""Read a single line from the stream"""
		return self.stream.readline()

	def get_context(self, lineno, context = LIQUID_DEBUG_SOURCE_CONTEXT, baselineno = 1):
		"""
		Get the line of source code and its context
		@params:
			lineno  (int): Line number of current line
			context (int): How many lines of context to show
		@returns:
			list: The formated code with context
		"""
		self.rewind()
		ret     = []
		maxline = lineno + context
		nbit    = len(str(maxline)) + 1
		i       = baselineno
		line    = self.readline()
		while line:
			if i < lineno - context or i > maxline:
				pass
			else:
				ret.append("{} {} {}".format(
					'>' if i == lineno else  ' ', (str(i) + '.').ljust(nbit), line.rstrip()))
			i += 1
			line = self.readline()
		return ret

	def split(self, delimiter, limit = 0, trim = True,
		wraps = None, quotes = '"\'`', escape = '\\'):
		"""
		Split the string of the stream
		@params:
			delimiter (str): The delimiter
			limit (int): The max limit of the split
			trim (bool): Whether to trim each part or not
			wraps (list): A list of paired wraps to skip of the delimiter is wrapped by them
			quotes (str): A series of quotes to skip of the delimiter is wrapped by them
			escape (str): The escape character to see if any character is escaped
		@returns:
			list: The split strings
		"""
		wraps = ['{}', '[]', '()'] if wraps is None else wraps
		preceding, stop = self.until([delimiter], False, wraps, quotes, escape)
		ret = [preceding.strip() if trim else preceding]
		nsplit = 0
		while stop:
			nsplit += 1
			if limit and nsplit >= limit:
				rest = self.dump()
				ret.append(rest.strip() if trim else rest)
				break
			preceding, stop = self.until([delimiter], False, wraps, quotes, escape)
			ret.append(preceding.strip() if trim else preceding)
		return ret

	def until(self, words, greedy = True, wraps = None, quotes = '"\'`', escape = '\\'):
		"""
		Get the string until certain words
		For example:
		```
		s = LiquidStream.from_string("abcdefgh")
		s.until(["f", "fg"]) == "abcde", "fg"
		# cursor point to 'h'
		s.until(["f", "fg"], greedy = False) == "abcde", "f"
		# cursor point to 'g'
		s.until(["x", "xy"]) == "abcdefg", ""
		# cursor point to eos
		```
		@params:
			words (list): A list of words to search
			greedy (bool): Whether do a greedy search or not
				- Only effective when the words have prefices in common. For example
				- ['abc', 'ab'], then abc will be matched first
			wraps (list): A list of paired wraps to skip of the delimiter is wrapped by them
			quotes (str): A series of quotes to skip of the delimiter is wrapped by them
			escape (str): The escape character to see if any character is escaped
		@returns:
			str: The string that has been searched
			str: The matched word
		"""
		# pylint:disable=too-many-locals,too-many-nested-blocks,too-many-branches
		wraps = ['{}', '[]', '()'] if wraps is None else wraps
		ret               = ''
		matrix            = words_to_matrix(words)
		len_matrix        = len(matrix)
		matched_chars     = ''
		matched_candidate = None
		wrap_opens        = {wrap[0]:i for i, wrap in enumerate(wraps)
			if  not any(wrap[0] in mat for mat in matrix) and \
				not any(wrap[1] in mat for mat in matrix)}
		wrap_closes       = {wraps[i][1]:i for wrap_open,i in wrap_opens.items()}
		quote_index       = {quote:i for i, quote in enumerate(quotes)
			if not any(quote in mat for mat in matrix)}
		wrap_flags        = [0 for _ in range(len(wraps))]
		quote_flags       = [False for _ in range(len(quotes))]
		escape_flags      = False
		char              = self.next()
		#print('START', '-' * 10, matrix)
		while True:
			#print(char, end = ' ')
			if not char:
				return ret, matched_candidate
			if char == escape:
				if matched_candidate and escape not in matrix[len(matched_candidate)]:
					self.back()
					return ret, matched_candidate
				escape_flags = not escape_flags
				ret += matched_chars + char
				matched_chars = ''
			elif not escape_flags: # and char != escape
				if char in wrap_opens and not any(quote_flags):
					wrap_flags[wrap_opens[char]] += 1
				elif char in wrap_closes and not any(quote_flags):
					wrap_flags[wrap_closes[char]] -= 1
				elif char in quote_index and \
					not any(flag for i, flag in enumerate(quote_flags) if i != quote_index[char]):
					# make sure I am not quoted
					quote_flags[quote_index[char]] = not quote_flags[quote_index[char]]
				if sum(wrap_flags) > 0 or any(quote_flags):
					if matched_candidate:
						self.back()
						return ret, matched_candidate
					ret += matched_chars + char
					matched_chars = ''
				else:
					len_matched_chars = len(matched_chars)
					matching_dict = matrix[len_matched_chars]
					#print(' ', len_matched_chars, matching_dict)
					if char in matching_dict:
						matched_chars += char
						endings = matching_dict[char]
						if not greedy and endings:
							return ret, matched_chars
						if endings:
							matched_candidate = matched_chars
							#print('mm', matched_candidate)
							if len_matched_chars + 1 == len_matrix: # we have matched all chars
								return ret, matched_chars

					elif matched_candidate:
						self.back()
						return ret, matched_candidate
					else:
						ret += matched_chars + char
						matched_chars = ''
			else: # char == escape or escape_flags
				escape_flags = False
				ret += matched_chars + char
				matched_chars = ''

			char = self.next()


import ROOT as root
from rootpy.plotting import Hist, Canvas
from math import log, exp
import array, sys
import numpy as np

lLimits = [0, 6000]

dTacDelays = {0:{'height':2.68665e-01, 'width':5.28788e-03},
			  8:{'height':3.06615e-01, 'width':5.19475e-03},
			  16:{'height':3.43537e-01, 'width':4.54747e-03},
			  24:{'height':3.82137e-01, 'width':5.52308e-03},
			  32:{'height':4.19476e-01, 'width':5.43046e-03},
			  64:{'height':5.32018e-01, 'width':4.95080e-03},
			  96:{'height':6.82914e-01, 'width':5.30571e-03},
			  112:{'height':7.57700e-01, 'width':4.51142e-03},
			  }

lDelays = sorted(dTacDelays.keys())
aDelays = array.array('f', lDelays)
aDelaysErr = array.array('f', [0 for i in xrange(len(lDelays))])
aHeight = array.array('f')
aHeightErr = array.array('f')

for delay in lDelays:
	height = dTacDelays[delay]['height']
	heightErr = dTacDelays[delay]['width']
	aHeight.append(height)
	aHeightErr.append(heightErr)

c1 = Canvas()

gTacHeight = root.TGraphErrors(len(lDelays), aDelays, aHeight, aDelaysErr, aHeightErr)
gTacHeight.SetTitle('')
gTacHeight.GetXaxis().SetTitle('Delay [ns]')
gTacHeight.GetXaxis().SetTitleOffset(1.12)
gTacHeight.GetXaxis().SetRangeUser(-10, 120)
gTacHeight.GetYaxis().SetTitle('TAC Height [V]')

fCal = root.TF1('fG1', 'pol1', lLimits[0], lLimits[1])
#fG1.FixParameter(0, 0)

fCal.SetLineStyle(9)
fCal.SetLineWidth(1)

gTacHeight.Draw('ap')
gTacHeight.Fit('fG1')

fCal.Draw('same')

slope = fCal.GetParameter(1)*1000
slopeErr = fCal.GetParError(1)*1000
intercept = fCal.GetParameter(0)*1000
interceptErr = fCal.GetParError(0)*1000

sFitInfo1 = 'slope = %.2f +/- %.2f [mV/ns]' % (slope, slopeErr)
sFitInfo2 = 'intercept = %.2f +/- %.2f [mV]' % (intercept, interceptErr)
pt1 = root.TPaveText(.2,.65,.6,.75,'blNDC')
text1 = pt1.AddText(sFitInfo1)
text2 = pt1.AddText(sFitInfo2)
pt1.SetTextColor(root.kBlue)
pt1.SetFillStyle(0)
pt1.SetBorderSize(0)
pt1.Draw('same')

c1.Update()

raw_input('Press enter to continue...')


from config import data_path
from config import db, cur
import os, traceback
from update_jobs.cal_boardIndex import board_stock
from update_jobs.cal_boardIndex import board_change

def get_board_new_date():
    file_path = os.path.join(data_path, 'index_data.csv')
    date_list = []
    with open(file_path, 'r', encoding='utf-8') as file:
        for content in file:
            item = content.split(',')
            date = item[0]
            date_list.append(date)

    return date_list[-1]

def get_update_date_list(last_date):
    sql = "SELECT cal_date from trade_days where " \
          "cal_date > '%s' and is_open = 1 order by cal_date asc" % last_date
    trade_date = []
    try:
        cur.execute(sql)
        result = cur.fetchall()
        for item in result:
            date = item[0].strftime('%Y%m%d')
            trade_date.append(date)
        return trade_date
    except Exception:
        traceback.print_exc()

def update_board_change():
    last_date = get_board_new_date()
    date_list = get_update_date_list(last_date)
    board_list = board_stock.board_to_stock().keys()
    out_put = data_path + '/index_data.csv'
    with open('%s' % out_put, 'a', encoding='utf-8') as file:
        print(board_list)
        for date in date_list[:5]:
            change_line = []
            for board in board_list:
                change = board_change.board_date_change(board, date)
                print(date, board, change)
                change_line.append(str(change))
            change_str = ','.join(change_line)
            value = date + ',' + change_str + '\n'
            print(value)
            file.write(value)

if __name__ == "__main__":
    # r = get_update_date_list('20081010')
    r = update_board_change()
    print(r)


import math

# Die n-stelligen Zahlen liegen zwischen 10**(n-1) und 10**n - 1.
# Die größte n-te Potenz, die wir uns ansehen müssen, ist also 9**n.
# Aus der Ungleichung 10**(n-1) <= 9**n erhalten wir
# n <= (log(9) / log(10/9)) + 1 = 21,85434...
# Also betrachten wir maximal 21-te Potenzen.

def numberOfDigits(n):
  sn = str(n)
  return len(sn)

totalCount = 0

for n in range(1, 21+1):
  count = 0
  for x in range(1, 9 + 1):
    p = x**n
    if numberOfDigits(p) == n:
      count += 1
  totalCount += count
  print(n, count)

print('Total count is', totalCount)


import numpy as np
import cPickle as pickle
from ..fitter.template import Template
from scipy.interpolate import splev
from ..dir_list import template_path

Msun = 1.9891e33 #unit: gram
Mpc = 3.08567758e24 #unit: cm
m_H = 1.6726219e-24 #unit: gram
r0 = 1.1  # pc

fp = open(template_path+"Cat3d_H.tmplt")
tp_cat3d_H = pickle.load(fp)
fp.close()
tcat3d_H = Template(**tp_cat3d_H)
waveLim = [1.0, 1e4]

def Cat3d_H(a, h, N0, i, logL, DL, z, wave, frame="rest", t=tcat3d_H, waveLim=waveLim):
    """
    This function generates the modified CLUMPY torus radiation from Garcia-Gonzalez et al. 2017.

    Parameters
    ----------
    a : float
    	The index a of the radial dust cloud distribution power law.
    N0 : float
        The number of clouds along an equatorial line-of-sight.
    theta : degree
        The half-opening angle.
    i : degree
        Inclination angle.
    logL : float
        UV luminosity erg/s in log.
    DL : float
        The luminosity distance.
    z : float
        The redshift.
    wave : float array
        The wavelengths of the output flux.
    frame : string
        "rest" for the rest frame SED and "obs" for the observed frame.
    t : Template object
        The template of DL07 model provided by user.
    waveLim : list
        The min and max of the wavelength covered by the template.

    Returns
    -------
    flux : float array
        The flux density of the model.

    Notes
    -----
    None.
    """
    fltr = (wave > waveLim[0]) & (wave < waveLim[1])
    if np.sum(fltr) == 0:
        return np.zeros_like(wave)
    para = [a, h, N0, i]
    if frame == "rest":
        idx = 2.0
    elif frame == "obs":
        idx = 1.0
    else:
        raise ValueError("The frame '{0}' is not recognised!".format(frame))
    f0 = (1 + z)**idx * 10**(logL - 46) * (r0 / DL * 1e-6)**2
    flux = np.zeros_like(wave)
    flux[fltr] = f0 * t(wave[fltr], para) * 1e29  # unit: mJy
    return flux


#calss header
class _AXIOM():
	def __init__(self,):
		self.name = "AXIOM"
		self.definitions = [u'a statement or principle that is generally accepted to be true, but need not be so: ', u'a formal statement or principle in mathematics, science, etc., from which other statements can be obtained: ']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


"""Motor Imagery contexts"""

import numpy as np
from .base import WithinSubjectContext, BaseContext
from mne import Epochs, find_events
from mne.epochs import concatenate_epochs, equalize_epoch_counts
from sklearn.model_selection import cross_val_score, LeaveOneGroupOut, KFold

class BaseMotorImagery(BaseContext):
    """Base Motor imagery context

    Parameters
    ----------
    datasets : List of Dataset instances.
        List of dataset instances on which the pipelines will be evaluated.
    pipelines : Dict of pipelines instances.
        Dictionary of pipelines. Keys identifies pipeline names, and values
        are scikit-learn pipelines instances.
    fmin : float | None, (default 7.)
        Low cut-off frequency in Hz. If None the data are only low-passed.
    fmax : float | None, (default 35)
        High cut-off frequency in Hz. If None the data are only high-passed.

    See Also
    --------
    MotorImageryTwoClasses
    """

    def __init__(self, datasets, pipelines, fmin=7., fmax=35.):
        self.fmin = fmin
        self.fmax = fmax
        super().__init__(datasets, pipelines)

    def _epochs(self, dataset, subjects, event_id):
        """epoch data"""
        raws = dataset.get_data(subjects=subjects)
        raws = raws[0]
        ep = []
        # we process each run independently
        for raw in raws:

            # find events
            events = find_events(raw, shortest_event=0, verbose=False)

            # pick some channels
            raw.pick_types(meg=True, eeg=True, stim=False,
                           eog=False, exclude='bads')

            # filter data
            raw.filter(self.fmin, self.fmax, method='iir')

            # epoch data
            epochs = Epochs(raw, events, event_id, dataset.tmin, dataset.tmax,
                            proj=False, baseline=None, preload=True,
                            verbose=False)
            ep.append(epochs)
        return ep

class MotorImageryMultiClasses(BaseMotorImagery, WithinSubjectContext):
    """Motor Imagery for multi class classification

    Multiclass motor imagery context. Evaluation is done in Randomized KFold or
    LeaveOneGroupOut (depending on the group variable, can be run or session)
    with accuracy as a metric. Epochs count are equalized.

    Parameters
    ----------
    datasets : List of Dataset instances.
        List of dataset instances on which the pipelines will be evaluated.
    pipelines : Dict of pipelines instances.
        Dictionary of pipelines. Keys identifies pipeline names, and values
        are scikit-learn pipelines instances.
    fmin : float | None, (default 7.)
        Low cut-off frequency in Hz. If None the data are only low-passed.
    fmax : float | None, (default 35)
        High cut-off frequency in Hz. If None the data are only high-passed.

    See Also
    --------
    MotorImageryTwoClasses
    """

    def prepare_data(self, dataset, subjects):
        """Prepare data for classification."""
        if len(dataset.event_id) < 3:
            # multiclass, pick two first classes
            raise(ValueError("Dataset %s only contains two classes" %
                             dataset.name))

        event_id = dataset.event_id
        epochs = self._epochs(dataset, subjects, event_id)
        groups = []
        full_epochs = []

        for ii, epoch in enumerate(epochs):
            epochs_list = [epoch[k] for k in event_id]
            # equalize for accuracy
            equalize_epoch_counts(epochs_list)
            ep = concatenate_epochs(epochs_list)
            groups.extend([ii] * len(ep))
            full_epochs.append(ep)

        epochs = concatenate_epochs(full_epochs)
        X = epochs.get_data()*1e6
        y = epochs.events[:, -1]
        groups = np.asarray(groups)
        return X, y, groups

    def score(self, clf, X, y, groups, n_jobs=1):
        """get the score"""
        if len(np.unique(groups)) > 1:
            # if group as different values, use group
            cv = LeaveOneGroupOut()
        else:
            # else use kfold
            cv = KFold(5, shuffle=True, random_state=45)

        auc = cross_val_score(clf, X, y, groups=groups, cv=cv,
                              scoring='accuracy', n_jobs=n_jobs)
        return auc.mean()

class MotorImageryTwoClasses(BaseMotorImagery, WithinSubjectContext):
    """Motor Imagery for binary classification

    Binary motor imagery context. Evaluation is done in Randomized KFold or
    LeaveOneGroupOut (depending on the group variable, can be run or session)
    with AUC as a metric.

    Parameters
    ----------
    datasets : List of Dataset instances.
        List of dataset instances on which the pipelines will be evaluated.
    pipelines : Dict of pipelines instances.
        Dictionary of pipelines. Keys identifies pipeline names, and values
        are scikit-learn pipelines instances.
    fmin : float | None, (default 7.)
        Low cut-off frequency in Hz. If None the data are only low-passed.
    fmax : float | None, (default 35)
        High cut-off frequency in Hz. If None the data are only high-passed.

    See Also
    --------
    MotorImageryTwoClasses
    """

    def prepare_data(self, dataset, subjects):
        """Prepare data for classification."""

        if len(dataset.event_id) > 2:
            # multiclass, pick two first classes
            raise(ValueError("Dataset %s contain more than two classes" %
                             dataset.name))

        event_id = dataset.event_id
        epochs = self._epochs(dataset, subjects, event_id)
        groups = []
        for ii, ep in enumerate(epochs):
            groups.extend([ii] * len(ep))
        epochs = concatenate_epochs(epochs)
        X = epochs.get_data()*1e6
        y = epochs.events[:, -1]
        y = np.asarray(y == np.max(y), dtype=np.int32)

        groups = np.asarray(groups)
        return X, y, groups

    def score(self, clf, X, y, groups, n_jobs=1):
        """get the score"""
        if len(np.unique(groups)) > 1:
            # if group as different values, use group
            cv = LeaveOneGroupOut()
        else:
            # else use kfold
            cv = KFold(5, shuffle=True, random_state=45)

        auc = cross_val_score(clf, X, y, groups=groups, cv=cv,
                              scoring='roc_auc', n_jobs=n_jobs)
        return auc.mean()


#!/usr/bin/env python

import time
import logging
import threading

import kubernetes as kub

from .multiproc import multiproc

logger = logging.getLogger("Monitor.{}".format(__name__))

class kubernetes_watch(multiproc):

    def __init__(self, settings, **args):
        multiproc.__init__(self, settings, **args)

        # pods/nodes
        self.watch = self.getconfig("watch", default='pods')
        self.timeout = self.getconfig("timeout", default=30)
        # {'label_selector': 'app=test'}
        # {'field_selector': 'metadata.namespace=test'}
        self.selectors = self.getconfig("selectors", default={})

        self.functions = self.getconfig("functions", default=[{'fnc': 'print',
                                                               'args': [],
                                                               'kwargs': {}
                                                               }])

        self.data = {}
        self.output = ''
        self.error = ''
        self.status = 0
        self.stop_all_threads = False

    def watch_kub(self):

        try:
            kub.config.load_incluster_config()
            logger.info('Loaded from in cluster config')
        except Exception:
            kub.config.load_kube_config()
            logger.info('Loaded from kube config')

        self.kub_api = kub.client.CoreV1Api()

        if self.watch == 'nodes':
            watch = self.kub_api.list_node
        elif self.watch == 'pods':
            watch = self.kub_api.list_pod_for_all_namespaces
        elif self.watch == 'namespaces':
            watch = self.kub_api.list_namespace

        for event in kub.watch.Watch().stream(watch, _request_timeout=self.timeout, **self.selectors):
            # et = event['type']
            # node = event['object'].metadata.name
            # raw_object = event['raw_object']
            yield event

    def watcher(self):

        for fnc in self.functions:

            fnc_name = 'kubernetes_{}'.format(fnc['fnc'])
            fnc_path = 'hc.multiproc.kubernetes_fnc.{}'.format(fnc_name)

            fnc['fnc_imported'] = vars(__import__(fnc_path, fromlist=['']))[fnc_name]()

        for event in self.watch_kub():
            logger.debug(event['type'])
            for fnc in self.functions:
                self.data[fnc.get('name', fnc['fnc'])] = fnc['fnc_imported'].resolve(
                    self.kub_api, event, fnc.get('args', []), fnc.get('kwargs', {}))

    def ticker(self):

        while 1:
            self.proc_save(self.status, self.data, self.output, self.error)
            if self.stop_all_threads:
                return

    def run_proc(self):

        self.proc_init()

        threads = []
        for fnc in [self.watcher, self.ticker]:
            thread = threading.Thread(target=fnc)
            threads.append(thread)
            thread.start()

        while 1:
            logger.debug([thr.is_alive for thr in threads])
            for thr in threads:
                if not thr.is_alive():
                    logger.error('Stopping all threads.')
                    self.stop_all_threads = True
                    return

            time.sleep(self._period)


# ------------------------------------------------
# terminal_effects.py
#
# terminal_effects
# Author: 	Evan Wilde <etcwilde@uvic.ca>
# Date:		Aug 05 2014
# ------------------------------------------------
'''
Gives access to some simple commandline effects
'''

class TerminalColors:
    '''
    Class conaining some terminal colors
    '''
    WARNING = '\033[93m'
    FAILURE = '\033[91m'
    END = '\033[0m'

    def disable(self):
        self.WARNING = ''
        self.FAILURE = ''
        self.END = ''

class CException(Exception):
    '''
    Throws colorful exceptions
    '''
    def __init__(self, message, values, critical=True):
        '''
        Create an exception
        '''
        self.critical = critical
        if critical:
            self.message = TerminalColors.FAILURE + "" + message
        else:
            self.message = TerminalColors.WARNING + "" + message
        self.values = values

    def is_critical(self):
        return self.critical

    def __str__(self):
        '''
        Throw message
        '''
        if self.values:
            if isinstance(self.values, str):
                values = self.values
            else:
                values = ""
                for elem in self.values:
                    values += elem + " "
            return self.message + " : " + values + "" + TerminalColors.END
        else:
            return self.message + TerminalColors.END

class NotImplementedError(CException):
    '''
    Thrown when a function or method is not implemented
    '''
    def __init__(self):
        super().__init__("Not Implemented", None)


#!/usr/bin/env python3

import hashlib
from binascii import hexlify
from itertools import product
from sys import argv
import time

POSSIBLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

# Number of characters to consider in the hash function, starting at the
# beginning. e.g. 6 means we use the first 6 bytes of sha1.
HASHCHARS = 12

# Supported algoritms
SUPORTED_ALGOS = ["sha1","sha224","sha256","sha384","sha512","md5"]

# Default ALGO
ALGO = "sha1"

def genhash(s):
    ''' Returns the shortened sha1 hash of s. If the input is bytes, they
        will be hashed directly; otherwise they will be encoded to ascii
        before being hashed.
    '''
    if type(s) is bytes:
        # Already encoded, just hash the bytes.
        if ALGO == "sha1":
            return hashlib.sha1(s).hexdigest()[:HASHCHARS]

        if ALGO == "sha224":
            return hashlib.sha224(s).hexdigest()[:HASHCHARS]

        if ALGO == "sha256":
            return hashlib.sha256(s).hexdigest()[:HASHCHARS]

        if ALGO == "sha384":
            return hashlib.sha384(s).hexdigest()[:HASHCHARS]

        if ALGO == "sha512":
            return hashlib.sha512(s).hexdigest()[:HASHCHARS]

        if ALGO == "md5":
            return hashlib.md5(s).hexdigest()[:HASHCHARS]
    else:
        # Convert it to ascii, then hash.
        if ALGO == "sha1":
            return hashlib.sha1(s.encode('ascii')).hexdigest()[:HASHCHARS]

        if ALGO == "sha224":
            return hashlib.sha224(s.encode('ascii')).hexdigest()[:HASHCHARS]

        if ALGO == "sha256":
            return hashlib.sha256(s.encode('ascii')).hexdigest()[:HASHCHARS]

        if ALGO == "sha384":
            return hashlib.sha384(s.encode('ascii')).hexdigest()[:HASHCHARS]

        if ALGO == "sha512":
            return hashlib.sha512(s.encode('ascii')).hexdigest()[:HASHCHARS]

        if ALGO == "md5":
            return hashlib.md5(s.encode('ascii')).hexdigest()[:HASHCHARS]

def show(orig_str, collision_str, duration):
    ''' Print the original string, the collision string, and then recompute
        the hashes of each of them and print those, to prove that we found
        a collision.
    '''
    # Do the encoding to ascii for bytes output
    orig_ascii = orig_str.encode('ascii')
    collision_ascii = collision_str.encode('ascii')

    # Print stuff.
    print()
    print('Collision found!')

    print()
    print("%r bytes collision found using %r algorithm in %r seconds" % (HASHCHARS,ALGO,duration))
    print ("Validate it")
    print ()
    print ("echo -n %r | %r | cut -c1-%d  && echo -n %r | %r | cut -c1-%r" %(orig_str,ALGO+"sum",HASHCHARS,collision_str,ALGO+"sum",HASHCHARS))

    print()
    print(orig_str
            + ' (bytes: ' + str(hexlify(orig_ascii)) + ')'
            + ' hashes to ' + str(genhash(orig_ascii))
            + ', but ' + collision_str
            + ' (bytes: ' + str(hexlify(collision_ascii)) + ')'
            + ' also hashes to ' + str(genhash(collision_ascii)))

def is_collision(trial, orig_hash):
    ''' Returns true if the hash of trial is the same as orig_hash.
    '''
    h = genhash(trial)
    return h == orig_hash

def collide(startnumber):
    ''' Search for collisions in the hash. Start with the possible match
        at index startnumber and look for collisions by searching upward
        from there.
        Note that this means if you choose a large value (e.g. 400000) this
        will not look for collisions on possibilities 0 <= x <= 400001, so
        choose a low number unless you want this to run for quite a while.
    '''
    start = int(time.time())

    # Iterator that yields possible characters.
    possible = product(POSSIBLE, repeat=100)

    # Iterate over the product until we reach the specified startnumber
    for i in range(startnumber):
        possible.__next__()

    # This is our collision target
    orig = ''.join([e for e in possible.__next__()]).lstrip('0')
    orig_hash = genhash(orig)

    # Iterate over the possible options
    for trial in possible:

        # Convert the tuple from itertools.product into a string
        trial = ''.join([e for e in trial])
        # Strip the leading zeros (who cares about zeros!)
        trial = trial.lstrip('0')

        # Exit if we found a collision
        if is_collision(trial, orig_hash):
            duration = int(time.time()) - start
            show(orig, trial, duration)
            break

if __name__ == '__main__':
    if len(argv) > 1:
        n = int(argv[1])
        if len(argv) > 2:
            HASHCHARS=int(argv[2])
        if len(argv) > 3:
            if argv[3] not in SUPORTED_ALGOS:
                print()
                print ("Using default algo sha256")
            ALGO = argv[3]
        collide(n)
    else:
        print('Please pass an integer as the argument to this program.')
        print()
        print('This number will be used as the starting offset on the string')
        print('search function. It is recommended not to use something like')
        print('"0", since this will end up searching for collisions on the')
        print('empty string ("").')
        print()
        print('e.g.:')
        print('    $ python3 sha.py 300 sha1')
        print('    $ python3 sha.py 300 sha224')
        print('    $ python3 sha.py 300 sha256')
        print('    $ python3 sha.py 300 sha384')
        print('    $ python3 sha.py 300 sha512')
        print('    $ python3 sha.py 300 md5')
        print()
        print('This will search for collisions on the string "4Q", the 301st')
        print('iteration of our cartesian product search ALGOrithm on a')
        print('subset of the ASCII alphabet.')


import os, json
from os import path as osp
from ffmpy import FFmpeg
import csv

classIndFile=open('soccerdb_class_ind.json')
classIndJSON=json.load(classIndFile)
classIndFile.close()
soccerdbVideosPath='/media/opekta/CopaEurope/SoccerDB/videos'
targetFolder='/home/opekta/copaeurope/mmaction2/data/soccernet/extractedFrames'

outputExtension='.jpg'

with open('soccerdb_seg_info.csv', newline='') as csvfile:
    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
    for row in spamreader:
        if (row[0][:6]=='seg_id'):
            continue
        # print('\n',row)
        # print(len(row))
        splitted=[]
        for rowPart in row:
            element=rowPart.split(',')
            for x in element:
                splitted.append(x)
        # print(splitted)
        # print(len(splitted))
        if (splitted[1][-4:] != '.mp4'):
            continue
        event_labels=[]
        for i in range(6, len(splitted)-1):
            event_labels.append(splitted[i])
            if (classIndJSON['events'][splitted[i]] == "Penalty"):
                print(classIndJSON['events'][splitted[i]])
                start=3600*int(splitted[4][:2])+60*int(splitted[4][3:5])+int(splitted[4][6:8])
                print(start)
                end=3600*int(splitted[5][:2])+60*int(splitted[5][3:5])+int(splitted[5][6:8])
                print(end)
                print(splitted[1][:-4],'\n')
                for root, dirs, files in os.walk(soccerdbVideosPath):
                    for file in files:
                        if (file[:30] == splitted[1][:-4]):
                            videoPath=osp.join(root,file)
                            targetVideoFolder=osp.join(targetFolder,classIndJSON['events'][splitted[i]]+'/'+file[:30])
                            os.makedirs(targetVideoFolder, exist_ok=True)
                            targetImage = osp.join(targetVideoFolder, "img_%05d"
                                            + outputExtension)
                            print(videoPath)
                            print(targetImage)
                            ff = FFmpeg(
                                inputs={videoPath: ['-y', '-ss',
                                                    str(start), '-to', str(end)
                                                    ]},
                                outputs={targetImage: ['-qmin', '1', '-qscale:v', '2', '-vf', 'scale=398:224']}
                            )
                            ff.run()


import graphene
from graphql_relay.node.node import from_global_id

from accounts.decorators import login_required
from accounts.permissions import has_permission
from db.models_graphql import Document
from backend.mutations import Mutation
from .models import Vote as VoteModel
from .models_graphql import Vote, Voting

def _set_vote(input, request, info):
    gid_type, gid = from_global_id(input.get('parent'))
    parent = Document._meta.model.objects.get(pk=gid)

    try:
        vote = VoteModel.objects.get(parent=parent,
                                     author=request.user.document)
    except VoteModel.DoesNotExist:
        vote = VoteModel(parent=parent, author=request.user.document)

    vote.value = input.get('value')
    vote.save(request=request)

    voting = Voting.get_node(info, id=gid)

    return {
        'vote': vote,
        'voting': voting
    }

class VoteSet(Mutation):
    class Input:
        parent = graphene.ID(required=True)
        value = graphene.Int(required=True)

    vote = graphene.Field(Vote)
    voting = graphene.Field(Voting)

    @classmethod
    @login_required
    def mutate_and_get_payload(cls, root, info, **input):
        return VoteSet(**_set_vote(input, info.context, info))

class VoteDelete(Mutation):
    class Input:
        id = graphene.ID(required=True)

    voteDeletedID = graphene.ID(required=True)
    voting = graphene.Field(Voting)

    @classmethod
    @login_required
    def mutate_and_get_payload(cls, root, info, **input):
        gid_type, gid = from_global_id(input.get('id'))
        vote = VoteModel.objects.get(document_id=gid,
                                     author=info.context.user.document)

        error = has_permission(cls, info.context, vote, 'delete')
        if error:
            return error

        parent_id = vote.parent_id
        vote.delete(request=info.context)

        voting = Voting.get_node(info, id=parent_id)

        return VoteDelete(voteDeletedID=input.get('id'), voting=voting)


from random import randint
from time import sleep

hp_dragon = 100
hp_player = 100

def health_status():
    global hp_dragon
    global hp_player
    print(f"You have {hp_player} hp, the dragon now has {hp_dragon} hp.")

def dragon():
    global hp_dragon
    global hp_player
    print("You now fight the dragon!...")
    sleep(2)
    print("The dragon has three different types of attacks. Each is most successfully paraded by a move of yours...")
    while (hp_dragon > 0 and hp_player > 0):
        attack = randint(0,2)
        if attack == 0:
            bite()
            input("> ")
        elif attack == 1:
            hit_w_tail()
            input("> ")
        elif attack ==2:
            spit_fire()
            input("> ")
        else:
            print("Error #1 this is not an attack of the dragon that should be possible")
    print(f"{hp_dragon} and {hp_player}")

def bite():
    global hp_dragon
    global hp_player
    print("The dragon tries to bite you!")
    choice = input("> How do you react? 'leap' backwards or do you 'block' the bite? ")
    if 'leap' in choice:
        print("You leap backwards!")
        effect = randint(0,99)
        if effect < 70:
            print("Successfully! The dragon bites in the air, but you are already safely on the side. You can even stab the dragon and damage him!")
            hp_dragon -= randint(10, 20)
            health_status()
        else:
            print("You leap backwards, but the dragon reaches you anyway, unlucky you!")
            hp_player -= randint(15,20)
            health_status()
    elif 'block' in choice:
        print("You try to block the dragon's attack!")
        effect = randint(0,99)
        if effect < 20:
            print("You block successfully! Wow! That is very difficult to achieve! ...You even give him a hit!")
            hp_dragon -= randint(10, 20)
            health_status()
        else:
            print("You want to block the bite, but that's very difficult! You fail this time...The dragon hurts you!")
            hp_player -= randint(20,30)
            health_status()
    else:
        print("You can choose between 'leap' backwards to avoid the bite or you try to 'block' the bite")

def hit_w_tail():
    global hp_dragon
    global hp_player
    print("The dragon tries hit you with his tail!")
    choice = input("> How do you react? 'stab' the tail with your sword or 'jump' on the tail? ")
    if 'stab' in choice:
        print("You try to stab the tail as it flies towards you!")
        effect = randint(0,99)
        if effect > 49:
            print("You stab the tail right in the middle! Dragon blood spills on you as the dragon howls loudly")
            hp_dragon -= randint(10,20)
            health_status()
        else:
            print("You try to stab the tail, but the dragon moves too fast! Instad of stabbing it, he hits you and slamms you against the wall!")
            hp_player -= randint(10,20)
            health_status()
    elif 'jump' in choice:
        print("You try to jump on the tail!")
        effect = randint(0,99)
        if effect > 85:
            print("You jump exactly at the right moment and land the the tail!! The dragon is furious! From there you can bury your sword deep in the beasts flesh!")
            hp_dragon -= randint(30,35)
            health_status()
        else:
            print("You try to jump on the tail, but you slip and miss it...at least you don't get hurt")
            health_status()
    else:
        print("You can 'stab' or 'jump' at this point.")

def spit_fire():
    global hp_dragon
    global hp_player
    print("The dragon spits fire!")
    choice = input("> How do you react? 'run' away or use a 'magic' spell to block it and throw the fire back at the dragon? ")
    if 'run' in choice:
        print("You run away! Smart move, dragon fire burns like hell!")
        print("The dragon looks like he got a little bit weaker through his attack of his")
        hp_dragon -= randint(1,5)
        health_status()
    elif 'magic' in choice:

        print("You use the magic spell!")

        effect = randint(0,99)

        if effect < 20:

            print("A powerful white light shoots out of your wand! You block the fire and it heads back to the dragon, hurting him badly!")

            hp_dragon -= randint(20,25)

            health_status()

        else:

            print("The spell and the fire hit on each other and with a loud 'bang!' both you and the dragon are being thrown backwards!")

            hp_dragon -= 10

            hp_player -= 5

            health_status()

    else:
        print("You can 'run' or use a 'magic' spell at this point.")

#dragon()
#bite()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 08 October, 2018
Testing suite for Network export_to_shp function
@author: J. Vicente Perez
@email: geolovic@hotmail.com
@date: 23 October, 2018
"""

from topopy import Network, DEM, Flow, Grid
import matplotlib.pyplot as plt
import numpy as np
infolder = "../data/in"
outfolder = "../data/out"

dem = DEM(infolder + "/morocco.tif")
fd = Flow(dem)
net = Network(dem, fd, 1500)

streams = net.get_streams()
streams.save(outfolder + "/canales_orig.tif")

outlet = np.array([579213, 504282]).reshape(1, 2)

basin = fd.get_drainage_basins(net.snap_points(outlet), asgrid=False)
c1 = basin.max(axis=0).argmax()
r1 = basin.max(axis=1).argmax()
c2 = basin.shape[1] - np.fliplr(basin).max(axis=0).argmax()
r2 = basin.shape[0] - np.flipud(basin).max(axis=1).argmax()
basin_cl = basin[r1:r2, c1:c2]

nrow = basin_cl.shape[0]
ncol = basin_cl.shape[1]
outgrid = Grid()

outgrid._size = (ncol, nrow)
outgrid._dims = (nrow, ncol)
geot = net._geot
ULx = geot[0] + geot[1] * c1
ULy = geot[3] + geot[5] * r1
outgrid._geot = (ULx, geot[1], 0.0, ULy, 0.0, geot[5])
outgrid._cellsize = geot[1]
outgrid._proj = net._proj
outgrid._ncells = basin_cl.size
outgrid._nodata = 0
outgrid._array = basin_cl
outgrid._tipo = str(basin.dtype)

fd.get_drainage_basins(net.snap_points(outlet)).save(outfolder + "/basin_orig.tif")
outgrid.save(outfolder + "/basin_test.tif")

# GET ONLY POINTS INSIDE BASIN
basin = basin.astype(np.bool).ravel()
I = basin[net._ix]

ix = net._ix[I]
ixc = net._ixc[I]

# Get grid channel cells
w = np.zeros(net._ncells, dtype=np.int8)
w[ix] = 1
w[ixc] = 1
w = w.reshape(net._dims)

plt.imshow(w)

#I = basin.astype(np.bool).ravel()[net._ix]
#
#ix = net._ix[I]
#ixc = net._ixc[I]
#
rowix, colix = net.ind_2_cell(ix)
nrowix = rowix - r1
ncolix = colix - c1
newix = outgrid.cell_2_ind(nrowix, ncolix)

#rowixc, colixc = net.ind_2_cell(ixc)
#rowixc -= r1
#colixc -= c1
#newixc = outgrid.cell_2_ind(rowixc, colixc)
#
w = np.zeros(basin_cl.size, dtype=np.int8)
w[newix] = 1
#w[newixc] = 1
w = w.reshape(outgrid._dims)

streams = Grid()
streams.copy_layout(outgrid)
streams._nodata = 0
streams._array = w
streams._tipo = str(w.dtype)

streams.save(outfolder + "/canales_prueba.tif")