from pickle import TRUE
import numpy as np
import pandas as pd
from scipy import signal

fix_char = '' #vcoi #TODO fix issue that arises with time array from first data point being removed i.e. set to nan

class quad_data_frame :
    def __init(self) :
        self.c = np.array()
        self.p = pd.DataFrame() #x,y,z,b pni axis/total field
        self.t = pd.DataFrame() #t,t,t,t timestamps for each axis
        self.td = pd.DataFrame() #td,td,td,td period between each measurement
        self.im = pd.DataFrame() #ax, ay, az, gx, gy, gz imu accelerometer/gyroscope
        self.tt = pd.DataFrame() #temperature
        self.sr = 0 #sampling rate
        self.cc = 0 #cycle count
        self.os = 1 #oversamples
        self.mn = -1

    def update_self(self) :
        #updates the calculated b field and sampling rate if the values of x,y,z or t have changed

        self.c[0].update_self()
        x = self.c[0].p.iloc[:, 0]
        y = self.c[0].p.iloc[:, 1]
        z = self.c[0].p.iloc[:, 2]
        t = self.c[0].t.iloc[:, 0]
        for i in range(1, len(self.c)) :
            self.c[i].update_self()
            x = x.add(self.c[i].p.iloc[:, 0])
            y = y.add(self.c[i].p.iloc[:, 1])
            z = z.add(self.c[i].p.iloc[:, 2])
            t = t.add(self.c[i].t.iloc[:, 0])

        #remove the offsets
        #
        #for i in range (0, len(self.c)) :
        #    x = x.sub(self.c[i].offset[0])
        #    y = y.sub(self.c[i].offset[1])
        #    z = z.sub(self.c[i].offset[2])

        x = x.div(4.0)
        y = y.div(4.0)
        z = z.div(4.0)
        t = t.div(4.0)

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = t.to_numpy()[1:]
        ty = t.to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)
        self.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
        self.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
        self.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        self.sr = 1/(td.mean())

class pni_data_frame :
    def __init(self) :
        self.p = pd.DataFrame() #x,y,z,b pni axis/total field
        self.t = pd.DataFrame() #t,t,t,t timestamps for each axis
        self.td = pd.DataFrame() #td,td,td,td period between each measurement
        self.sr = 0 #sampling rate
        self.cc = 0 #cycle count
        self.os = 1 #oversamples
        self.mn = 0 #mag identification number
        self.offset = [0, 0, 0] #just 3 offset values for x,y,z

    def update_self(self) :
        #updates the calculated b field and sampling rate if the values of x,y,z or t have changed

        self.calc_offset()

        self.t = self.t.sub(self.t.iloc[0])
        x = self.p.iloc[:, 0].sub(self.offset[0])
        y = self.p.iloc[:, 1].sub(self.offset[1])
        z = self.p.iloc[:, 2].sub(self.offset[2])

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = self.t.iloc[:,0].to_numpy()[1:]
        ty = self.t.iloc[:,0].to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)
        self.p = pd.DataFrame({'0': x,'1': y,'2': z,'3': b})
        self.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        self.sr = 1/(td.mean())

    def fix_self(self) :
        #fixes the datasets of this pni data frame
        a = fix_pni_dataset(self.t.iloc[:,0], self.p.iloc[:,0], fix_char)
        b = fix_pni_dataset(self.t.iloc[:,1], self.p.iloc[:,1], fix_char)
        c = fix_pni_dataset(self.t.iloc[:,2], self.p.iloc[:,2], fix_char)
        d = fix_pni_dataset(self.t.iloc[:,3], self.p.iloc[:,3], fix_char)
        self.t = pd.DataFrame({'0': a[0],'1':b[0],'2': c[0],'3': d[0]})
        self.p = pd.DataFrame({'0': a[1],'1':b[1],'2': c[1],'3': d[1]})
        self.update_self() #workaround for subtracting mean but not affecting quad data set

    def calc_offset(self) :
        #calculates the offset as the mean of the measured field
        #NOTE: assumes that the field is 0nT accross all axis

        self.offset[0] = self.p.iloc[:,0].mean()
        self.offset[1] = self.p.iloc[:,1].mean()
        self.offset[2] = self.p.iloc[:,2].mean()

        #self.offset = {0, 0 ,0} #alternative hardcode method

def pni_file_decode_quad(fn) :
    # def: decodes file based on quad mag python code formatting into a data_frame object and returns it
    # in: string fn (file to read from), int mn (mag identification)
    # out: quad_data_frame populated with file contents
    cdf = pd.read_csv(fn, skiprows=1, nrows=1, header=None)
    po = quad_data_frame()
    po.c = np.array([pni_data_frame(), pni_data_frame(), pni_data_frame(), pni_data_frame()]) #weird fix for bad python init
    po.cc = cdf.iloc[0,0]
    po.os = cdf.iloc[0,1]

    sf = float((1000/(0.3671 * po.cc + 1.5)) / po.os) #current accepted method
    #sf = float(1000/(0.3671 * cc * os + 1.5)) #alternative
    pdf = pd.read_csv(fn, skiprows=4, header=None) #read into pd dataframe, skipping file header and blank line

    #fix_corrupted_file(pdf)

    xtt = 0
    ytt = 0
    ztt = 0

    t = pdf.iloc[:,1]
    t = t.sub(t.iloc[0]) # start time from zero
    x = pdf.iloc[:,2].multiply(sf) #xpni readings
    xtt = x
    y = pdf.iloc[:,3].multiply(sf) #ypni readings
    ytt = y
    z = pdf.iloc[:,4].multiply(sf) #zpni readings
    ztt = z
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    tx = t.to_numpy()[1:]
    ty = t.to_numpy()[:-1]
    tz = tx - ty
    td = pd.Series(tz)
    po.c[0].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[0].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[0].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[0].sr = 1/(td.mean())
    po.c[0].cc = po.cc
    po.c[0].os = po.os
    po.c[0].mn = 0
    po.c[0].offset = [0, 0, 0]
    po.c[0].calc_offset()

    x = pdf.iloc[:,5].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,6].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,7].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[1].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[1].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[1].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[1].sr = 1/(td.mean())
    po.c[1].cc = po.cc
    po.c[1].os = po.os
    po.c[1].mn = 1
    po.c[1].offset = [0, 0, 0]
    po.c[1].calc_offset()

    x = pdf.iloc[:,8].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,9].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,10].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[2].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[2].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[2].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[2].sr = 1/(td.mean())
    po.c[2].cc = po.cc
    po.c[2].os = po.os
    po.c[2].mn = 2
    po.c[2].offset = [0, 0, 0]
    po.c[2].calc_offset()

    x = pdf.iloc[:,11].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,12].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,13].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[3].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[3].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[3].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[3].sr = 1/(td.mean())
    po.c[3].cc = po.cc
    po.c[3].os = po.os
    po.c[3].mn = 3
    po.c[3].offset = [0, 0, 0]
    po.c[3].calc_offset()

    #remove the offsets
    for i in range (0, 4) :
        xtt = xtt.sub(po.c[i].offset[0])
        ytt = ytt.sub(po.c[i].offset[1])
        ztt = ztt.sub(po.c[i].offset[2])

    xtt = xtt.div(4.0)
    ytt = ytt.div(4.0)
    ztt = ztt.div(4.0)
    b = xtt.pow(2)+ytt.pow(2)+ztt.pow(2) #bpni readings
    b = b.pow(0.5)

    po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.p = pd.DataFrame({'0': xtt,'1':ytt,'2': ztt,'3': b})
    po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.sr = 1/(td.mean())
    po.mn = 'quad'

    pni_describe(po)

    return po

def pni_file_decode_sean(fn, os, cc, mn) :
    # def: decodes file based on seans python code formatting into a data_frame object and returns it
    # in: string fn (file to read from), int os (number of oversamples), int cc (cycle count), int mn (mag identification)
    # out: pni_data_frame populated with file contents
    sf = float((1000/(0.3671 * cc + 1.5)) / os) #current accepted method
    #sf = float(1000/(0.3671 * cc * os + 1.5)) #alternative
    po = pni_data_frame()
    pdf = pd.read_csv(fn, skiprows=3, header=None) #read into pd dataframe, skipping file header
    t = pdf.iloc[:,0]
    x = pdf.iloc[:,1].multiply(sf) #xpni readings
    y = pdf.iloc[:,2].multiply(sf) #ypni readings
    z = pdf.iloc[:,3].multiply(sf) #zpni readings
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)
    tx = t.to_numpy()[1:]
    ty = t.to_numpy()[:-1]
    tz = tx - ty
    td = pd.Series(tz)
    po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.sr = 1/(td.mean())
    po.cc = cc
    po.os = os
    po.mn = mn

    return po

def fix_pni_dataset(t, d, f):
    # def: removes outliers from your time series and data series based on f args/crops percentage of the data
    # in: panda series t (time series), panda series d (data series), string f (flags)
    # out: nothing
    # flags:
    #   'v' is verbose ie print whats happening
    #   'c' is crop ie chop first cper and last cper of data points
    #   'o' is remove outliers
    #   'i' is interpolate data

    if f == '' :
        return [t,d]

    cper = 10.0 #change this to percent of data you want to remove from start and end of dataset

    ct = True if f.find('c') != -1 else False
    vt = True if f.find('v') != -1 else False
    ot = True if f.find('o') != -1 else False
    it = True if f.find('i') != -1 else False

    co = 0
    cl = d.size
    cp = int(cl * (cper/100.0))

    if ct :
        co = cp*2
        #d.iloc[:(cp+1)] = np.nan
        #d.iloc[(cl-cp):] = np.nan
        #t.iloc[:(cp+1)] = np.nan
        #t.iloc[(cl-cp):] = np.nan
        t = t.iloc[cp+1:cl-cp] #actually removes the datapoint now instead of nan
        d = d.iloc[cp+1:cl-cp]
        if vt :
            print("Removed data points (crop) 0 to " + str(cp) + " and data points (crop) " + str(cl-cp) + " to " + str(cl))

    dm = d.mean()
    ds = d.std()
    no = 0
    zdf = ((d.sub(dm)).div(ds)).abs()

    if vt and ot :
        for i,j in enumerate(zdf) :
            if j != np.nan and j > 4 : #outliers are > 4 std
                print("Removed data point (outlier) " + str(round(d[i],3)) + " at index " + str(i))
                t.iloc[i] = t.iloc[i] if it else np.nan
                d.iloc[i] = np.nan
                no+=1

    elif ot :
        for i,j in enumerate(zdf) :
            if j != np.nan and j > 4 :
                t.iloc[i] = t.iloc[i] if it else np.nan
                d.iloc[i] = np.nan
                no+=1

    if it : #interpolate
        #if ct:
        #    nsn, xx = nan_helper(d[cp+1:cl-cp].to_numpy()) #only interpolate data not cropped
        #    d[cp+1:cl-cp][nsn]= np.interp(xx(nsn), xx(~nsn), d[cp+1:cl-cp][~nsn])
        #else :
        nsn, xx = nan_helper(d.to_numpy())
        d[nsn]= np.interp(xx(nsn), xx(~nsn), d[~nsn])

    if vt :
        print("\n***********************************************************")
        print("The mean of this data set before outlier removal was " + str(dm) + " with standard deviation " + str(ds))
        print("The mean of this data set after outlier removal is " + str(d.mean()) + " with standard deviation " + str(d.std()))
        print("\nRemoved " + str(no+co) + " data points out of " + str(cl) + " total")
        print("Removed " + str(no) + " outlier data points and cropped " + str(co) + " data points\n")
        print("***********************************************************\n")

    return [t,d]

def nan_helper(a) :
    #def:
    #in:
    #out:
    return np.isnan(a), lambda z: z.nonzero()[0]

def decimate_helper(do, dsr) :
    #def: decimates data to 1hz using scipy decimate function
    #in: do is pni data object, dsr is the sample rate desired (must be int factor of original sr or close to)
    #out: modified pni data object with updated dsr sample rate and corresponding data

    nt = pd.DataFrame()
    nd = pd.DataFrame()

    while(do.sr > float(dsr*4.0)) :
        a = pd.Series(signal.decimate(do.t.iloc[:,0], 2))
        b = pd.Series(signal.decimate(do.p.iloc[:,0], 2))
        c = pd.Series(signal.decimate(do.p.iloc[:,1], 2))
        d = pd.Series(signal.decimate(do.p.iloc[:,2], 2))
        e = pd.Series(signal.decimate(do.p.iloc[:,3], 2))
        nt = pd.DataFrame({'0': a,'1':a,'2': a,'3': a})
        nd = pd.DataFrame({'0': b,'1':c,'2': d,'3': e})
        do.t = nt
        do.p = nd
        do.update_self()

    dsf = do.sr / dsr

    a = pd.Series(signal.decimate(do.t.iloc[:,0], int(dsf)))
    b = pd.Series(signal.decimate(do.p.iloc[:,0], int(dsf)))
    c = pd.Series(signal.decimate(do.p.iloc[:,1], int(dsf)))
    d = pd.Series(signal.decimate(do.p.iloc[:,2], int(dsf)))
    e = pd.Series(signal.decimate(do.p.iloc[:,3], 2))
    nt = pd.DataFrame({'0': a,'1':a,'2': a,'3': a})
    nd = pd.DataFrame({'0': b,'1':c,'2': d,'3': e})
    do.t = nt
    do.p = nd
    do.update_self()

    return do

def window_helper(do, wl) :
    #def:
    #in:
    #out:

    ns = [[],[],[],[]]
    nt = []

    for i in range(0, len(do.t.iloc[:,0])-wl, wl) :
        ns[0].append(do.p.iloc[i:i+wl,0].mean())
        ns[1].append(do.p.iloc[i:i+wl,1].mean())
        ns[2].append(do.p.iloc[i:i+wl,2].mean())
        ns[3].append(do.p.iloc[i:i+wl,2].mean())
        nt.append(do.t.iloc[i,0])

    do.t = pd.DataFrame({'0': pd.Series(nt),'1':pd.Series(nt),'2': pd.Series(nt),'3': pd.Series(nt)})
    do.p = pd.DataFrame({'0': pd.Series(ns[0]),'1':pd.Series(ns[1]),'2': pd.Series(ns[2]),'3': pd.Series(ns[3])})

    do.update_self()

    return do

def pni_describe(pdf) :
    # def: prints descriptive data to cl for a pni_data_frame
    # in: pdf pni_data_frame (data to describe)
    # out: none
    print("***Sampling Information***")
    print("Mag Identification Number: " +  str(pdf.mn))
    print("Cycle Count: " + str(pdf.cc) + "\nNumber of Oversamples: " + str(pdf.os))
    print("Sampling Rate (Hz): " + str(round(pdf.sr, 3)))
    print('X Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 0].std(), 3)))
    print('Y Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 1].std(), 3)))
    print('Z Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 2].std(), 3)))
    print('|B|-field Stdev (nT): ' + str(round(pdf.p.iloc[:, 3].std(), 3)))
    print("\n")

def find_best_window(pdf, wl, f, n=1) :
    # def: finds the |B| field lowest stdev for a wl window length in a pni_data_frame
    # in: pdf pni_data_frame (data), wl int window length
    # out: pdf pni_data_frame containing data from best wl length window

    # window average the data and update lowest stdev
    # note -> assumes that stdev at some point will be less than 1000000
    aod = True if f.find('a') != -1 else False

    mpdf = pdf

    if aod :

        por = []

        for j in range(0, n) :

            istd = 0
            lstd = 1000000
            for i in range(0, (len(mpdf.p.iloc[:,3]) - wl)) :
                s = mpdf.p.iloc[i:i+wl,3].std()
                istd = i if s < lstd else istd
                lstd = s if s < lstd else lstd

            po = quad_data_frame()
            po.c = np.array([pni_data_frame(), pni_data_frame(), pni_data_frame(), pni_data_frame()]) #weird fix for bad python init

            for i in range(0, 4) :
                t = mpdf.c[i].t.iloc[istd:istd+wl, 0]
                x = mpdf.c[i].p.iloc[istd:istd+wl, 0]
                y = mpdf.c[i].p.iloc[istd:istd+wl, 1]
                z = mpdf.c[i].p.iloc[istd:istd+wl, 2]

                b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
                b = b.pow(0.5)

                tx = mpdf.c[i].t.iloc[istd:istd+wl,0].to_numpy()[1:]
                ty = mpdf.c[i].t.iloc[istd:istd+wl,0].to_numpy()[:-1]
                tz = tx - ty
                td = pd.Series(tz)

                po.c[i].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
                po.c[i].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
                po.c[i].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
                po.c[i].sr = 1/(td.mean())
                po.c[i].cc = mpdf.c[i].cc
                po.c[i].os = mpdf.c[i].os
                po.c[i].mn = mpdf.c[i].mn
                po.c[i].offset = [0, 0, 0] #just 3 offset values for x,y,z
                po.c[i].calc_offset()

            po.cc = mpdf.cc
            po.os = mpdf.os
            po.mn = mpdf.mn
            po.update_self()

            if n == 1 :
                return po

            for k in range(0,4) :
                x = pd.Series(np.delete(mpdf.c[k].p.iloc[:,0].to_numpy(), slice(istd, istd+wl)))
                y = pd.Series(np.delete(mpdf.c[k].p.iloc[:,1].to_numpy(), slice(istd, istd+wl)))
                z = pd.Series(np.delete(mpdf.c[k].p.iloc[:,2].to_numpy(), slice(istd, istd+wl)))
                t = pd.Series(np.delete(mpdf.c[k].t.iloc[:,0].to_numpy(), slice(istd, istd+wl)))

                mpdf.c[k].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
                mpdf.c[k].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': z})

            mpdf.update_self()

            por.append(po)

            print('Found Best Window ' + str(j) + '!\n')

        return por

    else :

        istd = 0
        lstd = 1000000
        for i in range(0, (len(pdf.p.iloc[:,3]) - wl)) :
            s = pdf.p.iloc[i:i+wl,3].std()
            istd = i if s < lstd else istd
            lstd = s if s < lstd else lstd

        po = pni_data_frame()

        t = pdf.t.iloc[istd:istd+wl, 0]
        x = pdf.p.iloc[istd:istd+wl, 0]
        y = pdf.p.iloc[istd:istd+wl, 1]
        z = pdf.p.iloc[istd:istd+wl, 2]

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = pdf.t.iloc[istd:istd+wl,0].to_numpy()[1:]
        ty = pdf.t.iloc[istd:istd+wl,0].to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)

        po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
        po.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
        po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        po.sr = 1/(td.mean())
        po.cc = pdf.cc
        po.os = pdf.os
        po.mn = pdf.mn

        return po

def fix_corrupted_file(pdf) :
    #
    #
    #

    for i in range(0, len(pdf.iloc[:,0])) :
        if str(pdf.iloc[:,0].iloc[i]) != '4' :
            pdf.drop(labels=i, axis=0, inplace=True)
            print("Found corrupted data at line " + str(i) + '\n' + 'with value')

    return pdf


import serial
import serial.tools.list_ports
import configparser

import urllib3

urllib3.disable_warnings()

from utils import take_photo, prediction, send_prediction, empty_folders, checkInfectedMicrocontrollers, set_micro_light

import time

class Bridge():

	def __init__(self):
		self.config = configparser.ConfigParser()
		self.config.read('config.ini')
		self.setupSerial()

	def setupSerial(self):
		# open serial port
		self.ser = None

		if self.config.get("Serial","UseDescription", fallback=False):
			self.portname = self.config.get("Serial","PortName", fallback="COM3")
		else:
			print("list of available ports: ")
			ports = serial.tools.list_ports.comports()

			for port in ports:
				print (port.device)
				print (port.description)
				if self.config.get("Serial","PortDescription", fallback="arduino").lower() \
						in port.description.lower():
					self.portname = port.device

		try:
			if self.portname is not None:
				print ("connecting to " + self.portname)
				self.ser = serial.Serial(self.portname, 9600, timeout=0)
		except:
			self.ser = None

		# internal input buffer from serial
		self.inbuffer = []

	def loop(self):
		# infinite loop for serial managing
		#
		while (True):
			# attuazione
			# check if micro_id is in infected list from check routine 1
			# funzione in utils che se ritorna a true si invia un pacchetto seriale al microcontrollore per accendere il led
			mic_state = checkInfectedMicrocontrollers() #assegno vero se la varibile è presente nella lista
			set_micro_light(mic_state, self.ser) #accendo il led se mic_state è True

			#poi si fa la lettura del seriale per vedere le foto ricevute
			#look for a byte from serial
			if not self.ser is None:

				if self.ser.in_waiting>0:
					# data available from the serial port
					lastchar=self.ser.read(1)

					if lastchar==b'\xfe': #EOL
						print("\nValue received")
						self.useData()
						self.inbuffer = []
					else:
						# append
						self.inbuffer.append(lastchar)

	def useData(self):

		if len(self.inbuffer)<2:   # at least 2 bytes
			return False

		# split parts
		if self.inbuffer[0] != b'\xff':
			return False

		if self.inbuffer[1] != b'\xfa':
			return False
		else:

			print(self.inbuffer)
			start_time = time.time()

			print('take a photo!')
			take_photo()
			print('make prediction!')
			data = prediction()
			response = send_prediction(data)
			if response == 200:
				print('data sent!')
			print('empty folders!')
			empty_folders()

			end_time = time.time()
			elapsed_time = end_time - start_time
			print(f"Code took {elapsed_time} seconds to execute.")

if __name__ == '__main__':

	br = Bridge()
	br.loop()


import insider
import sector
import pandas as pd
import pandas_datareader as dr
from datetime import datetime,date,timedelta
startTime = datetime.now()

# insider csv creation
insider.scrape_to_csv() #uncomment to refresh the csv file
df = pd.read_csv('Insider_trade.csv',index_col = 0)

# sector csv creation
sector.sector_to_csv()

# updating insider csv with additional information
hashmap = {}
periods = 180

def get_info(ticker,n):

    try:
        tickerdf = dr.data.get_data_yahoo(ticker,start = date.today() - timedelta(300) , end = date.today())
        currentprice = tickerdf.iloc[-1]['Close']
        MA = pd.Series(tickerdf['Close'].rolling(n, min_periods=0).mean(), name='MA')
        currentma = MA[-1]
        print(f"data gathered for {ticker}")
        return (currentprice,currentma)
    except:
        return ('na','na')

def prices():
    periods = 180
    for ticker in df.Symbol.unique():
        #hashmap[ticker] = get_info(ticker, periods)
        if ticker not in hashmap.keys():
            hashmap[ticker] = get_info(ticker, periods)
    return hashmap

df['currentprice'] = df.apply (lambda row: prices()[row['Symbol']][0], axis=1)
print("Prices gathered")

df['movingaverage'] = df.apply (lambda row: prices()[row['Symbol']][1], axis=1)
print("movingaverages gathered")

df.to_csv('Insider_trade.csv')

print(f'Execution Time: {datetime.now() - startTime}')


from home.models import Professors
from django.db.models import Q, F, Count, Avg, FloatField, Max, Min, Case, When

import nltk, spacy, json, string, re
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag
from nltk.chunk import conlltags2tree, tree2conlltags
from pprint import pprint
from spacy import displacy
from collections import Counter, OrderedDict
from nltk.corpus import stopwords, words, wordnet
from nameparser.parser import HumanName
from itertools import chain

stop_words = stopwords.words('english')
english_words = set(words.words())

def get_uni_regex_common_words(university):
    with open("static/jsons/find_courses.json") as f:
        data = json.load(f)
        data = data[university]
        uni_courses_regex = data["regex"]
        uni_courses_regex_list = data["regex_list"]
        common_words = data["common_words"]
        prof_order = data["prof_order"]
        prof_name_order = data["prof_name_order"]
        f.close()
    return uni_courses_regex, uni_courses_regex_list, common_words, prof_order, prof_name_order

def get_prof_names(university):
    with open("static/jsons/prof_names.json") as f:
        data = json.load(f)
        first_names = data[university.lower()]['first_names']
        last_names = data[university.lower()]['last_names']
        f.close()
    return set(first_names), set(last_names)

def get_instructors(university, instructors, num):

    num_pairs = num # number of pairs means firstname and lastnames
    p = index = complete_pairs = 0
    instructor_pairs = {}
    professors = Professors.objects.filter(university__iexact=university)
    for _ in range(len(instructors)):
        if complete_pairs == num_pairs:
            break
        if index + 1 <= len(instructors) - 1:
            first_name = instructors[index]
            last_name = instructors[index+1]
            if professors.filter(first_name__iexact=first_name, last_name__iexact=last_name).exists():
                prof = professors.filter(first_name__iexact=first_name, last_name__iexact=last_name).first()
                instructor_pairs[p] = {
                    "first":prof.first_name.capitalize(),
                    "last":prof.last_name.capitalize(),
                }
                index += 2
                p += 1
                complete_pairs += 1
            else:
                instructor_pairs[p] = {
                    "first":first_name,
                    "last":'N/A',
                }
                index += 1
                p += 1

    if p > num_pairs:
        c = 0
        remove_indices = []
        for p in instructor_pairs:
            if instructor_pairs[p]["last"] == 'N/A':
                if c >= 1 or instructor_pairs[p]["first"].lower() in english_words:
                    remove_indices.append(p)
                c+= 1
    instructors = {}
    for i in remove_indices:
        del instructor_pairs[i]
    for i, p in enumerate(instructor_pairs):
        instructors[i] = instructor_pairs[p]
    return instructors

def find_courses_and_profs(text, university, first_name=None, last_name=None):
    text = text.replace(first_name,'')
    text = text.replace(last_name,'')
    days = ['Sun','Sunday','Mon', 'Monday','Tue','Tuesday', 'Wed', 'Wednesday','Thurs','Thursday','Fri','Friday','Sat','Saturday','Su','Mo','Tu','We','Th','Fr','Sa']
    uni_courses_regex, uni_courses_regex_list, UniCommonWords, prof_order, prof_name_order = get_uni_regex_common_words(university=university)
    prof_first_names, prof_last_names = get_prof_names(university)
    r = re.compile(uni_courses_regex_list)
    table = str.maketrans(dict.fromkeys(string.punctuation))
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(uni_courses_regex, '\\1\\2\\3', text)
    words = word_tokenize(text)
    words = filter(lambda ThisWord: not (re.match('^(?:(?:[0-9]{2}[:\/,]){2}[0-9]{2,4}|am|pm|PM|AM)$', ThisWord) or re.match(r'^\d+(?:(?:\.|:\d+)[ ]*(?:am|AM|pm|PM)?|(?:am|AM|pm|PM))(?:[ ]*-[ ]*\d+(?:(?:\.|:\d+)[ ]*(?:am|AM|pm|PM)?|(?:am|AM|pm|PM)))*$', ThisWord )), words)

    excluded_set = set(chain(stop_words, map(str.lower, UniCommonWords)))
    text_f = " ".join(list(words))
    text_f = text_f.translate(table)
    text_f = text_f.split()
    filtered_words = []
    for word in text_f:
        words = str(word)
        if (word.lower().strip() in map(str.lower, prof_first_names)) or (word.lower().strip() in map(str.lower, prof_last_names)) or (re.match(uni_courses_regex_list, word.strip()) is not None):
            filtered_words.append(word)
    filtered_words = [w for w in filtered_words if w.lower() not in excluded_set]
    text_f = " ".join(filtered_words).translate(table)
    text_f = re.sub(r"$\d+\W+|\b\d+\b|\W+\d+$", " ", text_f)
    text_f = re.sub(r'\s+', ' ', text_f)
    text_f = list(OrderedDict.fromkeys(text_f.split()))
    courses = list(filter(r.match, text_f))
    instructors = [x for x in text_f if x not in courses]
    instructors = get_instructors(university, instructors, len(courses))
    result = {}
    if prof_name_order == "fl":
        for i, c in enumerate(courses):
            course_pair = {}
            course_pair["code"] = c
            course_pair["instructor_first_name"] = instructors[i]["first"]
            course_pair["instructor_last_name"] =  instructors[i]["last"]
            result[f'course_{i}'] = course_pair
    elif prof_name_order == 'lf':
        for i, c in enumerate(courses):
            course_pair = {}
            course_pair["code"] = courses[i]
            course_pair["instructor_first_name"] = f
            course_pair["instructor_last_name"] = l
            result[f'pair_{i}'] = course_pair

    result = json.dumps(result)
    print(result)
    return result


from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import pandas as pd
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

options = webdriver.ChromeOptions()
options.add_argument("start-maximized")

options.add_experimental_option('excludeSwitches', ['enable-logging'])

driver = webdriver.Chrome(r'C:\Users\Mountain\Documents\Proyectos\selenium_con_python-master\\chromedriver.exe')

driver.get('https://www.eltiempo.es/')

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.CSS_SELECTOR,
                                      'button.didomi-components-button didomi-button didomi-dismiss-button didomi-components-button--color didomi-button-highlight highlight-button'.replace(' ', '.'))))\
    .click()

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.CSS_SELECTOR,
                                      'input#inputSearch')))\
    .send_keys('Madrid')

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.CSS_SELECTOR,
                                      'i.icon.icon-search')))\
    .click()

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.CSS_SELECTOR,
                                      'i.icon_weather_s.icon.icon-local')))\
    .click()

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.XPATH,
                                      '/html/body/div[7]/main/div[4]/div/section[4]/section/div/article/section/ul/li[2]/a')))\
    .click()

WebDriverWait(driver, 5)\
    .until(EC.element_to_be_clickable((By.XPATH,
                                      '/html/body/div[7]/main/div[4]/div/section[4]/section/div[1]/ul')))

texto_columnas = driver.find_element_by_xpath('/html/body/div[7]/main/div[4]/div/section[4]/section/div[1]/ul')
texto_columnas = texto_columnas.text

tiempo_hoy = texto_columnas.split('Mañana')[0].split('\n')[1:-1]

horas = list()
temp = list()
v_viento = list()

for i in range(0, len(tiempo_hoy), 4):
    horas.append(tiempo_hoy[i])
    temp.append(tiempo_hoy[i+1])
    v_viento.append(tiempo_hoy[i+2])

df = pd.DataFrame({'Horas': horas, 'Temperatura': temp, 'V_viento(km_h)':v_viento})
print(df)
df.to_csv('tiempo_hoy.csv', index=False)

driver.quit()


import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-t", "--template", nargs="+", type=str, help="Template PDB files")
parser.add_argument("-n", "--number", nargs="+", type=int, help="Repeat number of template molecules")
parser.add_argument("-x", "--xyz", type=str, help="Coordinate XYZ file")
parser.add_argument("-o", "--output", type=str, help="Output file name")
args = parser.parse_args()

with open(args.xyz, "r") as f:
    text = f.readlines()
nxyz = int(text[0].strip())
xyz = text[2:nxyz+2]
xyz = [i.strip().split()[-3:] for i in xyz]
xyz = [[float(j) for j in i] for i in xyz]
crdline = """HETATM{serial:>5d} {name:^4} {resname:>3}  {rindex:>4d}    {x:>8.3f}{y:>8.3f}{z:>8.3f}                      {elem:>2}\n"""
assert len(args.template) == len(args.number)

natom = 1
nres = 1
atomlist = []
bondlist = []

for n,p in enumerate(args.number):
    with open(args.template[n], "r") as f:
        text = f.readlines()
        atext = [i[6:].strip().split() for i in text if "HETATM" in i]
        btext = [i.strip().split() for i in text if "CONECT" in i]
        pdbtemp = [[i[1],i[2],i[-1]] for i in atext]
        bondtemp = [[int(j) for j in i[1:]] for i in btext]

    for nrep in range(p):
        ishift = natom - 1
        for brep in bondtemp:
            tmp = "CONECT"
            for bi in brep:
                tmp += "{i:>5d}".format(i=bi+ishift)
            tmp +="\n"
            bondlist.append(tmp)
        for arep in range(len(pdbtemp)):
            x, y, z = xyz[natom-1]
            a_name, res_name, elem = pdbtemp[arep]
            atomlist.append(crdline.format(serial=natom, name=a_name, resname=res_name, rindex=nres, x=x, y=y, z=z, elem=elem))
            natom += 1
        nres += 1
with open(args.output, "w") as fout:
    for l in atomlist:
        fout.write(l)
    for l in bondlist:
        fout.write(l)
    fout.write("END\n")


import requests
from bs4 import BeautifulSoup
import json
import oss2
import modles

headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) '
                         'Chrome/60.0.3112.113 Safari/537.36'}

ALBUM_INFO_URL = 'https://c.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&g_tk=5381' \
                 '&jsonpCallback=getAlbumInfoCallback&loginUin=0&hostUin=0&format=jsonp&inCharset=utf8' \
                 '&outCharset=utf-8&notice=0&platform=yqq&needNewCode=0'

OSS_SONG_PATH = 'http://source.edian66.com/'
SONG_STORE = 'prosong/'

IMAGE_URL_START = 'http://y.gtimg.cn/music/photo_new/'
IMAGE_NAME_BEG = 'T002R300x300M000'
IMAGE_NAME_END = '.jpg'

# 设置oss
auth = oss2.Auth('LTAIgKpditaGe91a', '0IbKHUZFhz1rOGzXi76PWFjUdLxGT8')
bucket = oss2.Bucket(auth, 'http://oss-cn-shenzhen.aliyuncs.com/', 'shujustore')

# 下载歌曲图片
def get_file(r_content, file_name):
    try:
        bucket.put_object(file_name, r_content)
    except Exception as e:
        print('song detail exception:', e)

def get_song_detail(album_mid, singer_id):
    req_url = ALBUM_INFO_URL % album_mid
    try:
        r = requests.get(req_url)
    except Exception as e:
        r = None
        print('song detail exception:', e)

    if r is None or r.status_code != 200:
        return init_song_detail()

    res_content = r.text
    if res_content.find('data') <= 0 or res_content.rfind('message') <= 0:
        return init_song_detail()

    response_dict = json.loads(res_content[res_content.find('data') + 6: res_content.rfind('message') - 2])
    song_time = response_dict.get('aDate')
    genre_info = response_dict.get('genre')

    lab1, lab2, lab3, lab4, lab5 = get_label_val(genre_info)

    lan = response_dict.get('lan')

    song_image = download_image(album_mid, str(singer_id))

    song_detail_dict = {'song_time': song_time}
    song_detail_dict['song_lan'] = lan
    song_detail_dict['song_image'] = song_image
    song_detail_dict['song_lable1'] = lab1
    song_detail_dict['song_lable2'] = lab2
    song_detail_dict['song_lable3'] = lab3
    song_detail_dict['song_lable4'] = lab4
    song_detail_dict['song_lable5'] = lab5

    return song_detail_dict

def init_song_detail():
    song_detail_dict = {'song_time': None}
    song_detail_dict['song_lan'] = None
    song_detail_dict['song_image'] = None
    song_detail_dict['song_lable1'] = None
    song_detail_dict['song_lable2'] = None
    song_detail_dict['song_lable3'] = None
    song_detail_dict['song_lable4'] = None
    song_detail_dict['song_lable5'] = None
    return song_detail_dict

def get_label_val(genre_info):
    genre_list = genre_info.split(' ')

    lab1 = None
    lab2 = None
    lab3 = None
    lab4 = None
    lab5 = None

    if genre_list.__len__() >= 1:
        lab1 = genre_list[0]
    if genre_list.__len__() >= 2:
        lab2 = genre_list[1]
    if genre_list.__len__() >= 3:
        lab3 = genre_list[2]
    if genre_list.__len__() >= 4:
        lab4 = genre_list[3]
    if genre_list.__len__() >= 5:
        lab5 = genre_list[4]

    return lab1, lab2, lab3, lab4, lab5

def download_image(album_mid, singer_id_str):
    file_name = SONG_STORE + singer_id_str + '/' + album_mid + IMAGE_NAME_END
    image_web_url = IMAGE_URL_START + IMAGE_NAME_BEG + album_mid + IMAGE_NAME_END
    try:
        req = requests.get(image_web_url, headers=headers)
        status_code = req.status_code
    except Exception as e:
        status_code = 404

    if status_code == 200:
        get_file(req.content, file_name)
    else:
        return None

    song_image = OSS_SONG_PATH + file_name;

    return song_image

# query_result = modles.query_songqq()
# for item in query_result:
#     album_mid, singer_id_str = item[0], str(item[1])
#     download_image(album_mid, singer_id_str)
#     print(album_mid, singer_id_str)
#     break
# result = get_song_detail('004XWK8X2snvBQ', 560462)
# print(result)


# -*- coding: utf-8 -*-
"""
Created on Mon Jan  6 15:15:02 2020

@author: gottince

@description: script permettant d'extraire des sequences proteiques et
nucleotidiques d'un fasta genomique a partir d'un gff.
"""
import argparse
import csv
from Bio import SeqIO
from Bio.Seq import translate, reverse_complement

#----------------------------------#
#    PARAMETRES
#----------------------------------#
# - File name of genomic fasta
# - output filename
# - output sequence type dna/rna/protein

parser = argparse.ArgumentParser()

parser.add_argument("-f","--fasta", type=str, help="Exact path of genomic fasta file")
parser.add_argument("-g","--gff", type=str, help="Exact path of gff file")
parser.add_argument("-o","--output", type=str, help="Output file name")
parser.add_argument("-t","--type", type=str, choices=["gene","cdna","prot","FScdna","FSprot","exon"], help="type of extracting sequence : <gene> extract genomique dna sequence with intron, <cdna> extract reconstruct coding nucleotide sequence with frameshift, <prot> extract reconstruct protein sequence, <exons> extract isolated coding exons, <cds> extract reconstruct coding nucleotide sequence without frameshift.")

args = parser.parse_args()

#----------------------------------#
#            FUNCTIONS
#----------------------------------#

# 1. extracting complete gene
def extract_gene(fasta,gff) :
    gff_reader = csv.reader(gff, delimiter='\t')

    sid=""
    allseq=[]
    for row in gff_reader :
        if(row[2]=="gene") :
            tmp=row[8].split(';')
            sid=tmp[0][3:]
            subseq=chr_dict[row[0]][int(row[3])-1:int(row[4])]
            if(row[6]=="-") :
                dna=reverse_complement(str(subseq.seq))
                allseq.append((sid,dna))
            else :
                dna=str(subseq.seq)
                allseq.append((sid,dna))

    return allseq

# 2. extracting coding sequence : prot and cdna without FS recoded
## the protein sequence should end at the first stop codon
def extract_coding(fasta,gff,typeseq) :
    gff_reader = csv.reader(gff, delimiter='\t')

    dna= ""
    sid=""
    allseq=[]

    for row in gff_reader :
        if(row[2]=="gene") :
            if(len(dna)>0) :
            # Export sequence
                if(typeseq=="prot") :
                    #dnaRec = SeqRecord(Seq(dna).translate(to_stop=True)+"*", id=sid, description="")
                    prot=translate(dna, to_stop=True)+"*"
                    allseq.append((sid,prot))
                else :
                    allseq.append((sid,dna))

            dna = ""
            tmp=row[8].split(';')
            sid=tmp[0][3:]
        elif(row[2]=="exon") :
        # Extract sequence
            subseq=chr_dict[row[0]][int(row[3])-1:int(row[4])]
            if(row[6]=="-") :
                #dna=dna+str(subseq.reverse_complement().seq)
                dna=str(subseq.reverse_complement().seq)+dna
            else :
                dna=dna+str(subseq.seq)

    ## dernier gene
    if(typeseq=="prot") :
        #dnaRec = SeqRecord(Seq(dna).translate(to_stop=True)+"*", id=sid, description="")
        prot=translate(dna, to_stop=True)+"*"
        allseq.append((sid, prot))
    else :
        allseq.append((sid,dna))

    return allseq

# 3. extracting individual cds fragment
def extract_exons(fasta,gff) :
    gff_reader = csv.reader(gff, delimiter='\t')

    sid=""
    dna=""
    allseq=[]
    for row in gff_reader :
        if(row[2]=="CDS" or row[2]=="cds") :
            tmp=row[8].split(";") #store first field corresponding to ID=...
            sid=tmp[0][3:] #remove "ID=" from sequence id
            subseq=chr_dict[row[0]][int(row[3])-1:int(row[4])]
            if(row[6]=="-") :
                dna=str(subseq.reverse_complement().seq)
            else :
                dna=str(subseq.seq)
            allseq.append((sid,dna))

    return allseq

#4. extracting cdna or prot with frameshift completing with "!"
def extract_frameshift(fasta, gff, typeseq) :
    gff_reader = csv.reader(gff, delimiter='\t')

    dna = ""
    sid = ""
    allseq = []  ## list de tuple (id, seq)
    lastStop = 0

    for row in gff_reader :
        if(row[2]=="gene") :
            if(len(dna)>0) :
            # Export sequence
                if(typeseq=="FSprot"):
                    ## change "!" to "N" to allow auto translation
                    prot = translate(dna.replace("!","N"))
                    allseq.append((sid,prot))
                else:
                    allseq.append((sid,dna))

            dna = ""
            tmp = row[8].split(';')
            sid = tmp[0][3:]
            lastStop = 0
        elif(row[2]=="CDS" or row[2]=="cds") :
        # Extract sequence
            # check short intron
            if((int(row[3])-1)<=(lastStop+15)) :
                #check if one or two "!" is needed
                if ((int(row[3])-1-lastStop)%3 == 1):
                    comp="!!"
                elif ((int(row[3])-1-lastStop)%3 == 2) :
                    comp="!"
                # integration of the short intron in the sequence
                if row[6]=="+" :
                    subseq=chr_dict[row[0]][lastStop:int(row[3])-1].lower()+comp+chr_dict[row[0]][int(row[3])-1:int(row[4])].upper()
                else :
                    subseq=comp+chr_dict[row[0]][lastStop:int(row[3])-1].lower()+chr_dict[row[0]][int(row[3])-1:int(row[4])].upper()
            else :
                subseq=chr_dict[row[0]][int(row[3])-1:int(row[4])].upper()

            lastStop=int(row[4])
            if(row[6]=="-") :
                dna=reverse_complement(str(subseq.seq))+dna
            else :
                dna=dna+str(subseq.seq)

    ## dernier gene
    if(typeseq=="FSprot"):
        ## change "!" to "N" to allow auto translation
        prot = translate(dna.replace("!","N"))
        allseq.append((sid,prot))
    else:
        allseq.append((sid,dna))

    return allseq

def write_fasta(mylist, myfile) :
    #mylist --> [(id,seq),(id,seq),...]
    fastafile=open(myfile, mode='w')
    for seq in mylist :
        #print(">"+seq[0]+"\n"+seq[1])
        line=">"+seq[0]+"\n"+seq[1]+"\n"
        fastafile.write(line)
    fastafile.close()

#----------------------------------#
#              MAIN
#----------------------------------#

# 1. Importing Fasta Genome
#=============================================
chr_dict = SeqIO.to_dict(SeqIO.parse(args.fasta, "fasta"))

# 2. Reading GFF and processing
#=============================================
gff=open(args.gff, mode='r')

seq_list=[]

if(args.type=="gene") :
    #full length gene
    seq_list=extract_gene(chr_dict,gff)

elif(args.type=="exon") :
    #indivudual exon
    seq_list=extract_exons(chr_dict,gff)

elif(args.type=="cdna" or args.type=="prot") :
    #complete coding sequence with insertions and frameshifts
    seq_list=extract_coding(chr_dict, gff, args.type)

elif(args.type=="FScdna" or args.type=="FSprot") :
    #exact cDNA or protein sequence with frameshift completed with "!"
    seq_list=extract_frameshift(chr_dict, gff, args.type)

else :
    print("argument error : unknown type -t "+args.type)

gff.close()

# 3. Export sequence
#=======================================
write_fasta(seq_list, args.output)


from logging import Logger

from kink import inject

from surebets_finder.bet.application.bet_finder_factory import BetFinderFactory
from surebets_finder.bet.domain.errors import BetNotFoundError
from surebets_finder.bet.domain.repositories import BetRepository
from surebets_finder.raw_content.facade import RawContentFacade

@inject
class BetImporter:
    def __init__(self, repository: BetRepository, logger: Logger) -> None:
        self._repository = repository
        self._logger = logger
        self._facade = RawContentFacade()  # type: ignore

    def import_all(self) -> None:
        self._logger.info("Bets importer has started!")

        for raw_content_dto in self._facade.get_all_unprocessed_raw_contents():
            finder = BetFinderFactory.create(raw_content_dto.provider)

            bets = finder.find_bets(raw_content_dto.content, raw_content_dto.category)

            for bet in bets:
                try:
                    existing_bet = self._repository.find_one(
                        {"opponent_1": bet.opponent_1, "opponent_2": bet.opponent_2, "date": bet.date}
                    )
                    self._logger.info(
                        f"Bet with with params"
                        f"`opponent_1={bet.opponent_1}, opponent_2={bet.opponent_2}, date={bet.date}` "
                        f"already exist! Updating values"
                    )

                    existing_bet.odds_1 = bet.odds_1
                    existing_bet.odds_2 = bet.odds_2
                    existing_bet.url = bet.url
                    self._repository.save(existing_bet)
                except BetNotFoundError:
                    self._logger.info("Creating new bet!")

                    self._repository.create(bet)

            self._facade.mark_raw_content_as_processed(raw_content_dto.id)

            self._logger.info(
                f"Found {len(bets)} from RawContent with id={raw_content_dto.id} where provider={raw_content_dto.provider.value}!"
            )


# %%
# Import packages
import autograd.numpy as np
import numpy.polynomial as polynomial
import autograd.scipy.stats as stats
import autograd.scipy.linalg as linalg
import autograd.scipy.special as special

# from autograd.extend import primitive, defvjp
# from autograd.numpy.numpy_vjps import unbroadcast_f

import global_vars as gv

# %%
# @primitive
# def logdiffexp(x, y):
#     """Numerically stable log(exp(x) - exp(y))"""
#     max_xy = np.maximum(x, y)
#     return max_xy + np.log(np.exp(x - max_xy) - np.exp(y - max_xy))

# def logdiffexp_vjp(ans, x, y):
#     x_shape = x.shape
#     y_shape = y.shape
#     max_xy = np.maximum(x, y)
#     return unbroadcast_f(x, lambda g: np.full(x_shape, g) * np.exp((x - max_xy) / (np.exp(x - max_xy) - np.exp(y - max_xy)))), unbroadcast_f(y, lambda g: -np.full(y_shape, g) * np.exp((y - max_xy) / (np.exp(x - max_xy) - np.exp(y - max_xy))))

# defvjp(logdiffexp, logdiffexp_vjp)

def approx_logPhi_k_special_2d(a, b, D, V, gh_deg=10):
    """
        Return the log of integral from a to b of a multivariate normal with a correlation matrix that has reduced rank of 2, approximated using Gauss-Hermite quadrature

    Parameters
    ----------
        a : ndarray
            (k,) array of lower bounds
        b : ndarray
            (k,) array of upper bounds
        D : ndarray
            (k,k) array of diagonal matrix
        V : ndarray
            (k,2) array of reduced rank matrix (could expand to arbitrary dimension, but 2 is all I will need)
        gh_deg : int
            number of points to sample for Gauss-Hermite quadrature

    Returns
    -------
        Phi_k : float
            approximated integral
    """

    # Determine reduced rank
    l = V.shape[1]

    # Reduce D to vector
    d = np.diag(D)

    # Sample points and weights for Gauss Hermite quadrature
    x, w = polynomial.hermite.hermgauss(gh_deg)

    log_Phi_b = stats.norm.logcdf((b[np.newaxis,np.newaxis,:] - np.sqrt(2.0) * (V[:,0] * x[:,np.newaxis,np.newaxis] + V[:,1] * x[np.newaxis,:,np.newaxis])) / np.sqrt(d[np.newaxis,np.newaxis,:]))
    log_Phi_a = stats.norm.logcdf((a[np.newaxis,np.newaxis,:] - np.sqrt(2.0) * (V[:,0] * x[:,np.newaxis,np.newaxis] + V[:,1] * x[np.newaxis,:,np.newaxis])) / np.sqrt(d[np.newaxis,np.newaxis,:]))
    log_Phi_combined = np.concatenate((log_Phi_b[:,:,:,np.newaxis], 1.0j * np.pi + log_Phi_a[:,:,:,np.newaxis]), axis=3) # i * pi is equivalent to a weight of -1
    log_diff = np.real(special.logsumexp(log_Phi_combined, axis=3))
#     log_diff = logdiffexp(log_Phi_b, log_Phi_a)
    sum_log_diff = np.sum(log_diff, axis=2)

    # Integrate from -infinity to +infinity using Gauss-Hermite quadrature
    weights = w[:,np.newaxis] * w[np.newaxis,:]
    logPhi_k = np.log(1.0 / np.pi) + special.logsumexp(sum_log_diff, b=weights)

    return logPhi_k

def process_theta(theta, X_eps, X_lQbar, X_dwind, specification, sources):
    """
        Return the (possibly transformed) parameters from the model parameter array

    Parameters
    ----------
        theta : ndarray
            (K,) array of model parameters
        X_eps : ndarray
            (G,T,L_eps) array of covariates of marginal cost parameters
        X_lQbar : ndarray
            (T,L_lQbar) array of covariates of Qbar
        X_dwind : ndarray
            (G_wind,T,L_dwind) array of covariates of wind capacity factors
        specification : int
            determines which specification the likelihood is running
        sources : ndarray
            (G,) array of which energy source each generator uses

    Returns
    -------
        zeta_1_sigma, zeta_2, beta_eps, sigma_lQbar, beta_lQbar, sigma_dwind, beta_dwind, rho_dwind_dwind, rho_dwind_lQbar, p1_dcoal, p1_dgas : float
            model parameters
    """

    # Determine whether to distribute to generators by source
    num_unique = X_eps.shape[0] if gv.include_indiv_generators[specification] else 2
    _, sources_idx = np.unique(sources, return_inverse=True)

    # Cost shock distribution
    next_idx = 0
    end_idx = next_idx + num_unique
    zeta_1_sigma = theta[next_idx:end_idx]

    next_idx = end_idx
    end_idx = next_idx + num_unique
    zeta_2 = theta[next_idx:end_idx]

    next_idx = end_idx
    if gv.include_corr_eps[specification]:
        rho_coal_coal = np.exp(theta[next_idx + 0]) / (1.0 + np.exp(theta[next_idx + 0]))
        rho_gas_gas = np.exp(theta[next_idx + 1]) / (1.0 + np.exp(theta[next_idx + 1]))
        rho_coal_gas = (2.0 * np.exp(theta[next_idx + 2]) / (1.0 + np.exp(theta[next_idx + 2])) - 1.0) * np.sqrt(rho_coal_coal * rho_gas_gas)
        next_idx = next_idx + 3
    else:
        rho_coal_coal = 0.0
        rho_gas_gas = 0.0
        rho_coal_gas = 0.0
    end_idx = next_idx + num_unique
    beta_eps = theta[next_idx:end_idx]

    # log(Qbar) and delta_wind distribution
    next_idx = end_idx
    sigma_lQbar = theta[next_idx + 0]
    next_idx = next_idx + 1
    end_idx = next_idx + X_lQbar.shape[1]
    beta_lQbar = theta[next_idx:end_idx]
    next_idx = end_idx
    sigma_dwind = theta[next_idx + 0]
    next_idx = next_idx + 1
    end_idx = next_idx + X_dwind.shape[2]
    beta_dwind = theta[next_idx:end_idx]
    next_idx = end_idx
    if gv.include_corr_lQbar_dwind[specification]:
        rho_dwind_dwind = np.exp(theta[next_idx + 0]) / (1.0 + np.exp(theta[next_idx + 0]))
        rho_dwind_lQbar = (2.0 * np.exp(theta[next_idx + 1]) / (1.0 + np.exp(theta[next_idx + 1])) - 1.0) * rho_dwind_dwind
        next_idx = next_idx + 2
    else:
        rho_dwind_dwind = 0.0
        rho_dwind_lQbar = 0.0

    # non-wind capacity factor distribution
    p1_dcoal = theta[next_idx]
    p1_dgas = theta[next_idx + 1]

    # Repeat across generators if parameters not unique to generator
    if not gv.include_indiv_generators[specification]:
        zeta_1_sigma = zeta_1_sigma[sources_idx]
        zeta_2 = zeta_2[sources_idx]
        #beta_eps = beta_eps[sources_idx] - not necessary b/c gets multiplied by X_eps

    return zeta_1_sigma, zeta_2, rho_coal_coal, rho_gas_gas, rho_coal_gas, beta_eps, sigma_lQbar, beta_lQbar, sigma_dwind, beta_dwind, rho_dwind_dwind, rho_dwind_lQbar, p1_dcoal, p1_dgas

def tobit_llh(theta, firms, sources, q, X_eps, K, P, Kbar, uKbar, Qbar, X_lQbar, Q_wind, deltas_nonwind, deltas_wind, X_dwind, specification, print_msg=False):
    """
        Return the log-likelihood of the data according to the wholesale market model

    Parameters
    ----------
        theta : ndarray
            (K,) array of likelihood parameters
        firms : ndarray
            (G,) array of which firm each generator belongs to
        sources : ndarray
            (G,) array of which energy source each generator uses
        q : ndarray
            (G,T) array of how much (non-wind) generators produced
        X_eps : ndarray
            (G,T,L_eps) array of covariates of marginal cost parameters
        P : ndarray
            (T,) array of wholesale prices
        Kbar : ndarray
            (G,T) array of effective maximum capacities
        uKbar : ndarray
            (G,T) array of effective minimum (positive) production levels
        Qbar : ndarray
            (T,) array of market demand
        X_lQbar : ndarray
            (T,L_lQbar) array of covariates of Qbar
        Q_wind : ndarray
            (T,) array of total wind production
        deltas_nonwind : ndarray
            (G_nonwind,T) array of non-wind capacity factors
        deltas_wind : ndarray
            (G_wind,T) array of wind capacity factors
        X_dwind : ndarray
            (G_wind,T,L_dwind) array of covariates of wind capacity factors
        specification : int
            determines which specification the likelihood is running
        print_msg : bool
            determines whether or not to print input and output of function

    Returns
    -------
        llh : float
            log-likelihood
    """

    sigma_eps, zeta_2, rho_coal_coal, rho_gas_gas, rho_coal_gas, beta_eps, sigma_lQbar, beta_lQbar, sigma_dwind, beta_dwind, rho_dwind_dwind, rho_dwind_lQbar, p1_dcoal, p1_dgas = process_theta(theta, X_eps, X_lQbar, X_dwind, specification, sources)

    # Process theta
    if print_msg:
        print(f"sigma_eps: {sigma_eps}", flush=True)
        print(f"zeta_2: {zeta_2}", flush=True)
        if gv.include_corr_eps[specification]:
            print(f"eps corr: {np.array([rho_coal_coal, rho_gas_gas, rho_coal_gas])}", flush=True)
        print(f"beta_eps: {beta_eps}", flush=True)
        print(f"sigma_lQbar: {sigma_lQbar}", flush=True)
        print(f"beta_lQbar: {beta_lQbar}", flush=True)
        print(f"sigma_dwind: {sigma_dwind}", flush=True)
        print(f"beta_dwind: {beta_dwind}", flush=True)
        if gv.include_corr_lQbar_dwind[specification]:
            print(f"lQbar-dwind corr: {np.array([rho_dwind_dwind, rho_dwind_lQbar])}", flush=True)
        print(f"p outage: {np.array([p1_dcoal, p1_dgas])}", flush=True)

    # Combine lQbar and wind
    beta_lQbar_dwind = np.concatenate((beta_lQbar, beta_dwind))
    X_lQbar_zeros = np.concatenate((X_lQbar, np.zeros((X_lQbar.shape[0], X_dwind.shape[2]))), axis=1) # add zeros for dwind characteristics
    X_dwind_zeros = np.concatenate((np.zeros((X_dwind.shape[0], X_dwind.shape[1], X_lQbar.shape[1])), X_dwind), axis=2) # add zeros for lQbar characteristics
    X_lQbar_dwind = np.concatenate((X_lQbar_zeros[np.newaxis,:,:], X_dwind_zeros), axis=0)

    # Determine which sources each generator belongs to
    _, sources_idx = np.unique(sources, return_inverse=True)

    # Convert theta parameters into distribution arrays
    mu_eps = np.zeros(sources.shape)

    F_eps = np.diag(sigma_eps)
    rho_coal_1 = np.sqrt(rho_coal_coal)
    rho_coal_2 = 0.0
    rho_gas_1 = rho_coal_gas / rho_coal_1 if rho_coal_1 != 0.0 else 0.0
    rho_gas_2 = np.sqrt(rho_gas_gas - rho_gas_1**2.0)
    rhos_eps_1 = np.array([rho_coal_1, rho_gas_1])
    rhos_eps_2 = np.array([rho_coal_2, rho_gas_2])
    V_eps = np.concatenate((rhos_eps_1[:,np.newaxis], rhos_eps_2[:,np.newaxis]), axis=1)
    V_eps = V_eps[sources_idx,:]
    D_eps = np.diag(np.diag(1.0 - V_eps @ V_eps.T))
    cov_eps = F_eps @ (D_eps + V_eps @ V_eps.T) @ F_eps

    num_wind = deltas_wind.shape[0]
    mu_lQbar_deltas = np.zeros((num_wind + 1,))

    sigma_lQbar_deltas = np.concatenate((np.array([sigma_lQbar]), np.repeat(np.array([sigma_dwind]), num_wind)))
    rho_dwind = np.sqrt(rho_dwind_dwind)
    rho_lQbar = rho_dwind_lQbar / rho_dwind if rho_dwind != 0.0 else 0.0
    rhos_lQbar_deltas = np.concatenate((np.array([rho_lQbar]), np.repeat(np.array([rho_dwind]), num_wind)))
    F_lQbar_deltas = np.diag(sigma_lQbar_deltas)
    D_lQbar_deltas = np.diag(1.0 - rhos_lQbar_deltas**2.0)
    cov_lQbar_deltas = F_lQbar_deltas @ (D_lQbar_deltas + np.outer(rhos_lQbar_deltas, rhos_lQbar_deltas)) @ F_lQbar_deltas

    p_nonwind_deltas = np.array([p1_dcoal, p1_dgas])
    p_nonwind_deltas = p_nonwind_deltas[sources_idx]

    # Determine whether competitive or strategic
    competitive = firms == "Competitive"
    strategic = ~competitive

    # Determine whether constrained
    buffer_fraction = 0.03 # 3% buffer
    constrained_below = q <= (1.0 + buffer_fraction) * uKbar
    constrained_above = q >= (1.0 - buffer_fraction) * Kbar
    unconstrained = ~constrained_below & ~constrained_above
    constrained = constrained_below | constrained_above
    constrained_below[constrained_below & constrained_above] = False # can happen if, e.g., Kbar = 0, uKbar = 0; assume it's constrained above

    # Determine number of periods
    T = q.shape[1]

    # Construct firm ownership matrix
    firm_matrix = firms[:,np.newaxis] == firms[np.newaxis,:]
    firm_matrix_strategic = firm_matrix[np.ix_(strategic, strategic)]
    Xi = firm_matrix + 1.0 - np.identity(firms.shape[0]) * 2.0

    # Construct likelihood period-by-period
    llh = 0.0 # initialize llh
    for t in range(T):
        # Determine which generators are in market in this time
        in_market_t = ~np.isnan(q[:,t])
        deltas_wind_in_market_t = ~np.isnan(deltas_wind[:,t])

        # Competitive supply
        gammas = 1.0 / (2.0 * zeta_2[unconstrained[:,t] & in_market_t] * K[unconstrained[:,t] & in_market_t]**-2.0) * (competitive[unconstrained[:,t] & in_market_t])
        c = -gammas
        phi = np.sum(-gammas * (X_eps[unconstrained[:,t] & in_market_t,t,:] @ beta_eps))
        beta = np.sum(gammas)

        # Strategic residual demand
        psi = (Qbar[t] - Q_wind[t] - np.sum(q[competitive & constrained_above[:,t] & in_market_t,t]) - phi) / beta
        d = -1.0 / beta * c
        b = 1.0 / beta

        # Strategic firm equilibrium quantities
        Xi_GuGu = Xi[np.ix_(strategic & unconstrained[:,t] & in_market_t, strategic & unconstrained[:,t] & in_market_t)]
        Xi_GuGplus = Xi[np.ix_(strategic & unconstrained[:,t] & in_market_t, strategic & constrained_above[:,t] & in_market_t)]
        ratio = 1.0 / (2.0 * b + 2.0 * zeta_2[strategic & unconstrained[:,t] & in_market_t] * K[strategic & unconstrained[:,t] & in_market_t]**-2.0)
        Psi = np.identity(np.sum(strategic & unconstrained[:,t] & in_market_t)) + (b  * ratio)[:,np.newaxis] * Xi_GuGu
        Psi_inv = np.linalg.inv(Psi)
        E = Psi_inv @ (ratio[:,np.newaxis] * (d[np.newaxis,:] - np.identity(np.sum(unconstrained[:,t] & in_market_t))[strategic[unconstrained[:,t] & in_market_t],:]))
        f = Psi_inv @ (ratio * (psi - b * Xi_GuGplus @ Kbar[strategic & constrained_above[:,t] & in_market_t,t] - X_eps[strategic & unconstrained[:,t] & in_market_t,t,:] @ beta_eps))

        # Equilibrium price
        g = d - b * E.T @ np.ones(np.sum(unconstrained[strategic,t] & in_market_t[strategic]))
        j = psi - b * (np.sum(q[strategic & constrained[:,t] & in_market_t,t]) + np.sum(f))

        # Competitive firm equilibrium quantities
        ratio = 1.0 / (2.0 * zeta_2[competitive & unconstrained[:,t] & in_market_t] * K[competitive & unconstrained[:,t] & in_market_t]**-2.0)
        K_ = ratio[:,np.newaxis] * (g[np.newaxis,:] - np.identity(np.sum(unconstrained[:,t] & in_market_t))[competitive[unconstrained[:,t] & in_market_t],:])
        l = ratio * (j - X_eps[competitive & unconstrained[:,t] & in_market_t,t,:] @ beta_eps)

        # Combine competitive and strategic
        M = np.concatenate((K_, E), axis=0)
        n = np.concatenate((l, f))
        arange_M = np.arange(np.sum(unconstrained[:,t] & in_market_t))
        order_M = np.concatenate((arange_M[competitive[unconstrained[:,t] & in_market_t]], arange_M[strategic[unconstrained[:,t] & in_market_t]]))
        argsort_order_M = np.argsort(order_M)
        M = M[argsort_order_M,:]
        n = n[argsort_order_M]

        # Remove a linearly dependent row, add on price
        q_tilde = np.concatenate((q[unconstrained[:,t] & in_market_t,t][1:], np.array([P[t]])))
        M_tilde = np.concatenate((M[1:,:], g[np.newaxis,:]), axis=0)
        n_tilde = np.concatenate((n[1:], np.array([j])))

        # Invert to get epsilons
        epsilons_unconstrained = np.linalg.solve(M_tilde, q_tilde - n_tilde)

#         print(f"t = {t}: {epsilons_unconstrained.shape[0]} unconstrained generators", end=", ")

        # Solve for likelihood of q_tilde (*not* unconstrained epsilons, if I did would improperly incorporate covariance)
        mu_unconstrained_t = mu_eps[unconstrained[:,t] & in_market_t]
        cov_unconstrained_t = cov_eps[np.ix_(unconstrained[:,t] & in_market_t, unconstrained[:,t] & in_market_t)]
        mu_unconstrained_t_q_tilde = n_tilde + M_tilde @ mu_unconstrained_t
        cov_unconstrained_t_q_tilde = M_tilde @ cov_unconstrained_t @ M_tilde.T
        llh = llh + stats.multivariate_normal.logpdf(q_tilde, mu_unconstrained_t_q_tilde, cov_unconstrained_t_q_tilde)

        # Get bounds on constrained competitive epsilons
        limit_constrained_competitive = uKbar[competitive & constrained[:,t] & in_market_t,t] * constrained_below[competitive & constrained[:,t] & in_market_t,t] + Kbar[competitive & constrained[:,t] & in_market_t,t] * constrained_above[competitive & constrained[:,t] & in_market_t,t]
        epsilons_competitive_constrained = P[t] - X_eps[competitive & constrained[:,t] & in_market_t,t,:] @ beta_eps - zeta_2[competitive & constrained[:,t] & in_market_t] * 2.0 * limit_constrained_competitive / K[competitive & constrained[:,t] & in_market_t]**2.0

        # Get bounds on constrained strategic epsilons
        a = psi + np.dot(d, epsilons_unconstrained)
        Q_other = np.nansum(q[strategic & in_market_t,t][np.newaxis,:] * ~firm_matrix_strategic[np.ix_(in_market_t[strategic], in_market_t[strategic])], axis=1)
        Q_own = np.nansum(q[strategic & in_market_t,t][np.newaxis,:] * firm_matrix_strategic[np.ix_(in_market_t[strategic], in_market_t[strategic])], axis=1)
        MR = a - b * (Q_other + 2.0 * Q_own)
        limit_constrained_strategic = uKbar[strategic & constrained[:,t] & in_market_t,t] * constrained_below[strategic & constrained[:,t] & in_market_t,t] + Kbar[strategic & constrained[:,t] & in_market_t,t] * constrained_above[strategic & constrained[:,t] & in_market_t,t]
        epsilons_strategic_constrained = MR[constrained[strategic & in_market_t,t]] - X_eps[strategic & constrained[:,t] & in_market_t,t,:] @ beta_eps - zeta_2[strategic & constrained[:,t] & in_market_t] * 2.0 * limit_constrained_strategic / K[strategic & constrained[:,t] & in_market_t]**2.0

        # Combine into eta
        eta = np.concatenate((epsilons_competitive_constrained, epsilons_strategic_constrained)) # combine into one array
        arange_eps = np.arange(np.sum(constrained[:,t] & in_market_t))
        order_eps = np.concatenate((arange_eps[competitive[constrained[:,t] & in_market_t]], arange_eps[strategic[constrained[:,t] & in_market_t]])) # b/c create eta as competitive then strategic
        eta = eta[np.argsort(order_eps)] # reorder rows to match the generator indices in q
#         print(f"{np.sum(constrained_below[constrained[:,t] & in_market_t,t])} constrained below generators, {np.sum(constrained_above[constrained[:,t] & in_market_t,t])} constrained above generators, {eta.shape[0]} constrained (total)")

        # Solve for likelihood of constrained epsilons
        if eta.shape[0] > 0:
            # Construct constrained conditional distribution - see "Proofs" section in appendix in paper for derivation of formulas below
            cov_unconstrained_inv = np.linalg.inv(cov_unconstrained_t)
            mu_constrained_t = mu_eps[constrained[:,t] & in_market_t] + cov_eps[np.ix_(constrained[:,t] & in_market_t, unconstrained[:,t] & in_market_t)] @ cov_unconstrained_inv @ (epsilons_unconstrained - mu_unconstrained_t)
            # cov_constrained_t = cov_eps[np.ix_(constrained[:,t] & in_market_t, constrained[:,t] & in_market_t)] - (cov_eps[np.ix_(constrained[:,t] & in_market_t, unconstrained[:,t] & in_market_t)] @ cov_unconstrained_inv @ cov_eps[np.ix_(unconstrained[:,t] & in_market_t, constrained[:,t] & in_market_t)])
            G = np.identity(V_eps.shape[1]) - V_eps[unconstrained[:,t] & in_market_t,:].T @ np.linalg.inv(D_eps[np.ix_(unconstrained[:,t] & in_market_t, unconstrained[:,t] & in_market_t)] + V_eps[unconstrained[:,t] & in_market_t,:] @ V_eps[unconstrained[:,t] & in_market_t,:].T) @ V_eps[unconstrained[:,t] & in_market_t,:]
            F_tilde_t = linalg.sqrtm(np.diag(np.diag(D_eps[np.ix_(constrained[:,t] & in_market_t, constrained[:,t] & in_market_t)] + V_eps[constrained[:,t] & in_market_t,:] @ G @ V_eps[constrained[:,t] & in_market_t,:].T))) @ F_eps[np.ix_(constrained[:,t] & in_market_t, constrained[:,t] & in_market_t)]
            V_tilde_t = np.linalg.inv(F_tilde_t) @ F_eps[np.ix_(constrained[:,t] & in_market_t, constrained[:,t] & in_market_t)] @ V_eps[constrained[:,t] & in_market_t,:] @ linalg.sqrtm(G)
            D_tilde_t = np.diag(np.diag(1.0 - V_tilde_t @ V_tilde_t.T))

            # Solve for eta_ubar and eta_bar
            eta_ubar_t = np.concatenate((eta[constrained_below[constrained[:,t] & in_market_t,t]], np.ones(np.sum(constrained_above[:,t] & in_market_t)) * -99999999999999.0))
            order_ubar = np.concatenate((arange_eps[constrained_below[constrained[:,t] & in_market_t,t]], arange_eps[constrained_above[constrained[:,t] & in_market_t,t]]))
            eta_ubar_t = eta_ubar_t[np.argsort(order_ubar)]
            eta_bar_t = np.concatenate((eta[constrained_above[constrained[:,t] & in_market_t,t]], np.ones(np.sum(constrained_below[:,t] & in_market_t)) * 99999999999999.0))
            order_bar = np.concatenate((arange_eps[constrained_above[constrained[:,t] & in_market_t,t]], arange_eps[constrained_below[constrained[:,t] & in_market_t,t]]))
            eta_bar_t = eta_bar_t[np.argsort(order_bar)]

#             constrained_etas_print = ""
#             for i in range(eta_ubar_t.shape[0]):
#                 constrained_etas_print += f"[{np.round(eta_ubar_t[i], 3)} - {np.round(eta_bar_t[i], 3)}]"
#             print(f"\t Unconstrained: {np.round(epsilons_unconstrained, 3)},\n \t Constrained: {constrained_etas_print}")
#             print(f"{sources[unconstrained[:,t] & in_market_t]}")

            # Convert to standardized form
            F_diag_t = np.diag(F_tilde_t)
            F_inv_diag_t = 1.0 / F_diag_t # this is the inverse because F_tilde_t is a diagonal matrix
            a_t = F_inv_diag_t * (eta_ubar_t - mu_constrained_t)
            b_t = F_inv_diag_t * (eta_bar_t - mu_constrained_t)

            # Find Pr(eps^- <= eta_ubar & eps^+ >= eta_bar | eps^u = eta^u)
            llh = llh + approx_logPhi_k_special_2d(a_t, b_t, D_tilde_t, V_tilde_t)

        # Transform Qbar
        lQbar_t = np.array([np.log(Qbar[t])])

        # Transform wind deltas
        deltas_wind_t = deltas_wind[deltas_wind_in_market_t,t]
        inv_deltas_wind_t = np.log(deltas_wind_t / (1.0 - deltas_wind_t))

        # Add likelihood of Qbar and deltas_wind
        lQbar_deltas_wind_in_market_t = np.concatenate((np.array([True]), deltas_wind_in_market_t))
        lQbar_inv_deltas_wind_t = np.concatenate((lQbar_t, inv_deltas_wind_t)) - X_lQbar_dwind[lQbar_deltas_wind_in_market_t,t,:] @ beta_lQbar_dwind
        llh = llh + stats.multivariate_normal.logpdf(lQbar_inv_deltas_wind_t, mu_lQbar_deltas[lQbar_deltas_wind_in_market_t], cov_lQbar_deltas[np.ix_(lQbar_deltas_wind_in_market_t, lQbar_deltas_wind_in_market_t)])

        # Add non-wind generator likelihood
        deltas_nonwind_t = deltas_nonwind[in_market_t,t] # these are either 0 or 1
        llh = llh + np.sum(deltas_nonwind_t * np.log(p_nonwind_deltas[in_market_t]) + (1.0 - deltas_nonwind_t) * np.log(1.0 - p_nonwind_deltas[in_market_t]))

#         print(f"t = {t}: {llh}")

    # Return llh
    if np.isinf(llh): # can happen when cdf ≈ 0
        llh = llh * np.nan
    if print_msg:
        print(f"llh: {np.round(llh, 5)}", flush=True)
    return llh


import logging
from collections import namedtuple
import json
import pathlib
import struct

import numpy as np

from gripy import g2pylib
from gripy import tables, grids

Section3Data = namedtuple('section3', [
            'lensect', 'section_num', 'grid_source', 'ndpts', 'num_oct',
            'interpret', 'template_num'
        ])

class Grib2File:
    def __init__(self, filename):
        self.file_obj = open(filename, 'rb')
        self.grib_msgs = []
        self.fetch_messages()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.file_obj.close()

    def fetch_messages(self):
        # loop over grib messages, read section 0, get entire grib message.
        while True:
            # find next occurence of string 'GRIB' (or EOF).
            nbyte = self.file_obj.tell()
            while True:
                self.file_obj.seek(nbyte)
                start = self.file_obj.read(4).decode('ascii', 'ignore')
                if start == '' or start == 'GRIB':
                    break
                nbyte = nbyte + 1
            if start == '':
                break  # at EOF
            # otherwise, start (='GRIB') contains indicator message (section 0)
            startpos = self.file_obj.tell() - 4
            try:
                self.grib_msgs.append(Grib2Message(self.file_obj, startpos))
            except IOError:
                print('Skipping Grib1 Message')
                continue
        # if no grib messages found, nmsg is still 0 and it's not GRIB.
        if not self.grib_msgs:
            raise IOError('not a GRIB file')

class Section1:
    """ IDENTIFICATION SECTION:
    This section includes center, date, production status and
    which grib tables are used to decode the data """
    def __init__(self, data, *_):
        if data[1] != 1:
            raise ValueError(data)
        self.__data = data
        self.decoded_data = {}
        self.__center = data[2]
        self.subcenter = data[3]
        self.gMasterVersion = data[4]
        self.gLocalVersion = data[5]
        self.__significanceOfReferenceTime = data[6]
        self.year = data[7]
        self.month = data[8]
        self.day = data[9]
        self.hour = data[10]
        self.minute = data[11]
        self.second = data[12]
        self.__productionStatusOfProcessedData = data[13]
        self.__typeOfProcessedData = data[14]

    def __repr__(self, ):
        return (
            "Section 1:\n"
            f"  Originating Center : {self.center} - {self.subcenter} \n"
            f"  Grib Master Version: {self.gMasterVersion}.{self.gLocalVersion}\n"
            f"  Date               : {self.year}-{self.month:02}-{self.day:02}\n"
            f"  Time               : {self.hour:02}:{self.minute:02}:{self.second:02}\n"
            f"  Prod. Status       : {self.__productionStatusOfProcessedData}\n"
            f"  Data Type          : {self.__typeOfProcessedData}\n")

    @property
    def center(self, ):
        if 'center' not in self.decoded_data:
            self.decoded_data['center'] = tables.centers[self.__center]
        return self.decoded_data['center']

    @property
    def significanceOfReferenceTime(self, ):
        if 'significanceOfReferenceTime' not in self.decoded_data:
            self.decoded_data['significanceOfReferenceTime'] = get_table_value(
                '1.2', self.__significanceOfReferenceTime)[0]
        return self.decoded_data['significanceOfReferenceTime']

    @property
    def productionStatusOfProcessedData(self, ):
        if 'productionStatusOfProcessedData' not in self.decoded_data:
            self.decoded_data[
                'productionStatusOfProcessedData'] = get_table_value(
                    '1.3', self.__productionStatusOfProcessedData)[0]
        return self.decoded_data['productionStatusOfProcessedData']

    @property
    def typeOfProcessedData(self, ):
        if 'typeOfProcessedData' not in self.decoded_data:
            self.decoded_data['typeOfProcessedData'] = get_table_value(
                '1.4', self.__typeOfProcessedData)[0]
        return self.decoded_data['typeOfProcessedData']

class Section2:
    """ Local Use Section:
        Who knows what goes on in here, we'll just keep quiet and move on """
    def __init__(self, *_):
        pass

class Section3:
    """ GRID DEFINITION SECTION:
    Includes metadata about the grid for the data. This is mainly just descriptors rather than
    the actual grid points.  This info will tell how to generate the grid locations"""
    def __init__(self, io, pos, length):
        self.file_obj = io
        self.start_pos = pos
        self.section_length = length
        self._decoded = False
        self._template_data = None

    @property
    def byte_array(self, ):
        self.file_obj.seek(self.start_pos)
        return self.file_obj.read(self.section_length)

    def decode_section(self, ):
        data = struct.unpack_from('>IBBIBBH', self.byte_array, 0)
        self.data = Section3Data(*data)
        if self.data.section_num != 3:
            raise ValueError(
                f"Found Section {self.data.section_num} while initialising section 3"
            )

    def check_decode(self):
        if not self._decoded:
            self.decode_section()

    @property
    def grid_source(self, ):
        self.check_decode()
        return self.data.grid_source

    @property
    def ndpts(self, ):
        self.check_decode()
        return self.data.ndpts

    @property
    def template_num(self, ):
        self.check_decode()
        return self.data.template_num

    @property
    def template(self):
        gdtmpl = g2pylib.grid_template(self.template_num)
        if gdtmpl is None:
            raise ValueError(
                "Grib Table 3.{tnum} not found".format(tnum=self.template_num))
            gdtmpl = {
                'name': 'Unknown Grid',
                'mapdrs': [],
                'mapgridlen': 1,
                'needext': 0
            }
        return gdtmpl

    @property
    def template_data(self, ):
        if self._template_data is None:
            gdtmpl = self.template
            fmt = gdtmpl['mapdrs']
            off = 14
            self._template_data = []
            for nbyte in fmt:
                self._template_data.append(
                    g2pylib.get_pp_bits(self.byte_array, off, nbyte))
                off += abs(nbyte)
        return self._template_data

    @property
    def grid_type(self):
        return self.template['name']

    def latlon(self, ):
        if self.template_num == 0:
            return grids.generate_regular_latlon_grid(self.ndpts,
                                                      self.template_data)

    def __repr__(self):
        return ("Section 3:\n"
                f"  grid_source     : {self.grid_source}\n"
                f"  ndpts           : {self.ndpts}\n"
                f"  num_oct         : {self.data.num_oct}\n"
                f"  template_num    : {self.template_num}\n"
                f"  grid_template   : {self.template_data}\n")

class Section4:
    """PRODUCT DEFINITION SECTION:
    This includes details about the variable name, units, vertical coords.
    """
    def __init__(self, io, pos, length):
        self.file_obj = io
        self.start_pos = pos
        self.section_length = length
        self._decoded = False
        self._template_num = None
        self._pds_template = None

    @property
    def byte_array(self):
        self.file_obj.seek(self.start_pos)
        return self.file_obj.read(self.section_length)

    def decode_section(self):
        template, template_num, _, pos, data = g2pylib.unpack4(
            self.byte_array, 0, [])
        self._template_num = template_num
        self._template = template
        self.template_data = data

    @property
    def template_num(self, ):
        if not self._decoded:
            self.decode_section()
        return self._template_num

    @property
    def pds_template(self, ):
        if not self._decoded:
            self.decode_section()
        return self._template

    def __repr__(self):
        return ("Section 4:\n"
                f"  template number  = {self.template_num}\n"
                f"  template data    ={self.pds_template}\n")

class Section5:
    """DATA REPRESENTATION SECTION:
    Inludes data about how many data points to expect, type of data, and how to
    unpack the data"""
    def __init__(self, io, pos, length):
        self.file_obj = io
        self.start_pos = pos
        self.section_length = length
        self._decoded = False
        self._template_num = None
        self._drs_template = None
        self._ndpts = None

    @property
    def byte_array(self):
        self.file_obj.seek(self.start_pos)
        return self.file_obj.read(self.section_length)

    def decode_section(self):
        template, template_num, ndpts, pos = g2pylib.unpack5(
            self.byte_array, 0, [])
        self._template_num = template_num
        self._template = template
        self._ndpts = ndpts

    @property
    def template_num(self, ):
        if not self._decoded:
            self.decode_section()
        return self._template_num

    @property
    def drs_template(self):
        if not self._decoded:
            self.decode_section()
        return self._template

    @property
    def ndpts(self):
        if not self._decoded:
            self.decode_section()
        return self._ndpts

    def __repr__(self, ):
        return ("Section 5:\n"
                f" template_num     : {self.template_num}\n"
                f" drs template     : {self.drs_template}\n"
                f" defined data     : {self.ndpts}\n")

class Section6:
    """BIT MAP SECTION:
    If the field has missing values, this section contains a binary mask of where those
    missing values are.  This is used to drop missing values in section 7"""
    def __init__(self, io, pos, length, ndpts):
        self.file_obj = io
        self.start_pos = pos
        self.section_length = length
        self.ndpts = ndpts
        self._bitmap = None
        self._bitmapflag = None
        self._decoded = False

    @property
    def byte_array(self):
        self.file_obj.seek(self.start_pos)
        return self.file_obj.read(self.section_length)

    def decode_section(self):
        bmap, bmapf = g2pylib.unpack6(self.byte_array, self.ndpts, 0, [])
        if bmapf == 0:
            self._bitmap = bmap.astype('B')
        else:
            self._bitmap = []
        self._bitmapflag = bmapf
        self._decoded = True

    @property
    def bitmap(self):
        if not self._decoded:
            self.decode_section()
        return self._bitmap

    @property
    def bmapflag(self):
        if not self._decoded:
            self.decode_section()
        return self._bitmapflag

    def __repr__(self, ):
        return ("Section 6:\n"
                f"  bitmap length  : {len(self.bmap)}\n"
                f"  bitmap mask    : {len(self.bmap) - sum(self.bmap)}\n"
                f"  bitmap flag    :{self.bmapflag}\n")

class Section7:
    """DATA SECTION:
    This is where the data will be stored and unpacked from.
    """
    def __init__(self, io, pos, length):
        self.file_obj = io
        self.start_pos = pos
        self.section_length = length

    @property
    def byte_array(self):
        self.file_obj.seek(self.start_pos)
        return self.file_obj.read(self.section_length)

def _unpack2(*_):
    return (_, )

class Grib2Message:
    def __init__(self, io, startpos):
        self.file_obj = io
        self.start_byte = startpos
        self.lensect0 = 16
        self.lengrib = None
        self.section_lengths = {}
        self.read_section0()
        self.section1 = self.read_section_n(1)
        self.section2 = self.read_section_n(2)
        self.section3 = self.read_section3()
        self.section4 = self.read_section4()
        self.section5 = self.read_section5()
        self.section6 = self.read_section6()
        self.section7 = self.read_section7()
        self.decode_metadata()
        self.read_section8()

    def __repr__(self):
        return (
            f"\tlongname                = {self.longname}\n"
            f"\tunits                   = {self.units}\n"
            f"\tshortname               = {self.shortname}\n"
            f"\tforecast type           = {self.generating_process}\n"
            f"\tforecast time           = {self.forecast_time} {self.time_units}\n"
            f"\tGRIB2 Reference Time    = {self.section1.significanceOfReferenceTime}\n"
            f"\tGRIB2 Discipline        = {self.discipline_name()}\n"
            f"\tGRIB2 Category          = {self.category}\n"
            f"\tGRIB2 Production Status = {self.section1.productionStatusOfProcessedData}\n"
            f"\tGRIB2 Data Type         = {self.section1.typeOfProcessedData}\n"
        )

    def decode_metadata(self):
        pds_template = self.section4.pds_template
        self.category = get_table_value('4.1', pds_template[0])
        cat = pds_template[0]
        disc = self.discipline_int
        variable_data = get_table_value(f'4.2.{disc}.{cat}', pds_template[1])
        self.shortname = variable_data[2]
        self.units = variable_data[1]
        self.longname = variable_data[0]
        self.generating_process = get_table_value('4.3', pds_template[2])
        self.observation_hours = pds_template[5]
        self.observation_minutes = pds_template[6]
        self.time_units = get_table_value('4.4', pds_template[7])
        self.forecast_time = pds_template[8]
        self.surface_type1 = get_table_value('4.5', pds_template[9])
        surface_scale = pds_template[10]
        surface_level = pds_template[11]
        self.surface_level1 = surface_level * surface_scale
        self.surface_type2 = get_table_value('4.5', pds_template[12])
        surface_scale = pds_template[13]
        surface_level = pds_template[14]
        self.surface_level2 = surface_level * surface_scale

    def section_starting_position(self, section_num):
        pos = self.start_byte
        for n in range(section_num):
            pos += self.section_lengths[n]
        return pos

    def _get_section_bytes(self, startpos):
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.file_obj.seek(startpos)
        return self.file_obj.read(lensect), sectnum

    def discipline_name(self, ):
        # Code Table 0.0: Discipline of processed data in the GRIB message,
        # number of GRIB Master Table
        table0_0 = {
            0: 'Meteorological products',
            1: 'Hydrological products',
            2: 'Land surface products',
            3: 'Space products',
            4: 'Space weather products',
            10: 'Oceanographic products',
            209: 'Multi-Radar/Multi-Sensor (MRMS) products',
            255: 'Missing'
        }
        return table0_0.get(self.discipline_int, 'Missing')

    def read_section0(self, ):
        """ Section 0 'INDICATOR SECTION' describes the grib message
            This just includes the grib type (1/2), discipline and length of message"""
        startpos = self.section_starting_position(0) + 4
        self.file_obj.seek(startpos)
        # get discipline and version info.
        _, disc, vers, lengrib = struct.unpack('>HBBq', self.file_obj.read(12))
        if vers != 2:
            self.file_obj.seek(lengrib,1)
            raise IOError('not a GRIB2 file (version number %d)' % vers)
        # we read 16 octets above, and need 4 at the end of the message
        self.file_obj.seek(lengrib - 20, 1)
        # make sure the message ends with '7777'
        end = self.file_obj.read(4).decode('ascii', 'ignore')
        if end != '7777':
            raise IOError('partial GRIB message (no "7777" at end)')
        self.section_lengths[0] = 16
        self.lengrib = lengrib
        self.discipline_int = disc

    def read_section_n(self, n):
        section = {
            1: Section1,
            2: Section2,
        }
        unpack = {
            1: g2pylib.unpack1,
            2: _unpack2,
        }

        startpos = self.section_starting_position(n)
        section_bytes, sectnum = self._get_section_bytes(startpos)
        if sectnum != n:
            self.section_lengths[n] = 0
            return section[n]()
        self.section_lengths[n] = len(section_bytes)
        return section[n](*unpack[n](section_bytes, 0, []))

    def read_section3(self, ):
        n = 3
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.section_lengths[n] = lensect
        return Section3(self.file_obj, startpos, lensect)

    def read_section4(self, ):
        n = 4
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.section_lengths[n] = lensect
        return Section4(self.file_obj, startpos, lensect)

    def read_section5(self, ):
        n = 5
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.section_lengths[n] = lensect
        return Section5(self.file_obj, startpos, lensect)

    def read_section6(self, ):
        n = 6
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.section_lengths[n] = lensect
        return Section6(self.file_obj, startpos, lensect, self.section3.ndpts)

    def read_section7(self, ):
        n = 7
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        lensect, sectnum = struct.unpack('>IB', self.file_obj.read(5))
        self.section_lengths[n] = lensect
        return Section7(self.file_obj, startpos, lensect)

    def read_section8(self, ):
        """ END SECTION:
        Should just include '7777' to indicate the end of the grib message
        """
        n = 8
        startpos = self.section_starting_position(n)
        self.file_obj.seek(startpos)
        flag = self.file_obj.read(4).decode('ascii', 'ignore')
        if flag != '7777':
            raise IOError('Grib2 Message not finished yet')

    @property
    def values(self, ):
        fld = g2pylib.unpack7(gribmessage=self.section7.byte_array,
                              gdtnum=self.section3.template_num,
                              gdtmpl=self.section3.template_data,
                              drtnum=self.section5.template_num,
                              drtmpl=self.section5.drs_template,
                              ndpts=self.section5.ndpts)
        if self.section6.bmapflag == 0:
            vals = np.empty(self.section3.ndpts, 'f')
            vals[self.section6.bmap == 1] = fld
            vals[self.section6.bmap == 0] = np.nan
        else:
            vals = fld
        return vals

def get_table_value(table, key):
    table_data = tables.table4.get(table, None)
    if table_data is None:
        maj = table[0]
        table_dir = pathlib.Path(__file__).parent
        table_path = table_dir / f'tables/ncep/{maj}/{table}.json'
        if table_path.exists():
            with open(table_dir /
                      f'tables/ncep/{maj}/{table}.json') as json_file:
                table_data = json.load(json_file)
        else:
            logging.warning(f"{table_path} was not found.")
            table_data = {}
    tables.table4[table] = table_data
    attr_data = table_data.get(str(key), None)
    if attr_data is None:
        logging.warning(f"{key} not found, attributes will not be decoded")
        attr_data = str(key)
    return attr_data


import d2lzh as d2l
from mxnet import gluon, init, autograd
from mxnet.gluon import loss as gloss, nn
import sys
import time

def get_fashion_mnist_labels(labels):
    '''
    将数据包中的数值标签转成相应的文本标签
    :param labels:
    :return:
    '''
    text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
    return [text_labels[int(i)] for i in labels]

def show_fashion_mnist(images, labels):
    '''
    在一行中画出多张图像和对应标签的函数
    :param images:
    :param labels:
    :return:
    '''
    d2l.use_svg_display()
    _, figs = d2l.plt.subplots(1, len(images), figsize=(12, 12))
    for f, img, lbl in zip(figs, images, labels):
        f.imshow(img.reshape((28, 28)).asnumpy())
        f.set_title(lbl)
        f.axes.get_xaxis().set_visible(False)
        f.axes.get_yaxis().set_visible(False)

def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size,params=None, lr=None, trainer=None):
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
        for X, y in train_iter:
            with autograd.record():
                y_hat = net(X)
                l = loss(y_hat, y).sum()
            l.backward()
            if trainer is None:
                d2l.sgd(params, lr, batch_size)
            else:
                trainer.step(batch_size) # “softmax回归的简洁实现”
            y = y.astype('float32')
        train_l_sum += l.asscalar()
        train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()
        n += y.size
        test_acc = d2l.evaluate_accuracy(test_iter, net)
        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

if __name__ == '__main__':
    batch_size = 256
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)

    net = nn.Sequential() #sequential用来存储所有神经网络中的所有单元
    net.add(nn.Dense(10)) #设置网络输出个数为10
    net.initialize(init.Normal(sigma=0.01)) #初始化网络参数为从均值为0、标准差为0.01的正态分布的随机取样的值

    loss = gloss.SoftmaxCrossEntropyLoss() # softmax运算和交叉熵损失计算的函数

    trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) # 使⽤学习率为0.1的小批量随机梯度下降作为优化算法
    num_epochs = 10
    d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs,
                  batch_size, None, None, trainer) #训练并输出结果
    for X, y in test_iter:
        break
    true_labels = d2l.get_fashion_mnist_labels(y.asnumpy())
    pred_labels = d2l.get_fashion_mnist_labels(net(X).argmax(axis=1).asnumpy())
    print(true_labels)
    print(pred_labels)
    print(float(len([x for x in range(len(true_labels)) if true_labels[x] == pred_labels[x]]))/len(true_labels))


import json
import numpy as np
from PIL import Image

def load_json(file_path):
    '''
    Extract user variables from json dictionary.
    Args:
        file_path: <string> path to file
    Returns:
        dictionary: <dict> use variables dictionary
    '''
    with open(file_path, 'r') as file:
        return json.load(file)

def remove_unwanted(all_lines,
                    unwanted_character):
    '''
    Remove unwanted characters from lines, removes hidden newline characters.
    Args:
        all_lines: <array> read lines
        unwanted_character: <string> characters to remove
    Return:
        lines: <array> lines with characters and newline characters removed
    '''
    lines = [
        (line.translate({ord(unwanted_character): None}))[0: -1]
        for line in all_lines]
    return lines

def lines_to_parameters(lines):
    '''
    Pull important parameters into dictionary.
    Args:
        lines: <array> array of lines from txt file, stripped of '$' and \n
    Returns:
        parameters: <dict> parameter dictionary
    '''
    parameters = dict()
    for line in lines:
        parameter_label, *parameter_values = line.strip().split(' ')
        parameters[parameter_label] = parameter_values
    return parameters

def desired_JEOL_parameters(all_parameters):
    '''
    Pull important parameters into dictionary. Add more parameters by adding
    names and keys to the dictionary in this function.
    Args:
        all_parameters: <dict> dictionary of all parameters from txt file
    Returns:
        desired_parameters: <dict> dictionary of all desired parameters from
                            txt file
    '''
    desired_parameters = {
        'acceleration_voltage': float(all_parameters['CM_ACCEL_VOLT'][0]),
        'emission_current': float(all_parameters['SM_EMI_CURRENT'][0]),
        'brightness': int(all_parameters['CM_BRIGHTNESS'][0]),
        'contrast': int(all_parameters['CM_CONTRAST'][0]),
        'magnification': int(all_parameters['CM_MAG'][0]),
        'working_distance': float(all_parameters['SM_WD'][0]),
        'calibration_pixels': int(all_parameters['SM_MICRON_BAR'][0])}

    ''' Units given to 2 characters (um, nm, etc.) '''
    units_suffix = 2
    calibration_distance = all_parameters['SM_MICRON_MARKER'][0]
    desired_parameters['calibration_distance'] = int(
        calibration_distance[: -units_suffix])
    desired_parameters['distance_unit'] = calibration_distance[-units_suffix:]

    ''' CM_FULL_SIZE is [width, height] '''
    width, height = (0, 1)
    desired_parameters['image_width'] = int(
        all_parameters['CM_FULL_SIZE'][width])
    desired_parameters['image_height'] = int(
        all_parameters['CM_FULL_SIZE'][height])
    return desired_parameters

def read_SEM_log(file_path):
    '''
    Load JEOL-SEM txt file as a dictionary of key parameters.
    Args:
        file_path: <string> path to file
    Returns:
        parameters: <dict> desired parameter dictionary
    '''
    with open(file_path) as infile:
        all_lines = infile.readlines()
    stripped_lines = remove_unwanted(
        all_lines=all_lines,
        unwanted_character='$')
    JEOL_parameters = lines_to_parameters(lines=stripped_lines)
    parameters = desired_JEOL_parameters(all_parameters=JEOL_parameters)
    return parameters

def read_image(file_path):
    '''
    Load image file.
    Args:
        file_path: <string> path to file
    Returns:
        image: <array> numpy array of pixels
    '''
    image = Image.open(file_path)
    return np.array(image)

def convert(o):
    '''
    Check type of data string
    '''
    if isinstance(o, np.generic):
        return o.item()
    raise TypeError

def save_json_dicts(out_path,
                    dictionary):
    '''
    Save dictionary to json file.
    Args:
        out_path: <string> path to file, including file name and extension
        dictionary: <dict> python dictionary to save out
    Returns:
        None
    '''
    with open(out_path, 'w') as outfile:
        json.dump(
            dictionary,
            outfile,
            indent=2,
            default=convert)
        outfile.write('\n')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

''' Config pour Cx_Freeze '''

import sys, os
import cx_Freeze as cx
import main

build_options = {
    'path': sys.path,
    'includes': ['keyboard', 'encodings'],
    'optimize': 2,
    'silent': True
}

base = None
if sys.platform == "win32":
    options["include_msvcr"] = True
    base = "Win32GUI"

executable = cx.Executable(
    script='main.py',
    base=base
)

cx.setup(
    name = '2PQSTD',
    targetName = 'app',
    version = main.__version__,
    description = 'A 2D online multiplayer game',
    options = {'build_exe': build_options} ,
    executables = [executable]
)


'''
给定一个按非递减顺序排序的整数数组 A，返回每个数字的平方组成的新数组，要求也按非递减顺序排序。
示例 1：
输入：[-4,-1,0,3,10]
输出：[0,1,9,16,100]
示例 2：
输入：[-7,-3,2,3,11]
输出：[4,9,9,49,121]
'''
class Solution(object):
    # 双指针
    def sortedSquares(self, A):
        n = len(A)
        if n == 0:
            return []
        res = [0] * n
        i, j, pos = 0, n-1, n-1
        while i <= j:
            if A[i] * A[i] > A[j] * A[j]:
                res[pos] = A[i] * A[i]
                i += 1
            else:
                res[pos] = A[j] * A[j]
                j -= 1
            pos -= 1
        return res

if __name__ == '__main__':
    u = Solution()
    A1 = [-4,-1,0,3,10]
    A2 = [-7,-3,2,3,11]
    print(u.sortedSquares(A1))
    print(u.sortedSquares(A2))

Hot100之外的高频面试题/977-有序数组的平方.py

977-有序数组的平方.py


import json
import os
import sys
from time import sleep

import requests

sys.path.insert(0, os.path.abspath('..'))

from api import app
from api.bigchain import bdb_helper, airmate_keys
from api.db import Station, db
from station_adapter import OpenaqSatationAdapter

if __name__ == '__main__':
    API_URL = 'https://api.openaq.org/v1/latest?country={}&city={}'

    data_src = 'data_src.json'

    with app.app_context():
        while True:
            with open(data_src) as src:
                country_city = json.load(src)

            for sCountry, lCities in country_city.items():
                for sCity in lCities:
                    request_url = API_URL.format(sCountry, sCity)
                    data = requests.get(request_url)
                    json_data = data.json()

                    station_adapter = OpenaqSatationAdapter(json_data)

                    for station in station_adapter.get_stations():

                        if station.latitude is None or station.longitude is None:
                            print(f'Station with id={station.id} has no coordinates.')
                            continue

                        stored_station = Station.query.get(station.id)
                        if stored_station is not None:
                            if stored_station.updated_at == station.updated_at:
                                continue

                            stored_station.updated_at = station.updated_at
                        else:
                            db.session.add(station)
                            stored_station = station

                        txid = bdb_helper.save({'data_provider': 'openaq'}, station.data,
                                               airmate_keys.public_key,
                                               airmate_keys.private_key)

                        stored_station.last_txid = txid
                        db.session.commit()

                        print(f'Station with id={station.id} updated.')

                    sleep(5)