import ply.lex as lex

states = (
    ('STRING', 'exclusive'),
    ('HELP', 'exclusive'),
)

reserved = {
    'mainmenu': 'MAINMENU',
    'menuconfig': 'MENUCONFIG',
    'menu': 'MENU',
    'endmenu': 'ENDMENU',
    'if': 'IF',
    'endif': 'ENDIF',
    'bool': 'BOOL',
    'tristate': 'TRISTATE',
    'int': 'INT',
    'hex': 'HEX',
    'string': 'STRING',
    'choice': 'CHOICE',
    'endchoice': 'ENDCHOICE',
    'depends': 'DEPENDS',
    'on': 'ON',
    'select': 'SELECT',
    'default': 'DEFAULT',
    'imply': 'IMPLY',
    'range': 'RANGE',
    'visible': 'VISIBILE',
    'modules': 'MODULES',
    'source': 'SOURCE',
    'config': 'CONFIG',
    'comment': 'COMMENT',
    'def_bool': 'DEF_BOOL',
    'def_tristate': 'DEF_TRISTATE',
    'optional': 'OPTIONAL',
    'prompt': 'PROMPT',
    'path': 'PATH',
    'endpath': 'ENDPATH',
}

tokens = [
    'WORD',
    'QUOTE_WORD',
    "SP_WORD",
    # 'ASSIGN_VAL',
    # operator
    'OR',  # ||
    'AND',  # &&
    'EQUAL',  # =
    'UNEQUAL',  # !=
    'LESS',  # <
    'LESS_EQUAL',  # <=
    'GREATER',  # >
    'GREATER_EQUAL',  # >=
    'NOT',  # !
    'OPEN_PARENT',  # (
    'CLOSE_PARENT',  # )
    # 'COLON_EQUAL',      # :=
    # 'PLUS_EQUAL',       # +=
    'EOL',
    'HELP',
    'HELP_CONTEXT',
] + list(reserved.values())

t_OR = '\|\|'
t_AND = '&&'
t_EQUAL = '='
t_UNEQUAL = '!='
t_LESS = '<'
t_LESS_EQUAL = '<='
t_GREATER = '>'
t_GREATER_EQUAL = '>='
t_NOT = '!'
t_OPEN_PARENT = '\('
t_CLOSE_PARENT = '\)'
# t_COLON_EQUAL = ':='
# t_PLUS_EQUAL = '\+='

t_ignore = '[ \t]+'
t_ignore_commet = r'\# .*'

def t_SP_WORD(t):
    r'\$(\{\{(.*?)\}\})?'
    t.type = "SP_WORD"
    return t

def t_ANY_error(t):
    print(t.value[:50])
    print("Illegal charactor '%s'" % t.value[0])
    raise

def t_HELP(t):
    r'help'
    t.lexer.begin('HELP')
    t.type = 'HELP'
    return t

def t_WORD(t):
    r'[\.A-Za-z0-9_-]+'
    if t.value.count('.') > 0:
        t.type = 'QUOTE_WORD'
    else:
        t.type = reserved.get(t.value, 'WORD')
    return t

def t_EOL(t):
    r'\n+'
    t.lexer.lineno += len(t.value)
    t.type = 'EOL'
    return t

def t_NEXTLINE(t):
    r'\\\n'
    t.lexer.lineno += len(t.value)
    pass

def t_TO_STRING(t):
    r'\"'
    t.lexer.begin('STRING')

##########################  string  ##########################
t_STRING_ignore = ''

def t_STRING_CONTEXT(t):
    r'.*?"'
    t.value = '"' + t.value[:len(t.value) - 1] + '"'
    t.type = 'QUOTE_WORD'
    t.lexer.begin('INITIAL')
    return t

##########################   help   ##########################
t_HELP_ignore = ''
help_context = ""

def t_HELP_CONTEXT(t):
    r'[\t]+.*'
    global help_context
    help_context += t.value

def t_HELP_NL(t):
    r'\n+'
    global help_context
    if len(help_context) > 0:
        help_context += '\n'
    # 检查换行后是否有缩进
    for ch in t.lexer.lexdata[t.lexer.lexpos:]:
        if ch != '\t':
            t.type = "HELP_CONTEXT"
            t.value = help_context
            t.lexer.begin('INITIAL')
            help_context = ""
            return t
        else:
            break

lexer = lex.lex()


from myOwnfunc import IsPrime
def FindPrimeFactor(number):
    num = []
    i = 2
    t = number
    count = 0
    if number<=1:
        print("please enter a number>=2")
        return None
    else:
        while i < number:
            if IsPrime.isPrime(i) and t % i == 0:
                while t % i == 0:
                    t = t / i
                    num.append(i)
            i += 1

    return num


import math
from .byteutil import *
from .config import MidiConfig

################################################

# Note: There are 24 MIDI clocks per quarter-note, apparently...

################################################

class EventCategory:
    TIMESIGNATURE = 0
    TEMPOCHANGE = 1
    PROGRAMCHANGE = 2

    CONTROLCHANGE = 10

    NOTEOFF = 21 # note off BEFORE note on
    NOTEON = 22

################################################

class MidiWriter:

    def __init__(self,filename):
        self.file = open(filename,'wb')
        self.eventByteArray = bytearray(b'')
        # self.eventBytes = []
        self.evs = []
        self.__addTimeSignature()
        self.ch = 0
        self.endTicks = 0
        self.tempoMultiplier = 1
        self.__initForCh()

    def setTpb(self,tpb):
        MidiConfig.tpb = tpb # ukz's tpb
        self.tempoMultiplier = 1
        if tpb > 127:
            for i in range(2,1000):
                if tpb%i==0 and tpb//i <= 127:
                    self.tpb = tpb//i #header's tpb
                    self.tempoMultiplier = i
                    return
            raise Exception(f"Value tpb={tpb} is a bad one")
        else:
            self.tpb = tpb
            self.tempoMultiplier = 1

    def __addTimeSignature(self):
        # Time signature bytes: FF 58 04 nn dd cc bb
        # The fraction you see on a score would be nn / 2^dd
        #   (denominator always has to be power of 2)
        # cc = number of MIDI clocks in a metronome click.
        # bb = number of notated 32nd-notes in a MIDI quarter-note.
        #   bb was added in MIDI to support legacy(?)

        self.__addGenericEvent(0, EventCategory.TIMESIGNATURE, b'\xFF\x58\x04\x04\x02\x24\x08')

    # def getMaxEventBeats(self):
    #     maxTicks = max(map(lambda e: e[0], self.evs))
    #     return maxTicks / MidiConfig.tpb

    def getMaxEventTicks(self):
        return max(map(lambda e: e[0], self.evs))

    def __initForCh(self):
        chBytePos = self.ch << 16
        self.chNoteOn = 0x900000 | chBytePos
        self.chNoteOff = 0x800000 | chBytePos
        self.chPitchBend = 0xE00000 | chBytePos

    def switchToChannel(self,channelNumber):
        if channelNumber < 0 or channelNumber > 15:
            raise ValueError("Bad channel number")
        self.ch = channelNumber
        self.__initForCh()

    def __addGenericEvent(self,time,category,eventBytes):
        if isinstance(time,float):
            raise 'what!'
        timeTicks = time
        self.evs.append((timeTicks,category,eventBytes))

    # Argument tempo in beats per minute
    def addTempoChange(self,t,beatsPerMinute):
        if t < 0:
            raise ValueError("Negative time!")
        if beatsPerMinute < 1:
            raise Exception("That's a little too slow, man.")

        # One beat is 24 ticks.
        # bpm = 120  =>  µsecsPerBeat = 500,000
        microsecondsPerBeat = 60000000//beatsPerMinute
        # X beat/min == Y µsec/beat
        # 1 min = X beats = (X*Y) µsecs
        # 1 min = 60 million µsecs
        # 0xA120 in decimal is 500000

        self.addTempoChangeMicroseconds(t,microsecondsPerBeat)

    # Argument tempo in beats per minute
    def addTempoChangeMicroseconds(self,t,microsecondsPerBeat):
        if t < 0:
            raise ValueError("Negative time!")

        if self.tempoMultiplier > 1:
            microsecondsPerBeat = microsecondsPerBeat // self.tempoMultiplier

        if microsecondsPerBeat < 1:
            raise Exception("That's a little too fast, man.")
        if microsecondsPerBeat >= 2**24:
            raise Exception("That's a little too slow, man.")

        theBytes = bytearray(b'\xFF\x51') + intToLengthedBytes3(microsecondsPerBeat)
        self.__addGenericEvent(t, EventCategory.TEMPOCHANGE, theBytes)

    def addProgramChanges(self,t,channelPrograms):
        for ch,prog in channelPrograms:
            if t < 0:
                raise ValueError("Negative time!")
            if prog < 0 or prog > 127:
                raise ValueError("Bad program number")

            theBytes = bytearray([0xc0 | ch,prog])
            self.__addGenericEvent(0, EventCategory.PROGRAMCHANGE, theBytes)

    def addProgramChange(self,ch,t,prog):
        if t < 0:
            raise ValueError("Negative time!")
        if prog < 0 or prog > 127:
            raise ValueError("Bad program number")

        #theBytes = prog | ((0xc0 | self.ch)<<8).to_bytes(2)
        theBytes = bytearray([0xc0 | ch,prog])
        self.__addGenericEvent(0, EventCategory.PROGRAMCHANGE, theBytes)

################################################

    def addInitialVolumes(self,time,channelVolumes):
        for ch,vol in channelVolumes:
            self.addControllerEvent(ch,7,time,vol)
            self.addControllerEvent(ch,11,time,127)

    def addInitialVolume(self,ch,time,vol):
        self.addControllerEvent(ch,7,time,vol)
        self.addControllerEvent(ch,11,time,127)

    def addControllerEvent(self,ch,ctrl,time,val):
        if val < 0 or val > 127:
            raise Exception("Bend out of range!")
        if time < 0:
            raise Exception("Time negative!")

        theBytes = bytearray([0xB0 | ch, ctrl, val])
        self.__addGenericEvent(time, EventCategory.CONTROLCHANGE, theBytes)

    def addControllerEventStream(self,ctrl,channelTimeVals):
        evs = self.evs
        category = EventCategory.CONTROLCHANGE
        for ch,time,val in channelTimeVals:
            if val < 0 or val > 127:
                raise Exception("Bend out of range!")
            if time < 0:
                raise Exception("Time negative!")

            theBytes = bytearray([0xB0 | ch, ctrl, val])

            # self.__addGenericEvent(time, EventCategory.CONTROLCHANGE, theBytes)
            timeTicks = time
            evs.append((timeTicks,category,theBytes))

################################################

    # Assume this works like any other controller.
    # But here we map bend value to the range
    #   that MIDI's pitch-bend event expects.
    def addPitchBend(self,ch,time,bend):
        if bend < 0 or bend > 127:
            raise Exception("Bend out of range!")
        if time < 0:
            raise Exception("Time negative!")

        #theBytes = (self.chPitchBend | (bend<<8) | bend).to_bytes(3,'big')
        theBytes = bytearray([0xE0 | ch, bend, bend])
        self.__addGenericEvent(time, EventCategory.CONTROLCHANGE, theBytes)

    def addNotes(self,notes):
        for note in notes:

            if note.p < 0 or note.p > 127:
                raise Exception("Pitch out of range!")
            if note.t < 0:
                raise Exception("Time negative!")
            if note.d < 0:
                raise Exception("Duration negative!")
            if note.l < 0 or note.l > 127:
                raise Exception("Velocity out of range!")

            # 0x90 is status byte for noteOn
            # 0x80 is status byte for noteOff
            # theBytesOn = (self.chNoteOn | (pitch<<8) | vel).to_bytes(3,'big')
            theBytesOn = bytearray([0x90 | note.c, note.p, note.l])
            # theBytesOff = (self.chNoteOff | (pitch<<8) | vel).to_bytes(3,'big')
            theBytesOff = bytearray([0x80 | note.c, note.p, note.l])

            timeTicksOn = note.t
            timeTicksOff = note.t+note.d
            if isinstance(timeTicksOn,float) or isinstance(timeTicksOff,float):
                raise 'what!'
            self.evs.append((timeTicksOn, EventCategory.NOTEON, theBytesOn))
            self.evs.append((timeTicksOff, EventCategory.NOTEOFF, theBytesOff))

    def setEndTime(self,t):
        self.endTicks = t

    def __setEventBytes(self):
        self.evs.sort()
        curtimeTicks = 0
        self.eventByteArray = bytearray(b'')

        bs = bytearray()

        for timeTicks,_,ebytes in self.evs:
            deltaTicks = timeTicks-curtimeTicks

            # self.eventByteArray.extend(intToVarLengthBytes(deltaTicks))
            if deltaTicks==0:
                self.eventByteArray += b'\x00'
            else:
                bs.clear()
                bs.insert(0, deltaTicks % 128) # rightmost byte, bit 7 NOT set
                m = deltaTicks >> 7
                while m > 0:
                    bs.insert(0, (m % 128) | 0x80) # NOT rightmost byte, bit 7 IS set
                    m >>= 7
                self.eventByteArray += bs

            self.eventByteArray.extend(ebytes)
            curtimeTicks = timeTicks

        deltaTicks = self.endTicks - curtimeTicks
        self.eventByteArray.extend(intToVarLengthBytes(deltaTicks))
        self.eventByteArray.extend(b'\xFF\x2F\x00')
        # self.eventByteArray.extend(b'\x00\xFF\x2F\x00') # track end

    def writeFile(self):
        self.__writeHeaderChunk()
        self.__setEventBytes()
        self.__writeChunk(b'MTrk',self.eventByteArray)
        self.file.close()

    def __writeChunk(self,typenameBytes,chunkBytesArray):
        self.__writeBytes(typenameBytes)
        self.__writeBytes(intToBytes(len(chunkBytesArray), 4))
        self.__writeBytes(chunkBytesArray)

    def __writeHeaderChunk(self):
        headerBytes = bytearray(b'\x00\x00\x00\x01\x00')
        headerBytes += bytes([self.tpb])
        self.__writeChunk(b'MThd',headerBytes)

    def __writeBytes(self,theBytes):
        self.file.write(theBytes)


'''
Created on Oct 27, 2016

@author: s528358
'''
import re
import itertools
def getAnagramIndices(haystack, needle):
    # WRITE YOUR CODE HERE
    a = []
    test = ["".join(perm) for perm in itertools.permutations(needle)]
    for i in test:
        a = a + [n for n in range(len(haystack)) if haystack.find(i,n) == n]
    a.sort()
    return a


#!/usr/bin/env python
# coding: utf-8

# # Welcome to my EDA and Modeling kernel !
#
# It's a very cool opportunity to practice and learn from other kagglers about interesting feature encoding techniques and modelling;
#
# I hope you enjoy my work and bring me your feedback. If this kernel is useful for you, please don't forget to upvote the kernel
#
# NOTE: English is not my native language, so sorry for any mistake.

# # Competition description:
# Is there a cat in your dat?
#
# A common task in machine learning pipelines is encoding categorical variables for a given algorithm in a format that allows as much useful signal as possible to be captured.
#
# Because this is such a common task and important skill to master, we've put together a dataset that contains only categorical features, and includes:
#
# - binary features
# - low- and high-cardinality nominal features
# - low- and high-cardinality ordinal features
# - (potentially) cyclical features
#
# This Playground competition will give you the opportunity to try different encoding schemes for different algorithms to compare how they perform. We encourage you to share what you find with the community.
#
# If you're not sure how to get started, you can check out the Categorical Variables section of Kaggle's Intermediate Machine Learning course.
#
#
# Have Fun!

# # Objective:
# I want to do complete exploration to understand the data and after it I will build a Machine Learning Model.

# # Questions:
# Before I start handling the data, I am thinking about what I want to find here, like:
# - What's the target distribution and what data type it is?
# - What is the type of features we have and the name?! These names can tell us anything?
# - What's the most common values in all cateogory's we have?
# - How many binary features we have?
# - How many missing values?
# - Which type of transformations we need to apply?!

#
# And many more questions that will rise through the exploration;

# ## Importing Library's

# In[1]:

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import scipy as sp
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns

# Preprocessing, modelling and evaluating
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix, roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_score, KFold
from xgboost import XGBClassifier
import xgboost as xgb

## Hyperopt modules
from hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK, STATUS_RUNNING
from functools import partial

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.

import gc

# ## Some utilities functions

# In[2]:

def resumetable(df):
    print(f"Dataset Shape: {df.shape}")
    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
    summary = summary.reset_index()
    summary['Name'] = summary['index']
    summary = summary[['Name','dtypes']]
    summary['Missing'] = df.isnull().sum().values
    summary['Uniques'] = df.nunique().values
    summary['First Value'] = df.loc[0].values
    summary['Second Value'] = df.loc[1].values
    summary['Third Value'] = df.loc[2].values

    for name in summary['Name'].value_counts().index:
        summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)

    return summary

## Function to reduce the DF size
def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df

# In[3]:

df_train = pd.read_csv('./data/cat-in-the-dat/train.csv')
df_test = pd.read_csv('./data/cat-in-the-dat/test.csv')
submission = pd.read_csv('./data/cat-in-the-dat/sample_submission.csv', index_col='id')

# ### Knowing our data

# In[4]:

summary = resumetable(df_train)
summary

# Cool!! As we can note we don't have missing values and it's nice to us.

# I will start exploring the Target and trying to find some patterns that could explain it;

# # Target Feature
# - Let's see the distribution and if we can identify what is the nature of this feature

# In[5]:

total = len(df_train)
plt.figure(figsize=(12,6))

g = sns.countplot(x='target', data=df_train, color='green')
g.set_title("TARGET DISTRIBUTION", fontsize = 20)
g.set_xlabel("Target Vaues", fontsize = 15)
g.set_ylabel("Count", fontsize = 15)
sizes=[] # Get highest values in y
for p in g.patches:
    height = p.get_height()
    sizes.append(height)
    g.text(p.get_x()+p.get_width()/2.,
            height + 3,
            '{:1.2f}%'.format(height/total*100),
            ha="center", fontsize=14)
g.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights

plt.show()

# Cool;
# We can see that our target is a binary feature and as it is 0 or 1 we can't know what is about.

# Another interesting thing to note is that isn't so imbalanced:

# - Category 0 with 79.4%

# - Category 1 with 30.6
#
# Now, as we have much of them, let's explore the patterns of other binary features

# # Binary Features

# In[6]:

bin_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']

# In[7]:

#Looking the V's features
import matplotlib.gridspec as gridspec # to do the grid of plots
grid = gridspec.GridSpec(3, 2) # The grid of chart
plt.figure(figsize=(16,20)) # size of figure

# loop to get column and the count of plots
for n, col in enumerate(df_train[bin_cols]):
    ax = plt.subplot(grid[n]) # feeding the figure of grid
    sns.countplot(x=col, data=df_train, hue='target', palette='hls')
    ax.set_ylabel('Count', fontsize=15) # y axis label
    ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label
    ax.set_xlabel(f'{col} values', fontsize=15) # x axis label
    sizes=[] # Get highest values in y
    for p in ax.patches: # loop to all objects
        height = p.get_height()
        sizes.append(height)
        ax.text(p.get_x()+p.get_width()/2.,
                height + 3,
                '{:1.2f}%'.format(height/total*100),
                ha="center", fontsize=14)
    ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights

plt.show()

# We can't see any clear pattern to positive values in target.
# - It's interesting to see that in bin_3 the ratio of target true have the same size in both values; (~15%)
# - In the other binary features the pattern is very similar.

# # Working on binary Features
# - To work in machine learning models, we need numerical features.
# - As we can note, 3 of binary features are already integer features (0 or 1)
# - We will need to work on the features that isn't in numerical representation.
#
# I'm assuming that:
# - T is True and F is False
# - Y is Yes and N is No

# In[8]:

# dictionary to map the feature
bin_dict = {'T':1, 'F':0, 'Y':1, 'N':0}

# Maping the category values in our dict
df_train['bin_3'] = df_train['bin_3'].map(bin_dict)
df_train['bin_4'] = df_train['bin_4'].map(bin_dict)
df_test['bin_3'] = df_test['bin_3'].map(bin_dict)
df_test['bin_4'] = df_test['bin_4'].map(bin_dict)

# ## Checking the results

# In[9]:

df_train[['bin_3', 'bin_4']].head()

# Cool! Now, it's ok to model this feature into a machine learning algorithmn.

# Let's work in the other features

# # Nominal Features (with more than 2 and less than 15 values)
# - Let's see the distribution of the feature and target Ratio for each value in nominal features

# In[10]:

nom_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']

# In[11]:

def ploting_cat_fet(df, cols, vis_row=5, vis_col=2):

    grid = gridspec.GridSpec(vis_row,vis_col) # The grid of chart
    plt.figure(figsize=(17, 35)) # size of figure

    # loop to get column and the count of plots
    for n, col in enumerate(df_train[cols]):
        tmp = pd.crosstab(df_train[col], df_train['target'], normalize='index') * 100
        tmp = tmp.reset_index()
        tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)

        ax = plt.subplot(grid[n]) # feeding the figure of grid
        sns.countplot(x=col, data=df_train, order=list(tmp[col].values) , color='green')
        ax.set_ylabel('Count', fontsize=15) # y axis label
        ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label
        ax.set_xlabel(f'{col} values', fontsize=15) # x axis label

        # twinX - to build a second yaxis
        gt = ax.twinx()
        gt = sns.pointplot(x=col, y='Yes', data=tmp,
                           order=list(tmp[col].values),
                           color='black', legend=False)
        gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)
        gt.set_ylabel("Target %True(1)", fontsize=16)
        sizes=[] # Get highest values in y
        for p in ax.patches: # loop to all objects
            height = p.get_height()
            sizes.append(height)
            ax.text(p.get_x()+p.get_width()/2.,
                    height + 3,
                    '{:1.2f}%'.format(height/total*100),
                    ha="center", fontsize=14)
        ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights

    plt.subplots_adjust(hspace = 0.5, wspace=.3)
    plt.show()

# In[12]:

ploting_cat_fet(df_train, nom_cols, vis_row=5, vis_col=2)

# Nice!!! We can see clear different patterns between the nominal category values.
#
# Some summary of these features:
# - NOM_0 -
# Red (~35%) value have the highest % of positive values in the target;
#
# - NOM_1 -
# Triangle(~36%) value have the highest % of positive values in the target;
#
# - NOM_2 -
# Hamster(~36%) value have the highest % of positive values in the target;
#
# - NOM_3 -
# India(~36%) value have the highest % of positive values in the target;
#
# - NOM_4 -
# Theremin(~36%) value have the highest % of positive values in the target;
#
# Do you noted something?!

# All the values with highest % of True values on target, are the category's with lowest frequency on the nominal category's.
# We need to handle with all these features;

# As it is nominal and don't have so many values by category's we can transform it in dummy features;

# # Nominal Features Transformation

# In[13]:

df_test['target'] = 'test'
df = pd.concat([df_train, df_test], axis=0, sort=False )

# In[14]:

print(f'Shape before dummy transformation: {df.shape}')
df = pd.get_dummies(df, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'], drop_first=True)
print(f'Shape after dummy transformation: {df.shape}')

# In[15]:

df_train, df_test = df[df['target'] != 'test'], df[df['target'] == 'test'].drop('target', axis=1)
del df

# In[16]:

df_train.head()

# Nice, we can see that it worked;

# Now, let's see the remaining columns with few category values;

# # Ordinal Features (with more than 2 and less than 15 values)

# In[17]:

ord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3']

# ### Ploting ordinal features

# In[18]:

ploting_cat_fet(df_train, ord_cols, vis_row=5, vis_col=2)

# Cool! In Ordinal features, the rule of nominal isn't real.

# We can see that only on ord_0 the highest ratio in target in the less common category;
#
# As the "ord_4" and "ord_5" have highest cardinality I will plot it separated
#
# ## Ord_4 and ord_5

# In[19]:

df_train['ord_5_ot'] = 'Others'
df_train.loc[df_train['ord_5'].isin(df_train['ord_5'].value_counts()[:25].sort_index().index), 'ord_5_ot'] = df_train['ord_5']

# In[20]:

tmp = pd.crosstab(df_train['ord_4'], df_train['target'], normalize='index') * 100
tmp = tmp.reset_index()
tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)
plt.figure(figsize=(15,12))

plt.subplot(211)
ax = sns.countplot(x='ord_4', data=df_train, order=list(tmp['ord_4'].values) , color='green')
ax.set_ylabel('Count', fontsize=17) # y axis label
ax.set_title('ord_4 Distribution with Target %ratio', fontsize=20) # title label
ax.set_xlabel('ord_4 values', fontsize=17) # x axis label
# twinX - to build a second yaxis
gt = ax.twinx()
gt = sns.pointplot(x='ord_4', y='Yes', data=tmp,
                   order=list(tmp['ord_4'].values),
                   color='black', legend=False)
gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)
gt.set_ylabel("Target %True(1)", fontsize=16)

tmp = pd.crosstab(df_train['ord_5_ot'], df_train['target'], normalize='index') * 100
tmp = tmp.reset_index()
tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)

plt.subplot(212)
ax1 = sns.countplot(x='ord_5_ot', data=df_train,
                   order=list(df_train['ord_5_ot'].value_counts().sort_index().index) ,
                   color='green')
ax1.set_ylabel('Count', fontsize=17) # y axis label
ax1.set_title('TOP 25 ord_5 and "others" Distribution with Target %ratio', fontsize=20) # title label
ax1.set_xlabel('ord_5 values', fontsize=17) # x axis label
# twinX - to build a second yaxis
gt = ax1.twinx()
gt = sns.pointplot(x='ord_5_ot', y='Yes', data=tmp,
                   order=list(df_train['ord_5_ot'].value_counts().sort_index().index),
                   color='black', legend=False)
gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)
gt.set_ylabel("Target %True(1)", fontsize=16)

plt.subplots_adjust(hspace = 0.4, wspace=.3)

plt.show()

# Cool!

# - In ord_4, as it is a ordinal feature, we can see a clearly linear relationship. As it is the alphabet order, it will be easy to encode
# - In ord_5 we have many category's, the top category is:

# 1 - "od" with 5019 entries,

# 2 - "f0" with 3435 entries,

# 3 - "Zq" with 2926

#
# I will explore further the ord_5 Feature. I will explore the distribution of the values and how it is.

# In[21]:

ord_5_count = df_train['ord_5'].value_counts().reset_index()['ord_5'].values

# ## Distribution of ord_5 features

# In[22]:

plt.figure(figsize=(12,6))

g = sns.distplot(ord_5_count, bins= 50)
g.set_title("Frequency of ord_5 category values", fontsize=22)
g.set_xlabel("Total of entries in ord_5 category's", fontsize=18)
g.set_ylabel("Density", fontsize=18)

plt.show()

# Cool!

# We can see that many values in ord_5 have ~2100 entries;

# Also, the data has many category's with ~100 entries;

# ## Ordinal Features Transformation
# - Only ord_0 is numerical values;
# - We need to transform ord_1, ord_2 and ord_3 to set it in the correctly order to feed the machine learning model;

# In[23]:

# Importing categorical options of pandas
from pandas.api.types import CategoricalDtype

# seting the orders of our ordinal features
ord_1 = CategoricalDtype(categories=['Novice', 'Contributor','Expert',
                                     'Master', 'Grandmaster'], ordered=True)
ord_2 = CategoricalDtype(categories=['Freezing', 'Cold', 'Warm', 'Hot',
                                     'Boiling Hot', 'Lava Hot'], ordered=True)
ord_3 = CategoricalDtype(categories=['a', 'b', 'c', 'd', 'e', 'f', 'g',
                                     'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'], ordered=True)
ord_4 = CategoricalDtype(categories=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
                                     'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
                                     'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], ordered=True)

# In[24]:

# Transforming ordinal Features
df_train.ord_1 = df_train.ord_1.astype(ord_1)
df_train.ord_2 = df_train.ord_2.astype(ord_2)
df_train.ord_3 = df_train.ord_3.astype(ord_3)
df_train.ord_4 = df_train.ord_4.astype(ord_4)

# test dataset
df_test.ord_1 = df_test.ord_1.astype(ord_1)
df_test.ord_2 = df_test.ord_2.astype(ord_2)
df_test.ord_3 = df_test.ord_3.astype(ord_3)
df_test.ord_4 = df_test.ord_4.astype(ord_4)

# In[25]:

df_train.ord_3.head()

# Ok, now it's ordered and we can get the codes that corresponds to each value in the nominal feature;
#
# ### Geting codes of nominal feature

# In[26]:

# Geting the codes of ordinal categoy's - train
df_train.ord_1 = df_train.ord_1.cat.codes
df_train.ord_2 = df_train.ord_2.cat.codes
df_train.ord_3 = df_train.ord_3.cat.codes
df_train.ord_4 = df_train.ord_4.cat.codes

# Geting the codes of ordinal categoy's - test
df_test.ord_1 = df_test.ord_1.cat.codes
df_test.ord_2 = df_test.ord_2.cat.codes
df_test.ord_3 = df_test.ord_3.cat.codes
df_test.ord_4 = df_test.ord_4.cat.codes

# In[27]:

df_train[['ord_0', 'ord_1', 'ord_2', 'ord_3']].head()

# # Date Features
# - Let's see the date distributions
# - What's the target ratio of each value
# - Do we need do any transformation?

# In[28]:

date_cols = ['day', 'month']

# Calling the plot function with date columns
ploting_cat_fet(df_train, date_cols, vis_row=5, vis_col=2)

# Curiously, the data have two values that have few entries;
# - In the day column, the value 6 (maybe saturday?!) have less entries;
# - In the month column, the value 6 (maybe the holidays?!) have less entries;
#
# At this momment, I will not transform it in dummy features, but could be a good solution to test;

# # Encoding Date features

# In[29]:

# Transfer the cyclical features into two dimensional sin-cos features
# https://www.kaggle.com/avanwyk/encoding-cyclical-features-for-deep-learning

def date_cyc_enc(df, col, max_vals):
    df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
    df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)
    return df

df_train = date_cyc_enc(df_train, 'day', 7)
df_test = date_cyc_enc(df_test, 'day', 7)

df_train = date_cyc_enc(df_train, 'month', 12)
df_test = date_cyc_enc(df_test, 'month', 12)

# NOTE, I discovered it on: kaggle.com/gogo827jz/catboost-baseline-with-feature-importance

# # Ordinal Feature - High Cardinality Features
# - I will start by the ord_5 feature
# - It's an sparse ordinal feature

# In[30]:

### Credit of this features to:
## https://www.kaggle.com/gogo827jz/catboost-baseline-with-feature-importance

import string

# Then encode 'ord_5' using ACSII values

# Option 1: Add up the indices of two letters in string.ascii_letters
df_train['ord_5_oe_add'] = df_train['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))
df_test['ord_5_oe_add'] = df_test['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))

# Option 2: Join the indices of two letters in string.ascii_letters
df_train['ord_5_oe_join'] = df_train['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))
df_test['ord_5_oe_join'] = df_test['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))

# Option 3: Split 'ord_5' into two new columns using the indices of two letters in string.ascii_letters, separately
df_train['ord_5_oe1'] = df_train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
df_test['ord_5_oe1'] = df_test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))

df_train['ord_5_oe2'] = df_train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
df_test['ord_5_oe2'] = df_test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))

for col in ['ord_5_oe1', 'ord_5_oe2', 'ord_5_oe_add', 'ord_5_oe_join']:
    df_train[col]= df_train[col].astype('float64')
    df_test[col]= df_test[col].astype('float64')

# In[31]:

df_train[['ord_5', 'ord_5_oe_add', 'ord_5_oe_join', 'ord_5_oe1', 'ord_5_oe2']].head()

# Cool, now, let's work on nominal features

# # Nominal Features - High Cardinality Features
#
# ## Feature Hasher

# In[32]:

high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']

# ### Nominal Features before hash transformation

# In[33]:

resumetable(df_train[high_card_feats])

# We can see high-cardinality in all this columns. The fewer cardinality is 222 values and the highest is 11981 unique values;
#
# To work with these features, I will try the Hash Trick Solution; Let's see if it give us good results.

# ## Hash Trick (Feature Hash)
# - I got this solution from @Giba

# In[34]:

for col in high_card_feats:
    df_train[f'hash_{col}'] = df_train[col].apply( lambda x: hash(str(x)) % 5000 )
    df_test[f'hash_{col}'] = df_test[col].apply( lambda x: hash(str(x)) % 5000 )

#

# ## Encoding with the Frequency

# In[35]:

for col in high_card_feats:
    enc_nom_1 = (df_train.groupby(col).size()) / len(df_train)
    df_train[f'freq_{col}'] = df_train[col].apply(lambda x : enc_nom_1[x])
    #df_test[f'enc_{col}'] = df_test[col].apply(lambda x : enc_nom_1[x])

#

# ## Label Encoder

# In[36]:

from sklearn.preprocessing import LabelEncoder

# Label Encoding
for f in ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']:
    if df_train[f].dtype=='object' or df_test[f].dtype=='object':
        lbl = preprocessing.LabelEncoder()
        lbl.fit(list(df_train[f].values) + list(df_test[f].values))
        df_train[f'le_{f}'] = lbl.transform(list(df_train[f].values))
        df_test[f'le_{f}'] = lbl.transform(list(df_test[f].values))

# Cool

# After all transformations done, we can see the summary of the features and what type differences we got

# In[37]:

new_feat = ['hash_nom_5', 'hash_nom_6', 'hash_nom_7', 'hash_nom_8',
            'hash_nom_9', 'freq_nom_5', 'freq_nom_6', 'freq_nom_7',
            'freq_nom_8', 'freq_nom_9', 'le_nom_5', 'le_nom_6',
            'le_nom_7', 'le_nom_8', 'le_nom_9']

resumetable(df_train[high_card_feats + new_feat])

# ### Looking nominal features after hash trick transformation

# In[38]:

df_train[['nom_5', 'hash_nom_5', 'freq_nom_5', 'le_nom_5']].head()

# Cool! Now, We can run our model. Let's do it.

# # Pre Processing
# - I will keep only the frequency encoding to evaluate the results.

#
# Feel free to change if you would;

# In[39]:

df_train.drop(['ord_5_ot', 'ord_5',
               'hash_nom_5', 'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9',
               #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',
               'freq_nom_5', 'freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',
               'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'
              ], axis=1, inplace=True)

df_test.drop(['ord_5',
              'hash_nom_5', 'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9',
              #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',
              #'freq_nom_5', 'freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',
              'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',
              ], axis=1, inplace=True)

#'feq_nom_5', 'feq_nom_6', 'feq_nom_7', 'feq_nom_8', 'feq_nom_9',

# ### Seting X and Y

# In[40]:

df_train = reduce_mem_usage(df_train)
df_test = reduce_mem_usage(df_test)

# In[41]:

X_train = df_train.drop(["id","target"], axis=1)
y_train = df_train["target"]
y_train = y_train.astype(bool)
X_test = df_test.drop(["id"],axis=1)

X_train.shape

# ## Libraries to Machine Learning Modeling

# In[42]:

#Importing the auxiliar and preprocessing librarys
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.pipeline import Pipeline

from sklearn.model_selection import train_test_split, KFold, cross_validate
from sklearn.metrics import accuracy_score

#Models
import warnings
warnings.filterwarnings("ignore")

from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.svm import SVC
from sklearn.linear_model import RidgeClassifier, SGDClassifier, LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from xgboost import XGBClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier, BaggingClassifier, VotingClassifier, RandomTreesEmbedding

# # Creating pipeline to evaluate different models

# In[ ]:

clfs = []
seed = 42

clfs.append(("LogReg",
             Pipeline([("Scaler", StandardScaler()),
                       ("LogReg", LogisticRegression())])))

clfs.append(("XGBClassifier", XGBClassifier()))

# clfs.append(("KNN",
#              Pipeline([("Scaler", StandardScaler()),
#                        ("KNN", KNeighborsClassifier(n_neighbors=5))])))

clfs.append(("DecisionTreeClassifier", DecisionTreeClassifier()))

clfs.append(("RandomForestClassifier", RandomForestClassifier(n_estimators=100)))

clfs.append(("GradientBoostingClassifier", GradientBoostingClassifier(n_estimators=100)))

clfs.append(("RidgeClassifier",
             Pipeline([("Scaler", StandardScaler()),
                       ("RidgeClassifier", RidgeClassifier())])))

clfs.append(("BaggingClassifier",
             Pipeline([("Scaler", StandardScaler()),
                       ("BaggingClassifier", BaggingClassifier())])))

clfs.append(("ExtraTreesClassifier",ExtraTreesClassifier()))

#'neg_mean_absolute_error', 'neg_mean_squared_error','r2'
scoring = 'roc_auc'
n_folds = 7

results, names  = [], []

for name, model  in clfs:
    kfold = KFold(n_splits=n_folds, shuffle=False, random_state=seed)

    cv_results = cross_val_score(model,
                                 X_train.values, y_train,
                                 cv= kfold, scoring=scoring,
                                 n_jobs=-1)
    names.append(name)
    results.append(cv_results)
    msg = "%s: %f (+/- %f)" % (name, cv_results.mean(),
                               cv_results.std())
    print(msg)

# boxplot algorithm comparison
fig = plt.figure(figsize=(15,6))
fig.suptitle('Classifier Algorithm Comparison', fontsize=22)
ax = fig.add_subplot(111)
sns.boxplot(x=names, y=results)
ax.set_xticklabels(names)
ax.set_xlabel("Algorithmn", fontsize=20)
ax.set_ylabel("Accuracy of Models", fontsize=18)
ax.set_xticklabels(ax.get_xticklabels(),rotation=45)

plt.show()

# Nice! We can see that Xgb is the best model. Let's use it with hyperopt

# # Logistic Regression - HyperOpt Optimization

# In[ ]:

from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
from xgboost import plot_importance
from sklearn.metrics import make_scorer

import time
def objective(params):
    time1 = time.time()
    params = {
        'max_depth': int(params['max_depth']),
        #'gamma': "{:.3f}".format(params['gamma']),
        #'subsample': "{:.2f}".format(params['subsample']),
        'reg_alpha': "{:.3f}".format(params['reg_alpha']),
        'reg_lambda': "{:.3f}".format(params['reg_lambda']),
        'learning_rate': "{:.3f}".format(params['learning_rate']),
        'num_leaves': '{:.3f}'.format(params['num_leaves']),
        'colsample_bytree': '{:.3f}'.format(params['colsample_bytree']),
        'min_child_samples': '{:.3f}'.format(params['min_child_samples']),
        'feature_fraction': '{:.3f}'.format(params['feature_fraction']),
        'bagging_fraction': '{:.3f}'.format(params['bagging_fraction'])
    }

    print("\n############## New Run ################")
    print(f"params = {params}")
    FOLDS = 12
    count=1
    kf = KFold(n_splits=FOLDS, shuffle=False, random_state=42)

    # tss = TimeSeriesSplit(n_splits=FOLDS)
    y_preds = np.zeros(submission.shape[0])
    # y_oof = np.zeros(X_train.shape[0])
    score_mean = 0
    for tr_idx, val_idx in kf.split(X_train, y_train):
        clf = xgb.XGBClassifier(
            n_estimators=500, random_state=4,
            verbose=True,
            tree_method='gpu_hist',
            **params
        )

        X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
        y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]

        clf.fit(X_tr, y_tr)
        #y_pred_train = clf.predict_proba(X_vl)[:,1]
        #print(y_pred_train)
        score = make_scorer(roc_auc_score, needs_proba=True)(clf, X_vl, y_vl)
        # plt.show()
        score_mean += score
        print(f'{count} CV - score: {round(score, 4)}')
        count += 1
    time2 = time.time() - time1
    print(f"Total Time Run: {round(time2 / 60,2)}")
    gc.collect()
    print(f'Mean ROC_AUC: {score_mean / FOLDS}')
    del X_tr, X_vl, y_tr, y_vl, clf, score

    return -(score_mean / FOLDS)

space = {
    # The maximum depth of a tree, same as GBM.
    # Used to control over-fitting as higher depth will allow model
    # to learn relations very specific to a particular sample.
    # Should be tuned using CV.
    # Typical values: 3-10
    'max_depth': hp.quniform('max_depth', 2, 8, 1),

    # reg_alpha: L1 regularization term. L1 regularization encourages sparsity
    # (meaning pulling weights to 0). It can be more useful when the objective
    # is logistic regression since you might need help with feature selection.
    'reg_alpha':  hp.uniform('reg_alpha', 0.01, 0.4),

    # reg_lambda: L2 regularization term. L2 encourages smaller weights, this
    # approach can be more useful in tree-models where zeroing
    # features might not make much sense.
    'reg_lambda': hp.uniform('reg_lambda', 0.01, .4),

    # eta: Analogous to learning rate in GBM
    # Makes the model more robust by shrinking the weights on each step
    # Typical final values to be used: 0.01-0.2
    'learning_rate': hp.uniform('learning_rate', 0.01, 0.15),

    # colsample_bytree: Similar to max_features in GBM. Denotes the
    # fraction of columns to be randomly samples for each tree.
    # Typical values: 0.5-1
    'colsample_bytree': hp.uniform('colsample_bytree', 0.3, 1),

    # A node is split only when the resulting split gives a positive
    # reduction in the loss function. Gamma specifies the
    # minimum loss reduction required to make a split.
    # Makes the algorithm conservative. The values can vary depending on the loss function and should be tuned.
    #'gamma': hp.uniform('gamma', 0.01, .7),

    # more increases accuracy, but may lead to overfitting.
    # num_leaves: the number of leaf nodes to use. Having a large number
    # of leaves will improve accuracy, but will also lead to overfitting.
    'num_leaves': hp.choice('num_leaves', list(range(20, 200, 5))),

    # specifies the minimum samples per leaf node.
    # the minimum number of samples (data) to group into a leaf.
    # The parameter can greatly assist with overfitting: larger sample
    # sizes per leaf will reduce overfitting (but may lead to under-fitting).
    'min_child_samples': hp.choice('min_child_samples', list(range(100, 250, 10))),

    # subsample: represents a fraction of the rows (observations) to be
    # considered when building each subtree. Tianqi Chen and Carlos Guestrin
    # in their paper A Scalable Tree Boosting System recommend
    #'subsample': hp.choice('subsample', [.5, 0.6, 0.7, .8]),

    # randomly select a fraction of the features.
    # feature_fraction: controls the subsampling of features used
    # for training (as opposed to subsampling the actual training data in
    # the case of bagging). Smaller fractions reduce overfitting.
    'feature_fraction': hp.uniform('feature_fraction', 0.4, .8),

    # randomly bag or subsample training data.
    'bagging_fraction': hp.uniform('bagging_fraction', 0.4, .9)

    # bagging_fraction and bagging_freq: enables bagging (subsampling)
    # of the training data. Both values need to be set for bagging to be used.
    # The frequency controls how often (iteration) bagging is used. Smaller
    # fractions and frequencies reduce overfitting.
}

# ## HyperOpt Run

# In[ ]:

best = fmin(fn=objective,
            space=space,
            algo=tpe.suggest,
            max_evals=45,
            # trials=trials
           )

# ## Best params

# In[ ]:

best_params = space_eval(space, best)
best_params['max_depth'] = int(best_params['max_depth'])
best_params

# ## Predicting with best params Xgb

# In[ ]:

clf = xgb.XGBClassifier(
    n_estimators=500,
    **best_params,
    tree_method='gpu_hist'
)

clf.fit(X_train, y_train)

y_preds = clf.predict_proba(X_test)[:,1]

# Cool! Now, lets see what's the features that have more importance to the classification
#

# ## Feature Importance of Xgb model
# # Weights

# In[ ]:

feature_important = clf.get_booster().get_score(importance_type="gain")
keys = list(feature_important.keys())
values = list(feature_important.values())

data = pd.DataFrame(data=values, index=keys, columns=["score"]).sort_values(by = "score", ascending=False)

# Top 10 features
data.head(20)

# # Gain features

# In[ ]:

feature_important = clf.get_booster().get_score(importance_type="weight")
keys = list(feature_important.keys())
values = list(feature_important.values())

data = pd.DataFrame(data=values, index=keys, columns=["score"]).sort_values(by = "score", ascending=False)

# Top 10 features
data.head(20)

# # Seting submission CSV

# In[ ]:

submission['target'] = y_preds
submission.to_csv('XGB_hypopt_model.csv')

# # I will keep working on this kernel; Stay tuned
#
# ## Give me your feedback and if you liked my kernel please UPVOTE

#


from sqlalchemy_mptt.mixins import BaseNestedSets

from flask import Flask
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView

from flask_sqlalchemy import SQLAlchemy

####################################################################
# extensions

db = SQLAlchemy()
# 因为依赖 bootstrap3， 所以一定要设置 template_mode='bootstrap3'
admin = Admin(template_mode='bootstrap3')

app = Flask(__name__)
db.init_app(app)
db.app = app
admin.init_app(app)

app.config['SECRET_KEY'] = '123456790'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data.sqlite'

####################################################################
# model

class Tree(db.Model, BaseNestedSets):
    """混入了 BaseNestedSets ， 使用“左右值树”数据结构来处理树状数据"""

    __tablename__ = 'tree'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(64))

    def __str__(self):
        return '{}'.format(self.name)

def tree2schema(tree):
    return {
        'id': tree.id,
        'name': tree.name,
        'left': tree.left,
        'right': tree.right,
        'parent_id': tree.parent_id,
        'tree_id': tree.tree_id,
    }

####################################################################
# admin view

class TreeTableConfigMixin(object):
    """树形表格专用配置混入类"""
    # 设置 list_template
    list_template = 'admin/model/tree_table.html'

    # 禁止分页，直接获取该数据表所有数据
    page_size = None

    # 禁止设置每页显示的数量
    can_set_page_size = False

    # 禁止任何字段的排序，以免干扰 treegrid.js 显示正确的数据顺序
    column_sortable_list = []

    # 默认以 left 字段排序，使 treegrid.js 能正确显示树形表格
    column_default_sort = 'left'

class TreeView(TreeTableConfigMixin, ModelView):
    """
    注意 TreeTableConfigMixin 要放在 ModelView 前面
    """
    column_display_pk = True
    column_list = ['id', 'name', 'left',
                   'right', 'parent_id', 'level', 'tree_id']

    can_view_details = True
    can_edit = False
    can_delete = True

admin.add_view(TreeView(Tree, db.session))

####################################################################
# initdata

def initdata(count=10):
    db.drop_all()
    db.create_all()

    trunk = Tree(name="Trunk")
    db.session.add(trunk)
    for i in range(count):
        branch = Tree()
        branch.name = "Branch " + str(i+1)
        branch.parent = trunk
        db.session.add(branch)
        for j in range(5):
            leaf = Tree()
            leaf.name = "Leaf " + str(j+1)
            leaf.parent = branch
            db.session.add(leaf)
            for k in range(3):
                item = Tree()
                item.name = "Item " + str(k+1)
                item.parent = leaf
                db.session.add(item)
    db.session.commit()

if __name__ == "__main__":
    initdata()
    app.run(debug=True)


from MemTrace import MemTrace
from PyQt5.QtCore import QObject, pyqtSignal

import sys
import warnings

warnings.simplefilter("ignore", UserWarning)
sys.coinit_flags = 2

# from pywinauto.application import Application

# import pywinauto
import time
import re
import sqlite3

from DBM import HistoryMgr
from BKLOG import DEBUG, INFO, ERROR

from WindowsObject import WindowsObject, ChildObject
from Config import CONFIG

# GLOBAL_WIN = "AGENT"
GLOBAL_WIN = "EDIT"

"""
class WindowsObject:
    def __init__(self, r_text=None):
        self.win_objs = []
        self.pattern = re.compile(r_text)
        win32gui.EnumWindows(self.__EnumWindowsHandler, None)
        if len(self.win_objs) < 1:
            raise ValueError("Windows Object를 발견하지 못하였습니다.")
        self.obj = self.win_objs[0]

    def getFirstObj(self):
        if len(self.win_objs) < 1:
            return None
        else:
            return self.win_objs[0]

    def __EnumWindowsHandler(self, hwnd, find_text):
        wintext = win32gui.GetWindowText(hwnd)
        if self.pattern.match(wintext):
            obj = {}
            obj["handle"] = hwnd
            obj["text"] = wintext
            self.win_objs.append(obj)

class ChildObject:
    def __init__(self, parent_hwnd=None, match_class=None):
        self.parent_hwnd = parent_hwnd
        self.match_class = match_class
        self.pattern = re.compile(self.match_class)
        self.win_objs = []
        win32gui.EnumChildWindows(parent_hwnd, self.__EnumChildWindowsHandler, None)
        if len(self.win_objs) < 1:
            raise ValueError("Windows Object를 발견하지 못하였습니다.")
        self.obj = self.win_objs[0]

    def getFirstObj(self):
        if len(self.win_objs) < 1:
            return None
        else:
            return self.win_objs[0]

    def __EnumChildWindowsHandler(self, hwnd, extra):
        ctrl_id = win32gui.GetDlgCtrlID(hwnd)
        win_text = win32gui.GetWindowText(hwnd)
        wnd_clas = win32gui.GetClassName(hwnd)

        if self.match_class:
            if self.pattern.match(wnd_clas):
                obj = {}
                obj["handle"] = hwnd
                obj["control_id"] = ctrl_id
                obj["class_name"] = wnd_clas
                obj["text"] = win_text
                self.win_objs.append(obj)

    def line_count(self):
        control_hwnd = self.obj['handle']
        linecnt = win32gui.SendMessage(control_hwnd, win32con.EM_GETLINECOUNT, 0, 0)
        return linecnt

    def line_length(self, line_index):
        control_hwnd = self.obj['handle']
        char_index = win32gui.SendMessage(control_hwnd, win32con.EM_LINEINDEX, line_index, 0)
        line_length = win32gui.SendMessage(control_hwnd, win32con.EM_LINELENGTH, char_index, 0)
        return line_length

    def get_line(self, line_index):
        control_hwnd = self.obj['handle']
        text_len = self.line_length(line_index)
        # create a buffer and set the length at the start of the buffer
        text = ctypes.create_unicode_buffer(text_len+3)
        text[0] = six.unichr(text_len)

        result = win32gui.SendMessage(control_hwnd, win32con.EM_GETLINE, line_index, text)
        return text.value
"""

class LogCaptureWin32Worker(QObject):
    started = pyqtSignal()
    running = pyqtSignal()
    finished = pyqtSignal()
    inserted = pyqtSignal(str)
    dupped = pyqtSignal(str, str)
    in_processing = pyqtSignal(int)

    def __init__(self):
        super().__init__()

        self.loop_flag = True

        try:
            if GLOBAL_WIN == "AGENT":
                # self.w = WindowsObject("KRC-EC100 에이전트 v1.2.5.0 학원번호 : test - [  ]")
                self.w = WindowsObject(
                    r"KRC-EC100 에이전트 v[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ 학원번호 : .* - \[.*\]"
                )
            else:
                self.w = WindowsObject(r".*sample.txt - Windows 메모장")

            DEBUG(f"w={self.w.win_objs}")
            DEBUG(
                "w.handle,text=[%08X][%s]" % (self.w.obj["handle"], self.w.obj["text"])
            )

            self.hwnd = self.w.obj["handle"]
            if GLOBAL_WIN == "AGENT":
                self.logwin = ChildObject(self.hwnd, CONFIG["class_name"])
            else:
                self.logwin = ChildObject(self.hwnd, r"Edit")

            """
            self.hwnd = self.w.obj["handle"]
            self.app = Application(backend="win32").connect(handle=self.hwnd)

            self.dig = self.app.window(handle=self.hwnd)
            # dig.print_control_identifiers()

            if GLOBAL_WIN == "AGENT":
                self.logwin = self.dig.child_window(
                    #class_name="WindowsForms10.RichEdit20W.app.0.1bb715_r7_ad1"
                    #class_name="WindowsForms10.RichEdit20W."
                    class_name=CONFIG["class_name"]
                )
            else:
                self.logwin = self.dig.child_window(class_name="Edit")
            """

        except ValueError as ve:
            self.loop_flag = False
            ERROR(f"Log Capture 생성 오류 [{ve}]")

    def stop(self):
        self.loop_flag = False

    def run(self):
        self.running.emit()
        processing_pattern = re.compile(
            r"\[(?P<dtm>[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2})\] 출입 확인 \(이름: (?P<name>.*), 체온: (?P<temper>[0-9.]{3,5}) 출입시간 : (?P<dtm2>.{19})\)"
        )

        if self.loop_flag:
            edit_control = self.logwin

        mt = MemTrace()

        while self.loop_flag:
            line_cnt = edit_control.line_count()
            INFO(f"line_cnt:[{line_cnt}]")

            for i in range(0, line_cnt):
                line = edit_control.get_line(i)
                self.log_processing(processing_pattern, line)
            DEBUG("")
            self.in_processing.emit(line_cnt)
            time.sleep(3)
            # mt.end_print(5)
            mt.print_mem_trace()

        self.finished.emit()

    def log_processing(self, compiled_pattern, strLog):
        history_mgr = HistoryMgr()
        DEBUG(f"strLog=[{strLog}]")
        # 외부로 이동
        # p = re.compile(r"\[(?P<dtm>[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2})\] 출입 확인 \(이름: (?P<name>.*), 체온: (?P<temper>[0-9.]{5}) 출입시간 : (?P<dtm2>.{19})\)")
        p = compiled_pattern
        m = p.match(strLog)

        try:
            DEBUG(f"m.group(dtm)    = [{m.group('dtm')}]")
            DEBUG(f"m.group(name)   = [{m.group('name')}]")
            DEBUG(f"m.group(temper) = [{m.group('temper')}]")
            DEBUG(f"m.group(dtm2)   = [{m.group('dtm2')}]")
        except AttributeError as ae:
            # 파싱 오류
            DEBUG(f"파싱 오류 무시 DB처리 하지 않음 [{ae}]")
            return

        # DB처리 (Cache 검증 및 insert)
        # 등록하려는 정보의 앞뒤 1기간 가량안에 동일 사용자의 로그가 있다면 등록하지 않고 무시
        sql = " \n".join(
            (
                f"SELECT count(*) cnt",
                f"FROM inout_history a",
                f"WHERE",
                f"name = '{m.group('name')}'",
                f"and datetime(a.dtm) BETWEEN datetime('{m.group('dtm')}', '-1 hours') and datetime('{m.group('dtm')}', '+1 hours')",
            )
        )

        datas = history_mgr.query(sql)

        dup_cnt = datas[0]["cnt"]

        if dup_cnt:
            DEBUG(f"sql = [{sql}]")
            DEBUG(f"datas = [{datas}]")
            DEBUG(f"dup_cnt = [{dup_cnt}][{type(dup_cnt)}]")

        if not dup_cnt:
            try:
                history_mgr.execute_param(
                    history_mgr.INSERT_HISTORY,
                    (
                        m.group("dtm"),
                        m.group("name"),
                        m.group("temper"),
                        m.group("dtm2"),
                    ),
                )
            except sqlite3.IntegrityError as si:
                # Insert 무결성은 무시
                self.dupped.emit(strLog, str(si))
                history_mgr.rollback()
                DEBUG(f"INSERT 무결성은 무시 [{si}]")
            else:
                history_mgr.commit()
                self.inserted.emit(strLog)
                INFO(f"로그등록:[{strLog}]")
        history_mgr.close()
        del history_mgr

if __name__ == "__main__":
    logCapture = LogCaptureWin32Worker()
    logCapture.run()
    # log_capture_main()


import cv2
import mediapipe as mp
from mediapipe.python.solutions.drawing_utils import draw_landmarks
import math
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
import numpy

Devi = AudioUtilities.GetSpeakers()
Inface = Devi.Activate(
    IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
volume = cast(Inface, POINTER(IAudioEndpointVolume))

cap = cv2.VideoCapture(0)
mpHands = mp.solutions.hands
hands = mpHands.Hands()
mpDraw = mp.solutions.drawing_utils

while True:
    success, img = cap.read()
    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    results = hands.process(imgRGB)

    if results.multi_hand_landmarks:
        for handLms in results.multi_hand_landmarks:
            lmList = []
            for id, lm in enumerate(handLms.landmark):
                h, w, c = img.shape
                cx, cy = int(lm.x*w), int(lm.y*h)
                lmList.append([id, cx, cy])

                mpDraw.draw_landmarks(img, handLms, mpHands.HAND_CONNECTIONS)

        if lmList:
            x1, y1 = lmList[4][1], lmList[4][2]
            x2, y2 = lmList[8][1], lmList[8][2]
            cv2.circle(img, (x1, y1), 15, (255, 0, 0), cv2.FILLED)
            cv2.circle(img, (x2, y2), 15, (255, 0, 0), cv2.FILLED)
            cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), 3)

            z1, z2 = (x1+x2)//2, (y1+y2)//2
            length = math.hypot(x2 - x1, y2 - y1)
            if length < 50:
                cv2.circle(img, (z1, z2), 15, (255, 255, 255), cv2.FILLED)

        volRange = volume.GetVolumeRange()
        minVol = volRange[0]
        maxVol = volRange[1]
        vol = numpy.interp(length, [50, 300], [minVol, maxVol])
        volBar = numpy.interp(length, [50, 300], [400, 150])
        volPer = numpy.interp(length, [50, 300], [0, 100])

        volume.SetMasterVolumeLevel(vol, None)
        cv2.rectangle(img, (50, 150), (85, 400), (123, 213, 122), 3)
        cv2.rectangle(img, (50, int(volBar)), (85, 400),
                      (0, 231, 23), cv2.FILLED)
        cv2.putText(img, str(int(volPer)), (40, 450),
                    cv2.FONT_HERSHEY_PLAIN, 4, (24, 34, 34), 3)

    cv2.imshow("Image", img)
    cv2.waitKey(1)


import pytest
import time
from selenium_js.base import Base

class TestJs(Base):
    @pytest.mark.skip
    def test_js_scroll(self):
        self.driver.get("https://www.baidu.com/")
        self.driver.find_element_by_id("kw").send_keys("selenium测试")
        element = self.driver.execute_script("return document.getElementById('su')")
        element.click()
        self.driver.execute_script("document.documentElement.scrollTop=1000")
        time.sleep(3)
        self.driver.find_element_by_xpath("//*[@id='page']/a[10]").click()
        time.sleep(3)
        # for code in [
        #     'return document.title', 'return JSON.stringify(performance.timing)'
        # ]:
        #     print(self.driver.execute_script(code))
        print(self.driver.execute_script("return document.title;return JSON.stringify(performance.timing)"))

    @pytest.mark.skip
    def test_datetime(self):
        self.driver.get("https://www.12306.cn/index/")
        time_element = self.driver.execute_script("a=document.getElementById('train_date');a.removeAttribute('readonly')")
        self.driver.execute_script("document.getElementById('train_date').value='2020-12-30'")
        time.sleep(3)

    # @pytest.mark.skip
    def test_time(self):
        self.driver.get("https://www.12306.cn/index/")
        self.driver.find_element_by_id("fromStationText").send_keys("西安")
        time.sleep(3)
        self.driver.find_element_by_id("toStationText").send_keys("宝鸡")
        # self.driver.find_element_by_id("train_date").send_keys("2020-12-30")
        # time.sleep(3)
        # self.driver.find_element_by_id("search_one").click()


from bacnet_config import config
from util import *
import sys
import logging
from building_depot import DataService, BDError
import re

class BDCreate(object):
    #Create new instance for every building
    def __init__(self, bd_dataservice, building):
        self.dataservice = bd_dataservice
        self.institution = config["institution"]
        self.campus = config["campus"]
        self.building = building
        self.config = config["buildings"][building]
        self.sensor_network = "BACNET"

    def create_sensorpoint_type(self, type_name = "PresentValue"):
        #TODO: Deal with other types of sensorpoints such as operator
        #override
        #Check if sensorpoint type exists first
        type_name = re.sub('[^a-zA-Z0-9_]', ' ', type_name)
        try:
            self.dataservice.view_sensorpoint_type(type_name)
            return
        except:
            e = sys.exc_info()
            logging.error("Could not locate sensorpoint %s. Error %s, %s",
                          type_name, e[0], e[1])
            #pass

        data = {
            "name": type_name,
            "description": "Shows the present value of sensor in BACnet",
            "data_type": "float", #Warning: there should be no data type.
                                  #Assuming float for now
        }
        try:
            response = self.dataservice.create_sensorpoint_type(**data)
            logging.info("Created sensorpoint type: " + str(response))
        except:
            e = sys.exc_info()
            logging.error("Could not create sensorpoint %s. Error %s, %s",
                          type_name, e[0], e[1])
            raise

    def create_sensor_template(self, sensor_template, sensorpoint_type = 'PresentValue'):
        #Check if sensor template exists first
        sensor_template = re.sub('[^a-zA-Z0-9_]', ' ', sensor_template)
        try:
            self.dataservice.view_sensor_template(sensor_template)
            return
        except:
            e = sys.exc_info()
            logging.info("Could not locate sensor template %s. Error %s %s",
                         sensor_template, e[0], e[1])
            pass

        template_data = {
            "name": sensor_template,
            "description": "BACnet Sensor",
            "sensorpoint_types":[sensorpoint_type]
        }
        try:
            response = self.dataservice.create_sensor_template(**template_data)
            logging.info("Created sensor template: "+str(response))
        except:
            e = sys.exc_info()
            logging.error("Could not create sensor template %s. Error %s, %s",
                          sensor_template, e[0], e[1])
            raise

    def add_sensorpoint_type(self, sensor_template, sensorpoint_type = 'PresentValue'):
        sensor_template = re.sub('[^a-zA-Z0-9_]', ' ', sensor_template)
        sensorpoint_type = re.sub('[^a-zA-Z0-9_]', ' ', sensorpoint_type)
        try:
            response = self.dataservice.view_sensor_template(sensor_template)
            if 'sensorpoint_types' in response:
                sensorpoint_type_dict = {spt['name']: spt['description']
                                            for spt in response['sensorpoint_types']}
                if sensorpoint_type not in sensorpoint_type_dict:
                    logging.info("%s sensorpoint type not in sensor template", sensorpoint_type)
                else:
                    return
            else:
                logging.info("%s sensorpoint type not in sensor template", sensorpoint_type)
        except:
            e = sys.exc_info()
            logging.error("Could not access sensorpoint_type %s. Error %s, %s",
                          sensorpoint_type, e[0], e[1])
            pass
        try:
            response = self.dataservice.add_sensor_template_sensorpoint_type(sensor_template,
                                                                 sensorpoint_type)
            logging.info("Added sensorpoint type %s to sensor template %s",
                         sensorpoint_type, sensor_template)
        except:
            e = sys.exc_info()
            logging.error("Could not add sensorpoint type %s to sensor template %s. Error %s, %s",
                          sensorpoint_type, sensor_template, e[0], e[1])
            raise

    def create_sensor_network(self, network_name='BACNET'):
        network_name = re.sub('[^a-zA-Z0-9_]', ' ', network_name)
        try:
            self.dataservice.view_sensor_network(network_name)
            return
        except:
            e = sys.exc_info()
            logging.error("Could not access network %s. Error %s, %s", network_name, e[0], e[1])
            pass
        network_data = {
            "name": network_name,
            "description": "BACnet network used by Building Management Systems"
        }
        try:
            response = self.dataservice.create_sensor_network(**network_data)
            logging.info("Created sensor network: %s", response)
        except:
            e = sys.exc_info()
            logging.error("Could not create sensor network %s. Error %s, %s",
                          network_name, e[0], e[1])
            raise

    def create_keyword(self, keyword = "system"):
        system = re.sub('[^a-zA-Z0-9_]', ' ', keyword)
        try:
            self.dataservice.view_keyword(keyword)
            return
        except:
            pass
        keyword_data = {
            "name": keyword,
            "description": "Part of building system"
        }
        try:
            response = self.dataservice.create_keyword(**keyword_data)
            logging.info("Created keyword: "+str(response))
        except:
            e = sys.exc_info()
            logging.error("Could not create keyword: %s. Error %s, %s",
                          keyword, e[0], e[1])
            raise

    def create_sensor(self, source_identifier, sensor_obj):
        sensor_type = sensor_obj["sensor_type"]
        sensor_type = (re.sub('[^a-zA-Z0-9_]', ' ', sensor_type)).strip()
        sensor_name = ("%s %s" % (sensor_type, self.building) )
        #Getting sensor context
        sensor_context = {
                    "institution": self.institution,
                    "campus": self.campus,
                    "building": self.building,
            }
        context_string = sensor_obj["object_desc_prop"]
        split_symbol = self.config["split_symbol"]
        split_names = self.config["split_names"]

        try:
            split_contexts = context_string.split(split_symbol)
        except ValueError as e:
            logging.error("Could not split sensor %s, with context %s. %s",
                          sensor_type, context_string, e)
            raise
        # Note: May need to revisit this. Too hard coded still
        split_context_dict = {}
        for index, context in enumerate(split_contexts):
            name = split_names[index]
            split_context_dict[name] = context

        logging.info("Split context: %s. Sensor type: %s",
                     split_context_dict, sensor_type)
        if "room" in split_context_dict:
            system = split_context_dict["room"]
            if contains_digits(system):
                room = system
                if room[3] == 'B': #hard coded
                    floor = 'Basement'
                else:
                    floor = 'Flr-' + room[3] #hard coded
                sensor_context["room"] = room
                sensor_context["floor"] = floor
                sensor_name = sensor_name + " " + room
            else:
                sensor_context["system"] = system
                sensor_name = sensor_name + " " + system

        ##Special case for handling Soft Thermostat. Update Source Identifier.
        if sensor_type == "Warm Cool Adjust":
            sensor_context["template"]=sensor_type
            try:
                response = self.dataservice.list_sensors(sensor_context)
            except:
                e = sys.exc_info()
                logging.error("Could not find sensor: %s. Error %s, %s",
                              sensor_name, e[0], e[1])
                return
            if len(response['sensors']) > 0: #sensor found
                sensor_uuid = response['sensors'][0]['uuid']
                update_data = {
                    'source_name': sensor_name,
                    'source_identifier': source_identifier,
                }
                try:
                    self.dataservice.update_sensor(sensor_uuid, **update_data)
                    logging.info("Updating sensor: %s", sensor_name)
                    return sensor_uuid
                except:
                    e = sys.exc_info()
                    logging.error("Could not update sensor: %s. Error %s, %s",
                                  sensor_name, e[0], e[1])
                    return
            else:
                logging.info("No sensor found. Response: %s", response['sensors'])
                del sensor_context["template"]

        ## Create sensor (name, source id, template, network, context - institution, campus, building, floor, room)
        sensor_data = {
                "source_name": sensor_name,
                "source_identifier": source_identifier,
                "template": sensor_type,
                "network": self.sensor_network,
                "context": sensor_context,
        }
        try:
            response = self.dataservice.create_sensor(**sensor_data)
            logging.info("Created sensor: "+str(response))
            return response
        except:
            e = sys.exc_info()
            logging.error("Could not create sensor: %s. Error %s, %s",
                          sensor_name, e[0], e[1])
            raise

    def create_bd_entities(self, source_identifier, sensor_obj, sensorpoint_type):
        '''Deal with all the different types of entities that need to
            be created - sensor, sensorpoint, sensor template, etc.
        '''
        try:
            #Steps for creating a sensor
            ## Create sensorpoint type if necessary
            self.create_sensorpoint_type()
            ## Create sensor template if necessary
            self.create_sensor_template(sensor_obj["sensor_type"])
            ## Add sensorpoint type to template if necessary
            self.add_sensorpoint_type(sensor_obj["sensor_type"])
            ## Use BACnet sensor network. Create if necessary
            self.create_sensor_network()
            ## Create keyword if necessary - room, floor, system, etc.
            self.create_keyword()
            ## Create sensor
            sensor_uuid = self.create_sensor(source_identifier, sensor_obj)
            return sensor_uuid
        except:
            e = sys.exc_info()
            logging.error("Could not create sensor %s. Error %s, %s",
                          sensor_obj, e[0], e[1])
            raise


class Node:
  def __init__(self, val):
    self.val = val
    self.next = None

a = Node("a")
b = Node("b")
c = Node("c")
d = Node("d")
e = Node("e")
f = Node("f")

a.next = b
b.next = c
c.next = d
d.next = e
e.next = f

def reverse_list(head, previous=None):
    if head == None:
        return previous
    next = head.next
    head.next = previous
    return reverse_list(next, head)

    # previous = None
    # current = head
    # while current != None:
        # next = current.next
        # current.next = previous
        # previous = current
        # current = next
    # return previous.val
reverse = reverse_list(a)

while reverse != None:
    print(reverse.val)


class Solution:
    def isPalindrome(self, x: int) -> bool:
        if x == 0:
            return True
        _to_str = str(x)
        _to_list = list(_to_str)
        if _to_list[0] == '-':
            return False
        if _to_list[-1] == '0':
            return False
        list_len = len(_to_list)
        for i in range(0, int(list_len/2)):
            if _to_list[i] != _to_list[list_len - 1 - i]:
                return False
        return True

Sol = Solution()
b_palindrome = Sol.isPalindrome(101)
print(b_palindrome)


import os
import json
from django.contrib import admin
from emergency_management.models import *
from django.utils.html import format_html
from System.settings import base_url

admin.site.site_header = '预案管理系统'
admin.site.site_title = '预案管理系统'

# 事件类型
@admin.register(EventType)
class EventTypeAdmin(admin.ModelAdmin):
    list_display = ["type_name", "add_time", "modify_time"]
    date_hierarchy = 'add_time'
    # search_fields = ["type_name"]
    list_filter = ['type_name', 'add_time']
    list_per_page = 30

# 信息提取
@admin.register(Events)
class EventsAdmin(admin.ModelAdmin):
    list_display = ["event_name", "event_name", "keyword", "people_number", "level", "create_time", "open_time"]
    search_fields = ["event_name", "event_type"]
    list_per_page = 50

# 信息接收
@admin.register(Message)
class MessageAdmin(admin.ModelAdmin):
    list_display = ["title_name", "create_user", "happend_time", "add_time", "show_firm_url"]
    search_fields = ["title_name", "status"]
    list_per_page = 50
    list_filter = ["status"]

    def change_view(self, request, object_id, form_url='', extra_context=None):
        print(request.user)
        print(dir(request.user))
        print(request.user.get_all_permissions())
        permissions = request.user.get_all_permissions()
        if 'emergency_management.change_message' not in permissions or 'emergency_management.add_message' not in permissions:
            MessageAdmin.fieldsets = (
                ("基本信息", {'fields': ['title_name', 'create_user', "status", "happend_time", "add_time"]}),
                ("介绍", {'fields': ['des_html']})
            )
        else:
            MessageAdmin.fieldsets = (
                ("基本信息", {'fields': ['title_name', 'create_user', "status", "happend_time", "add_time"]}),
                ("介绍", {'fields': ['des']})
            )
        return super().change_view(
            request, object_id, form_url, extra_context=extra_context,
        )

    def show_firm_url(self, obj):
        message = obj.status
        color = "blue"
        flag = True
        if obj.status == "1":
            message = "【未读】"
            color = "blue"
        elif obj.status == "2":
            message = "【代办】"
            color = "green"
        elif obj.status == "3":
            message = "【办理中】"
            color = "yellow"
        elif obj.status == "4":
            message = "【已处理】"
            flag = False
        if flag:
            return format_html(
                "{}".format(
                    base_url, obj.id, obj.status, color, message)
            )
        return message

    show_firm_url.short_description = "操作"
    readonly_fields = ["des_html"]


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 15 12:54:53 2019

@author: user
"""
import time
import torch

def valuate(env, policy):
    num_iter = 300
    num_try = 149
    reached = 0
    for i in range(num_iter):
        obs = env.reset()
        for j in range(num_try):
            obs = torch.from_numpy(obs).float().unsqueeze(0)
            action = policy(obs)
            obs, reward, done, _ = env.step(action.detach().squeeze().numpy())
            if done:
                reached += 1
                break
    return reached

def tic():
    return time.time()

def toc(x):
    return time.time() - x

def weighSync(target_model, source_model, tau=0.001):
    for target_param, source_param in zip(target_model.parameters(), source_model.parameters()):
            target_param.data.copy_(tau*source_param.data + (1.0-tau)*target_param.data)

def evaluate_policy(env, policy):
    obs = env.reset()
    env.render('human')
    steps = 0
    done = False
    time.sleep(3)
    while steps<10000:
        obs = torch.from_numpy(obs).float().unsqueeze(0)

        action = policy(obs).detach().squeeze().numpy()
        obs, r, done, info = env.step(action)
        if done:
            obs = env.reset()
        steps+=1
        env.render('human')
        time.sleep(0.1)


import torch
import torch.nn as nn
import MODEL.Layer as LYR

class ASPP_Y5(nn.Module) :
    def __init__(self, in_dim, act_fn):
        super(ASPP_Y5, self).__init__()
        self.main = nn.Sequential(
            LYR.ConvGN(in_dim, in_dim // 2, 1, 1, 0, act_fn),
            ASPP_UNIT(in_dim//2, act_fn),
            LYR.ConvGN(in_dim * 2, in_dim, 1, 1, 0, act_fn)
        )

    def forward(self, x):
        return self.main(x)

class ASPP_UNIT(nn.Module) :
    def __init__(self,dim,  act_fn):
        super(ASPP_UNIT, self).__init__()
        self.cp1 = LYR.ConvGN(dim, dim, 3, 1, 6, act_fn, 1, 6)
        self.cp2 = LYR.ConvGN(dim, dim, 3, 1, 12, act_fn, 1, 12)
        self.cp3 = LYR.ConvGN(dim, dim, 3, 1, 18, act_fn, 1, 18)
        self.cp4 = LYR.ConvGN(dim, dim, 3, 1, 24, act_fn, 1, 24)

    def forward(self, x):

        p1 = self.cp1(x)
        p2 = self.cp2(x)
        p3 = self.cp3(x)
        p4 = self.cp4(x)

        return torch.cat([p1,p2,p3,p4], dim=1)